Merge pull request #1581 from silx-kit/sanitize-bigints

Convert all bigints to number in `H5WasmProvider` without exceptions
silx-kit · Feb 28, 2024 · 689d589 · 689d589
2 parents ba18687 + 9187d32
commit 689d589
Show file tree

Hide file tree

Showing 12 changed files with 160 additions and 116 deletions.
diff --git a/create_h5_sample.py b/create_h5_sample.py
@@ -180,8 +180,17 @@ def print_h5t_class(dataset):
     add_scalar(
         h5,
         "compound_nested",
-        ((True, 1 + 2j),),
-        [("nested", [("bool", np.bool_), ("cplx", np.complex64)])],
+        ((True, 1 + 2j, 3),),
+        [
+            (
+                "nested",
+                [
+                    ("bool", np.bool_),
+                    ("cplx", np.complex64),
+                    ("bigint", np.int64),
+                ],
+            )
+        ],
     )
 
     comp = add_array(

diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts
@@ -22,10 +22,5 @@ export { assertEnvVar } from '@h5web/shared/guards';
 export { default as DataProvider } from './providers/DataProvider';
 export { DataProviderApi } from './providers/api';
 export type { ValuesStoreParams } from './providers/models';
-export {
-  flattenValue,
-  getNameFromPath,
-  sliceValue,
-  getValueOrError,
-} from './providers/utils';
+export { getNameFromPath, getValueOrError } from './providers/utils';
 export { assertNonNull } from '@h5web/shared/guards';
diff --git a/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap b/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap
@@ -1298,6 +1298,7 @@ exports[`test file matches snapshot 1`] = `
       "class": 6,
       "dtype": {
         "nested": {
+          "bigint": "<i8",
           "bool": "|b1",
           "cplx": "<c8",
         },
@@ -1306,10 +1307,18 @@ exports[`test file matches snapshot 1`] = `
         "nested": {
           "class": 6,
           "dtype": {
+            "bigint": "<i8",
             "bool": "|b1",
             "cplx": "<c8",
           },
           "members": {
+            "bigint": {
+              "class": 0,
+              "dtype": "<i8",
+              "order": 0,
+              "sign": 1,
+              "size": 8,
+            },
             "bool": {
               "base": {
                 "class": 0,
@@ -1346,10 +1355,10 @@ exports[`test file matches snapshot 1`] = `
               "size": 8,
             },
           },
-          "size": 9,
+          "size": 17,
         },
       },
-      "size": 9,
+      "size": 17,
     },
     "shape": [],
     "type": {
@@ -1358,6 +1367,11 @@ exports[`test file matches snapshot 1`] = `
         "nested": {
           "class": "Compound",
           "fields": {
+            "bigint": {
+              "class": "Integer",
+              "endianness": "little-endian",
+              "size": 64,
+            },
             "bool": {
               "class": "Boolean",
             },
@@ -1385,6 +1399,7 @@ exports[`test file matches snapshot 1`] = `
           1,
           2,
         ],
+        3,
       ],
     ],
   },

diff --git a/packages/app/src/providers/hsds/hsds-api.ts b/packages/app/src/providers/hsds/hsds-api.ts
@@ -20,7 +20,7 @@ import { buildEntityPath, getChildEntity } from '@h5web/shared/hdf5-utils';
 
 import { DataProviderApi } from '../api';
 import type { ExportFormat, ExportURL, ValuesStoreParams } from '../models';
-import { flattenValue, handleAxiosError } from '../utils';
+import { handleAxiosError } from '../utils';
 import type {
   BaseHsdsEntity,
   HsdsAttribute,
@@ -43,6 +43,7 @@ import {
   convertHsdsAttributes,
   convertHsdsShape,
   convertHsdsType,
+  flattenValue,
   isHsdsGroup,
 } from './utils';
 

diff --git a/packages/app/src/providers/hsds/utils.ts b/packages/app/src/providers/hsds/utils.ts
@@ -1,4 +1,4 @@
-import { isGroup } from '@h5web/shared/guards';
+import { assertArray, isGroup } from '@h5web/shared/guards';
 import type {
   ArrayShape,
   Attribute,
@@ -166,3 +166,14 @@ export function convertHsdsAttributes(attrs: HsdsAttribute[]): Attribute[] {
     type: convertHsdsType(attr.type),
   }));
 }
+
+export function flattenValue(
+  value: unknown,
+  dataset: Dataset<ArrayShape>,
+  selection?: string,
+): unknown[] {
+  assertArray(value);
+  const slicedDims = selection?.split(',').filter((s) => s.includes(':'));
+  const dims = slicedDims || dataset.shape;
+  return value.flat(dims.length - 1);
+}
diff --git a/packages/app/src/providers/mock/mock-api.ts b/packages/app/src/providers/mock/mock-api.ts
@@ -22,9 +22,8 @@ import axios from 'axios';
 
 import { DataProviderApi } from '../api';
 import type { ExportFormat, ExportURL, ValuesStoreParams } from '../models';
-import { sliceValue } from '../utils';
 import { makeMockFile } from './mock-file';
-import { findMockEntity } from './utils';
+import { findMockEntity, sliceValue } from './utils';
 
 export const SLOW_TIMEOUT = 3000;
 

diff --git a/packages/app/src/providers/mock/utils.ts b/packages/app/src/providers/mock/utils.ts
@@ -5,10 +5,18 @@ import {
   isGroup,
 } from '@h5web/shared/guards';
 import type {
+  ArrayShape,
+  Dataset,
+  DType,
   GroupWithChildren,
+  Primitive,
   ProvidedEntity,
+  ScalarShape,
 } from '@h5web/shared/hdf5-models';
 import { getChildEntity } from '@h5web/shared/hdf5-utils';
+import ndarray from 'ndarray';
+
+import { applyMapping } from '../../vis-packs/core/utils';
 
 export function findMockEntity(
   group: GroupWithChildren,
@@ -35,3 +43,18 @@ export function findMockEntity(
   }
   return child;
 }
+
+export function sliceValue<T extends DType>(
+  value: unknown,
+  dataset: Dataset<ArrayShape | ScalarShape, T>,
+  selection: string,
+): Primitive<T>[] {
+  const { shape, type } = dataset;
+  const dataArray = ndarray(value as Primitive<typeof type>[], shape);
+  const mappedArray = applyMapping(
+    dataArray,
+    selection.split(',').map((s) => (s === ':' ? s : Number.parseInt(s, 10))),
+  );
+
+  return mappedArray.data;
+}
diff --git a/packages/app/src/providers/utils.ts b/packages/app/src/providers/utils.ts
@@ -1,31 +1,17 @@
-import { assertArray, isNumericType } from '@h5web/shared/guards';
+import { isNumericType } from '@h5web/shared/guards';
 import type {
   ArrayShape,
   Dataset,
   DType,
-  Primitive,
   ScalarShape,
 } from '@h5web/shared/hdf5-models';
 import { DTypeClass } from '@h5web/shared/hdf5-models';
 import { AxiosError } from 'axios';
-import ndarray from 'ndarray';
 
-import { applyMapping } from '../vis-packs/core/utils';
 import type { DataProviderApi } from './api';
 
 export const CANCELLED_ERROR_MSG = 'Request cancelled';
 
-export function flattenValue(
-  value: unknown,
-  dataset: Dataset<ArrayShape>,
-  selection?: string,
-): unknown[] {
-  assertArray(value);
-  const slicedDims = selection?.split(',').filter((s) => s.includes(':'));
-  const dims = slicedDims || dataset.shape;
-  return value.flat(dims.length - 1);
-}
-
 export async function handleAxiosError<T>(
   func: () => Promise<T>,
   getErrorToThrow: (status: number, errorData: unknown) => string | undefined,
@@ -51,21 +37,6 @@ export function getNameFromPath(path: string) {
   return segments[segments.length - 1];
 }
 
-export function sliceValue<T extends DType>(
-  value: unknown,
-  dataset: Dataset<ArrayShape | ScalarShape, T>,
-  selection: string,
-): Primitive<T>[] {
-  const { shape, type } = dataset;
-  const dataArray = ndarray(value as Primitive<typeof type>[], shape);
-  const mappedArray = applyMapping(
-    dataArray,
-    selection.split(',').map((s) => (s === ':' ? s : Number.parseInt(s, 10))),
-  );
-
-  return mappedArray.data;
-}
-
 export function typedArrayFromDType(dtype: DType) {
   /* Adapted from https://github.com/ludwigschubert/js-numpy-parser/blob/v1.2.3/src/main.js#L116 */
   if (!isNumericType(dtype)) {

diff --git a/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap b/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap
@@ -1622,16 +1622,27 @@ exports[`test file matches snapshot 1`] = `
                   "type": 6,
                   "vlen": false,
                 },
+                {
+                  "cset": -1,
+                  "littleEndian": true,
+                  "name": "bigint",
+                  "offset": 9,
+                  "shape": [],
+                  "signed": true,
+                  "size": 8,
+                  "type": 0,
+                  "vlen": false,
+                },
               ],
-              "nmembers": 2,
+              "nmembers": 3,
             },
             "cset": -1,
             "littleEndian": true,
             "name": "nested",
             "offset": 0,
             "shape": [],
             "signed": false,
-            "size": 9,
+            "size": 17,
             "type": 6,
             "vlen": false,
           },
@@ -1641,7 +1652,7 @@ exports[`test file matches snapshot 1`] = `
       "cset": -1,
       "littleEndian": true,
       "signed": false,
-      "size": 9,
+      "size": 17,
       "total_size": 1,
       "type": 6,
       "vlen": false,
@@ -1653,6 +1664,11 @@ exports[`test file matches snapshot 1`] = `
         "nested": {
           "class": "Compound",
           "fields": {
+            "bigint": {
+              "class": "Integer",
+              "endianness": "little-endian",
+              "size": 64,
+            },
             "bool": {
               "class": "Boolean",
             },
@@ -1680,6 +1696,7 @@ exports[`test file matches snapshot 1`] = `
           1,
           2,
         ],
+        3,
       ],
     ],
   },
@@ -1769,8 +1786,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          152,
-          209,
+          248,
+          215,
           18,
           0,
         ],
@@ -1785,8 +1802,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          232,
-          231,
+          104,
+          238,
           18,
           0,
         ],
@@ -1801,8 +1818,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          0,
-          232,
+          128,
+          238,
           18,
           0,
         ],
@@ -2078,8 +2095,8 @@ exports[`test file matches snapshot 1`] = `
       0,
       0,
       0,
-      104,
-      178,
+      96,
+      210,
       18,
       0,
     ],
@@ -2109,24 +2126,24 @@ exports[`test file matches snapshot 1`] = `
       0,
       0,
       0,
-      0,
-      204,
+      176,
+      25,
       18,
       0,
       2,
       0,
       0,
       0,
-      72,
-      107,
-      14,
+      96,
+      12,
+      18,
       0,
       3,
       0,
       0,
       0,
-      80,
-      234,
+      240,
+      239,
       18,
       0,
     ],

diff --git a/packages/h5wasm/src/guards.ts b/packages/h5wasm/src/guards.ts
@@ -1,6 +1,3 @@
-import { isCompoundType } from '@h5web/shared/guards';
-import type { Dataset, DType } from '@h5web/shared/hdf5-models';
-import { DTypeClass } from '@h5web/shared/hdf5-models';
 import { Dataset as H5WasmDataset } from 'h5wasm';
 
 import type { H5WasmEntity } from './models';
@@ -12,18 +9,3 @@ export function assertH5WasmDataset(
     throw new TypeError('Expected H5Wasm entity to be dataset');
   }
 }
-
-function isInt64Type(type: DType): boolean {
-  return (
-    (type.class === DTypeClass.Integer || type.class === DTypeClass.Unsigned) &&
-    type.size === 64
-  );
-}
-
-export function hasInt64Type(dataset: Dataset) {
-  const { type } = dataset;
-  return (
-    isInt64Type(type) ||
-    (isCompoundType(type) && Object.values(type.fields).some(isInt64Type))
-  );
-}