diff --git a/create_h5_sample.py b/create_h5_sample.py index 269094b52..6a7093264 100644 --- a/create_h5_sample.py +++ b/create_h5_sample.py @@ -180,8 +180,17 @@ def print_h5t_class(dataset): add_scalar( h5, "compound_nested", - ((True, 1 + 2j),), - [("nested", [("bool", np.bool_), ("cplx", np.complex64)])], + ((True, 1 + 2j, 3),), + [ + ( + "nested", + [ + ("bool", np.bool_), + ("cplx", np.complex64), + ("bigint", np.int64), + ], + ) + ], ) comp = add_array( diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts index ee982098d..844d9eb42 100644 --- a/packages/app/src/index.ts +++ b/packages/app/src/index.ts @@ -22,10 +22,5 @@ export { assertEnvVar } from '@h5web/shared/guards'; export { default as DataProvider } from './providers/DataProvider'; export { DataProviderApi } from './providers/api'; export type { ValuesStoreParams } from './providers/models'; -export { - flattenValue, - getNameFromPath, - sliceValue, - getValueOrError, -} from './providers/utils'; +export { getNameFromPath, getValueOrError } from './providers/utils'; export { assertNonNull } from '@h5web/shared/guards'; diff --git a/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap b/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap index ef5f6f2c0..8098df706 100644 --- a/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap +++ b/packages/app/src/providers/h5grove/__snapshots__/h5grove-api.test.ts.snap @@ -1298,6 +1298,7 @@ exports[`test file matches snapshot 1`] = ` "class": 6, "dtype": { "nested": { + "bigint": ", + selection?: string, +): unknown[] { + assertArray(value); + const slicedDims = selection?.split(',').filter((s) => s.includes(':')); + const dims = slicedDims || dataset.shape; + return value.flat(dims.length - 1); +} diff --git a/packages/app/src/providers/mock/mock-api.ts b/packages/app/src/providers/mock/mock-api.ts index d067f715e..0307931f4 100644 --- a/packages/app/src/providers/mock/mock-api.ts +++ b/packages/app/src/providers/mock/mock-api.ts @@ -22,9 +22,8 @@ import axios from 'axios'; import { DataProviderApi } from '../api'; import type { ExportFormat, ExportURL, ValuesStoreParams } from '../models'; -import { sliceValue } from '../utils'; import { makeMockFile } from './mock-file'; -import { findMockEntity } from './utils'; +import { findMockEntity, sliceValue } from './utils'; export const SLOW_TIMEOUT = 3000; diff --git a/packages/app/src/providers/mock/utils.ts b/packages/app/src/providers/mock/utils.ts index 9c26b0c1f..756745c14 100644 --- a/packages/app/src/providers/mock/utils.ts +++ b/packages/app/src/providers/mock/utils.ts @@ -5,10 +5,18 @@ import { isGroup, } from '@h5web/shared/guards'; import type { + ArrayShape, + Dataset, + DType, GroupWithChildren, + Primitive, ProvidedEntity, + ScalarShape, } from '@h5web/shared/hdf5-models'; import { getChildEntity } from '@h5web/shared/hdf5-utils'; +import ndarray from 'ndarray'; + +import { applyMapping } from '../../vis-packs/core/utils'; export function findMockEntity( group: GroupWithChildren, @@ -35,3 +43,18 @@ export function findMockEntity( } return child; } + +export function sliceValue( + value: unknown, + dataset: Dataset, + selection: string, +): Primitive[] { + const { shape, type } = dataset; + const dataArray = ndarray(value as Primitive[], shape); + const mappedArray = applyMapping( + dataArray, + selection.split(',').map((s) => (s === ':' ? s : Number.parseInt(s, 10))), + ); + + return mappedArray.data; +} diff --git a/packages/app/src/providers/utils.ts b/packages/app/src/providers/utils.ts index 07a56891a..d6d056679 100644 --- a/packages/app/src/providers/utils.ts +++ b/packages/app/src/providers/utils.ts @@ -1,31 +1,17 @@ -import { assertArray, isNumericType } from '@h5web/shared/guards'; +import { isNumericType } from '@h5web/shared/guards'; import type { ArrayShape, Dataset, DType, - Primitive, ScalarShape, } from '@h5web/shared/hdf5-models'; import { DTypeClass } from '@h5web/shared/hdf5-models'; import { AxiosError } from 'axios'; -import ndarray from 'ndarray'; -import { applyMapping } from '../vis-packs/core/utils'; import type { DataProviderApi } from './api'; export const CANCELLED_ERROR_MSG = 'Request cancelled'; -export function flattenValue( - value: unknown, - dataset: Dataset, - selection?: string, -): unknown[] { - assertArray(value); - const slicedDims = selection?.split(',').filter((s) => s.includes(':')); - const dims = slicedDims || dataset.shape; - return value.flat(dims.length - 1); -} - export async function handleAxiosError( func: () => Promise, getErrorToThrow: (status: number, errorData: unknown) => string | undefined, @@ -51,21 +37,6 @@ export function getNameFromPath(path: string) { return segments[segments.length - 1]; } -export function sliceValue( - value: unknown, - dataset: Dataset, - selection: string, -): Primitive[] { - const { shape, type } = dataset; - const dataArray = ndarray(value as Primitive[], shape); - const mappedArray = applyMapping( - dataArray, - selection.split(',').map((s) => (s === ':' ? s : Number.parseInt(s, 10))), - ); - - return mappedArray.data; -} - export function typedArrayFromDType(dtype: DType) { /* Adapted from https://github.com/ludwigschubert/js-numpy-parser/blob/v1.2.3/src/main.js#L116 */ if (!isNumericType(dtype)) { diff --git a/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap b/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap index 9078d59f2..d4bf5a455 100644 --- a/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap +++ b/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap @@ -1622,8 +1622,19 @@ exports[`test file matches snapshot 1`] = ` "type": 6, "vlen": false, }, + { + "cset": -1, + "littleEndian": true, + "name": "bigint", + "offset": 9, + "shape": [], + "signed": true, + "size": 8, + "type": 0, + "vlen": false, + }, ], - "nmembers": 2, + "nmembers": 3, }, "cset": -1, "littleEndian": true, @@ -1631,7 +1642,7 @@ exports[`test file matches snapshot 1`] = ` "offset": 0, "shape": [], "signed": false, - "size": 9, + "size": 17, "type": 6, "vlen": false, }, @@ -1641,7 +1652,7 @@ exports[`test file matches snapshot 1`] = ` "cset": -1, "littleEndian": true, "signed": false, - "size": 9, + "size": 17, "total_size": 1, "type": 6, "vlen": false, @@ -1653,6 +1664,11 @@ exports[`test file matches snapshot 1`] = ` "nested": { "class": "Compound", "fields": { + "bigint": { + "class": "Integer", + "endianness": "little-endian", + "size": 64, + }, "bool": { "class": "Boolean", }, @@ -1680,6 +1696,7 @@ exports[`test file matches snapshot 1`] = ` 1, 2, ], + 3, ], ], }, @@ -1769,8 +1786,8 @@ exports[`test file matches snapshot 1`] = ` 0, 0, 0, - 152, - 209, + 248, + 215, 18, 0, ], @@ -1785,8 +1802,8 @@ exports[`test file matches snapshot 1`] = ` 0, 0, 0, - 232, - 231, + 104, + 238, 18, 0, ], @@ -1801,8 +1818,8 @@ exports[`test file matches snapshot 1`] = ` 0, 0, 0, - 0, - 232, + 128, + 238, 18, 0, ], @@ -2078,8 +2095,8 @@ exports[`test file matches snapshot 1`] = ` 0, 0, 0, - 104, - 178, + 96, + 210, 18, 0, ], @@ -2109,24 +2126,24 @@ exports[`test file matches snapshot 1`] = ` 0, 0, 0, - 0, - 204, + 176, + 25, 18, 0, 2, 0, 0, 0, - 72, - 107, - 14, + 96, + 12, + 18, 0, 3, 0, 0, 0, - 80, - 234, + 240, + 239, 18, 0, ], diff --git a/packages/h5wasm/src/guards.ts b/packages/h5wasm/src/guards.ts index 74d4cce70..584cdd976 100644 --- a/packages/h5wasm/src/guards.ts +++ b/packages/h5wasm/src/guards.ts @@ -1,6 +1,3 @@ -import { isCompoundType } from '@h5web/shared/guards'; -import type { Dataset, DType } from '@h5web/shared/hdf5-models'; -import { DTypeClass } from '@h5web/shared/hdf5-models'; import { Dataset as H5WasmDataset } from 'h5wasm'; import type { H5WasmEntity } from './models'; @@ -12,18 +9,3 @@ export function assertH5WasmDataset( throw new TypeError('Expected H5Wasm entity to be dataset'); } } - -function isInt64Type(type: DType): boolean { - return ( - (type.class === DTypeClass.Integer || type.class === DTypeClass.Unsigned) && - type.size === 64 - ); -} - -export function hasInt64Type(dataset: Dataset) { - const { type } = dataset; - return ( - isInt64Type(type) || - (isCompoundType(type) && Object.values(type.fields).some(isInt64Type)) - ); -} diff --git a/packages/h5wasm/src/h5wasm-api.ts b/packages/h5wasm/src/h5wasm-api.ts index 2e87488b8..09103238d 100644 --- a/packages/h5wasm/src/h5wasm-api.ts +++ b/packages/h5wasm/src/h5wasm-api.ts @@ -1,11 +1,6 @@ import type { ExportFormat, ExportURL, ValuesStoreParams } from '@h5web/app'; -import { - DataProviderApi, - flattenValue, - getNameFromPath, - sliceValue, -} from '@h5web/app'; -import { assertNonNull, hasArrayShape } from '@h5web/shared/guards'; +import { DataProviderApi, getNameFromPath } from '@h5web/app'; +import { assertNonNull } from '@h5web/shared/guards'; import type { ArrayShape, Dataset, @@ -21,13 +16,15 @@ import { } from 'h5wasm'; import { nanoid } from 'nanoid'; -import { assertH5WasmDataset, hasInt64Type } from './guards'; +import { assertH5WasmDataset } from './guards'; import type { H5WasmEntity } from './models'; import type { Plugin } from './utils'; import { - convertSelectionToRanges, + hasBigInts, parseEntity, PLUGINS_BY_FILTER_ID, + readSelectedValue, + sanitizeBigInts, } from './utils'; const PLUGINS_PATH = '/plugins'; // path to plugins on EMScripten virtual file system @@ -64,26 +61,8 @@ export class H5WasmApi extends DataProviderApi { // Ensure all filters are supported and loaded (if available) await this.processFilters(h5wDataset.filters); - /* h5wasm returns bigints for (u)int64 dtypes, so we use `to_array` to get numbers instead. - * We do this only for datasets that are supported by at least one visualization (other than `RawVis`), - * so for (u)int64 scalars/arrays, and for compound datasets with at least one (u)int64 field (`MatrixVis`). */ - if (hasInt64Type(dataset)) { - const rawValue = h5wDataset.to_array(); - - // `to_array` returns nested JS arrays for nD datasets, so we need to re-flatten them - const value = hasArrayShape(dataset) - ? flattenValue(rawValue, dataset) - : rawValue; - - return selection ? sliceValue(value, dataset, selection) : value; - } - - if (selection) { - const ranges = convertSelectionToRanges(h5wDataset, selection); - return h5wDataset.slice(ranges); - } - - return h5wDataset.value; + const value = readSelectedValue(h5wDataset, selection); + return hasBigInts(dataset.type) ? sanitizeBigInts(value) : value; } public async getAttrValues(entity: Entity) { diff --git a/packages/h5wasm/src/utils.ts b/packages/h5wasm/src/utils.ts index 52d7ddc58..7a0eae17c 100644 --- a/packages/h5wasm/src/utils.ts +++ b/packages/h5wasm/src/utils.ts @@ -7,7 +7,12 @@ import type { ProvidedEntity, Shape, } from '@h5web/shared/hdf5-models'; -import { Endianness, EntityKind, H5TClass } from '@h5web/shared/hdf5-models'; +import { + DTypeClass, + Endianness, + EntityKind, + H5TClass, +} from '@h5web/shared/hdf5-models'; import { arrayType, bitfieldType, @@ -246,18 +251,55 @@ function toEndianness(littleEndian: boolean): Endianness { return littleEndian ? Endianness.LE : Endianness.BE; } -export function convertSelectionToRanges( - dataset: H5WasmDataset, - selection: string, -): [number, number][] { - const { shape } = dataset; +export function readSelectedValue( + h5wDataset: H5WasmDataset, + selection: string | undefined, +) { + if (!selection) { + return h5wDataset.value; + } + + const { shape } = h5wDataset; const selectionMembers = selection.split(','); - return selectionMembers.map((member, i) => { + const ranges = selectionMembers.map<[number, number]>((member, i) => { if (member === ':') { return [0, shape[i]]; } return [Number(member), Number(member) + 1]; }); + + return h5wDataset.slice(ranges); +} + +export function hasBigInts(type: DType): boolean { + if (type.class === DTypeClass.Array || type.class === DTypeClass.Enum) { + return hasBigInts(type.base); + } + + if (type.class === DTypeClass.Compound) { + return Object.values(type.fields).some(hasBigInts); + } + + return ( + (type.class === DTypeClass.Integer || type.class === DTypeClass.Unsigned) && + type.size === 64 + ); +} + +export function sanitizeBigInts(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sanitizeBigInts); + } + + if (value instanceof BigInt64Array || value instanceof BigUint64Array) { + return [...value].map(Number); + } + + if (typeof value === 'bigint') { + return Number(value); + } + + return value; }