Load filters from virtual sources

silx-kit · Jun 6, 2024 · fbd0eef · fbd0eef
1 parent 30219c9
commit fbd0eef
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 40 deletions.
diff --git a/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap b/packages/h5wasm/src/__snapshots__/h5wasm-api.test.ts.snap
@@ -1727,8 +1727,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          80,
-          19,
+          112,
+          36,
           19,
           0,
         ],
@@ -1743,8 +1743,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          192,
-          41,
+          224,
+          58,
           19,
           0,
         ],
@@ -1759,8 +1759,8 @@ exports[`test file matches snapshot 1`] = `
           0,
           0,
           0,
-          216,
-          41,
+          248,
+          58,
           19,
           0,
         ],
@@ -2027,9 +2027,9 @@ exports[`test file matches snapshot 1`] = `
       0,
       0,
       0,
-      248,
-      238,
-      18,
+      128,
+      3,
+      19,
       0,
     ],
   },
@@ -2057,24 +2057,24 @@ exports[`test file matches snapshot 1`] = `
       0,
       0,
       0,
-      176,
-      244,
+      128,
+      115,
       18,
       0,
       2,
       0,
       0,
       0,
-      24,
-      67,
+      72,
+      68,
       18,
       0,
       3,
       0,
       0,
       0,
-      72,
-      43,
+      64,
+      61,
       19,
       0,
     ],

diff --git a/packages/h5wasm/src/h5wasm-api.ts b/packages/h5wasm/src/h5wasm-api.ts
@@ -1,6 +1,6 @@
 import type { ExportFormat, ExportURL, ValuesStoreParams } from '@h5web/app';
 import { DataProviderApi } from '@h5web/app';
-import { assertNonNull } from '@h5web/shared/guards';
+import { assertNonNull, isDefined } from '@h5web/shared/guards';
 import type {
   ArrayShape,
   Dataset,
@@ -9,7 +9,7 @@ import type {
   Value,
 } from '@h5web/shared/hdf5-models';
 import { getNameFromPath } from '@h5web/shared/hdf5-utils';
-import type { Filter, Module } from 'h5wasm';
+import type { Dataset as H5WasmDataset, Module } from 'h5wasm';
 import {
   File as H5WasmFile,
   Group as H5WasmGroup,
@@ -60,7 +60,7 @@ export class H5WasmApi extends DataProviderApi {
     assertH5WasmDataset(h5wDataset);
 
     // Ensure all filters are supported and loaded (if available)
-    await this.processFilters(h5wDataset.filters);
+    await this.processFilters(h5wDataset);
 
     try {
       const value = readSelectedValue(h5wDataset, selection);
@@ -163,23 +163,26 @@ export class H5WasmApi extends DataProviderApi {
     return h5wEntity;
   }
 
-  private async processFilters(filters: Filter[]): Promise<void> {
+  private async processFilters(h5wDataset: H5WasmDataset): Promise<void> {
     const h5Module = await this.h5wasm;
+    const { virtual_sources } = h5wDataset.metadata;
 
-    for await (const filter of filters) {
-      if (filter.id < h5Module.H5Z_FILTER_RESERVED) {
-        continue; // filter supported out of the box
-      }
+    // Retrieve filters of any local virtual source datasets
+    const localSources = await Promise.all(
+      (virtual_sources?.filter(({ file_name }) => file_name === '.') || []).map(
+        ({ dset_name }) => this.getH5WasmEntity(dset_name),
+      ),
+    );
+    const allFilters = [h5wDataset, ...localSources].flatMap((source) => {
+      assertH5WasmDataset(source);
+      return source.filters || [];
+    });
 
-      const plugin = PLUGINS_BY_FILTER_ID[filter.id];
-      if (!plugin) {
-        // eslint-disable-next-line no-console
-        console.warn(
-          `Compression filter ${filter.id} not supported (${filter.name})`,
-        );
-        continue;
-      }
+    const pluginsToLoad = allFilters
+      .map(({ id }) => PLUGINS_BY_FILTER_ID[id])
+      .filter(isDefined);
 
+    for await (const plugin of pluginsToLoad) {
       const pluginPath = `${PLUGINS_PATH}/libH5Z${plugin}.so`;
 
       if (h5Module.FS.analyzePath(pluginPath).exists) {

diff --git a/packages/h5wasm/src/local/h5wasm-local-file-api.ts b/packages/h5wasm/src/local/h5wasm-local-file-api.ts
@@ -1,13 +1,13 @@
 import type { ExportFormat, ExportURL, ValuesStoreParams } from '@h5web/app';
 import { DataProviderApi } from '@h5web/app';
-import { isDefined } from '@h5web/shared/guards';
+import { assertDataset, isDefined } from '@h5web/shared/guards';
 import type {
   ArrayShape,
   AttributeValues,
   Dataset,
   Entity,
-  Filter,
   ProvidedEntity,
+  ScalarShape,
   Value,
 } from '@h5web/shared/hdf5-models';
 import type { Remote } from 'comlink';
@@ -48,7 +48,7 @@ export class H5WasmLocalFileApi extends DataProviderApi {
     const { dataset, selection } = params;
     const fileId = await this.fileId;
 
-    await this.processFilters(dataset.filters);
+    await this.processFilters(dataset);
 
     try {
       const value = await this.remote.getValue(fileId, dataset.path, selection);
@@ -100,8 +100,23 @@ export class H5WasmLocalFileApi extends DataProviderApi {
     return this.remote.closeFile(await this.fileId);
   }
 
-  private async processFilters(filters: Filter[] = []): Promise<void> {
-    const pluginsToLoad = filters
+  private async processFilters(
+    dataset: Dataset<ScalarShape | ArrayShape>,
+  ): Promise<void> {
+    const fileId = await this.fileId;
+
+    // Retrieve filters of any local virtual source datasets
+    const localSources = await Promise.all(
+      (dataset.virtualSources?.filter(({ file }) => file === '.') || []).map(
+        ({ path }) => this.remote.getEntity(fileId, path),
+      ),
+    );
+    const allFilters = [dataset, ...localSources].flatMap((source) => {
+      assertDataset(source);
+      return source.filters || [];
+    });
+
+    const pluginsToLoad = allFilters
       .map(({ id }) => PLUGINS_BY_FILTER_ID[id])
       .filter(isDefined);
 

diff --git a/packages/h5wasm/src/local/worker.utils.ts b/packages/h5wasm/src/local/worker.utils.ts
@@ -3,9 +3,11 @@ import type {
   ChildEntity,
   Group,
   ProvidedEntity,
+  VirtualSource,
 } from '@h5web/shared/hdf5-models';
 import { EntityKind } from '@h5web/shared/hdf5-models';
 import { buildEntityPath, getNameFromPath } from '@h5web/shared/hdf5-utils';
+import type { Metadata } from 'h5wasm';
 import { ready as h5wasmReady, ready } from 'h5wasm';
 
 import { parseDType } from '../utils';
@@ -80,15 +82,16 @@ export function parseEntity(
 
   if (kind === h5wasm.H5G_DATASET) {
     const metadata = h5wasm.get_dataset_metadata(fileId, path);
-    const { chunks, maxshape, shape, ...rawType } = metadata; // keep `rawType` concise
+    const { chunks, maxshape, shape, virtual_sources, ...rawType } = metadata; // keep `rawType` concise
 
     return {
       ...baseEntity,
       kind: EntityKind.Dataset,
-      shape: metadata.shape,
+      shape,
       type: parseDType(metadata),
-      chunks: metadata.chunks ?? undefined,
+      chunks: chunks ?? undefined,
       filters: h5wasm.get_dataset_filters(fileId, path),
+      virtualSources: parseVirtualSources(metadata),
       attributes: parseAttributes(h5wasm, fileId, path),
       rawType,
     };
@@ -154,3 +157,10 @@ function parseAttributes(
     };
   });
 }
+
+function parseVirtualSources(metadata: Metadata): VirtualSource[] | undefined {
+  return metadata.virtual_sources?.map(({ file_name, dset_name }) => ({
+    file: file_name,
+    path: dset_name,
+  }));
+}
diff --git a/packages/h5wasm/src/utils.ts b/packages/h5wasm/src/utils.ts
@@ -12,6 +12,7 @@ import type {
   Group,
   ProvidedEntity,
   Shape,
+  VirtualSource,
 } from '@h5web/shared/hdf5-models';
 import { DTypeClass, EntityKind } from '@h5web/shared/hdf5-models';
 import {
@@ -99,7 +100,7 @@ export function parseEntity(
 
   if (h5wEntity instanceof H5WasmDataset) {
     const { metadata, filters } = h5wEntity;
-    const { chunks, maxshape, shape, ...rawType } = metadata; // keep `rawType` concise
+    const { chunks, maxshape, shape, virtual_sources, ...rawType } = metadata; // keep `rawType` concise
 
     return {
       ...baseEntity,
@@ -108,6 +109,7 @@ export function parseEntity(
       type: parseDType(metadata),
       chunks: metadata.chunks ?? undefined,
       filters,
+      virtualSources: parseVirtualSources(metadata),
       attributes: parseAttributes(h5wEntity.attrs),
       rawType,
     };
@@ -241,6 +243,13 @@ export function parseDType(metadata: Metadata): DType {
   return unknownType();
 }
 
+function parseVirtualSources(metadata: Metadata): VirtualSource[] | undefined {
+  return metadata.virtual_sources?.map(({ file_name, dset_name }) => ({
+    file: file_name,
+    path: dset_name,
+  }));
+}
+
 export function readSelectedValue(
   h5wDataset: H5WasmDataset,
   selection: string | undefined,

diff --git a/packages/shared/src/hdf5-models.ts b/packages/shared/src/hdf5-models.ts
@@ -51,6 +51,7 @@ export interface Dataset<S extends Shape = Shape, T extends DType = DType>
   rawType?: unknown;
   chunks?: number[];
   filters?: Filter[];
+  virtualSources?: VirtualSource[];
 }
 
 export interface Datatype<T = DType> extends Entity {
@@ -232,3 +233,8 @@ export interface Filter {
   id: number;
   name: string;
 }
+
+export interface VirtualSource {
+  file: string;
+  path: string;
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -1727,8 +1727,8 @@ exports[`test file matches snapshot 1`] = ` @@
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
             ],
@@ Expand All / @@ -1743,8 +1743,8 @@ exports[`test file matches snapshot 1`] = ` @@
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
             ],
@@ Expand All / @@ -1759,8 +1759,8 @@ exports[`test file matches snapshot 1`] = ` @@
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
             ],
@@ Expand Down Expand Up / @@ -2027,9 +2027,9 @@ exports[`test file matches snapshot 1`] = ` @@
 ,
 ,
 ,
-,
-,
-,
+,
+,
+,
 ,
         ],
       },
@@ Expand Down Expand Up / @@ -2057,24 +2057,24 @@ exports[`test file matches snapshot 1`] = ` @@
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
 ,
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
 ,
 ,
 ,
 ,
-,
-,
+,
+,
 ,
 ,
         ],
@@ Expand Down @@