From 479c2826c57cc48ae8226f6027dff42455a611a4 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:00:51 +0200 Subject: [PATCH 01/87] Rename generate_data to implementations --- .gitignore | 2 +- Makefile | 16 ++++++++-------- .../generate_reference_image.py | 2 +- doc/development_overview.md | 2 +- .../generate_pyn5.py | 0 .../generate_z5py.py | 0 .../generate_zarr.py | 0 .../generate_zarrita.py | 0 .../js/generate_data.sh | 0 .../js/package-lock.json | 0 .../js/package.json | 0 .../js/src/fsstore.js | 0 .../js/src/index.js | 0 .../jzarr/generate_data.sh | 0 {generate_data => implementations}/jzarr/pom.xml | 0 .../java/zarr_implementations/jzarr/App.java | 0 .../n5-java/generate_data.sh | 0 .../n5-java/pom.xml | 0 .../java/zarr_implementations/n5_java/App.java | 0 .../xtensor_zarr/CMakeLists.txt | 2 +- .../xtensor_zarr/generate_data.sh | 0 .../xtensor_zarr/modules/FindBlosc.cmake | 0 .../xtensor_zarr/modules/FindOIIO.cmake | 0 .../xtensor_zarr/src/main.cpp | 0 test/test_read_all.py | 4 ++-- 25 files changed, 14 insertions(+), 14 deletions(-) rename {generate_data => data}/generate_reference_image.py (82%) rename {generate_data => implementations}/generate_pyn5.py (100%) rename {generate_data => implementations}/generate_z5py.py (100%) rename {generate_data => implementations}/generate_zarr.py (100%) rename {generate_data => implementations}/generate_zarrita.py (100%) rename {generate_data => implementations}/js/generate_data.sh (100%) rename {generate_data => implementations}/js/package-lock.json (100%) rename {generate_data => implementations}/js/package.json (100%) rename {generate_data => implementations}/js/src/fsstore.js (100%) rename {generate_data => implementations}/js/src/index.js (100%) rename {generate_data => implementations}/jzarr/generate_data.sh (100%) rename {generate_data => implementations}/jzarr/pom.xml (100%) rename {generate_data => implementations}/jzarr/src/main/java/zarr_implementations/jzarr/App.java (100%) rename {generate_data => implementations}/n5-java/generate_data.sh (100%) rename {generate_data => implementations}/n5-java/pom.xml (100%) rename {generate_data => implementations}/n5-java/src/main/java/zarr_implementations/n5_java/App.java (100%) rename {generate_data => implementations}/xtensor_zarr/CMakeLists.txt (98%) rename {generate_data => implementations}/xtensor_zarr/generate_data.sh (100%) rename {generate_data => implementations}/xtensor_zarr/modules/FindBlosc.cmake (100%) rename {generate_data => implementations}/xtensor_zarr/modules/FindOIIO.cmake (100%) rename {generate_data => implementations}/xtensor_zarr/src/main.cpp (100%) diff --git a/.gitignore b/.gitignore index 03b3105c..2da36a79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target* __pycache__/ -generate_data/xtensor_zarr/build +implemnetations/xtensor_zarr/build node_modules/ report.* diff --git a/Makefile b/Makefile index e1888d6f..e024ec79 100644 --- a/Makefile +++ b/Makefile @@ -6,35 +6,35 @@ data/reference_image.png: .PHONY: jzarr jzarr: data/reference_image.png - bash generate_data/jzarr/generate_data.sh + bash implementations/jzarr/generate_data.sh .PHONY: n5java n5java: data/reference_image.png - bash generate_data/n5-java/generate_data.sh + bash implementations/n5-java/generate_data.sh .PHONY: pyn5 pyn5: data/reference_image.png - python generate_data/generate_pyn5.py + python implementations/generate_pyn5.py .PHONY: z5py z5py: data/reference_image.png - python generate_data/generate_z5py.py + python implementations/generate_z5py.py .PHONY: zarr zarr: data/reference_image.png - python generate_data/generate_zarr.py + python implementations/generate_zarr.py .PHONY: zarrita zarrita: data/reference_image.png - python generate_data/generate_zarrita.py + python implementations/generate_zarrita.py .PHONY: js js: data/reference_image.png - bash generate_data/js/generate_data.sh + bash implementations/js/generate_data.sh .PHONY: xtensor_zarr xtensor_zarr: data/reference_image.png - bash generate_data/xtensor_zarr/generate_data.sh + bash implementations/xtensor_zarr/generate_data.sh .PHONY: data data: jzarr n5java pyn5 z5py zarr js xtensor_zarr zarrita diff --git a/generate_data/generate_reference_image.py b/data/generate_reference_image.py similarity index 82% rename from generate_data/generate_reference_image.py rename to data/generate_reference_image.py index 1780d7b1..2d1cbe85 100644 --- a/generate_data/generate_reference_image.py +++ b/data/generate_reference_image.py @@ -3,7 +3,7 @@ def write_reference_image(): - path = '../data/reference_image.png' + path = 'reference_image.png' im = astronaut() imsave(path, im) diff --git a/doc/development_overview.md b/doc/development_overview.md index 2a503a04..cc34d634 100644 --- a/doc/development_overview.md +++ b/doc/development_overview.md @@ -44,7 +44,7 @@ conda activate zarr_impl_dev ## Data Generation All test data is currently generated from a common image file that gets -generated by the script `generate_data/generate_reference_image.py`. Each +generated by the script `data/generate_reference_image.py`. Each library that generates data currently has either a python script or a shell script that will generate datasets for the supported file types and codecs. Some implementations also generate versions using both flat and nested file diff --git a/generate_data/generate_pyn5.py b/implementations/generate_pyn5.py similarity index 100% rename from generate_data/generate_pyn5.py rename to implementations/generate_pyn5.py diff --git a/generate_data/generate_z5py.py b/implementations/generate_z5py.py similarity index 100% rename from generate_data/generate_z5py.py rename to implementations/generate_z5py.py diff --git a/generate_data/generate_zarr.py b/implementations/generate_zarr.py similarity index 100% rename from generate_data/generate_zarr.py rename to implementations/generate_zarr.py diff --git a/generate_data/generate_zarrita.py b/implementations/generate_zarrita.py similarity index 100% rename from generate_data/generate_zarrita.py rename to implementations/generate_zarrita.py diff --git a/generate_data/js/generate_data.sh b/implementations/js/generate_data.sh similarity index 100% rename from generate_data/js/generate_data.sh rename to implementations/js/generate_data.sh diff --git a/generate_data/js/package-lock.json b/implementations/js/package-lock.json similarity index 100% rename from generate_data/js/package-lock.json rename to implementations/js/package-lock.json diff --git a/generate_data/js/package.json b/implementations/js/package.json similarity index 100% rename from generate_data/js/package.json rename to implementations/js/package.json diff --git a/generate_data/js/src/fsstore.js b/implementations/js/src/fsstore.js similarity index 100% rename from generate_data/js/src/fsstore.js rename to implementations/js/src/fsstore.js diff --git a/generate_data/js/src/index.js b/implementations/js/src/index.js similarity index 100% rename from generate_data/js/src/index.js rename to implementations/js/src/index.js diff --git a/generate_data/jzarr/generate_data.sh b/implementations/jzarr/generate_data.sh similarity index 100% rename from generate_data/jzarr/generate_data.sh rename to implementations/jzarr/generate_data.sh diff --git a/generate_data/jzarr/pom.xml b/implementations/jzarr/pom.xml similarity index 100% rename from generate_data/jzarr/pom.xml rename to implementations/jzarr/pom.xml diff --git a/generate_data/jzarr/src/main/java/zarr_implementations/jzarr/App.java b/implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java similarity index 100% rename from generate_data/jzarr/src/main/java/zarr_implementations/jzarr/App.java rename to implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java diff --git a/generate_data/n5-java/generate_data.sh b/implementations/n5-java/generate_data.sh similarity index 100% rename from generate_data/n5-java/generate_data.sh rename to implementations/n5-java/generate_data.sh diff --git a/generate_data/n5-java/pom.xml b/implementations/n5-java/pom.xml similarity index 100% rename from generate_data/n5-java/pom.xml rename to implementations/n5-java/pom.xml diff --git a/generate_data/n5-java/src/main/java/zarr_implementations/n5_java/App.java b/implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java similarity index 100% rename from generate_data/n5-java/src/main/java/zarr_implementations/n5_java/App.java rename to implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java diff --git a/generate_data/xtensor_zarr/CMakeLists.txt b/implementations/xtensor_zarr/CMakeLists.txt similarity index 98% rename from generate_data/xtensor_zarr/CMakeLists.txt rename to implementations/xtensor_zarr/CMakeLists.txt index 7ee11e5e..619eebaf 100644 --- a/generate_data/xtensor_zarr/CMakeLists.txt +++ b/implementations/xtensor_zarr/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.8) if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - project(generate_data) + project(implementations) find_package(xtensor-zarr REQUIRED CONFIG) set(XTENSOR_ZARR_INCLUDE_DIR ${xtensor_zarr_INCLUDE_DIRS}) diff --git a/generate_data/xtensor_zarr/generate_data.sh b/implementations/xtensor_zarr/generate_data.sh similarity index 100% rename from generate_data/xtensor_zarr/generate_data.sh rename to implementations/xtensor_zarr/generate_data.sh diff --git a/generate_data/xtensor_zarr/modules/FindBlosc.cmake b/implementations/xtensor_zarr/modules/FindBlosc.cmake similarity index 100% rename from generate_data/xtensor_zarr/modules/FindBlosc.cmake rename to implementations/xtensor_zarr/modules/FindBlosc.cmake diff --git a/generate_data/xtensor_zarr/modules/FindOIIO.cmake b/implementations/xtensor_zarr/modules/FindOIIO.cmake similarity index 100% rename from generate_data/xtensor_zarr/modules/FindOIIO.cmake rename to implementations/xtensor_zarr/modules/FindOIIO.cmake diff --git a/generate_data/xtensor_zarr/src/main.cpp b/implementations/xtensor_zarr/src/main.cpp similarity index 100% rename from generate_data/xtensor_zarr/src/main.cpp rename to implementations/xtensor_zarr/src/main.cpp diff --git a/test/test_read_all.py b/test/test_read_all.py index c8e77049..493c7a7e 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -81,7 +81,7 @@ def read_with_jzarr(fpath, ds_name, nested=None): ds_name = "blosc/lz4" cmd = ( - f"generate_data/jzarr/generate_data.sh " + f"implementations/jzarr/generate_data.sh " f"-verify {str(fpath)} {ds_name}" ) @@ -137,7 +137,7 @@ def read_with_xtensor_zarr(fpath, ds_name, nested): fname = "a.npz" if os.path.exists(fname): os.remove(fname) - subprocess.check_call(["generate_data/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) + subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) return np.load(fname)["a"] From aba8bbdbfd1b35c5b12e6198cb69efec35a7b019 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:12:43 +0200 Subject: [PATCH 02/87] Parallelize and unify GHAs --- .github/workflows/build.yml | 31 +++++++++++++++++---- .github/workflows/zarr-dev.yml | 51 ---------------------------------- 2 files changed, 26 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/zarr-dev.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bec61101..04efcb93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,30 +4,51 @@ name: Build on: [push, pull_request] jobs: - test: + + get_implementations: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: List implementations + id: set-matrix + run: echo "::set-output name=matrix::$(cd implementations && ls -1" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + test_implementations: name: ${{ matrix.platform }} ${{ matrix.python-version }} runs-on: ${{ matrix.platform }} + needs: get_implementations strategy: fail-fast: false matrix: platform: [ubuntu-latest] - python-version: [3.7] + python-version: [3.9] + implementation: ${{ needs.get_implementations.outputs.matrix }} + zarr-python: [released, pre] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup miniconda uses: conda-incubator/setup-miniconda@v1 with: auto-update-conda: true channels: conda-forge,ome - environment-file: environment.yml + environment-file: ${{ implementation}}/environment.yml python-version: ${{ matrix.python-version }} env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true + - name: Install zarr dev (optional) + shell: bash -l {0} + if: matrix.config.zarr-python == "pre" + run: | + python -m pip install git+https://github.com/zarr-developers/zarr-python.git + - name: Cache local Maven repository uses: actions/cache@v2 with: @@ -38,4 +59,4 @@ jobs: - name: Run tests shell: bash -l {0} - run: make + run: make ${{ implementation }} diff --git a/.github/workflows/zarr-dev.yml b/.github/workflows/zarr-dev.yml deleted file mode 100644 index c9cc2025..00000000 --- a/.github/workflows/zarr-dev.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: Build with Zarr dev - -on: [push, pull_request] - -jobs: - test: - name: ${{ matrix.platform }} ${{ matrix.python-version }} - runs-on: ${{ matrix.platform }} - - strategy: - fail-fast: true - matrix: - platform: [ubuntu-latest] - python-version: [3.7] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,ome - environment-file: environment.yml - python-version: ${{ matrix.python-version }} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - - - name: Install zarr dev - shell: bash -l {0} - run: | - python -m pip install git+https://github.com/zarr-developers/zarr-python.git - - - name: Run tests - shell: bash -l {0} - run: make - - - name: Generate Report - shell: bash -l {0} - run: | - python test/test_read_all.py - - - name: Archive test report artifacts - uses: actions/upload-artifact@v2 - with: - name: test-report - path: | - report.md - report.html From eab673db4648a0761e324397b5ca7223984dab98 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:14:42 +0200 Subject: [PATCH 03/87] Move all implementations to subdirs --- implementations/{ => pyn5}/generate_pyn5.py | 0 implementations/{ => z5py}/generate_z5py.py | 0 implementations/{ => zarr-python}/generate_zarr.py | 0 implementations/{ => zarrita}/generate_zarrita.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename implementations/{ => pyn5}/generate_pyn5.py (100%) rename implementations/{ => z5py}/generate_z5py.py (100%) rename implementations/{ => zarr-python}/generate_zarr.py (100%) rename implementations/{ => zarrita}/generate_zarrita.py (100%) diff --git a/implementations/generate_pyn5.py b/implementations/pyn5/generate_pyn5.py similarity index 100% rename from implementations/generate_pyn5.py rename to implementations/pyn5/generate_pyn5.py diff --git a/implementations/generate_z5py.py b/implementations/z5py/generate_z5py.py similarity index 100% rename from implementations/generate_z5py.py rename to implementations/z5py/generate_z5py.py diff --git a/implementations/generate_zarr.py b/implementations/zarr-python/generate_zarr.py similarity index 100% rename from implementations/generate_zarr.py rename to implementations/zarr-python/generate_zarr.py diff --git a/implementations/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py similarity index 100% rename from implementations/generate_zarrita.py rename to implementations/zarrita/generate_zarrita.py From c5d521ffdaecaa5c46f95b8308bc91e8b4c0acb5 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:24:00 +0200 Subject: [PATCH 04/87] Add environment files for each implementation --- .gitignore | 1 + implementations/js/environment.yml | 5 +++++ implementations/jzarr/environment.yml | 6 ++++++ implementations/n5-java/environment.yml | 6 ++++++ implementations/pyn5/environment.yml | 15 +++++++++++++++ implementations/xtensor_zarr/.gitignore | 1 + implementations/zarr-python/environment.yml | 15 +++++++++++++++ implementations/zarrita/environment.yml | 15 +++++++++++++++ 8 files changed, 64 insertions(+) create mode 100644 implementations/js/environment.yml create mode 100644 implementations/jzarr/environment.yml create mode 100644 implementations/n5-java/environment.yml create mode 100644 implementations/pyn5/environment.yml create mode 100644 implementations/xtensor_zarr/.gitignore create mode 100644 implementations/zarr-python/environment.yml create mode 100644 implementations/zarrita/environment.yml diff --git a/.gitignore b/.gitignore index 2da36a79..65a5d48f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ report.* # ignore data subdirectories data/**/ +*iml diff --git a/implementations/js/environment.yml b/implementations/js/environment.yml new file mode 100644 index 00000000..4c3e61e9 --- /dev/null +++ b/implementations/js/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge + - defaults +dependencies: + - nodejs diff --git a/implementations/jzarr/environment.yml b/implementations/jzarr/environment.yml new file mode 100644 index 00000000..ea0509fe --- /dev/null +++ b/implementations/jzarr/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - openjdk + - maven diff --git a/implementations/n5-java/environment.yml b/implementations/n5-java/environment.yml new file mode 100644 index 00000000..ea0509fe --- /dev/null +++ b/implementations/n5-java/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - openjdk + - maven diff --git a/implementations/pyn5/environment.yml b/implementations/pyn5/environment.yml new file mode 100644 index 00000000..e6894551 --- /dev/null +++ b/implementations/pyn5/environment.yml @@ -0,0 +1,15 @@ +channels: + - conda-forge + - defaults +dependencies: + - openimageio + - python == 3.7.9 + - scikit-image + - pytest + - zarr >= 2.8.3 + - pip + - pandas + - tabulate + - pip: + - pyn5 + - git+https://github.com/grlee77/zarrita.git@codec_update diff --git a/implementations/xtensor_zarr/.gitignore b/implementations/xtensor_zarr/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/implementations/xtensor_zarr/.gitignore @@ -0,0 +1 @@ +build diff --git a/implementations/zarr-python/environment.yml b/implementations/zarr-python/environment.yml new file mode 100644 index 00000000..e6894551 --- /dev/null +++ b/implementations/zarr-python/environment.yml @@ -0,0 +1,15 @@ +channels: + - conda-forge + - defaults +dependencies: + - openimageio + - python == 3.7.9 + - scikit-image + - pytest + - zarr >= 2.8.3 + - pip + - pandas + - tabulate + - pip: + - pyn5 + - git+https://github.com/grlee77/zarrita.git@codec_update diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml new file mode 100644 index 00000000..e6894551 --- /dev/null +++ b/implementations/zarrita/environment.yml @@ -0,0 +1,15 @@ +channels: + - conda-forge + - defaults +dependencies: + - openimageio + - python == 3.7.9 + - scikit-image + - pytest + - zarr >= 2.8.3 + - pip + - pandas + - tabulate + - pip: + - pyn5 + - git+https://github.com/grlee77/zarrita.git@codec_update From ffb6e40cf3a98a4aeb9c4391825406e5e815765f Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:48:44 +0200 Subject: [PATCH 05/87] start re-working Makefile for parallelism --- Makefile | 70 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index e024ec79..f712cf26 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,62 @@ +IMPLEMENTATIONS=$(wildcard implementations/*) + + +ifeq ($(TEST),) +# If TEST is not set, by default build everything, generate +# data for all implementations, and then run all pytests. + +.PHONY: test test: data - pytest -v + pytest -v -k W + +else +# Otherwise, focus on a single implementation. + +.PHONY: test +test: $(TEST) + pytest -v -k $(TEST) + +endif data/reference_image.png: python generate_reference_image.py -.PHONY: jzarr -jzarr: data/reference_image.png - bash implementations/jzarr/generate_data.sh +.PHONY: report +report: data + python test/test_read_all.py + +# +# Implementations +# + +.PHONY: implementations/jzarr +implementations/jzarr: data/reference_image.png + bash implementations/jzarr/driver.sh -.PHONY: n5java -n5java: data/reference_image.png +.PHONY: implementations/n5java +implementations/n5java: data/reference_image.png bash implementations/n5-java/generate_data.sh -.PHONY: pyn5 -pyn5: data/reference_image.png +.PHONY: implementations/pyn5 +implementations/pyn5: data/reference_image.png python implementations/generate_pyn5.py -.PHONY: z5py -z5py: data/reference_image.png +.PHONY: implementations/z5py +implementations/z5py: data/reference_image.png python implementations/generate_z5py.py -.PHONY: zarr -zarr: data/reference_image.png +.PHONY: implementations/zarr-python +implementations/zarr-python: data/reference_image.png python implementations/generate_zarr.py -.PHONY: zarrita -zarrita: data/reference_image.png +.PHONY: implementations/zarrita +implementations/zarrita: data/reference_image.png python implementations/generate_zarrita.py -.PHONY: js -js: data/reference_image.png +.PHONY: implementations/js +implementations/js: data/reference_image.png bash implementations/js/generate_data.sh -.PHONY: xtensor_zarr -xtensor_zarr: data/reference_image.png +.PHONY: implementations/xtensor_zarr +implementations/xtensor_zarr: data/reference_image.png bash implementations/xtensor_zarr/generate_data.sh - -.PHONY: data -data: jzarr n5java pyn5 z5py zarr js xtensor_zarr zarrita - -.PHONY: test - -.PHONY: report -report: data - python test/test_read_all.py From f635ab8c1d4c1dfa067e885f4f964f2197310856 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 09:51:24 +0200 Subject: [PATCH 06/87] Use Makefile patterns --- Makefile | 49 ++++++++----------------------------------- test/test_read_all.py | 4 ++-- 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index f712cf26..aa7c3a52 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ IMPLEMENTATIONS=$(wildcard implementations/*) - ifeq ($(TEST),) # If TEST is not set, by default build everything, generate # data for all implementations, and then run all pytests. @@ -10,53 +9,23 @@ test: data pytest -v -k W else -# Otherwise, focus on a single implementation. +# Otherwise, focus on a single implementation, only generating +# its data and using the "-k W-" keyword to limit which pytests +# get run .PHONY: test -test: $(TEST) - pytest -v -k $(TEST) +test: implementations/$(TEST) + pytest -v -k W-$(TEST) endif data/reference_image.png: python generate_reference_image.py +.PHONY: $(IMPLEMENTATIONS) +implementations/%: data/reference_image.png + bash $^/driver.sh + .PHONY: report report: data python test/test_read_all.py - -# -# Implementations -# - -.PHONY: implementations/jzarr -implementations/jzarr: data/reference_image.png - bash implementations/jzarr/driver.sh - -.PHONY: implementations/n5java -implementations/n5java: data/reference_image.png - bash implementations/n5-java/generate_data.sh - -.PHONY: implementations/pyn5 -implementations/pyn5: data/reference_image.png - python implementations/generate_pyn5.py - -.PHONY: implementations/z5py -implementations/z5py: data/reference_image.png - python implementations/generate_z5py.py - -.PHONY: implementations/zarr-python -implementations/zarr-python: data/reference_image.png - python implementations/generate_zarr.py - -.PHONY: implementations/zarrita -implementations/zarrita: data/reference_image.png - python implementations/generate_zarrita.py - -.PHONY: implementations/js -implementations/js: data/reference_image.png - bash implementations/js/generate_data.sh - -.PHONY: implementations/xtensor_zarr -implementations/xtensor_zarr: data/reference_image.png - bash implementations/xtensor_zarr/generate_data.sh diff --git a/test/test_read_all.py b/test/test_read_all.py index 493c7a7e..f226d829 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -220,8 +220,8 @@ def create_params(): if write_attrs: write_attrs = ' (' + write_attrs + ')' ids.append( - f"read {writing_library}{write_attrs} {fmt} using " - f"{reading_library}, {codec}" + f"W-{writing_library}{write_attrs} {fmt}_R-" + f"{reading_library}_{codec}" ) return argnames, params, ids From d3daa8194fe1695262b8cc6e81e054a25fde3908 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:07:08 +0200 Subject: [PATCH 07/87] create driver.sh for each implementation --- Makefile | 30 ++++++++++++++----- implementations/.conda_driver.sh | 20 +++++++++++++ implementations/js/driver.sh | 12 ++++++++ implementations/js/generate_data.sh | 6 ---- .../jzarr/{generate_data.sh => driver.sh} | 14 +++++---- implementations/n5-java/driver.sh | 14 +++++++++ implementations/n5-java/generate_data.sh | 8 ----- implementations/pyn5/driver.sh | 13 ++++++++ implementations/pyn5/environment.yml | 10 ++----- .../{generate_data.sh => driver.sh} | 14 +++++++-- implementations/xtensor_zarr/environment.yml | 7 +++++ implementations/z5py/driver.sh | 13 ++++++++ implementations/z5py/environment.yml | 7 +++++ implementations/zarr-python/driver.sh | 13 ++++++++ implementations/zarr-python/environment.yml | 12 ++------ implementations/zarrita/driver.sh | 12 ++++++++ implementations/zarrita/environment.yml | 8 +---- 17 files changed, 160 insertions(+), 53 deletions(-) create mode 100644 implementations/.conda_driver.sh create mode 100755 implementations/js/driver.sh delete mode 100755 implementations/js/generate_data.sh rename implementations/jzarr/{generate_data.sh => driver.sh} (67%) create mode 100755 implementations/n5-java/driver.sh delete mode 100755 implementations/n5-java/generate_data.sh create mode 100755 implementations/pyn5/driver.sh rename implementations/xtensor_zarr/{generate_data.sh => driver.sh} (52%) create mode 100644 implementations/xtensor_zarr/environment.yml create mode 100755 implementations/z5py/driver.sh create mode 100644 implementations/z5py/environment.yml create mode 100755 implementations/zarr-python/driver.sh create mode 100755 implementations/zarrita/driver.sh diff --git a/Makefile b/Makefile index aa7c3a52..feca752e 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,46 @@ IMPLEMENTATIONS=$(wildcard implementations/*) -ifeq ($(TEST),) +ifeq ($(TEST),) ################################################# # If TEST is not set, by default build everything, generate # data for all implementations, and then run all pytests. -.PHONY: test +.PHONY: test data test: data pytest -v -k W +data: $(IMPLEMENTATIONS) + else # Otherwise, focus on a single implementation, only generating # its data and using the "-k W-" keyword to limit which pytests # get run -.PHONY: test +.PHONY: test data test: implementations/$(TEST) pytest -v -k W-$(TEST) -endif +data: implementations/$(TEST) + +endif ########################################################## + data/reference_image.png: python generate_reference_image.py -.PHONY: $(IMPLEMENTATIONS) -implementations/%: data/reference_image.png - bash $^/driver.sh +define mk-impl-target +# For each of the items in our "implementations" directory, +# create a target which depends on the reference data and +# calls the "driver.sh" script. +.PHONY: $1 +$1: data/reference_image.png + bash $1/driver.sh + +# Alias in case the trailing slash is included +.PHONE: $1/ +$1/: $1 + +endef +$(foreach impl,$(IMPLEMENTATIONS),$(eval $(call mk-impl-target,$(impl)))) .PHONY: report report: data diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh new file mode 100644 index 00000000..5c834961 --- /dev/null +++ b/implementations/.conda_driver.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# This is re-usable driver code for all of the implementations +# that make use of a conda environment file. +# + +set -e +set -o pipefail + +create_or_activate(){ + if { conda env list | grep $ENVNAME; } >/dev/null 2>&1; then + echo "Using $ENVNAME" + else + echo "Creating $ENVNAME" + conda env create -n $ENVNAME -f $IMPL/environment.yml + fi + eval "$(conda shell.bash hook)" + echo "Activating $ENVNAME" + conda activate $ENVNAME +} diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh new file mode 100755 index 00000000..fc7296c8 --- /dev/null +++ b/implementations/js/driver.sh @@ -0,0 +1,12 @@ +ENVNAME=ZI_js + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +cd "${IMPL}" + +npm install +npm start diff --git a/implementations/js/generate_data.sh b/implementations/js/generate_data.sh deleted file mode 100755 index c70e51f0..00000000 --- a/implementations/js/generate_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -# cd to this directory -# https://stackoverflow.com/a/6393573/2700168 -cd "${0%/*}" - -npm install -npm start diff --git a/implementations/jzarr/generate_data.sh b/implementations/jzarr/driver.sh similarity index 67% rename from implementations/jzarr/generate_data.sh rename to implementations/jzarr/driver.sh index c860ea03..12c50848 100755 --- a/implementations/jzarr/generate_data.sh +++ b/implementations/jzarr/driver.sh @@ -1,10 +1,15 @@ # cd to this directory # https://stackoverflow.com/a/6393573/2700168 -cd "${0%/*}" -set -e -set -u -set -x +ENVNAME=ZI_jzarr + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +cd "${IMPL}" MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} mvn "${MVN_FLAGS}" clean package @@ -16,4 +21,3 @@ java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && { echo jzarr failed exit 2 } - diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh new file mode 100755 index 00000000..c3546f60 --- /dev/null +++ b/implementations/n5-java/driver.sh @@ -0,0 +1,14 @@ +ENVNAME=ZI_n5_java + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +cd "${IMPL}" + +MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} +mvn "${MVN_FLAGS}" clean package + +java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App diff --git a/implementations/n5-java/generate_data.sh b/implementations/n5-java/generate_data.sh deleted file mode 100755 index 2607bea3..00000000 --- a/implementations/n5-java/generate_data.sh +++ /dev/null @@ -1,8 +0,0 @@ -# cd to this directory -# https://stackoverflow.com/a/6393573/2700168 -cd "${0%/*}" - -MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} -mvn "${MVN_FLAGS}" clean package - -java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh new file mode 100755 index 00000000..98fe7fa6 --- /dev/null +++ b/implementations/pyn5/driver.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# + +ENVNAME=ZI_pyn5 + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +python $IMPL/generate_pyn5.py diff --git a/implementations/pyn5/environment.yml b/implementations/pyn5/environment.yml index e6894551..27c88d00 100644 --- a/implementations/pyn5/environment.yml +++ b/implementations/pyn5/environment.yml @@ -2,14 +2,10 @@ channels: - conda-forge - defaults dependencies: - - openimageio - - python == 3.7.9 + - rust + - maturin + - python - scikit-image - - pytest - - zarr >= 2.8.3 - pip - - pandas - - tabulate - pip: - pyn5 - - git+https://github.com/grlee77/zarrita.git@codec_update diff --git a/implementations/xtensor_zarr/generate_data.sh b/implementations/xtensor_zarr/driver.sh similarity index 52% rename from implementations/xtensor_zarr/generate_data.sh rename to implementations/xtensor_zarr/driver.sh index b6e66582..8f13113c 100755 --- a/implementations/xtensor_zarr/generate_data.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -1,6 +1,14 @@ -# cd to this directory -# https://stackoverflow.com/a/6393573/2700168 -cd "${0%/*}" +ENVNAME=ZI_xtensor_zarr + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +cd "${IMPL}" + +set +u # Due to GDAL_DATA rm -rf build mkdir build diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml new file mode 100644 index 00000000..f71db734 --- /dev/null +++ b/implementations/xtensor_zarr/environment.yml @@ -0,0 +1,7 @@ +channels: + - conda-forge + - defaults +dependencies: + - make + - cmake + - xtensor-zarr >=0.0.8|=0.0.7=*_1 diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh new file mode 100755 index 00000000..90103f85 --- /dev/null +++ b/implementations/z5py/driver.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# + +ENVNAME=ZI_z5py + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +python $IMPL/generate_z5py.py diff --git a/implementations/z5py/environment.yml b/implementations/z5py/environment.yml new file mode 100644 index 00000000..71cadd41 --- /dev/null +++ b/implementations/z5py/environment.yml @@ -0,0 +1,7 @@ +channels: + - conda-forge + - defaults +dependencies: + - z5py + - python + - scikit-image diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh new file mode 100755 index 00000000..a5e0706a --- /dev/null +++ b/implementations/zarr-python/driver.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# + +ENVNAME=ZI_zarr_python + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +python $IMPL/generate_zarr.py diff --git a/implementations/zarr-python/environment.yml b/implementations/zarr-python/environment.yml index e6894551..59af0770 100644 --- a/implementations/zarr-python/environment.yml +++ b/implementations/zarr-python/environment.yml @@ -2,14 +2,6 @@ channels: - conda-forge - defaults dependencies: - - openimageio - - python == 3.7.9 + - python - scikit-image - - pytest - - zarr >= 2.8.3 - - pip - - pandas - - tabulate - - pip: - - pyn5 - - git+https://github.com/grlee77/zarrita.git@codec_update + - zarr diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh new file mode 100755 index 00000000..0e968b75 --- /dev/null +++ b/implementations/zarrita/driver.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# + +ENVNAME=ZI_zarritz + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +python $IMPL/generate_zarrita.py diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index e6894551..ed0c053f 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -2,14 +2,8 @@ channels: - conda-forge - defaults dependencies: - - openimageio - - python == 3.7.9 + - python - scikit-image - - pytest - - zarr >= 2.8.3 - pip - - pandas - - tabulate - pip: - - pyn5 - git+https://github.com/grlee77/zarrita.git@codec_update From 4a93ff4e8cf1023a9f46478b6e03557f36b08024 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:11:50 +0200 Subject: [PATCH 08/87] Fix GHA variable reference --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 04efcb93..8b833226 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: with: auto-update-conda: true channels: conda-forge,ome - environment-file: ${{ implementation}}/environment.yml + environment-file: ${{ matrix.implementation }}/environment.yml python-version: ${{ matrix.python-version }} env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -59,4 +59,4 @@ jobs: - name: Run tests shell: bash -l {0} - run: make ${{ implementation }} + run: make ${{ matrix.implementation }} From 07b0382d4d0db8679a2e65b9fd2cfe89b59ffdfa Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:14:46 +0200 Subject: [PATCH 09/87] fix if: clause in GHA --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b833226..46780954 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: - name: Install zarr dev (optional) shell: bash -l {0} - if: matrix.config.zarr-python == "pre" + if: ${{ matrix.config.zarr-python == "pre" }} run: | python -m pip install git+https://github.com/zarr-developers/zarr-python.git From 1e63230107d580f3ff1faa840f479a8ff9592a08 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:15:59 +0200 Subject: [PATCH 10/87] Use single quotes in GHA --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 46780954..bc394fcb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: - name: Install zarr dev (optional) shell: bash -l {0} - if: ${{ matrix.config.zarr-python == "pre" }} + if: ${{ matrix.config.zarr-python == 'pre' }} run: | python -m pip install git+https://github.com/zarr-developers/zarr-python.git From d98899577ea79f2c282eeb50b49792faab07290e Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:17:01 +0200 Subject: [PATCH 11/87] close quotes in GHA --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc394fcb..80a1bde8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: List implementations id: set-matrix - run: echo "::set-output name=matrix::$(cd implementations && ls -1" + run: echo "::set-output name=matrix::$(cd implementations && ls -1)" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From ee33ce24429a1b82b105a192c17c4db5101c55ea Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:22:33 +0200 Subject: [PATCH 12/87] Convert file-listing to JSON with jq --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80a1bde8..e312e247 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: List implementations id: set-matrix - run: echo "::set-output name=matrix::$(cd implementations && ls -1)" + run: echo "::set-output name=matrix::$(cd implementations && ls -1 | jq -ncR '[inputs]')" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -26,7 +26,7 @@ jobs: matrix: platform: [ubuntu-latest] python-version: [3.9] - implementation: ${{ needs.get_implementations.outputs.matrix }} + implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} zarr-python: [released, pre] steps: From ac90fd4813edd26ab4d4319f62ccedcc213ba63b Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:24:10 +0200 Subject: [PATCH 13/87] Add directory prefix --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e312e247..e8b7dd6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: with: auto-update-conda: true channels: conda-forge,ome - environment-file: ${{ matrix.implementation }}/environment.yml + environment-file: implementations/${{ matrix.implementation }}/environment.yml python-version: ${{ matrix.python-version }} env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -59,4 +59,4 @@ jobs: - name: Run tests shell: bash -l {0} - run: make ${{ matrix.implementation }} + run: make implementations/${{ matrix.implementation }} From f5e89c77c44a871acf8e84aa4ebaa1a479053d43 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:28:26 +0200 Subject: [PATCH 14/87] Add impl to matrix name --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e8b7dd6c..4dc97b07 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} test_implementations: - name: ${{ matrix.platform }} ${{ matrix.python-version }} + name: ${{ matrix.implementation }}-${{ matrix.platform }}-py${{ matrix.python-version }}-${{ matrix.zarr-python }} runs-on: ${{ matrix.platform }} needs: get_implementations From f777577684179c59d395ba91b6fe2df3f8bd7192 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 12 Oct 2022 11:47:34 +0200 Subject: [PATCH 15/87] Add blosc for java components --- implementations/jzarr/environment.yml | 1 + implementations/n5-java/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/implementations/jzarr/environment.yml b/implementations/jzarr/environment.yml index ea0509fe..a7a62cbb 100644 --- a/implementations/jzarr/environment.yml +++ b/implementations/jzarr/environment.yml @@ -4,3 +4,4 @@ channels: dependencies: - openjdk - maven + - blosc diff --git a/implementations/n5-java/environment.yml b/implementations/n5-java/environment.yml index ea0509fe..a7a62cbb 100644 --- a/implementations/n5-java/environment.yml +++ b/implementations/n5-java/environment.yml @@ -4,3 +4,4 @@ channels: dependencies: - openjdk - maven + - blosc From 54ac97f6b3b0b118325cce3e31b88fd89ca0f7c3 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 11:45:19 +0100 Subject: [PATCH 16/87] js: handle missing data dir --- implementations/js/src/index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/implementations/js/src/index.js b/implementations/js/src/index.js index 02998710..b3f3c2ec 100644 --- a/implementations/js/src/index.js +++ b/implementations/js/src/index.js @@ -38,7 +38,10 @@ async function generateZarrFormat(codecIds = ["gzip", "blosc", "zlib", null]) { const path = p.join("..", "..", "data", "js.zr"); const img = imread(p.join("..", "..", "data", "reference_image.png")); - fs.rmdirSync(path, { recursive: true, force: true }); + if (fs.existsSync(path)) { + fs.rmdirSync(path, { recursive: true, force: true }); + } + const grp = await open(path); for (const id of codecIds) { const config = id ? STR_TO_COMPRESSOR[id] : null; From 1afc30ac50b904e10ad7947d63ae7de6d945e7bc Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 12:00:52 +0100 Subject: [PATCH 17/87] gha: only build pre if released works --- .github/workflows/build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4dc97b07..53929df0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: platform: [ubuntu-latest] python-version: [3.9] implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} - zarr-python: [released, pre] + zarr-python: [released] steps: - name: Checkout @@ -60,3 +60,9 @@ jobs: - name: Run tests shell: bash -l {0} run: make implementations/${{ matrix.implementation }} + + test_implementations_pre: + uses: ./github/workflows/build.yml + needs: test_implementations + with: + python: [pre] From 85ee2d9d269062d096be7626f24f0c6db8dfff95 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 12:47:59 +0100 Subject: [PATCH 18/87] gha: split reusable component to impl.yml --- .github/workflows/build.yml | 64 +++++++++++++------------------------ .github/workflows/impl.yml | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/impl.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53929df0..d2478774 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,53 +16,35 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - test_implementations: - name: ${{ matrix.implementation }}-${{ matrix.platform }}-py${{ matrix.python-version }}-${{ matrix.zarr-python }} - runs-on: ${{ matrix.platform }} - needs: get_implementations - + test_implementations_released: strategy: fail-fast: false matrix: + implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} platform: [ubuntu-latest] python-version: [3.9] - implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} zarr-python: [released] + needs: get_implementations + uses: ./.github/workflows/impl.yml + with: + implementation: ${{ matrix.implementation }} + platform: ${{ matrix.platform }} + python-version: ${{ matrix.python-version }} + zarr-python: ${{ matrix.zarr-python }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v1 - with: - auto-update-conda: true - channels: conda-forge,ome - environment-file: implementations/${{ matrix.implementation }}/environment.yml - python-version: ${{ matrix.python-version }} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - - - name: Install zarr dev (optional) - shell: bash -l {0} - if: ${{ matrix.config.zarr-python == 'pre' }} - run: | - python -m pip install git+https://github.com/zarr-developers/zarr-python.git - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Run tests - shell: bash -l {0} - run: make implementations/${{ matrix.implementation }} - + # Re-run the same configurations but with a pre-release of Zarr test_implementations_pre: - uses: ./github/workflows/build.yml - needs: test_implementations + strategy: + fail-fast: false + matrix: + implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} + platform: [ubuntu-latest] + python-version: [3.9] + zarr-python: [pre] + needs: test_implementations_released + uses: ./.github/workflows/impl.yml with: - python: [pre] + implementation: ${{ matrix.implementation }} + platform: ${{ matrix.platform }} + python-version: ${{ matrix.python-version }} + zarr-python: ${{ matrix.zarr-python }} diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml new file mode 100644 index 00000000..877a96af --- /dev/null +++ b/.github/workflows/impl.yml @@ -0,0 +1,56 @@ +--- +name: Implementation tests + +on: + workflow_call: + inputs: + implementation: + required: true + type: string + platform: + required: true + type: string + python-version: + required: true + type: string + zarr-python: + required: true + type: string + +jobs: + + test_implementations: + name: ${{ inputs.implementation }}-${{ inputs.platform }}-py${{ inputs.python-version }}-${{ inputs.zarr-python }} + runs-on: ${{ inputs.platform }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v1 + with: + auto-update-conda: true + channels: conda-forge,ome + environment-file: implementations/${{ inputs.implementation }}/environment.yml + python-version: ${{ inputs.python-version }} + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + - name: Install zarr dev (optional) + shell: bash -l {0} + if: ${{ inputs.config.zarr-python == 'pre' }} + run: | + python -m pip install git+https://github.com/zarr-developers/zarr-python.git + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Run tests + shell: bash -l {0} + run: make implementations/${{ inputs.implementation }} From 5bda6450a876b19592998d17e7b272bc540ad69c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 13:11:43 +0100 Subject: [PATCH 19/87] gha: drop use of 'set-output' --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2478774..7c959a03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: List implementations id: set-matrix - run: echo "::set-output name=matrix::$(cd implementations && ls -1 | jq -ncR '[inputs]')" + run: echo "matrix=$(cd implementations && ls -1 | jq -ncR '[inputs]')" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From c710fbeb07bf277a48007e56800a70f5ab7e8da1 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 13:19:28 +0100 Subject: [PATCH 20/87] gha: Attempt (cached) mamba --- .github/workflows/impl.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index 877a96af..b19d66f2 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -27,12 +27,24 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Cache conda + uses: actions/cache@v2 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles( format('{0}/{1}/{2}', 'implementations/', inputs.implementation, '/environment.yml') ) }} + - name: Setup miniconda uses: conda-incubator/setup-miniconda@v1 with: auto-update-conda: true - channels: conda-forge,ome + channels: conda-forge,defaults + channel-priority: true environment-file: implementations/${{ inputs.implementation }}/environment.yml + mamba-version: "*" python-version: ${{ inputs.python-version }} env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true From fd686c38aff51163ca2350c06fb0d3c2dd161a6e Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 13:30:39 +0100 Subject: [PATCH 21/87] xtensor: attempt unpinning --- implementations/xtensor_zarr/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index f71db734..25ce02c7 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -4,4 +4,4 @@ channels: dependencies: - make - cmake - - xtensor-zarr >=0.0.8|=0.0.7=*_1 + - xtensor-zarr From 30e6633335b47a2725613d50d068a9f45320cb38 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 13:31:24 +0100 Subject: [PATCH 22/87] zarrita: install numcodecs from conda-forge --- implementations/zarrita/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index ed0c053f..8f72ae2a 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -4,6 +4,7 @@ channels: dependencies: - python - scikit-image + - numcodecs - pip - pip: - git+https://github.com/grlee77/zarrita.git@codec_update From 98a28f918a6e2eda071a5953ffd9e771d24ec8a4 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 14:01:23 +0100 Subject: [PATCH 23/87] pyn5: try installing from git --- implementations/pyn5/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/pyn5/environment.yml b/implementations/pyn5/environment.yml index 27c88d00..44fbebf7 100644 --- a/implementations/pyn5/environment.yml +++ b/implementations/pyn5/environment.yml @@ -8,4 +8,4 @@ dependencies: - scikit-image - pip - pip: - - pyn5 + - "git+https://github.com/pattonw/rust-pyn5@master#egg=pyn5" From e708ae2461468e31b6c495ff87dc99bedecd1c2e Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 14:07:34 +0100 Subject: [PATCH 24/87] xtensor: install gdal --- implementations/xtensor_zarr/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 25ce02c7..838a25be 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -4,4 +4,5 @@ channels: dependencies: - make - cmake + - gdal - xtensor-zarr From 11a2686a84af560c90fa6deffea31b1b0fe61a0f Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 14:26:00 +0100 Subject: [PATCH 25/87] Revert "xtensor: attempt unpinning" This reverts commit fd686c38aff51163ca2350c06fb0d3c2dd161a6e. Attempts to solve: ``` Looking for: ['make', 'cmake', 'gdal', 'xtensor-zarr'] Encountered problems while solving: - package xtensor-zarr-0.0.1-h0efe328_0 requires xtensor-io >=0.10.1,<0.11.0a0, but none of the providers can be installed ``` --- implementations/xtensor_zarr/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 838a25be..255bccfd 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -5,4 +5,4 @@ dependencies: - make - cmake - gdal - - xtensor-zarr + - xtensor-zarr >=0.0.8|=0.0.7=*_1 From 1bba956f9f182c627039d4eb1360f7b69df2fd81 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 14:40:08 +0100 Subject: [PATCH 26/87] pyn5: move up one more directory to find 'data' --- implementations/pyn5/generate_pyn5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/pyn5/generate_pyn5.py b/implementations/pyn5/generate_pyn5.py index 99b10c34..6861a8ce 100755 --- a/implementations/pyn5/generate_pyn5.py +++ b/implementations/pyn5/generate_pyn5.py @@ -9,7 +9,7 @@ def generate_n5_format(compressors=pyn5.CompressionType): here = Path(__file__).resolve().parent - data_dir = here.parent / "data" + data_dir = here.parent.parent / "data" path = data_dir / "pyn5.n5" im = imread(data_dir / "reference_image.png") From 61491e27d0c18e665e6680d3bfcce8b949ee9c9f Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 15:17:21 +0100 Subject: [PATCH 27/87] xtensor: pin gdal to 3.0.4 --- implementations/xtensor_zarr/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 255bccfd..11721422 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -4,5 +4,5 @@ channels: dependencies: - make - cmake - - gdal + - gdal ==3.0.4 - xtensor-zarr >=0.0.8|=0.0.7=*_1 From 3d2014e2cd27d12b4ae74f1d69168ba7c99bc109 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 15:37:14 +0100 Subject: [PATCH 28/87] xtensor: pin to py38 --- implementations/xtensor_zarr/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 11721422..8ce8d7a8 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -2,6 +2,7 @@ channels: - conda-forge - defaults dependencies: + - python ==3.8 - make - cmake - gdal ==3.0.4 From 2a2a8a160a78e049f918e9946f472b965f1731f1 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 1 Nov 2022 15:49:42 +0100 Subject: [PATCH 29/87] gha: add additional dep --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c959a03..6c157394 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: platform: [ubuntu-latest] python-version: [3.9] zarr-python: [released] - needs: get_implementations + needs: [get_implementations] uses: ./.github/workflows/impl.yml with: implementation: ${{ matrix.implementation }} @@ -41,7 +41,7 @@ jobs: platform: [ubuntu-latest] python-version: [3.9] zarr-python: [pre] - needs: test_implementations_released + needs: [get_implementations, test_implementations_released] uses: ./.github/workflows/impl.yml with: implementation: ${{ matrix.implementation }} From 5740527e81b6e0236cae7f7d582a297676588d57 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Mon, 27 Feb 2023 17:09:25 +0100 Subject: [PATCH 30/87] Start adding Rarr implementation --- implementations/Rarr/generate_Rarr.R | 23 +++++++++++++++++++++++ implementations/Rarr/generate_data.sh | 6 ++++++ implementations/Rarr/install_packages.R | 4 ++++ test/test_read_all.py | 15 +++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 implementations/Rarr/generate_Rarr.R create mode 100755 implementations/Rarr/generate_data.sh create mode 100644 implementations/Rarr/install_packages.R diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R new file mode 100644 index 00000000..360d6f12 --- /dev/null +++ b/implementations/Rarr/generate_Rarr.R @@ -0,0 +1,23 @@ +library(png) +library(Rarr) + +img <- png::readPNG(source = "../../data/reference_image.png") + +chunk_dim <- c(100, 100, 1) + +for(codec in c("blosc", "zlib")) { + + output_name <- file.path("../../data", sprintf("%s.zr", codec)) + + dir.create(output_name, recursive = TRUE) + + compressor <- switch (codec, + "blosc" = Rarr:::use_blosc(), + "zlib" = Rarr:::use_zlib() + ) + + write_zarr_array(x = img, + zarr_array_path = output_name, + chunk_dim = chunk_dim, + compressor = compressor) +} \ No newline at end of file diff --git a/implementations/Rarr/generate_data.sh b/implementations/Rarr/generate_data.sh new file mode 100755 index 00000000..5ba36464 --- /dev/null +++ b/implementations/Rarr/generate_data.sh @@ -0,0 +1,6 @@ +# cd to this directory +# https://stackoverflow.com/a/6393573/2700168 +cd "${0%/*}" + +Rscript install_packages.R +Rscript generate_data.R diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R new file mode 100644 index 00000000..63f5c330 --- /dev/null +++ b/implementations/Rarr/install_packages.R @@ -0,0 +1,4 @@ +required_pkgs <- c("remotes", "png") +install.packages(required_pkgs) + +remotes::install_github("grimbough/Rarr") diff --git a/test/test_read_all.py b/test/test_read_all.py index f226d829..75e6a2c4 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -73,8 +73,23 @@ "zarr-v3": ["blosc", "gzip", "raw", "zlib"], "N5": [], }, + "Rarr": { + "zarr": ["blosc"], + "zarr-v3": [], + "N5": [], + }, } +def read_with_Rarr(fpath, ds_name, nested=None): + + cmd = ( + f"implementations/jzarr/generate_data.sh " + f"-verify {str(fpath)} {ds_name}" + ) + + # will raise subprocess.CalledProcessError if return code is not 0 + subprocess.check_output(cmd, shell=True) + return None def read_with_jzarr(fpath, ds_name, nested=None): if ds_name == "blosc": From 7834df07bedb626431008be1e726116aec5da768 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 10:30:43 +0100 Subject: [PATCH 31/87] Add driver.sh for Rarr --- implementations/Rarr/driver.sh | 12 ++++++++++++ implementations/Rarr/environment.yml | 6 ++++++ implementations/Rarr/generate_Rarr.R | 6 +++--- implementations/Rarr/generate_data.sh | 6 ------ 4 files changed, 21 insertions(+), 9 deletions(-) create mode 100755 implementations/Rarr/driver.sh create mode 100644 implementations/Rarr/environment.yml delete mode 100755 implementations/Rarr/generate_data.sh diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh new file mode 100755 index 00000000..26f0f257 --- /dev/null +++ b/implementations/Rarr/driver.sh @@ -0,0 +1,12 @@ +ENVNAME=ZI_Rarr + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) +. $ROOT/.conda_driver.sh +create_or_activate + +cd "${IMPL}" + +Rscript install_packages.R +Rscript generate_data.R diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml new file mode 100644 index 00000000..c1f8a6c3 --- /dev/null +++ b/implementations/Rarr/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - r-base + diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R index 360d6f12..61142484 100644 --- a/implementations/Rarr/generate_Rarr.R +++ b/implementations/Rarr/generate_Rarr.R @@ -7,9 +7,9 @@ chunk_dim <- c(100, 100, 1) for(codec in c("blosc", "zlib")) { - output_name <- file.path("../../data", sprintf("%s.zr", codec)) + output_name <- file.path("../../data", "Rarr.zr", codec) - dir.create(output_name, recursive = TRUE) + dir.create(output_name, recursive = TRUE, showWarnings = FALSE) compressor <- switch (codec, "blosc" = Rarr:::use_blosc(), @@ -20,4 +20,4 @@ for(codec in c("blosc", "zlib")) { zarr_array_path = output_name, chunk_dim = chunk_dim, compressor = compressor) -} \ No newline at end of file +} diff --git a/implementations/Rarr/generate_data.sh b/implementations/Rarr/generate_data.sh deleted file mode 100755 index 5ba36464..00000000 --- a/implementations/Rarr/generate_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -# cd to this directory -# https://stackoverflow.com/a/6393573/2700168 -cd "${0%/*}" - -Rscript install_packages.R -Rscript generate_data.R From 21373572f9b4fe84bc19e5c13af5a834c44a498e Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 10:58:55 +0100 Subject: [PATCH 32/87] Add R version to environment --- implementations/Rarr/environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index c1f8a6c3..29efcacc 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -2,5 +2,4 @@ channels: - conda-forge - defaults dependencies: - - r-base - + - r-base=4.2.2 \ No newline at end of file From c5854ed60fc445c33ad189ed368e53fd7dae5b1e Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 11:19:44 +0100 Subject: [PATCH 33/87] Install packages using conda --- implementations/Rarr/environment.yml | 4 +++- implementations/Rarr/install_packages.R | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index 29efcacc..ce04ada2 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -2,4 +2,6 @@ channels: - conda-forge - defaults dependencies: - - r-base=4.2.2 \ No newline at end of file + - r-base=4.2.2 + - r-png + - r-remotes diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 63f5c330..6e47ad5a 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -1,4 +1,2 @@ -required_pkgs <- c("remotes", "png") -install.packages(required_pkgs) - +chooseCRANmirror(ind = 1) remotes::install_github("grimbough/Rarr") From 8a75823b87f0e3ec087cddde31bcb0cc7d9d5aab Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 11:32:07 +0100 Subject: [PATCH 34/87] No need for PAT --- implementations/Rarr/install_packages.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 6e47ad5a..3540a773 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -1,2 +1,2 @@ chooseCRANmirror(ind = 1) -remotes::install_github("grimbough/Rarr") +remotes::install_git("https://git.bioconductor.org/packages/Rarr") From e4e5565c2a4b2acbad0b79edd7f52931edcf650e Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 12:23:05 +0100 Subject: [PATCH 35/87] Install r package binaries --- implementations/Rarr/driver.sh | 2 +- implementations/Rarr/environment.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index 26f0f257..95ed84ac 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -9,4 +9,4 @@ create_or_activate cd "${IMPL}" Rscript install_packages.R -Rscript generate_data.R +Rscript generate_Rarr.R diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index ce04ada2..b5e87953 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -2,6 +2,19 @@ channels: - conda-forge - defaults dependencies: + ## required for generating data - r-base=4.2.2 - r-png + ## R-pkgs needed for installing Rarr - will be simplified once in Bioc - r-remotes + - r-stringr + - r-httr + - r-jsonlite + - r-paws.storage + - r-r.utils + - r-curl + ## compilers for Rarr codecs + - gxx + - gcc +variables: + LC_ALL: C.UTF-8 \ No newline at end of file From ec5708ae9409b02796728fc96941a9a7611912af Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 28 Feb 2023 20:47:59 +0100 Subject: [PATCH 36/87] Functions to verify data --- implementations/Rarr/environment.yml | 2 ++ implementations/Rarr/install_packages.R | 2 +- implementations/Rarr/verify_data.R | 17 +++++++++++++++++ test/test_read_all.py | 21 +++++++++------------ 4 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 implementations/Rarr/verify_data.R diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index b5e87953..8314148e 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -5,6 +5,8 @@ dependencies: ## required for generating data - r-base=4.2.2 - r-png + ## require for checking data + - r-reticulate ## R-pkgs needed for installing Rarr - will be simplified once in Bioc - r-remotes - r-stringr diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 3540a773..938168b5 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -1,2 +1,2 @@ chooseCRANmirror(ind = 1) -remotes::install_git("https://git.bioconductor.org/packages/Rarr") +remotes::install_git("https://git.bioconductor.org/packages/Rarr", upgrade = "never") diff --git a/implementations/Rarr/verify_data.R b/implementations/Rarr/verify_data.R new file mode 100644 index 00000000..7f1ceb1f --- /dev/null +++ b/implementations/Rarr/verify_data.R @@ -0,0 +1,17 @@ +args <- commandArgs(trailingOnly = TRUE) + +stopifnot(length(args) == 2) + +fpath <- args[1] +ds_name <- args[2] + +library(Rarr) +library(reticulate) +np <- import("numpy") + +zarr_array <- file.path(fpath, ds_name) + +x <- read_zarr_array(zarr_array_path = zarr_array) + +a <- np$array(x) +np$savez("a.npz", a) diff --git a/test/test_read_all.py b/test/test_read_all.py index 75e6a2c4..53bcce5a 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -74,23 +74,12 @@ "N5": [], }, "Rarr": { - "zarr": ["blosc"], + "zarr": ["blosc", "zlib"], "zarr-v3": [], "N5": [], }, } -def read_with_Rarr(fpath, ds_name, nested=None): - - cmd = ( - f"implementations/jzarr/generate_data.sh " - f"-verify {str(fpath)} {ds_name}" - ) - - # will raise subprocess.CalledProcessError if return code is not 0 - subprocess.check_output(cmd, shell=True) - return None - def read_with_jzarr(fpath, ds_name, nested=None): if ds_name == "blosc": ds_name = "blosc/lz4" @@ -155,6 +144,13 @@ def read_with_xtensor_zarr(fpath, ds_name, nested): subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) return np.load(fname)["a"] +def read_with_Rarr(fpath, ds_name, nested=None): + + fname = "a.npz" + if os.path.exists(fname): + os.remove(fname) + subprocess.check_call(["Rscript", "implementations/Rarr/verify_data.R", fpath, ds_name]) + return np.load(fname)["a"] EXTENSIONS = {"zarr": ".zr", "N5": ".n5", "zarr-v3": ".zr3"} HERE = Path(__file__).resolve().parent @@ -252,6 +248,7 @@ def _get_read_fn(reading_library): "z5py": read_with_z5py, "zarrita": read_with_zarrita, "xtensor_zarr": read_with_xtensor_zarr, + "Rarr": read_with_Rarr, }[reading_library] return read_fn From 4a565059af8fffebc77166a89b3d539af96752b6 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Wed, 1 Mar 2023 10:11:31 +0100 Subject: [PATCH 37/87] Upgrade actions versions to use nodejs16 --- .github/workflows/impl.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index b19d66f2..413059e5 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v3 - name: Cache conda - uses: actions/cache@v2 + uses: actions/cache@v3 env: # Increase this value to reset cache if etc/example-environment.yml has not changed CACHE_NUMBER: 0 @@ -38,7 +38,7 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles( format('{0}/{1}/{2}', 'implementations/', inputs.implementation, '/environment.yml') ) }} - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v1 + uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true channels: conda-forge,defaults @@ -56,7 +56,7 @@ jobs: python -m pip install git+https://github.com/zarr-developers/zarr-python.git - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} From ae6d21f66a3f5a16191584ce3128b6180ee15e6d Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Wed, 1 Mar 2023 16:06:15 +0100 Subject: [PATCH 38/87] Update Rarr validation script --- implementations/Rarr/generate_Rarr.R | 11 ++++----- implementations/Rarr/install_packages.R | 5 ++++- implementations/Rarr/verify_data.R | 2 +- implementations/Rarr/verify_data_internal.R | 25 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 implementations/Rarr/verify_data_internal.R diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R index 61142484..61958b39 100644 --- a/implementations/Rarr/generate_Rarr.R +++ b/implementations/Rarr/generate_Rarr.R @@ -1,23 +1,24 @@ -library(png) +library(loder) library(Rarr) -img <- png::readPNG(source = "../../data/reference_image.png") +img <- loder::readPng("../../data/reference_image.png") chunk_dim <- c(100, 100, 1) -for(codec in c("blosc", "zlib")) { +for(codec in c("blosc/lz4", "zlib")) { output_name <- file.path("../../data", "Rarr.zr", codec) dir.create(output_name, recursive = TRUE, showWarnings = FALSE) compressor <- switch (codec, - "blosc" = Rarr:::use_blosc(), + "blosc/lz4" = Rarr:::use_blosc(), "zlib" = Rarr:::use_zlib() ) write_zarr_array(x = img, zarr_array_path = output_name, chunk_dim = chunk_dim, - compressor = compressor) + compressor = compressor, + order = "C") } diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 938168b5..38728f0b 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -1,2 +1,5 @@ chooseCRANmirror(ind = 1) -remotes::install_git("https://git.bioconductor.org/packages/Rarr", upgrade = "never") +if (!require("Rarr", quietly = TRUE)) { + #remotes::install_git("https://git.bioconductor.org/packages/Rarr", upgrade = "never") + remotes::install_git("https://github.com/grimbough/Rarr.git/", ref = "writing-overhangs", upgrade = "never") +} diff --git a/implementations/Rarr/verify_data.R b/implementations/Rarr/verify_data.R index 7f1ceb1f..8b5fde72 100644 --- a/implementations/Rarr/verify_data.R +++ b/implementations/Rarr/verify_data.R @@ -14,4 +14,4 @@ zarr_array <- file.path(fpath, ds_name) x <- read_zarr_array(zarr_array_path = zarr_array) a <- np$array(x) -np$savez("a.npz", a) +np$save("a.npy", a) diff --git a/implementations/Rarr/verify_data_internal.R b/implementations/Rarr/verify_data_internal.R new file mode 100644 index 00000000..5063ba86 --- /dev/null +++ b/implementations/Rarr/verify_data_internal.R @@ -0,0 +1,25 @@ +args <- commandArgs(trailingOnly = TRUE) + +stopifnot(length(args) == 2) + +fpath <- args[1] +ds_name <- args[2] + +library(loder) +library(Rarr) + +## Read the reference image. We strip the "loder" class so it is a +## regular array, making the comparison easier +reference_img <- loder::readPng("data/reference_image.png") +class(reference_img) <- "array" + +## read the zarr input +zarr_array <- file.path(fpath, ds_name) +x <- read_zarr_array(zarr_array_path = zarr_array) + +## if the values are different quit +if(!isTRUE(all.equal(reference_img, x, check.attributes = FALSE))) { + stop("Input and reference image are different") +} + +quit(save = "no", status = 0) From f262aa1c36322d22aed26e6f3c6a407e03f3cefe Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 12:38:12 +0100 Subject: [PATCH 39/87] Update environment files for Rarr --- environment.yml | 9 +++++++++ implementations/Rarr/environment.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8f651a0f..e8079a27 100644 --- a/environment.yml +++ b/environment.yml @@ -20,3 +20,12 @@ dependencies: - pip: - pyn5 - git+https://github.com/grlee77/zarrita.git@codec_update + - r-base + - r-loder + - r-remotes + - r-stringr + - r-httr + - r-jsonlite + - r-paws.storage + - r-r.utils + - r-curl diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index 8314148e..408921f4 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -4,7 +4,7 @@ channels: dependencies: ## required for generating data - r-base=4.2.2 - - r-png + - r-loder ## require for checking data - r-reticulate ## R-pkgs needed for installing Rarr - will be simplified once in Bioc From bcecb024f7777cfba9591161c1f073da55fd21db Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 12:38:28 +0100 Subject: [PATCH 40/87] Add Rarr to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd0bda30..3b8fb655 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi * https://github.com/gzuidhof/zarr.js * https://github.com/zarr-developers/zarr-python * https://github.com/constantinpape/z5 - +* https://gihutb.com/grimbough/Rarr ## Other implementations From ffec3bf34248df1b9df9f408e60f07faf7919d50 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 13:18:18 +0100 Subject: [PATCH 41/87] r-loder causes conda conflicts --- environment.yml | 1 - implementations/Rarr/install_packages.R | 3 +++ implementations/Rarr/verify_data_internal.R | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e8079a27..add5b86a 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,6 @@ dependencies: - pyn5 - git+https://github.com/grlee77/zarrita.git@codec_update - r-base - - r-loder - r-remotes - r-stringr - r-httr diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 38728f0b..174695ed 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -1,4 +1,7 @@ chooseCRANmirror(ind = 1) +if (!require("loder", quietly = TRUE)) { + install.packages("loder", ) +} if (!require("Rarr", quietly = TRUE)) { #remotes::install_git("https://git.bioconductor.org/packages/Rarr", upgrade = "never") remotes::install_git("https://github.com/grimbough/Rarr.git/", ref = "writing-overhangs", upgrade = "never") diff --git a/implementations/Rarr/verify_data_internal.R b/implementations/Rarr/verify_data_internal.R index 5063ba86..99f3f266 100644 --- a/implementations/Rarr/verify_data_internal.R +++ b/implementations/Rarr/verify_data_internal.R @@ -5,6 +5,8 @@ stopifnot(length(args) == 2) fpath <- args[1] ds_name <- args[2] +source("implementations/Rarr/install_packages.R") + library(loder) library(Rarr) From f2d12ae974a18ebe1b3fd1296991efd30f714e6b Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 15:04:50 +0100 Subject: [PATCH 42/87] Update Rarr for nested writing --- implementations/Rarr/environment.yml | 3 +- implementations/Rarr/generate_Rarr.R | 38 +++++++++++++++---------- implementations/Rarr/install_packages.R | 2 +- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index 408921f4..be58dcca 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -5,8 +5,6 @@ dependencies: ## required for generating data - r-base=4.2.2 - r-loder - ## require for checking data - - r-reticulate ## R-pkgs needed for installing Rarr - will be simplified once in Bioc - r-remotes - r-stringr @@ -16,6 +14,7 @@ dependencies: - r-r.utils - r-curl ## compilers for Rarr codecs + - make - gxx - gcc variables: diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R index 61958b39..942a2a40 100644 --- a/implementations/Rarr/generate_Rarr.R +++ b/implementations/Rarr/generate_Rarr.R @@ -5,20 +5,28 @@ img <- loder::readPng("../../data/reference_image.png") chunk_dim <- c(100, 100, 1) -for(codec in c("blosc/lz4", "zlib")) { +for (sep in c("_flat", "_nested")) { + for (codec in c("blosc/lz4", "zlib")) { + + dir_name <- paste0("Rarr", sep, ".zr") + output_name <- file.path("../../data", dir_name, codec) - output_name <- file.path("../../data", "Rarr.zr", codec) - - dir.create(output_name, recursive = TRUE, showWarnings = FALSE) - - compressor <- switch (codec, - "blosc/lz4" = Rarr:::use_blosc(), - "zlib" = Rarr:::use_zlib() - ) - - write_zarr_array(x = img, - zarr_array_path = output_name, - chunk_dim = chunk_dim, - compressor = compressor, - order = "C") + dir.create(output_name, recursive = TRUE, showWarnings = FALSE) + + dim_sep <- ifelse(sep == "_flat", yes = ".", no = "/") + + compressor <- switch(codec, + "blosc/lz4" = Rarr:::use_blosc(), + "zlib" = Rarr:::use_zlib() + ) + + write_zarr_array( + x = img, + zarr_array_path = output_name, + chunk_dim = chunk_dim, + compressor = compressor, + dimension_separator = dim_sep, + order = "C" + ) + } } diff --git a/implementations/Rarr/install_packages.R b/implementations/Rarr/install_packages.R index 174695ed..651f46b6 100644 --- a/implementations/Rarr/install_packages.R +++ b/implementations/Rarr/install_packages.R @@ -4,5 +4,5 @@ if (!require("loder", quietly = TRUE)) { } if (!require("Rarr", quietly = TRUE)) { #remotes::install_git("https://git.bioconductor.org/packages/Rarr", upgrade = "never") - remotes::install_git("https://github.com/grimbough/Rarr.git/", ref = "writing-overhangs", upgrade = "never") + remotes::install_git("https://github.com/grimbough/Rarr.git/", ref = "main", upgrade = "never") } From f995439293a0d615dd98f9e72b72773577379f22 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 15:08:03 +0100 Subject: [PATCH 43/87] Update read_with_Rarr --- test/test_read_all.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/test/test_read_all.py b/test/test_read_all.py index 53bcce5a..177fec1f 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -136,21 +136,19 @@ def read_with_zarrita(fpath, ds_name, nested): return h["/" + ds_name][:] def read_with_xtensor_zarr(fpath, ds_name, nested): +def read_with_Rarr(fpath, ds_name, nested): if ds_name == "blosc": ds_name = "blosc/lz4" - fname = "a.npz" - if os.path.exists(fname): - os.remove(fname) - subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) - return np.load(fname)["a"] - -def read_with_Rarr(fpath, ds_name, nested=None): - - fname = "a.npz" - if os.path.exists(fname): - os.remove(fname) - subprocess.check_call(["Rscript", "implementations/Rarr/verify_data.R", fpath, ds_name]) - return np.load(fname)["a"] + + cmd = ( + f"Rscript implementations/Rarr/verify_data_internal.R " + f"{str(fpath)} {ds_name}" + ) + + # will raise subprocess.CalledProcessError if return code is not 0 + subprocess.check_output(cmd, shell=True) + return None + EXTENSIONS = {"zarr": ".zr", "N5": ".n5", "zarr-v3": ".zr3"} HERE = Path(__file__).resolve().parent From 20b3208e48a464a23ccce57022c6a40de117f75c Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 2 Mar 2023 15:15:00 +0100 Subject: [PATCH 44/87] Didn't mean to remove xtensor_zarr test --- test/test_read_all.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_read_all.py b/test/test_read_all.py index 177fec1f..84ec6fb2 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -136,7 +136,16 @@ def read_with_zarrita(fpath, ds_name, nested): return h["/" + ds_name][:] def read_with_xtensor_zarr(fpath, ds_name, nested): + if ds_name == "blosc": + ds_name = "blosc/lz4" + fname = "a.npz" + if os.path.exists(fname): + os.remove(fname) + subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) + return np.load(fname)["a"] + def read_with_Rarr(fpath, ds_name, nested): + if ds_name == "blosc": ds_name = "blosc/lz4" From 24328d51bf125002ea6ed507a73ba9d703abb05f Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Fri, 3 Mar 2023 16:43:57 +0100 Subject: [PATCH 45/87] Messed up indentation --- implementations/Rarr/generate_Rarr.R | 5 +++-- test/test_read_all.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R index 942a2a40..846b9d92 100644 --- a/implementations/Rarr/generate_Rarr.R +++ b/implementations/Rarr/generate_Rarr.R @@ -6,7 +6,7 @@ img <- loder::readPng("../../data/reference_image.png") chunk_dim <- c(100, 100, 1) for (sep in c("_flat", "_nested")) { - for (codec in c("blosc/lz4", "zlib")) { + for (codec in c("blosc/lz4", "zlib", "raw")) { dir_name <- paste0("Rarr", sep, ".zr") output_name <- file.path("../../data", dir_name, codec) @@ -17,7 +17,8 @@ for (sep in c("_flat", "_nested")) { compressor <- switch(codec, "blosc/lz4" = Rarr:::use_blosc(), - "zlib" = Rarr:::use_zlib() + "zlib" = Rarr:::use_zlib(), + "raw" = NULL ) write_zarr_array( diff --git a/test/test_read_all.py b/test/test_read_all.py index 84ec6fb2..8c1380f9 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -137,7 +137,7 @@ def read_with_zarrita(fpath, ds_name, nested): def read_with_xtensor_zarr(fpath, ds_name, nested): if ds_name == "blosc": - ds_name = "blosc/lz4" + ds_name = "blosc/lz4" fname = "a.npz" if os.path.exists(fname): os.remove(fname) From dc82cd8e930747adb8528ebc951f74011ae243c7 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Fri, 3 Mar 2023 16:47:05 +0100 Subject: [PATCH 46/87] Rarr can now read/write without compression --- test/test_read_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_read_all.py b/test/test_read_all.py index 8c1380f9..27fd141f 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -74,7 +74,7 @@ "N5": [], }, "Rarr": { - "zarr": ["blosc", "zlib"], + "zarr": ["blosc", "raw", "zlib"], "zarr-v3": [], "N5": [], }, From 88f0236cd8a5fef84ea1f401a9959519a2237c11 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Tue, 7 Mar 2023 16:51:14 +0100 Subject: [PATCH 47/87] Rarr will read/write gzip compressed chunks --- implementations/Rarr/generate_Rarr.R | 4 ++-- test/test_read_all.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/implementations/Rarr/generate_Rarr.R b/implementations/Rarr/generate_Rarr.R index 846b9d92..d41e7c7a 100644 --- a/implementations/Rarr/generate_Rarr.R +++ b/implementations/Rarr/generate_Rarr.R @@ -6,8 +6,7 @@ img <- loder::readPng("../../data/reference_image.png") chunk_dim <- c(100, 100, 1) for (sep in c("_flat", "_nested")) { - for (codec in c("blosc/lz4", "zlib", "raw")) { - + for (codec in c("blosc/lz4", "zlib", "gzip", "raw")) { dir_name <- paste0("Rarr", sep, ".zr") output_name <- file.path("../../data", dir_name, codec) @@ -18,6 +17,7 @@ for (sep in c("_flat", "_nested")) { compressor <- switch(codec, "blosc/lz4" = Rarr:::use_blosc(), "zlib" = Rarr:::use_zlib(), + "gzip" = Rarr::use_gzip(), "raw" = NULL ) diff --git a/test/test_read_all.py b/test/test_read_all.py index 27fd141f..e15f6b5e 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -74,7 +74,7 @@ "N5": [], }, "Rarr": { - "zarr": ["blosc", "raw", "zlib"], + "zarr": ["blosc", "gzip", "raw", "zlib"], "zarr-v3": [], "N5": [], }, From a23424a89996608b942ba4902346483a5da4bb2c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 9 Mar 2023 13:30:22 +0100 Subject: [PATCH 48/87] Try rolling back the top-level env changes --- environment.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/environment.yml b/environment.yml index add5b86a..8f651a0f 100644 --- a/environment.yml +++ b/environment.yml @@ -20,11 +20,3 @@ dependencies: - pip: - pyn5 - git+https://github.com/grlee77/zarrita.git@codec_update - - r-base - - r-remotes - - r-stringr - - r-httr - - r-jsonlite - - r-paws.storage - - r-r.utils - - r-curl From 02bfe6b4e7fc2c6cfe11a33c88d796eaf1ecfeb5 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 9 Mar 2023 13:30:36 +0100 Subject: [PATCH 49/87] Allow multiple commands in the drivers --- Makefile | 11 ++++++++-- implementations/.conda_driver.sh | 11 +++++++++- implementations/Rarr/driver.sh | 16 ++++++++++----- implementations/js/driver.sh | 16 ++++++++++----- implementations/jzarr/driver.sh | 28 ++++++++++++++++---------- implementations/n5-java/driver.sh | 17 ++++++++++------ implementations/pyn5/driver.sh | 11 +++++++--- implementations/xtensor_zarr/driver.sh | 28 ++++++++++++++++---------- implementations/z5py/driver.sh | 11 +++++++--- implementations/zarr-python/driver.sh | 11 +++++++--- implementations/zarrita/driver.sh | 11 +++++++--- 11 files changed, 118 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index feca752e..c76dbf2e 100644 --- a/Makefile +++ b/Makefile @@ -31,14 +31,21 @@ define mk-impl-target # For each of the items in our "implementations" directory, # create a target which depends on the reference data and # calls the "driver.sh" script. -.PHONY: $1 +# +.PHONY: $1 $1-destroy + $1: data/reference_image.png - bash $1/driver.sh + bash $1/driver.sh run # Alias in case the trailing slash is included .PHONE: $1/ $1/: $1 +$1-destroy: + bash $1/driver.sh destroy + +clean: $1-destroy + endef $(foreach impl,$(IMPLEMENTATIONS),$(eval $(call mk-impl-target,$(impl)))) diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index 5c834961..bfeee21d 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -7,7 +7,7 @@ set -e set -o pipefail -create_or_activate(){ +create_or_activate(){ if { conda env list | grep $ENVNAME; } >/dev/null 2>&1; then echo "Using $ENVNAME" else @@ -18,3 +18,12 @@ create_or_activate(){ echo "Activating $ENVNAME" conda activate $ENVNAME } + +destroy(){ + if { conda env list | grep $ENVNAME; } >/dev/null 2>&1; then + echo "Destroying $ENVNAME" + conda env remove -y -n $ENVNAME + else + echo "No known env: $ENVNAME" + fi +} diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index 95ed84ac..00327247 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -3,10 +3,16 @@ ENVNAME=ZI_Rarr # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -cd "${IMPL}" +run(){ + create_or_activate + + cd "${IMPL}" -Rscript install_packages.R -Rscript generate_Rarr.R + Rscript install_packages.R + Rscript generate_Rarr.R +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh index fc7296c8..e332cc8e 100755 --- a/implementations/js/driver.sh +++ b/implementations/js/driver.sh @@ -3,10 +3,16 @@ ENVNAME=ZI_js # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -cd "${IMPL}" +run(){ + create_or_activate + + cd "${IMPL}" -npm install -npm start + npm install + npm start +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/jzarr/driver.sh b/implementations/jzarr/driver.sh index 12c50848..67650c77 100755 --- a/implementations/jzarr/driver.sh +++ b/implementations/jzarr/driver.sh @@ -6,18 +6,24 @@ ENVNAME=ZI_jzarr # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -cd "${IMPL}" +run(){ + create_or_activate + + cd "${IMPL}" -MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} -mvn "${MVN_FLAGS}" clean package + MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} + mvn "${MVN_FLAGS}" clean package -java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && { - # Workaround for: https://github.com/bcdev/jzarr/issues/25 - find ../../data/jzarr* -name .zarray -exec sed -ibak 's/>u1/|u1/' {} \; -} || { - echo jzarr failed - exit 2 + java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && { + # Workaround for: https://github.com/bcdev/jzarr/issues/25 + find ../../data/jzarr* -name .zarray -exec sed -ibak 's/>u1/|u1/' {} \; + } || { + echo jzarr failed + exit 2 + } } + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh index c3546f60..28f11949 100755 --- a/implementations/n5-java/driver.sh +++ b/implementations/n5-java/driver.sh @@ -3,12 +3,17 @@ ENVNAME=ZI_n5_java # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -cd "${IMPL}" +run(){ + create_or_activate + + cd "${IMPL}" + MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} + mvn "${MVN_FLAGS}" clean package -MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} -mvn "${MVN_FLAGS}" clean package + java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App +} -java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh index 98fe7fa6..30fe3d24 100755 --- a/implementations/pyn5/driver.sh +++ b/implementations/pyn5/driver.sh @@ -7,7 +7,12 @@ ENVNAME=ZI_pyn5 # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -python $IMPL/generate_pyn5.py +run(){ + create_or_activate + python $IMPL/generate_pyn5.py +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/xtensor_zarr/driver.sh b/implementations/xtensor_zarr/driver.sh index 8f13113c..a5ceb87d 100755 --- a/implementations/xtensor_zarr/driver.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -3,17 +3,23 @@ ENVNAME=ZI_xtensor_zarr # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -cd "${IMPL}" +run(){ + create_or_activate + + cd "${IMPL}" -set +u # Due to GDAL_DATA + set +u # Due to GDAL_DATA -rm -rf build -mkdir build -cd build -export LDFLAGS="${LDFLAGS} -Wl,-rpath,$CONDA_PREFIX/lib -Wl,-rpath,$PWD" -export LINKFLAGS="${LDFLAGS}" -cmake .. -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_INSTALL_LIBDIR=lib -make run + rm -rf build + mkdir build + cd build + export LDFLAGS="${LDFLAGS} -Wl,-rpath,$CONDA_PREFIX/lib -Wl,-rpath,$PWD" + export LINKFLAGS="${LDFLAGS}" + cmake .. -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_INSTALL_LIBDIR=lib + make run +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh index 90103f85..ddabe3a5 100755 --- a/implementations/z5py/driver.sh +++ b/implementations/z5py/driver.sh @@ -7,7 +7,12 @@ ENVNAME=ZI_z5py # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -python $IMPL/generate_z5py.py +run(){ + create_or_activate + python $IMPL/generate_z5py.py +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index a5e0706a..4b4fed23 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -7,7 +7,12 @@ ENVNAME=ZI_zarr_python # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -python $IMPL/generate_zarr.py +run(){ + create_or_activate + python $IMPL/generate_zarr.py +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 0e968b75..36b5ed6a 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -6,7 +6,12 @@ ENVNAME=ZI_zarritz # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -. $ROOT/.conda_driver.sh -create_or_activate -python $IMPL/generate_zarrita.py +run(){ + create_or_activate + python $IMPL/generate_zarrita.py +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" From a73cf477ac61c8d871924edb3b18eef3df158736 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 9 Mar 2023 17:39:48 +0100 Subject: [PATCH 50/87] Try adding tbb to get xtensor to pass --- implementations/xtensor_zarr/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 8ce8d7a8..02183f06 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -6,4 +6,5 @@ dependencies: - make - cmake - gdal ==3.0.4 + - tbb - xtensor-zarr >=0.0.8|=0.0.7=*_1 From 2c4e954949fd8bc9fbbf1719f17017583d8b3c47 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 9 Mar 2023 17:41:16 +0100 Subject: [PATCH 51/87] Filter out xtensor for the moment --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index c76dbf2e..3a4bf72f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ IMPLEMENTATIONS=$(wildcard implementations/*) +IMPLEMENTATIONS=$(filter-out implementations/xtensor_zarr, $(IMPEMENTATIONS)) ifeq ($(TEST),) ################################################# # If TEST is not set, by default build everything, generate From 27686a6c6e53c6445a8bd9e4a6fd7d0179099ae7 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 9 Mar 2023 21:45:38 +0100 Subject: [PATCH 52/87] Update xtensor-zarr environment.yml - this works locally --- implementations/xtensor_zarr/environment.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/implementations/xtensor_zarr/environment.yml b/implementations/xtensor_zarr/environment.yml index 02183f06..a991ba1a 100644 --- a/implementations/xtensor_zarr/environment.yml +++ b/implementations/xtensor_zarr/environment.yml @@ -2,9 +2,8 @@ channels: - conda-forge - defaults dependencies: - - python ==3.8 - - make + - python=3.8 - cmake - - gdal ==3.0.4 - - tbb - - xtensor-zarr >=0.0.8|=0.0.7=*_1 + - xtensor-zarr>=0.0.7 + - gdal>=3.0.4 + - tbb=2020.2 From 986675e9a96fbffdda832d9122b5c084e9fa31ca Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 9 Mar 2023 21:46:16 +0100 Subject: [PATCH 53/87] Create initial enviroment with ZI_* name to save creating it twice --- .github/workflows/impl.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index 413059e5..058145a2 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -46,6 +46,7 @@ jobs: environment-file: implementations/${{ inputs.implementation }}/environment.yml mamba-version: "*" python-version: ${{ inputs.python-version }} + activate-environment: ZI_${{ inputs.implementation }} env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true From 813b387ae6919f20ace09e0ba440eac829292e42 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 9 Mar 2023 21:58:27 +0100 Subject: [PATCH 54/87] I think these are typos in the enivronment names. They match the directory names now. --- implementations/zarr-python/driver.sh | 2 +- implementations/zarrita/driver.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index 4b4fed23..2570a8b8 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -2,7 +2,7 @@ # # -ENVNAME=ZI_zarr_python +ENVNAME=ZI_zarr-python # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 36b5ed6a..756671ae 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -ENVNAME=ZI_zarritz +ENVNAME=ZI_zarrita # Standard bootstrapping IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) From ada020e339639405ce0b91cbd84a6c399fa54216 Mon Sep 17 00:00:00 2001 From: Mike Smith Date: Thu, 9 Mar 2023 23:26:24 +0100 Subject: [PATCH 55/87] Use python 3.8 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c157394..1e16b86a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: matrix: implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} platform: [ubuntu-latest] - python-version: [3.9] + python-version: [3.8] zarr-python: [released] needs: [get_implementations] uses: ./.github/workflows/impl.yml @@ -39,7 +39,7 @@ jobs: matrix: implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} platform: [ubuntu-latest] - python-version: [3.9] + python-version: [3.8] zarr-python: [pre] needs: [get_implementations, test_implementations_released] uses: ./.github/workflows/impl.yml From 0beec6c7c91b5a88a0fcb4b430fa6c83ba289d95 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 09:58:43 +0100 Subject: [PATCH 56/87] Add missing bash driver --- implementations/.bash_driver.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 implementations/.bash_driver.sh diff --git a/implementations/.bash_driver.sh b/implementations/.bash_driver.sh new file mode 100644 index 00000000..0fbe4c05 --- /dev/null +++ b/implementations/.bash_driver.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# This is re-usable driver code for all of the implementations. +# + +set -e +set -o pipefail + +argparse(){ + case "${1}" in + run) + echo "Running driver..." + run;; + destroy) + echo "Tearing down..." + destroy;; + *) + echo "Unknown command: ${1}" + exit 2;; + esac +} From f52abd79fa84bf1d79f4d90a5affce4e29e80fc0 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 09:58:54 +0100 Subject: [PATCH 57/87] Minor cleanups --- README.md | 1 - doc/development_overview.md | 2 +- test/test_read_all.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3b8fb655..fa3e33d1 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi ## Implementations currently tested - * https://github.com/bcdev/jzarr * https://github.com/saalfeldlab/n5-zarr * https://github.com/aschampion/rust-n5 diff --git a/doc/development_overview.md b/doc/development_overview.md index cc34d634..0d7d7d43 100644 --- a/doc/development_overview.md +++ b/doc/development_overview.md @@ -2,7 +2,7 @@ ## Overview -This repository contains scripts to generate datasets in zarr-v2, zarr-v3 and +This repository contains scripts to generate datasets in Zarr-v2, Zarr-v3 and N5 formats via a number of different implementations. Once generated, the test suite will attempt to have each library read all datasets with a supported format. diff --git a/test/test_read_all.py b/test/test_read_all.py index e15f6b5e..52a31bb3 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -153,7 +153,7 @@ def read_with_Rarr(fpath, ds_name, nested): f"Rscript implementations/Rarr/verify_data_internal.R " f"{str(fpath)} {ds_name}" ) - + # will raise subprocess.CalledProcessError if return code is not 0 subprocess.check_output(cmd, shell=True) return None From bed9ade885d2d9802e9da93a68341c53e2fe8170 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 10:43:25 +0100 Subject: [PATCH 58/87] Rename 'run' to 'write' --- .github/workflows/build.yml | 6 +++--- Makefile | 7 ++++++- implementations/.bash_driver.sh | 6 +++--- implementations/Rarr/driver.sh | 2 +- implementations/js/driver.sh | 2 +- implementations/jzarr/driver.sh | 2 +- implementations/n5-java/driver.sh | 2 +- implementations/pyn5/driver.sh | 2 +- implementations/xtensor_zarr/driver.sh | 2 +- implementations/z5py/driver.sh | 2 +- implementations/zarr-python/driver.sh | 2 +- implementations/zarrita/driver.sh | 2 +- 12 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e16b86a..03580e0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - test_implementations_released: + write_released: strategy: fail-fast: false matrix: @@ -33,7 +33,7 @@ jobs: zarr-python: ${{ matrix.zarr-python }} # Re-run the same configurations but with a pre-release of Zarr - test_implementations_pre: + write_pre: strategy: fail-fast: false matrix: @@ -41,7 +41,7 @@ jobs: platform: [ubuntu-latest] python-version: [3.8] zarr-python: [pre] - needs: [get_implementations, test_implementations_released] + needs: [get_implementations, write_released] uses: ./.github/workflows/impl.yml with: implementation: ${{ matrix.implementation }} diff --git a/Makefile b/Makefile index 3a4bf72f..e792610a 100644 --- a/Makefile +++ b/Makefile @@ -28,15 +28,20 @@ endif ########################################################## data/reference_image.png: python generate_reference_image.py +.PHONY: write + define mk-impl-target # For each of the items in our "implementations" directory, # create a target which depends on the reference data and # calls the "driver.sh" script. # + +write: $1 + .PHONY: $1 $1-destroy $1: data/reference_image.png - bash $1/driver.sh run + bash $1/driver.sh write # Alias in case the trailing slash is included .PHONE: $1/ diff --git a/implementations/.bash_driver.sh b/implementations/.bash_driver.sh index 0fbe4c05..444530de 100644 --- a/implementations/.bash_driver.sh +++ b/implementations/.bash_driver.sh @@ -8,9 +8,9 @@ set -o pipefail argparse(){ case "${1}" in - run) - echo "Running driver..." - run;; + write) + echo "Generating data..." + write;; destroy) echo "Tearing down..." destroy;; diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index 00327247..2a831ae8 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -4,7 +4,7 @@ ENVNAME=ZI_Rarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh index e332cc8e..d2bbc7e9 100755 --- a/implementations/js/driver.sh +++ b/implementations/js/driver.sh @@ -4,7 +4,7 @@ ENVNAME=ZI_js IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/jzarr/driver.sh b/implementations/jzarr/driver.sh index 67650c77..6965d90d 100755 --- a/implementations/jzarr/driver.sh +++ b/implementations/jzarr/driver.sh @@ -7,7 +7,7 @@ ENVNAME=ZI_jzarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh index 28f11949..a8ba97ba 100755 --- a/implementations/n5-java/driver.sh +++ b/implementations/n5-java/driver.sh @@ -4,7 +4,7 @@ ENVNAME=ZI_n5_java IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh index 30fe3d24..f0d83d25 100755 --- a/implementations/pyn5/driver.sh +++ b/implementations/pyn5/driver.sh @@ -8,7 +8,7 @@ ENVNAME=ZI_pyn5 IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate python $IMPL/generate_pyn5.py } diff --git a/implementations/xtensor_zarr/driver.sh b/implementations/xtensor_zarr/driver.sh index a5ceb87d..2399e702 100755 --- a/implementations/xtensor_zarr/driver.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -4,7 +4,7 @@ ENVNAME=ZI_xtensor_zarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh index ddabe3a5..184735d4 100755 --- a/implementations/z5py/driver.sh +++ b/implementations/z5py/driver.sh @@ -8,7 +8,7 @@ ENVNAME=ZI_z5py IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate python $IMPL/generate_z5py.py } diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index 2570a8b8..dd36a6f7 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -8,7 +8,7 @@ ENVNAME=ZI_zarr-python IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate python $IMPL/generate_zarr.py } diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 756671ae..53381c59 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -7,7 +7,7 @@ ENVNAME=ZI_zarrita IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -run(){ +write(){ create_or_activate python $IMPL/generate_zarrita.py } From d5647f11b7b23191fe341867a6d7e4436526ca42 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 10:44:46 +0100 Subject: [PATCH 59/87] Fix minor typos --- .github/workflows/impl.yml | 2 +- .gitignore | 2 +- Makefile | 2 +- implementations/Rarr/environment.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index 058145a2..000c21cd 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -52,7 +52,7 @@ jobs: - name: Install zarr dev (optional) shell: bash -l {0} - if: ${{ inputs.config.zarr-python == 'pre' }} + if: ${{ inputs.zarr-python == 'pre' }} run: | python -m pip install git+https://github.com/zarr-developers/zarr-python.git diff --git a/.gitignore b/.gitignore index 65a5d48f..c9cee769 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target* __pycache__/ -implemnetations/xtensor_zarr/build +implmenetations/xtensor_zarr/build node_modules/ report.* diff --git a/Makefile b/Makefile index e792610a..a66bb73c 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ $1: data/reference_image.png bash $1/driver.sh write # Alias in case the trailing slash is included -.PHONE: $1/ +.PHONY: $1/ $1/: $1 $1-destroy: diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index be58dcca..bf18c46a 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -18,4 +18,4 @@ dependencies: - gxx - gcc variables: - LC_ALL: C.UTF-8 \ No newline at end of file + LC_ALL: C.UTF-8 From 5447f137260593dd19472952322e196288545e69 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 10 Mar 2023 13:13:53 +0100 Subject: [PATCH 60/87] fixes for zarrita --- implementations/zarrita/environment.yml | 2 +- implementations/zarrita/generate_zarrita.py | 63 ++++++++++++--------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index 8f72ae2a..040e419e 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -7,4 +7,4 @@ dependencies: - numcodecs - pip - pip: - - git+https://github.com/grlee77/zarrita.git@codec_update + - git+https://github.com/scalableminds/zarrita.git@v3 diff --git a/implementations/zarrita/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py index 2765dea4..51ba26e0 100644 --- a/implementations/zarrita/generate_zarrita.py +++ b/implementations/zarrita/generate_zarrita.py @@ -1,42 +1,53 @@ import zarrita -import numcodecs from skimage.data import astronaut # choose chunks s.t. we do have overhanging edge-chunks -CHUNKS = (100, 100, 1) -STR_TO_COMPRESSOR = { - "gzip": numcodecs.GZip, - "blosc": numcodecs.Blosc, - "zlib": numcodecs.Zlib, +CHUNK_SHAPE = (100, 100, 1) +SHARD_SHAPE = (1000, 1000, 3) +STR_TO_CODEC = { + "gzip": zarrita.codecs.gzip_codec(), + "blosc": zarrita.codecs.blosc_codec(cname="lz4"), } -COMPRESSION_OPTIONS = {"blosc": {"cname": "lz4"}} -def generate_zr3_format(compressors=['gzip', 'blosc', 'zlib', None], - nested=True): +def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=True): im = astronaut() + fname = "zarrita" if nested: - chunk_separator = '/' - fname = 'data/zarrita_nested.zr3' + chunk_separator = "/" + fname += "_nested" else: - chunk_separator = '.' - fname = 'data/zarrita.zr3' - h = zarrita.create_hierarchy(fname) - for compressor in compressors: - copts = COMPRESSION_OPTIONS.get(compressor, {}) - if compressor is None: + chunk_separator = "." + if sharded: + fname += "_sharded" + store = zarrita.FileSystemStore("file://./data") + g = zarrita.Group.create(store, fname) + for codec in codecs: + if codec is None: name = "raw" - elif compressor == "blosc": - name = "%s/%s" % (compressor, copts.get("cname")) + elif codec == "blosc": + name = f"{codec}/{STR_TO_CODEC[codec].configuration.cname}" else: - name = compressor - compressor_impl = STR_TO_COMPRESSOR[compressor](**copts) if compressor is not None else None - a = h.create_array('/' + name, shape=im.shape, chunk_shape=CHUNKS, - chunk_separator=chunk_separator, dtype=im.dtype, - compressor=compressor_impl) + name = codec + codecs_impl = [STR_TO_CODEC[codec]] if codec is not None else [] + if sharded: + codecs_impl = [ + zarrita.codecs.sharding_codec( + chunk_shape=CHUNK_SHAPE, codecs=codecs_impl + ) + ] + a = g.create_array( + name, + shape=im.shape, + chunk_shape=SHARD_SHAPE if sharded else CHUNK_SHAPE, + chunk_key_encoding=("default", chunk_separator), + dtype=im.dtype, + codecs=codecs_impl, + ) a[...] = im -if __name__ == '__main__': +if __name__ == "__main__": for nested in [False, True]: - generate_zr3_format(nested=nested) + for sharded in [False, True]: + generate_zr3_format(nested=nested, sharded=sharded) From 113cc3929a06f1f5751107cf7696b9b6eee040c6 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 13:15:30 +0100 Subject: [PATCH 61/87] Simplify top-level environment.yml --- environment.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/environment.yml b/environment.yml index 8f651a0f..d986de51 100644 --- a/environment.yml +++ b/environment.yml @@ -2,21 +2,10 @@ channels: - conda-forge - defaults dependencies: - - openjdk - - maven - make - - cmake - - xtensor-zarr >=0.0.8|=0.0.7=*_1 - - openimageio - - nodejs - - z5py >= 2.0.10 - python == 3.7.9 - scikit-image - pytest - - zarr >= 2.8.3 - pip - pandas - tabulate - - pip: - - pyn5 - - git+https://github.com/grlee77/zarrita.git@codec_update From ccd36365144e9d69682df542f7e2c7b9807042a3 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 13:19:25 +0100 Subject: [PATCH 62/87] Update GHA to write then read in parallel --- .dockerignore | 5 +++++ .github/workflows/build.yml | 8 ++++++++ .github/workflows/impl.yml | 27 ++++++++++++++++++++++--- Dockerfile | 2 +- Makefile | 39 +++++++++++++++++++++++-------------- 5 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d3397f9d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +data/**/ +target* +.netcdf-java +.tensorstore +.git diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03580e0b..2aefd7a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,3 +48,11 @@ jobs: platform: ${{ matrix.platform }} python-version: ${{ matrix.python-version }} zarr-python: ${{ matrix.zarr-python }} + + read_released: + needs: [write_pre, write_released] + with: + implementation: ${{ matrix.implementation }} + platform: ${{ matrix.platform }} + python-version: ${{ matrix.python-version }} + zarr-python: ${{ matrix.zarr-python }} diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index 000c21cd..068450e5 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -16,10 +16,13 @@ on: zarr-python: required: true type: string + action: + required: true + type: string jobs: - test_implementations: + impl: name: ${{ inputs.implementation }}-${{ inputs.platform }}-py${{ inputs.python-version }}-${{ inputs.zarr-python }} runs-on: ${{ inputs.platform }} @@ -64,6 +67,24 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - - name: Run tests + # + # Choose read or write based on {{ action }} + # + + - name: Download previous output for testing + uses: actions/download-artifact@v3 + if: ${{ inputs.action == 'read' }} + with: + name: ${{ inputs.implementation }}-${{ inputs.zarr-python }}-data + path: data + + - name: Run build shell: bash -l {0} - run: make implementations/${{ inputs.implementation }} + run: make implementations/${{ inputs.implementation }}-${{ inputs.action }} + + - name: Save output for testing + uses: actions/upload-artifact@v3 + if: ${{ inputs.action == 'write' }} + with: + name: ${{ inputs.implementation }}-${{ inputs.zarr-python }}-data + path: data diff --git a/Dockerfile b/Dockerfile index ee308a08..d551d51d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM continuumio/miniconda3 WORKDIR /src COPY environment.yml /src/environment.yml -RUN apt-get update -y && apt install -y freeglut3-dev +RUN apt-get update -y && apt install -y freeglut3-dev # Unsure why freeglut is here RUN conda env create -f environment.yml -n z SHELL ["conda", "run", "-n", "z", "/bin/bash", "-c"] diff --git a/Makefile b/Makefile index a66bb73c..d07afb12 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,23 @@ IMPLEMENTATIONS=$(wildcard implementations/*) -IMPLEMENTATIONS=$(filter-out implementations/xtensor_zarr, $(IMPEMENTATIONS)) + +# +# The default target: +# (1) creates an environment for all implementations (if needed), +# (2) generates sample data for all implementations, and +# (3) runs tests against that sample data for all implementations. +# + +.PHONY: report + +report: data + python test/test_read_all.py + +.PHONY: test data ifeq ($(TEST),) ################################################# # If TEST is not set, by default build everything, generate # data for all implementations, and then run all pytests. -.PHONY: test data test: data pytest -v -k W @@ -16,7 +28,6 @@ else # its data and using the "-k W-" keyword to limit which pytests # get run -.PHONY: test data test: implementations/$(TEST) pytest -v -k W-$(TEST) @@ -28,25 +39,27 @@ endif ########################################################## data/reference_image.png: python generate_reference_image.py -.PHONY: write define mk-impl-target # For each of the items in our "implementations" directory, -# create a target which depends on the reference data and -# calls the "driver.sh" script. -# +# create targets which depend on the reference data and +# call the "driver.sh" script as necessary. -write: $1 +.PHONY: read write $1 $1/ $1-read $1-write $1-destroy clean -.PHONY: $1 $1-destroy +read: $1-read +write: $1-write -$1: data/reference_image.png +$1-write: data/reference_image.png bash $1/driver.sh write +# Alias for read & write +$1: $1-write $1-read + # Alias in case the trailing slash is included -.PHONY: $1/ $1/: $1 +# Additional target to cleanup the environment $1-destroy: bash $1/driver.sh destroy @@ -54,7 +67,3 @@ clean: $1-destroy endef $(foreach impl,$(IMPLEMENTATIONS),$(eval $(call mk-impl-target,$(impl)))) - -.PHONY: report -report: data - python test/test_read_all.py From 51c0f4484481d3750c37eea69e186342c52eb0fe Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 13:43:49 +0100 Subject: [PATCH 63/87] Fix new GHA setup --- .github/workflows/build.yml | 11 +++++++++++ .github/workflows/impl.yml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2aefd7a1..8ffd8db0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: platform: ${{ matrix.platform }} python-version: ${{ matrix.python-version }} zarr-python: ${{ matrix.zarr-python }} + action: write # Re-run the same configurations but with a pre-release of Zarr write_pre: @@ -48,11 +49,21 @@ jobs: platform: ${{ matrix.platform }} python-version: ${{ matrix.python-version }} zarr-python: ${{ matrix.zarr-python }} + action: write read_released: + strategy: + fail-fast: false + matrix: + implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} + platform: [ubuntu-latest] + python-version: [3.8] + zarr-python: [pre] needs: [write_pre, write_released] + uses: ./.github/workflows/impl.yml with: implementation: ${{ matrix.implementation }} platform: ${{ matrix.platform }} python-version: ${{ matrix.python-version }} zarr-python: ${{ matrix.zarr-python }} + action: read diff --git a/.github/workflows/impl.yml b/.github/workflows/impl.yml index 068450e5..3287f3df 100644 --- a/.github/workflows/impl.yml +++ b/.github/workflows/impl.yml @@ -23,7 +23,7 @@ on: jobs: impl: - name: ${{ inputs.implementation }}-${{ inputs.platform }}-py${{ inputs.python-version }}-${{ inputs.zarr-python }} + name: ${{ inputs.implementation }}-${{ inputs.action }}-${{ inputs.platform }}-py${{ inputs.python-version }}-${{ inputs.zarr-python }} runs-on: ${{ inputs.platform }} steps: From 1a11b06d72b33ddb1705edcfefafff6fa941d895 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 16:23:13 +0100 Subject: [PATCH 64/87] Run read regardless --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ffd8db0..6858f7e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,9 @@ jobs: platform: [ubuntu-latest] python-version: [3.8] zarr-python: [pre] + # Run the read job regardless once writing is complete needs: [write_pre, write_released] + if: ${{ always() }} uses: ./.github/workflows/impl.yml with: implementation: ${{ matrix.implementation }} From 53ac12fcc6ecabfa13773947c03bb89a1888b5d6 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 16:41:49 +0100 Subject: [PATCH 65/87] Disable write_pre --- .github/workflows/build.yml | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6858f7e5..b035ef27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,23 +33,24 @@ jobs: zarr-python: ${{ matrix.zarr-python }} action: write - # Re-run the same configurations but with a pre-release of Zarr - write_pre: - strategy: - fail-fast: false - matrix: - implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} - platform: [ubuntu-latest] - python-version: [3.8] - zarr-python: [pre] - needs: [get_implementations, write_released] - uses: ./.github/workflows/impl.yml - with: - implementation: ${{ matrix.implementation }} - platform: ${{ matrix.platform }} - python-version: ${{ matrix.python-version }} - zarr-python: ${{ matrix.zarr-python }} - action: write +# TMP: disabling to speed up testing +# # Re-run the same configurations but with a pre-release of Zarr +# write_pre: +# strategy: +# fail-fast: false +# matrix: +# implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} +# platform: [ubuntu-latest] +# python-version: [3.8] +# zarr-python: [pre] +# needs: [get_implementations, write_released] +# uses: ./.github/workflows/impl.yml +# with: +# implementation: ${{ matrix.implementation }} +# platform: ${{ matrix.platform }} +# python-version: ${{ matrix.python-version }} +# zarr-python: ${{ matrix.zarr-python }} +# action: write read_released: strategy: @@ -60,7 +61,8 @@ jobs: python-version: [3.8] zarr-python: [pre] # Run the read job regardless once writing is complete - needs: [write_pre, write_released] + # TMP needs: [write_pre, write_released] + needs: [write_released] if: ${{ always() }} uses: ./.github/workflows/impl.yml with: From 6ffc3e236ee2c3a6f807c7da9c26bda0614ae544 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 10 Mar 2023 20:16:12 +0100 Subject: [PATCH 66/87] Update implementations/zarrita/environment.yml Co-authored-by: Norman Rzepka --- implementations/zarrita/environment.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index 040e419e..d86507f2 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -2,9 +2,8 @@ channels: - conda-forge - defaults dependencies: - - python - - scikit-image - - numcodecs + - python==3.10 - pip - pip: - - git+https://github.com/scalableminds/zarrita.git@v3 + - scikit-image + - zarrita==0.1.0a1 From 245e79705dad183fd4fadd88858ea636d39307a0 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Mon, 13 Mar 2023 09:26:35 +0100 Subject: [PATCH 67/87] Update implementations/zarrita/environment.yml Co-authored-by: Norman Rzepka --- implementations/zarrita/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index d86507f2..05426b70 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -2,7 +2,7 @@ channels: - conda-forge - defaults dependencies: - - python==3.10 + - python - pip - pip: - scikit-image From 32dfa6ca6200ca1807a42613e4d581ee799494b2 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:12:21 +0200 Subject: [PATCH 68/87] Permit skipping an implementation by touching a .skip file --- Makefile | 2 +- README.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d07afb12..8d7c8ddf 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ read: $1-read write: $1-write $1-write: data/reference_image.png - bash $1/driver.sh write + @if test -e $1/.skip; then echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh write; fi # Alias for read & write $1: $1-write $1-read diff --git a/README.md b/README.md index fa3e33d1..6a741205 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,12 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi * [Zarr.jl](https://github.com/meggart/Zarr.jl) #42 * https://github.com/freeman-lab/zarr-js * [GDAL >= 3.4](https://gdal.org/drivers/raster/zarr.html) + +## Running locally + +* Create an environment: `conda env create -n ZI -f environment.yml` +* Run `make` + +## Other features + +* `touch implementations/{impl}/.skip` to disable an implementation From a7ffd0959e412859900fa768cc5e5ad5965ced30 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:14:47 +0200 Subject: [PATCH 69/87] Fix zarr-python, zarrita, js --- environment.yml | 2 +- implementations/js/package-lock.json | 54 +++++++++++++++------ implementations/zarr-python/environment.yml | 1 + implementations/zarrita/environment.yml | 2 +- implementations/zarrita/generate_zarrita.py | 37 ++++++++------ 5 files changed, 64 insertions(+), 32 deletions(-) diff --git a/environment.yml b/environment.yml index d986de51..f3dad3f1 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - defaults dependencies: - make - - python == 3.7.9 + - python - scikit-image - pytest - pip diff --git a/implementations/js/package-lock.json b/implementations/js/package-lock.json index c03611b7..59c9ea77 100644 --- a/implementations/js/package-lock.json +++ b/implementations/js/package-lock.json @@ -1,59 +1,81 @@ { + "name": "js", + "lockfileVersion": 3, "requires": true, - "lockfileVersion": 1, - "dependencies": { - "eventemitter3": { + "packages": { + "": { + "dependencies": { + "pngjs": "^6.0.0", + "zarr": "^0.4.0" + } + }, + "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" }, - "numcodecs": { + "node_modules/numcodecs": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.1.1.tgz", "integrity": "sha512-UjKulZ6GIFKLdBIczEbsoXNZQmiHafpoIdo39YcdecHVGyMKh0+azsfHTrybXm5RZwepqLZv24mkjqGdZGm24Q==", - "requires": { + "dependencies": { "pako": "^1.0.11" } }, - "p-finally": { + "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "engines": { + "node": ">=4" + } }, - "p-queue": { + "node_modules/p-queue": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.2.0.tgz", "integrity": "sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg==", - "requires": { + "dependencies": { "eventemitter3": "^4.0.0", "p-timeout": "^3.1.0" + }, + "engines": { + "node": ">=8" } }, - "p-timeout": { + "node_modules/p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "requires": { + "dependencies": { "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" } }, - "pako": { + "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" }, - "pngjs": { + "node_modules/pngjs": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", - "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==" + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "engines": { + "node": ">=12.13.0" + } }, - "zarr": { + "node_modules/zarr": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.4.0.tgz", "integrity": "sha512-zvxdX3aRWxjy6H3OtA7R05NNZvRKxn/7bkNJhUsVKKbNoJ3DBqYERQfzI4WfAV1OTcclqvlYwkQ7DWsGJA5QEw==", - "requires": { + "dependencies": { "numcodecs": "^0.1.0", "p-queue": "6.2.0" + }, + "engines": { + "node": ">=8.0.0" } } } diff --git a/implementations/zarr-python/environment.yml b/implementations/zarr-python/environment.yml index 59af0770..1fd13515 100644 --- a/implementations/zarr-python/environment.yml +++ b/implementations/zarr-python/environment.yml @@ -5,3 +5,4 @@ dependencies: - python - scikit-image - zarr + - fsspec diff --git a/implementations/zarrita/environment.yml b/implementations/zarrita/environment.yml index 05426b70..66901648 100644 --- a/implementations/zarrita/environment.yml +++ b/implementations/zarrita/environment.yml @@ -6,4 +6,4 @@ dependencies: - pip - pip: - scikit-image - - zarrita==0.1.0a1 + - zarrita diff --git a/implementations/zarrita/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py index 51ba26e0..bb254455 100644 --- a/implementations/zarrita/generate_zarrita.py +++ b/implementations/zarrita/generate_zarrita.py @@ -6,7 +6,7 @@ SHARD_SHAPE = (1000, 1000, 3) STR_TO_CODEC = { "gzip": zarrita.codecs.gzip_codec(), - "blosc": zarrita.codecs.blosc_codec(cname="lz4"), + "blosc": zarrita.codecs.blosc_codec(cname="lz4", typesize=10), } @@ -20,8 +20,8 @@ def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=Tru chunk_separator = "." if sharded: fname += "_sharded" - store = zarrita.FileSystemStore("file://./data") - g = zarrita.Group.create(store, fname) + store = zarrita.LocalStore("./data") + g = zarrita.Group.create(store / fname, exists_ok=True) for codec in codecs: if codec is None: name = "raw" @@ -29,22 +29,31 @@ def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=Tru name = f"{codec}/{STR_TO_CODEC[codec].configuration.cname}" else: name = codec - codecs_impl = [STR_TO_CODEC[codec]] if codec is not None else [] + + codecs_impl = [zarrita.codecs.bytes_codec()] + if codec is not None: + codecs_impl.append(STR_TO_CODEC[codec]) + if sharded: codecs_impl = [ zarrita.codecs.sharding_codec( chunk_shape=CHUNK_SHAPE, codecs=codecs_impl - ) + ), ] - a = g.create_array( - name, - shape=im.shape, - chunk_shape=SHARD_SHAPE if sharded else CHUNK_SHAPE, - chunk_key_encoding=("default", chunk_separator), - dtype=im.dtype, - codecs=codecs_impl, - ) - a[...] = im + try: + a = g.create_array( + name, + shape=im.shape, + chunk_shape=SHARD_SHAPE if sharded else CHUNK_SHAPE, + chunk_key_encoding=("default", chunk_separator), + dtype=im.dtype, + codecs=codecs_impl, + exists_ok=True, + ) + except: + print(f"Failed on n:{nested}/s:{sharded}: {codecs_impl}") + raise + a[:, :, :]= im if __name__ == "__main__": From f14f92f88c5e4f3189b957361ab0b4f9265dba29 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:16:08 +0200 Subject: [PATCH 70/87] Also support mamba --- implementations/.conda_driver.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index bfeee21d..125d3e5c 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -7,22 +7,37 @@ set -e set -o pipefail +## Setup based on mamba versus conda installation +if command -v mamba &> /dev/null +then + COMMAND=mamba +else + COMMAND=conda +fi + create_or_activate(){ - if { conda env list | grep $ENVNAME; } >/dev/null 2>&1; then + + if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then echo "Using $ENVNAME" else echo "Creating $ENVNAME" - conda env create -n $ENVNAME -f $IMPL/environment.yml + $COMMAND env create -n $ENVNAME -f $IMPL/environment.yml fi - eval "$(conda shell.bash hook)" + export MAMBA_ROOT_PREFIX=$(mamba info --base -q) + export MAMBA_EXE=${MAMBA_ROOT_PREFIX}/bin/mamba + export CONDA_EXE=${MAMBA_ROOT_PREFIX}/bin/conda + . $MAMBA_ROOT_PREFIX/etc/profile.d/conda.sh + . $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh + echo "Activating $ENVNAME" - conda activate $ENVNAME + $COMMAND activate $ENVNAME } destroy(){ - if { conda env list | grep $ENVNAME; } >/dev/null 2>&1; then + + if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then echo "Destroying $ENVNAME" - conda env remove -y -n $ENVNAME + $COMMAND env remove -y -n $ENVNAME else echo "No known env: $ENVNAME" fi From e3826e43d26d00dde112ca79f0b52443808e3db3 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:16:42 +0200 Subject: [PATCH 71/87] Rarr: use bioconda package (untested; amd64) --- implementations/Rarr/environment.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/implementations/Rarr/environment.yml b/implementations/Rarr/environment.yml index bf18c46a..36a94516 100644 --- a/implementations/Rarr/environment.yml +++ b/implementations/Rarr/environment.yml @@ -1,21 +1,7 @@ channels: - - conda-forge - defaults + - bioconda dependencies: - ## required for generating data - - r-base=4.2.2 - - r-loder - ## R-pkgs needed for installing Rarr - will be simplified once in Bioc - - r-remotes - - r-stringr - - r-httr - - r-jsonlite - - r-paws.storage - - r-r.utils - - r-curl - ## compilers for Rarr codecs - - make - - gxx - - gcc + - bioconductor-rarr variables: LC_ALL: C.UTF-8 From 7cb742a1beea1ff2d829b543796dbf2e134102a7 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:17:59 +0200 Subject: [PATCH 72/87] Add tensorstore implementation (.skipped) --- implementations/tensorstore/.skip | 0 implementations/tensorstore/driver.sh | 18 +++ implementations/tensorstore/environment.yml | 10 ++ .../tensorstore/generate_tensorstore.py | 103 ++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 implementations/tensorstore/.skip create mode 100755 implementations/tensorstore/driver.sh create mode 100644 implementations/tensorstore/environment.yml create mode 100755 implementations/tensorstore/generate_tensorstore.py diff --git a/implementations/tensorstore/.skip b/implementations/tensorstore/.skip new file mode 100644 index 00000000..e69de29b diff --git a/implementations/tensorstore/driver.sh b/implementations/tensorstore/driver.sh new file mode 100755 index 00000000..7c806820 --- /dev/null +++ b/implementations/tensorstore/driver.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# + +ENVNAME=ZI_tensorstore + +# Standard bootstrapping +IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$( dirname $IMPL) + +write(){ + create_or_activate + python $IMPL/generate_tensorstore.py +} + +. $ROOT/.conda_driver.sh +. $ROOT/.bash_driver.sh +argparse "$@" diff --git a/implementations/tensorstore/environment.yml b/implementations/tensorstore/environment.yml new file mode 100644 index 00000000..a0ebaddc --- /dev/null +++ b/implementations/tensorstore/environment.yml @@ -0,0 +1,10 @@ +channels: + - conda-forge +dependencies: + - tensorstore + # Test generation + - numcodecs + - python + - scikit-image + - zarr + - fsspec diff --git a/implementations/tensorstore/generate_tensorstore.py b/implementations/tensorstore/generate_tensorstore.py new file mode 100755 index 00000000..de268b08 --- /dev/null +++ b/implementations/tensorstore/generate_tensorstore.py @@ -0,0 +1,103 @@ +import tensorstore as ts +import numcodecs +from skimage.data import astronaut + +# choose chunks s.t. we do have overhanging edge-chunks +CHUNKS = (100, 100, 1) +STR_TO_COMPRESSOR = { + "gzip": numcodecs.GZip, + "blosc": numcodecs.Blosc, + "zlib": numcodecs.Zlib, +} +COMPRESSION_OPTIONS = {"blosc": {"cname": "lz4"}} + +im = astronaut() +SIZE = im.shape + + +def n5_metadata(compression: str): + return { + 'compression': { + 'type': compression, + }, + 'dataType': 'uint32', + 'dimensions': SIZE, + 'blockSize': CHUNKS, + } + +def zr_metadata(): + chunk_grid = {"name": "regular", "configuration": {"chunk_shape": chunks}} # read size + + sharding_codec = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": chunks, # write size + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "blosc", "configuration": {"cname": "zstd", "clevel": 5}}], + "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "crc32c"}], + "index_location": "end" + } + } + codecs = [sharding_codec] + else: + # Alternative without sharding... + chunk_grid = {"name": "regular", "configuration": {"chunk_shape": chunks}} + blosc_codec = {"name": "blosc", "configuration": { "cname": "lz4", "clevel": 5}} + codecs = [blosc_codec] + + base_config = { + "driver": "zarr3", + "kvstore": CONFIGS[1], + "metadata": { + "shape": shape, + "chunk_grid": chunk_grid, + "chunk_key_encoding": {"name": "default"}, # "configuration": {"separator": "/"}}, + "codecs": codecs, + "data_type": read.dtype, + "dimension_names": dimension_names, + } + } + + +def ts_write(driver: str, path: str, metadata: dict, data: List): + arr = ts.open({ + 'driver': driver, + 'kvstore': { + 'driver': 'file', + 'path': path, + }, + 'metadata': metadata, + 'create': True, + 'delete_existing': True, + }).result() + write_future = arr.write(data) + write_future.result() + + +# TODO use more compressors from numcodecs and more blosc filter_ids +def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): + for compressor in compressors: + copts = COMPRESSION_OPTIONS.get(compressor, {}) + if compressor is None: + name = "raw" + elif compressor == "blosc": + name = "%s/%s" % (compressor, copts.get("cname")) + else: + name = compressor + compressor_impl = STR_TO_COMPRESSOR[compressor](**copts) if compressor is not None else None + # V2. TODO: add method for v3 everywhere + ts_write('zarr', f'data/tensorstore.zr/{name}', zr_metadata(), im) + + +def generate_n5_format(compressors=['gzip', None]): + im = astronaut() + for compressor in compressors: + name = compressor if compressor is not None else 'raw' + compressor_impl = STR_TO_COMPRESSOR[compressor]() if compressor is not None else None + ts_write('zarr', f'data/tensorstore.n5/{name}', n5_metadata(), im) + + +if __name__ == '__main__': + generate_zarr_format() + generate_n5_format() From 41f35df64cd502a45b1e4beab9789963e2f53a3d Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 20:22:38 +0200 Subject: [PATCH 73/87] Remove pre-release GH config; use an implementation --- .github/workflows/build.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b035ef27..a5c6372f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,25 +33,6 @@ jobs: zarr-python: ${{ matrix.zarr-python }} action: write -# TMP: disabling to speed up testing -# # Re-run the same configurations but with a pre-release of Zarr -# write_pre: -# strategy: -# fail-fast: false -# matrix: -# implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} -# platform: [ubuntu-latest] -# python-version: [3.8] -# zarr-python: [pre] -# needs: [get_implementations, write_released] -# uses: ./.github/workflows/impl.yml -# with: -# implementation: ${{ matrix.implementation }} -# platform: ${{ matrix.platform }} -# python-version: ${{ matrix.python-version }} -# zarr-python: ${{ matrix.zarr-python }} -# action: write - read_released: strategy: fail-fast: false From 36147f039ae14b3480ec172fb33efe5bce1f543a Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 21:04:53 +0200 Subject: [PATCH 74/87] Put read skeleton in place --- Makefile | 8 +++++++- implementations/.bash_driver.sh | 8 ++++++-- implementations/.conda_driver.sh | 2 +- implementations/Rarr/driver.sh | 6 +++++- implementations/js/driver.sh | 6 +++++- implementations/jzarr/driver.sh | 14 +++++++++++++- implementations/n5-java/driver.sh | 6 +++++- implementations/pyn5/driver.sh | 6 +++++- implementations/tensorstore/driver.sh | 6 +++++- implementations/xtensor_zarr/driver.sh | 6 +++++- implementations/z5py/driver.sh | 6 +++++- implementations/zarr-python/driver.sh | 6 +++++- implementations/zarrita/driver.sh | 6 +++++- 13 files changed, 72 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 8d7c8ddf..3f5ec6e2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ IMPLEMENTATIONS=$(wildcard implementations/*) +CURRENT_DIR = $(shell pwd) # # The default target: @@ -39,7 +40,6 @@ endif ########################################################## data/reference_image.png: python generate_reference_image.py - define mk-impl-target # For each of the items in our "implementations" directory, # create targets which depend on the reference data and @@ -47,12 +47,18 @@ define mk-impl-target .PHONY: read write $1 $1/ $1-read $1-write $1-destroy clean +read-fast: $1-read-fast read: $1-read write: $1-write $1-write: data/reference_image.png @if test -e $1/.skip; then echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh write; fi +$1-read: write $1-read-fast + +$1-read-fast: + @if test -e $1/.skip; then echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh read $(CURRENT_DIR)/data/zarr_FSStore_flat.zr blosc/lz4; fi + # Alias for read & write $1: $1-write $1-read diff --git a/implementations/.bash_driver.sh b/implementations/.bash_driver.sh index 444530de..6799cb86 100644 --- a/implementations/.bash_driver.sh +++ b/implementations/.bash_driver.sh @@ -8,12 +8,16 @@ set -o pipefail argparse(){ case "${1}" in + read) + shift; + echo "Reading data..." + zi_read "$@";; write) echo "Generating data..." - write;; + zi_write;; destroy) echo "Tearing down..." - destroy;; + zi_destroy;; *) echo "Unknown command: ${1}" exit 2;; diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index 125d3e5c..00dbed99 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -33,7 +33,7 @@ create_or_activate(){ $COMMAND activate $ENVNAME } -destroy(){ +zi_destroy(){ if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then echo "Destroying $ENVNAME" diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index 2a831ae8..6c48d929 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -4,7 +4,11 @@ ENVNAME=ZI_Rarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh index d2bbc7e9..dfc890e2 100755 --- a/implementations/js/driver.sh +++ b/implementations/js/driver.sh @@ -4,7 +4,11 @@ ENVNAME=ZI_js IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/jzarr/driver.sh b/implementations/jzarr/driver.sh index 6965d90d..af5958b6 100755 --- a/implementations/jzarr/driver.sh +++ b/implementations/jzarr/driver.sh @@ -7,7 +7,19 @@ ENVNAME=ZI_jzarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + create_or_activate + + cd "${IMPL}" + + MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} + mvn "${MVN_FLAGS}" clean package + #notrap_outerr + + java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App -verify "$@" +} + +zi_write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh index a8ba97ba..69797f6b 100755 --- a/implementations/n5-java/driver.sh +++ b/implementations/n5-java/driver.sh @@ -4,7 +4,11 @@ ENVNAME=ZI_n5_java IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh index f0d83d25..998c1e57 100755 --- a/implementations/pyn5/driver.sh +++ b/implementations/pyn5/driver.sh @@ -8,7 +8,11 @@ ENVNAME=ZI_pyn5 IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate python $IMPL/generate_pyn5.py } diff --git a/implementations/tensorstore/driver.sh b/implementations/tensorstore/driver.sh index 7c806820..9c4432a2 100755 --- a/implementations/tensorstore/driver.sh +++ b/implementations/tensorstore/driver.sh @@ -8,7 +8,11 @@ ENVNAME=ZI_tensorstore IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate python $IMPL/generate_tensorstore.py } diff --git a/implementations/xtensor_zarr/driver.sh b/implementations/xtensor_zarr/driver.sh index 2399e702..e2832bd2 100755 --- a/implementations/xtensor_zarr/driver.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -4,7 +4,11 @@ ENVNAME=ZI_xtensor_zarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate cd "${IMPL}" diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh index 184735d4..3a33c271 100755 --- a/implementations/z5py/driver.sh +++ b/implementations/z5py/driver.sh @@ -8,7 +8,11 @@ ENVNAME=ZI_z5py IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate python $IMPL/generate_z5py.py } diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index dd36a6f7..a00cda02 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -8,7 +8,11 @@ ENVNAME=ZI_zarr-python IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate python $IMPL/generate_zarr.py } diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 53381c59..3d69dc6e 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -7,7 +7,11 @@ ENVNAME=ZI_zarrita IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -write(){ +zi_read(){ + echo "skipping read" +} + +zi_write(){ create_or_activate python $IMPL/generate_zarrita.py } From e34dc306176e27380cd4e663064017f208b131b2 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 23 Jul 2024 22:40:59 +0200 Subject: [PATCH 75/87] Add `make list` targets Additional changes: * Add `make help` default target * Quiet `mvn` builds * Send status messages to stderr * Run all commands from the IMPL directory --- Makefile | 38 ++++++++++++----- implementations/.bash_driver.sh | 15 ++++--- implementations/.conda_driver.sh | 10 ++--- implementations/Rarr/driver.sh | 22 ++++++++-- implementations/js/driver.sh | 18 ++++++-- implementations/js/package-lock.json | 10 +++++ implementations/js/package.json | 1 + implementations/js/src/index.js | 32 +++++++++++---- implementations/jzarr/driver.sh | 41 ++++++++++++------- .../java/zarr_implementations/jzarr/App.java | 22 +++++++--- implementations/n5-java/driver.sh | 28 ++++++++++--- .../zarr_implementations/n5_java/App.java | 9 +++- implementations/pyn5/driver.sh | 17 ++++++-- implementations/pyn5/generate_pyn5.py | 21 +++++++--- implementations/tensorstore/driver.sh | 17 ++++++-- .../tensorstore/generate_tensorstore.py | 26 +++++++++--- implementations/xtensor_zarr/driver.sh | 14 +++++-- implementations/z5py/driver.sh | 17 ++++++-- implementations/z5py/generate_z5py.py | 30 ++++++++++---- implementations/zarr-python/driver.sh | 17 ++++++-- implementations/zarr-python/generate_zarr.py | 35 +++++++++++----- implementations/zarrita/driver.sh | 17 ++++++-- implementations/zarrita/generate_zarrita.py | 32 ++++++++++----- 23 files changed, 361 insertions(+), 128 deletions(-) diff --git a/Makefile b/Makefile index 3f5ec6e2..ad7e9ddd 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,25 @@ IMPLEMENTATIONS=$(wildcard implementations/*) CURRENT_DIR = $(shell pwd) -# -# The default target: -# (1) creates an environment for all implementations (if needed), -# (2) generates sample data for all implementations, and -# (3) runs tests against that sample data for all implementations. -# +define HELP_TEXT +make [target] + +Target(s): + write generate data for all implementations + list print directory\tpath pairs for all written data + read read all data written by all implementations + read-fast like read, but do not re-run `write` + +Notes: + - Each implementation has a conda environment named ZI_ + +endef + +export HELP_TEXT + +.PHONY: help +help: + @echo "$$HELP_TEXT" .PHONY: report @@ -45,19 +58,24 @@ define mk-impl-target # create targets which depend on the reference data and # call the "driver.sh" script as necessary. -.PHONY: read write $1 $1/ $1-read $1-write $1-destroy clean +.PHONY: write list read-fast read $1 $1/ $1-list $1-read-fast $1-read $1-write $1-destroy clean + +write: $1-write +list: $1-list read-fast: $1-read-fast read: $1-read -write: $1-write $1-write: data/reference_image.png - @if test -e $1/.skip; then echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh write; fi + @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh write; fi + +$1-list: + @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh list; fi $1-read: write $1-read-fast $1-read-fast: - @if test -e $1/.skip; then echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh read $(CURRENT_DIR)/data/zarr_FSStore_flat.zr blosc/lz4; fi + @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh read $(CURRENT_DIR)/data/zarr_FSStore_flat.zr blosc/lz4; fi # Alias for read & write $1: $1-write $1-read diff --git a/implementations/.bash_driver.sh b/implementations/.bash_driver.sh index 6799cb86..0903ec03 100644 --- a/implementations/.bash_driver.sh +++ b/implementations/.bash_driver.sh @@ -8,18 +8,21 @@ set -o pipefail argparse(){ case "${1}" in + write) + >&2 echo "Generating data..." + zi_write;; + list) + shift; + zi_list "$@";; read) shift; - echo "Reading data..." + >&2 echo "Verifying data..." zi_read "$@";; - write) - echo "Generating data..." - zi_write;; destroy) - echo "Tearing down..." + >&2 echo "Tearing down..." zi_destroy;; *) - echo "Unknown command: ${1}" + >&2 echo "Unknown command: ${1}" exit 2;; esac } diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index 00dbed99..47f82de4 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -18,9 +18,9 @@ fi create_or_activate(){ if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then - echo "Using $ENVNAME" + >&2 echo "Using $ENVNAME" else - echo "Creating $ENVNAME" + >&2 echo "Creating $ENVNAME" $COMMAND env create -n $ENVNAME -f $IMPL/environment.yml fi export MAMBA_ROOT_PREFIX=$(mamba info --base -q) @@ -29,16 +29,16 @@ create_or_activate(){ . $MAMBA_ROOT_PREFIX/etc/profile.d/conda.sh . $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh - echo "Activating $ENVNAME" + >&2 echo "Activating $ENVNAME" $COMMAND activate $ENVNAME } zi_destroy(){ if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then - echo "Destroying $ENVNAME" + >&2 echo "Destroying $ENVNAME" $COMMAND env remove -y -n $ENVNAME else - echo "No known env: $ENVNAME" + >&2 echo "No known env: $ENVNAME" fi } diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index 6c48d929..e31271eb 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -4,10 +4,6 @@ ENVNAME=ZI_Rarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ create_or_activate @@ -17,6 +13,24 @@ zi_write(){ Rscript generate_Rarr.R } +zi_list(){ + create_or_activate + + cd "${IMPL}" + + Rscript install_packages.R + Rscript generate_Rarr.R -list +} + +zi_read(){ + create_or_activate + + cd "${IMPL}" + + Rscript install_packages.R + Rscript verify_data.R "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh index dfc890e2..fd19aced 100755 --- a/implementations/js/driver.sh +++ b/implementations/js/driver.sh @@ -4,10 +4,6 @@ ENVNAME=ZI_js IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ create_or_activate @@ -17,6 +13,20 @@ zi_write(){ npm start } +zi_list(){ + create_or_activate + + cd "${IMPL}" + + npm install --silent + npm run --silent start -- --list "$@" +} + +zi_read(){ + cd "${IMPL}" + npm run start -- --verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/js/package-lock.json b/implementations/js/package-lock.json index 59c9ea77..1128d74e 100644 --- a/implementations/js/package-lock.json +++ b/implementations/js/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "minimist": "1.2.8", "pngjs": "^6.0.0", "zarr": "^0.4.0" } @@ -14,6 +15,15 @@ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/numcodecs": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.1.1.tgz", diff --git a/implementations/js/package.json b/implementations/js/package.json index 60d38ffc..7d67cecb 100644 --- a/implementations/js/package.json +++ b/implementations/js/package.json @@ -4,6 +4,7 @@ "start": "node src/index.js" }, "dependencies": { + "minimist": "1.2.8", "pngjs": "^6.0.0", "zarr": "^0.4.0" } diff --git a/implementations/js/src/index.js b/implementations/js/src/index.js index b3f3c2ec..da1d68b6 100644 --- a/implementations/js/src/index.js +++ b/implementations/js/src/index.js @@ -1,6 +1,7 @@ import fs from "fs"; import p from "path"; import pkg from "pngjs"; +import minimist from "minimist"; const { PNG } = pkg; import { openGroup, NestedArray, slice } from "zarr"; @@ -34,11 +35,11 @@ function getName(config) { return config.id; } -async function generateZarrFormat(codecIds = ["gzip", "blosc", "zlib", null]) { +async function generateZarrFormat(listOnly, codecIds = ["gzip", "blosc", "zlib", null]) { const path = p.join("..", "..", "data", "js.zr"); const img = imread(p.join("..", "..", "data", "reference_image.png")); - if (fs.existsSync(path)) { + if (!listOnly && fs.existsSync(path)) { fs.rmdirSync(path, { recursive: true, force: true }); } @@ -46,12 +47,27 @@ async function generateZarrFormat(codecIds = ["gzip", "blosc", "zlib", null]) { for (const id of codecIds) { const config = id ? STR_TO_COMPRESSOR[id] : null; const name = getName(config); - grp.createDataset(name, undefined, img, { - compressor: config, - chunks: CHUNKS, - fillValue: 0, - }); + if (listOnly) { + console.log(path + "\t" + name); + } else { + grp.createDataset(name, undefined, img, { + compressor: config, + chunks: CHUNKS, + fillValue: 0, + }); + } } } -generateZarrFormat(); +function main(){ + var argv = minimist(process.argv.slice(2), { + boolean: ["list", "verify"], + }); + if (argv.verify) { + verifyZarrFormat(verify); + } else { + generateZarrFormat(argv.list); + } +} + +main(); diff --git a/implementations/jzarr/driver.sh b/implementations/jzarr/driver.sh index af5958b6..68450fb4 100755 --- a/implementations/jzarr/driver.sh +++ b/implementations/jzarr/driver.sh @@ -7,25 +7,13 @@ ENVNAME=ZI_jzarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - create_or_activate - - cd "${IMPL}" - - MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} - mvn "${MVN_FLAGS}" clean package - #notrap_outerr - - java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App -verify "$@" -} - zi_write(){ create_or_activate cd "${IMPL}" - MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} - mvn "${MVN_FLAGS}" clean package + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && { # Workaround for: https://github.com/bcdev/jzarr/issues/25 @@ -36,6 +24,31 @@ zi_write(){ } } +zi_list(){ + create_or_activate + + cd "${IMPL}" + + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package + #notrap_outerr + + java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App -list +} + +zi_read(){ + create_or_activate + + cd "${IMPL}" + + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package + #notrap_outerr + + java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App -verify "$@" +} + + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java b/implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java index 4cc39d15..2a8a90b7 100644 --- a/implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java +++ b/implementations/jzarr/src/main/java/zarr_implementations/jzarr/App.java @@ -80,11 +80,14 @@ private static int[] getArrayData(ZarrArray zarr) throws Exception { public static void main(String args[]) throws Exception { - if (args.length != 0 && args.length != 3) { + if (args.length != 0 && args.length != 1 && args.length != 3) { System.out.println("usage: App"); + System.out.println("usage: App -list"); System.out.println("usage: App -verify fpath dsname"); System.exit(2); // EARLY EXIT - } else if (args.length == 3) { + } + + if (args.length == 3) { String fpath = args[1]; String dsname = args[2]; ZarrArray verification = ZarrGroup.open(fpath).openArray(dsname); @@ -105,6 +108,11 @@ public static void main(String args[]) throws Exception { return; // EARLY EXIT } + boolean listOnly = false; + if (args.length == 1) { + listOnly = true; + } + int[] data = getTestData(); final ZarrGroup container = ZarrGroup.create(OUT_PATH); @@ -121,9 +129,13 @@ public static void main(String args[]) throws Exception { dsname = "blosc/lz4"; // FIXME: better workaround? } Path subdir = OUT_PATH.resolve(dsname); - ZarrArray zArray = ZarrArray.create(subdir, arrayParams); - // final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); - zArray.write(data, SHAPE, new int[]{0, 0, 0}); + if (listOnly) { + System.out.println(OUT_PATH + "\t" + dsname); + } else { + ZarrArray zArray = ZarrArray.create(subdir, arrayParams); + // final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); + zArray.write(data, SHAPE, new int[]{0, 0, 0}); + } } } } diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh index 69797f6b..e3693409 100755 --- a/implementations/n5-java/driver.sh +++ b/implementations/n5-java/driver.sh @@ -4,20 +4,36 @@ ENVNAME=ZI_n5_java IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ create_or_activate cd "${IMPL}" - MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"} - mvn "${MVN_FLAGS}" clean package + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App } +zi_list(){ + create_or_activate + + cd "${IMPL}" + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package + + java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App -list +} + +zi_read(){ + create_or_activate + + cd "${IMPL}" + MVN_FLAGS=${MVN_FLAGS:-"-q --no-transfer-progress"} + mvn ${MVN_FLAGS} package + + java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java b/implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java index bc6202c3..eea10f9d 100644 --- a/implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java +++ b/implementations/n5-java/src/main/java/zarr_implementations/n5_java/App.java @@ -58,13 +58,18 @@ private static RandomAccessibleInterval getData() throws IOExc public static void main(String args[]) throws IOException { + boolean listOnly = (args.length == 1 && args[0].equals("-list")); RandomAccessibleInterval data = getData(); final N5FSWriter container = new N5FSWriter(OUT_PATH); for (final Compression compression : getCompressions()) { final DatasetAttributes attrs = new DatasetAttributes(Intervals.dimensionsAsLongArray(data), BLOCK_SIZE, DataType.UINT8, compression); final String dataset = compression.getType(); - container.createDataset(dataset, attrs); - N5Utils.save(data, container, dataset, BLOCK_SIZE, compression); + if (listOnly) { + System.out.println(OUT_PATH + "\t" + dataset); + } else { + container.createDataset(dataset, attrs); + N5Utils.save(data, container, dataset, BLOCK_SIZE, compression); + } } } diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh index 998c1e57..d13ea64c 100755 --- a/implementations/pyn5/driver.sh +++ b/implementations/pyn5/driver.sh @@ -8,15 +8,24 @@ ENVNAME=ZI_pyn5 IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ + cd "${IMPL}" create_or_activate python $IMPL/generate_pyn5.py } +zi_list(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_pyn5.py -list +} + +zi_read(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_pyn5.py -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/pyn5/generate_pyn5.py b/implementations/pyn5/generate_pyn5.py index 6861a8ce..5e976dbc 100755 --- a/implementations/pyn5/generate_pyn5.py +++ b/implementations/pyn5/generate_pyn5.py @@ -7,9 +7,8 @@ CHUNKS = (100, 100, 1) -def generate_n5_format(compressors=pyn5.CompressionType): - here = Path(__file__).resolve().parent - data_dir = here.parent.parent / "data" +def generate_n5_format(list_only: bool, compressors=pyn5.CompressionType): + data_dir = Path("../..") / "data" path = data_dir / "pyn5.n5" im = imread(data_dir / "reference_image.png") @@ -17,8 +16,20 @@ def generate_n5_format(compressors=pyn5.CompressionType): f = pyn5.File(path, pyn5.Mode.CREATE_TRUNCATE) for compressor in compressors: name = str(compressor) - f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) + if list_only: + print(f"{path}\t{name}") + else: + f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) if __name__ == '__main__': - generate_n5_format() + import sys + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-list", action="store_true") + parser.add_argument("-verify", action="store_true") + ns = parser.parse_args() + if ns.verify: + verify_n5_format() + else: + generate_n5_format(ns.list) diff --git a/implementations/tensorstore/driver.sh b/implementations/tensorstore/driver.sh index 9c4432a2..7d4c0951 100755 --- a/implementations/tensorstore/driver.sh +++ b/implementations/tensorstore/driver.sh @@ -8,15 +8,24 @@ ENVNAME=ZI_tensorstore IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ + cd "${IMPL}" create_or_activate python $IMPL/generate_tensorstore.py } +zi_list(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_tensorstore.py -list +} + +zi_read(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_tensorstore.py -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/tensorstore/generate_tensorstore.py b/implementations/tensorstore/generate_tensorstore.py index de268b08..7df20d0e 100755 --- a/implementations/tensorstore/generate_tensorstore.py +++ b/implementations/tensorstore/generate_tensorstore.py @@ -76,7 +76,7 @@ def ts_write(driver: str, path: str, metadata: dict, data: List): # TODO use more compressors from numcodecs and more blosc filter_ids -def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): +def generate_zarr_format(list_only:bool, compressors=['gzip', 'blosc', 'zlib', None]): for compressor in compressors: copts = COMPRESSION_OPTIONS.get(compressor, {}) if compressor is None: @@ -87,17 +87,31 @@ def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): name = compressor compressor_impl = STR_TO_COMPRESSOR[compressor](**copts) if compressor is not None else None # V2. TODO: add method for v3 everywhere - ts_write('zarr', f'data/tensorstore.zr/{name}', zr_metadata(), im) + if list_only: + print(f"data/tensorstore.zr\t{name}") + else: + ts_write('zarr', f'data/tensorstore.zr/{name}', zr_metadata(), im) -def generate_n5_format(compressors=['gzip', None]): +def generate_n5_format(list_only:bool, compressors=['gzip', None]): im = astronaut() for compressor in compressors: name = compressor if compressor is not None else 'raw' compressor_impl = STR_TO_COMPRESSOR[compressor]() if compressor is not None else None - ts_write('zarr', f'data/tensorstore.n5/{name}', n5_metadata(), im) + if list_only: + print(f"data/tensorstore.n5\t{name}") + else: + ts_write('zarr', f'data/tensorstore.n5/{name}', n5_metadata(), im) if __name__ == '__main__': - generate_zarr_format() - generate_n5_format() + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-list", action="store_true") + parser.add_argument("-verify", action="store_true") + ns = parser.parse_args() + if ns.verify: + verify_format(ns.known_args) + else: + generate_zarr_format(ns.list) + generate_n5_format(ns.list) diff --git a/implementations/xtensor_zarr/driver.sh b/implementations/xtensor_zarr/driver.sh index e2832bd2..0f3dd0ee 100755 --- a/implementations/xtensor_zarr/driver.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -4,10 +4,6 @@ ENVNAME=ZI_xtensor_zarr IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ create_or_activate @@ -24,6 +20,16 @@ zi_write(){ make run } +zi_list(){ + echo "skipping list" + exit 1 +} + +zi_read(){ + echo "skipping read" + exit 1 +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh index 3a33c271..566bdb98 100755 --- a/implementations/z5py/driver.sh +++ b/implementations/z5py/driver.sh @@ -8,15 +8,24 @@ ENVNAME=ZI_z5py IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ + cd "${IMPL}" create_or_activate python $IMPL/generate_z5py.py } +zi_list(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_z5py.py -list +} + +zi_read(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_z5py.py -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/z5py/generate_z5py.py b/implementations/z5py/generate_z5py.py index 8106cf71..00d711fa 100644 --- a/implementations/z5py/generate_z5py.py +++ b/implementations/z5py/generate_z5py.py @@ -11,8 +11,8 @@ # TODO support more compressors: # - more compressors in numcodecs # - more blosc codecs -def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', 'raw']): - path = 'data/z5py.zr' +def generate_zarr_format(list_only:bool, compressors=['gzip', 'blosc', 'zlib', 'raw']): + path = '../../data/z5py.zr' im = astronaut() f = z5py.File(path, mode='w') @@ -23,19 +23,33 @@ def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', 'raw']): if compressor != "blosc" else "%s/%s" % (compressor, copts.get("codec")) ) - f.create_dataset(name, data=im, compression=compressor, chunks=CHUNKS, **copts) + if list_only: + print(f"{path}\t{name}") + else: + f.create_dataset(name, data=im, compression=compressor, chunks=CHUNKS, **copts) -def generate_n5_format(compressors=['gzip', 'raw']): - path = 'data/z5py.n5' +def generate_n5_format(list_only:bool, compressors=['gzip', 'raw']): + path = '../../data/z5py.n5' im = astronaut() f = z5py.File(path, mode='w') for compressor in compressors: name = compressor - f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) + if list_only: + print(f"{path}\t{name}") + else: + f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) if __name__ == '__main__': - generate_zarr_format() - generate_n5_format() + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-list", action="store_true") + parser.add_argument("-verify", action="store_true") + ns = parser.parse_args() + if ns.verify: + verify_format(ns.known_args) + else: + generate_zarr_format(ns.list) + generate_n5_format(ns.list) diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index a00cda02..15a3aaff 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -8,15 +8,24 @@ ENVNAME=ZI_zarr-python IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ + cd "${IMPL}" create_or_activate python $IMPL/generate_zarr.py } +zi_list(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_zarr.py -list +} + +zi_read(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_zarr.py -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/zarr-python/generate_zarr.py b/implementations/zarr-python/generate_zarr.py index a0e0c4ca..83ddd49e 100644 --- a/implementations/zarr-python/generate_zarr.py +++ b/implementations/zarr-python/generate_zarr.py @@ -13,7 +13,7 @@ # TODO use more compressors from numcodecs and more blosc filter_ids -def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): +def generate_zarr_format(list_only:bool, compressors=['gzip', 'blosc', 'zlib', None]): for nested, StoreClass, store_kwargs in [ (False, zarr.storage.DirectoryStore, {}), @@ -24,7 +24,7 @@ def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): ]: nested_str = '_nested' if nested else '_flat' - path = f'data/zarr_{StoreClass.__name__}{nested_str}.zr' + path = f'../../data/zarr_{StoreClass.__name__}{nested_str}.zr' store = StoreClass(path, **store_kwargs) im = astronaut() @@ -38,21 +38,36 @@ def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): else: name = compressor compressor_impl = STR_TO_COMPRESSOR[compressor](**copts) if compressor is not None else None - f.create_dataset(name, data=im, chunks=CHUNKS, - compressor=compressor_impl) + if list_only: + print(f"{path}\t{name}") + else: + f.create_dataset(name, data=im, chunks=CHUNKS, + compressor=compressor_impl) -def generate_n5_format(compressors=['gzip', None]): +def generate_n5_format(list_only:bool, compressors=['gzip', None]): im = astronaut() - f = zarr.open('data/zarr.n5', mode='w') + path = "../../data/zarr.n5" + f = zarr.open(path, mode='w') for compressor in compressors: name = compressor if compressor is not None else 'raw' compressor_impl = STR_TO_COMPRESSOR[compressor]() if compressor is not None else None - f.create_dataset(name, data=im, chunks=CHUNKS, - compressor=compressor_impl) + if list_only: + print(f"{path}\t{name}") + else: + f.create_dataset(name, data=im, chunks=CHUNKS, + compressor=compressor_impl) if __name__ == '__main__': - generate_zarr_format() - generate_n5_format() + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-list", action="store_true") + parser.add_argument("-verify", action="store_true") + ns = parser.parse_args() + if ns.verify: + verify_format(ns.known_args) + else: + generate_zarr_format(ns.list) + generate_n5_format(ns.list) diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 3d69dc6e..2da78393 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -7,15 +7,24 @@ ENVNAME=ZI_zarrita IMPL=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) ROOT=$( dirname $IMPL) -zi_read(){ - echo "skipping read" -} - zi_write(){ + cd "${IMPL}" create_or_activate python $IMPL/generate_zarrita.py } +zi_list(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_zarrita.py -list +} + +zi_read(){ + cd "${IMPL}" + create_or_activate + python $IMPL/generate_zarrita.py -verify "$@" +} + . $ROOT/.conda_driver.sh . $ROOT/.bash_driver.sh argparse "$@" diff --git a/implementations/zarrita/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py index bb254455..afa31dd0 100644 --- a/implementations/zarrita/generate_zarrita.py +++ b/implementations/zarrita/generate_zarrita.py @@ -10,7 +10,7 @@ } -def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=True): +def generate_zr3_format(list_only:bool, codecs=["gzip", "blosc", None], nested=True, sharded=True): im = astronaut() fname = "zarrita" if nested: @@ -20,8 +20,10 @@ def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=Tru chunk_separator = "." if sharded: fname += "_sharded" - store = zarrita.LocalStore("./data") - g = zarrita.Group.create(store / fname, exists_ok=True) + + path = f"../../data/{fname}" + store = zarrita.LocalStore(path) + g = zarrita.Group.create(store, exists_ok=True) for codec in codecs: if codec is None: name = "raw" @@ -40,7 +42,10 @@ def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=Tru chunk_shape=CHUNK_SHAPE, codecs=codecs_impl ), ] - try: + + if list_only: + print(f"{path}\t{name}") + else: a = g.create_array( name, shape=im.shape, @@ -50,13 +55,18 @@ def generate_zr3_format(codecs=["gzip", "blosc", None], nested=True, sharded=Tru codecs=codecs_impl, exists_ok=True, ) - except: - print(f"Failed on n:{nested}/s:{sharded}: {codecs_impl}") - raise - a[:, :, :]= im + a[:, :, :]= im if __name__ == "__main__": - for nested in [False, True]: - for sharded in [False, True]: - generate_zr3_format(nested=nested, sharded=sharded) + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-list", action="store_true") + parser.add_argument("-verify", action="store_true") + ns = parser.parse_args() + if ns.verify: + verify_format(ns.known_args) + else: + for nested in [False, True]: + for sharded in [False, True]: + generate_zr3_format(ns.list, nested=nested, sharded=sharded) From f747875946cf3ad2c561b7af70b4cbe341410f00 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 00:23:40 +0200 Subject: [PATCH 76/87] Add read loop over files in 'formats' file --- Makefile | 24 ++++++++++++++++---- implementations/zarr-python/generate_zarr.py | 7 +++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ad7e9ddd..1b758e20 100644 --- a/Makefile +++ b/Makefile @@ -67,15 +67,31 @@ read-fast: $1-read-fast read: $1-read $1-write: data/reference_image.png - @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh write; fi + @if test -e $1/.skip; \ + then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \ + else \ + bash $1/driver.sh write; \ + fi $1-list: - @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh list; fi + @if test -e $1/.skip; \ + then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \ + else \ + bash $1/driver.sh list; \ + fi -$1-read: write $1-read-fast +formats := $(shell cat formats) $1-read-fast: - @if test -e $1/.skip; then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; else bash $1/driver.sh read $(CURRENT_DIR)/data/zarr_FSStore_flat.zr blosc/lz4; fi + @if test -e $1/.skip; \ + then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \ + else \ + bash verify.sh $1; \ + fi + + +$1-read: write $1-read-fast + # Alias for read & write $1: $1-write $1-read diff --git a/implementations/zarr-python/generate_zarr.py b/implementations/zarr-python/generate_zarr.py index 83ddd49e..85681a1a 100644 --- a/implementations/zarr-python/generate_zarr.py +++ b/implementations/zarr-python/generate_zarr.py @@ -60,14 +60,19 @@ def generate_n5_format(list_only:bool, compressors=['gzip', None]): compressor=compressor_impl) +def verify_format(path: str, group: str): + print(path, group) + + if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument("-list", action="store_true") parser.add_argument("-verify", action="store_true") + parser.add_argument("args", nargs="*") ns = parser.parse_args() if ns.verify: - verify_format(ns.known_args) + verify_format(*ns.args) else: generate_zarr_format(ns.list) generate_n5_format(ns.list) From b27cd726dda74edc68dffa2792fd1b5a3a72649f Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 09:33:53 +0200 Subject: [PATCH 77/87] Remove dangling 'formats' command in Makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1b758e20..ad31904a 100644 --- a/Makefile +++ b/Makefile @@ -80,13 +80,11 @@ $1-list: bash $1/driver.sh list; \ fi -formats := $(shell cat formats) - $1-read-fast: @if test -e $1/.skip; \ then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \ else \ - bash verify.sh $1; \ + bash $1/driver.sh read $(DIR) $(DATASET); \ fi From 79ddfd6ce1238386f15d94e8c59c913d77343afc Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 09:21:11 +0200 Subject: [PATCH 78/87] Add env call to all python scripts --- data/generate_reference_image.py | 2 ++ implementations/tensorstore/generate_tensorstore.py | 2 ++ implementations/z5py/generate_z5py.py | 2 ++ implementations/zarr-python/generate_zarr.py | 2 ++ implementations/zarrita/generate_zarrita.py | 2 ++ test/test_read_all.py | 1 + 6 files changed, 11 insertions(+) diff --git a/data/generate_reference_image.py b/data/generate_reference_image.py index 2d1cbe85..dec8da3d 100644 --- a/data/generate_reference_image.py +++ b/data/generate_reference_image.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from skimage.data import astronaut from skimage.io import imsave diff --git a/implementations/tensorstore/generate_tensorstore.py b/implementations/tensorstore/generate_tensorstore.py index 7df20d0e..ccbda352 100755 --- a/implementations/tensorstore/generate_tensorstore.py +++ b/implementations/tensorstore/generate_tensorstore.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import tensorstore as ts import numcodecs from skimage.data import astronaut diff --git a/implementations/z5py/generate_z5py.py b/implementations/z5py/generate_z5py.py index 00d711fa..1c2d30a9 100644 --- a/implementations/z5py/generate_z5py.py +++ b/implementations/z5py/generate_z5py.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import z5py from skimage.data import astronaut diff --git a/implementations/zarr-python/generate_zarr.py b/implementations/zarr-python/generate_zarr.py index 85681a1a..b3b81ec4 100644 --- a/implementations/zarr-python/generate_zarr.py +++ b/implementations/zarr-python/generate_zarr.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import zarr import numcodecs from skimage.data import astronaut diff --git a/implementations/zarrita/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py index afa31dd0..f34a8494 100644 --- a/implementations/zarrita/generate_zarrita.py +++ b/implementations/zarrita/generate_zarrita.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import zarrita from skimage.data import astronaut diff --git a/test/test_read_all.py b/test/test_read_all.py index 52a31bb3..67787350 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ Usage ===== From 225e71a073758a16de99148174328b81a0225ba5 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 09:41:29 +0200 Subject: [PATCH 79/87] unify bash script headers --- implementations/Rarr/driver.sh | 1 + implementations/js/driver.sh | 1 + implementations/jzarr/driver.sh | 4 +--- implementations/n5-java/driver.sh | 1 + implementations/pyn5/driver.sh | 3 --- implementations/tensorstore/driver.sh | 3 --- implementations/xtensor_zarr/driver.sh | 1 + implementations/z5py/driver.sh | 3 --- implementations/zarr-python/driver.sh | 3 --- implementations/zarrita/driver.sh | 2 -- 10 files changed, 5 insertions(+), 17 deletions(-) diff --git a/implementations/Rarr/driver.sh b/implementations/Rarr/driver.sh index e31271eb..9858ec4b 100755 --- a/implementations/Rarr/driver.sh +++ b/implementations/Rarr/driver.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash ENVNAME=ZI_Rarr # Standard bootstrapping diff --git a/implementations/js/driver.sh b/implementations/js/driver.sh index fd19aced..1821c3eb 100755 --- a/implementations/js/driver.sh +++ b/implementations/js/driver.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash ENVNAME=ZI_js # Standard bootstrapping diff --git a/implementations/jzarr/driver.sh b/implementations/jzarr/driver.sh index 68450fb4..df8a830f 100755 --- a/implementations/jzarr/driver.sh +++ b/implementations/jzarr/driver.sh @@ -1,6 +1,4 @@ -# cd to this directory -# https://stackoverflow.com/a/6393573/2700168 - +#!/usr/bin/env bash ENVNAME=ZI_jzarr # Standard bootstrapping diff --git a/implementations/n5-java/driver.sh b/implementations/n5-java/driver.sh index e3693409..912e111f 100755 --- a/implementations/n5-java/driver.sh +++ b/implementations/n5-java/driver.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash ENVNAME=ZI_n5_java # Standard bootstrapping diff --git a/implementations/pyn5/driver.sh b/implementations/pyn5/driver.sh index d13ea64c..290d5a1a 100755 --- a/implementations/pyn5/driver.sh +++ b/implementations/pyn5/driver.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash -# -# - ENVNAME=ZI_pyn5 # Standard bootstrapping diff --git a/implementations/tensorstore/driver.sh b/implementations/tensorstore/driver.sh index 7d4c0951..2abbfc57 100755 --- a/implementations/tensorstore/driver.sh +++ b/implementations/tensorstore/driver.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash -# -# - ENVNAME=ZI_tensorstore # Standard bootstrapping diff --git a/implementations/xtensor_zarr/driver.sh b/implementations/xtensor_zarr/driver.sh index 0f3dd0ee..8cf4ea6b 100755 --- a/implementations/xtensor_zarr/driver.sh +++ b/implementations/xtensor_zarr/driver.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash ENVNAME=ZI_xtensor_zarr # Standard bootstrapping diff --git a/implementations/z5py/driver.sh b/implementations/z5py/driver.sh index 566bdb98..ffc9ae6f 100755 --- a/implementations/z5py/driver.sh +++ b/implementations/z5py/driver.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash -# -# - ENVNAME=ZI_z5py # Standard bootstrapping diff --git a/implementations/zarr-python/driver.sh b/implementations/zarr-python/driver.sh index 15a3aaff..5c98dc29 100755 --- a/implementations/zarr-python/driver.sh +++ b/implementations/zarr-python/driver.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash -# -# - ENVNAME=ZI_zarr-python # Standard bootstrapping diff --git a/implementations/zarrita/driver.sh b/implementations/zarrita/driver.sh index 2da78393..480b393d 100755 --- a/implementations/zarrita/driver.sh +++ b/implementations/zarrita/driver.sh @@ -1,6 +1,4 @@ #!/usr/bin/env bash -# - ENVNAME=ZI_zarrita # Standard bootstrapping From 1dc6f47ae7bc24eba76d59aee198ae902c985be1 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 18:32:07 +0200 Subject: [PATCH 80/87] Locally working test_read_all.py --- README.md | 1 + implementations/.bash_driver.sh | 8 +- implementations/.conda_driver.sh | 8 +- implementations/pyn5/generate_pyn5.py | 8 +- .../tensorstore/generate_tensorstore.py | 3 +- implementations/z5py/generate_z5py.py | 8 +- implementations/zarrita/generate_zarrita.py | 3 +- test/test_read_all.py | 84 ++++++++----------- 8 files changed, 61 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 6a741205..5b1a22d7 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,4 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi ## Other features * `touch implementations/{impl}/.skip` to disable an implementation +* Use `make NODEBUG=1 ...` to quiet output diff --git a/implementations/.bash_driver.sh b/implementations/.bash_driver.sh index 0903ec03..f52650b0 100644 --- a/implementations/.bash_driver.sh +++ b/implementations/.bash_driver.sh @@ -9,20 +9,20 @@ set -o pipefail argparse(){ case "${1}" in write) - >&2 echo "Generating data..." + [[ -z "${NODEBUG}" ]] && >&2 echo "Generating data..." zi_write;; list) shift; zi_list "$@";; read) shift; - >&2 echo "Verifying data..." + [[ -z "${NODEBUG}" ]] && >&2 echo "Verifying data..." zi_read "$@";; destroy) - >&2 echo "Tearing down..." + [[ -z "${NODEBUG}" ]] && >&2 echo "Tearing down..." zi_destroy;; *) - >&2 echo "Unknown command: ${1}" + [[ -z "${NODEBUG}" ]] && >&2 echo "Unknown command: ${1}" exit 2;; esac } diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index 47f82de4..b364b3d8 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -18,9 +18,9 @@ fi create_or_activate(){ if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then - >&2 echo "Using $ENVNAME" + [[ -z "${NODEBUG}" ]] && >&2 echo "Using $ENVNAME" else - >&2 echo "Creating $ENVNAME" + [[ -z "${NODEBUG}" ]] && >&2 echo "Creating $ENVNAME" $COMMAND env create -n $ENVNAME -f $IMPL/environment.yml fi export MAMBA_ROOT_PREFIX=$(mamba info --base -q) @@ -29,14 +29,14 @@ create_or_activate(){ . $MAMBA_ROOT_PREFIX/etc/profile.d/conda.sh . $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh - >&2 echo "Activating $ENVNAME" + [[ -z "${NODEBUG}" ]] && >&2 echo "Activating $ENVNAME" $COMMAND activate $ENVNAME } zi_destroy(){ if { $COMMAND env list | grep $ENVNAME; } >/dev/null 2>&1; then - >&2 echo "Destroying $ENVNAME" + [[ -z "${NODEBUG}" ]] && >&2 echo "Destroying $ENVNAME" $COMMAND env remove -y -n $ENVNAME else >&2 echo "No known env: $ENVNAME" diff --git a/implementations/pyn5/generate_pyn5.py b/implementations/pyn5/generate_pyn5.py index 5e976dbc..f375b5d2 100755 --- a/implementations/pyn5/generate_pyn5.py +++ b/implementations/pyn5/generate_pyn5.py @@ -22,14 +22,20 @@ def generate_n5_format(list_only: bool, compressors=pyn5.CompressionType): f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) +def verify_format(directory: str, dataset: str): + f = pyn5.File(f"{directory}/{dataset}")# TODO, mode="r") + return f[:] + + if __name__ == '__main__': import sys import argparse parser = argparse.ArgumentParser() parser.add_argument("-list", action="store_true") parser.add_argument("-verify", action="store_true") + parser.add_argument("args", nargs="*") ns = parser.parse_args() if ns.verify: - verify_n5_format() + verify_format(*ns.args) else: generate_n5_format(ns.list) diff --git a/implementations/tensorstore/generate_tensorstore.py b/implementations/tensorstore/generate_tensorstore.py index ccbda352..9e490428 100755 --- a/implementations/tensorstore/generate_tensorstore.py +++ b/implementations/tensorstore/generate_tensorstore.py @@ -111,9 +111,10 @@ def generate_n5_format(list_only:bool, compressors=['gzip', None]): parser = argparse.ArgumentParser() parser.add_argument("-list", action="store_true") parser.add_argument("-verify", action="store_true") + parser.add_argument("args", nargs="*") ns = parser.parse_args() if ns.verify: - verify_format(ns.known_args) + verify_format(*ns.args) else: generate_zarr_format(ns.list) generate_n5_format(ns.list) diff --git a/implementations/z5py/generate_z5py.py b/implementations/z5py/generate_z5py.py index 1c2d30a9..4746ae60 100644 --- a/implementations/z5py/generate_z5py.py +++ b/implementations/z5py/generate_z5py.py @@ -44,14 +44,20 @@ def generate_n5_format(list_only:bool, compressors=['gzip', 'raw']): f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) +def verify_format(directory: str, dataset: str): + f = z5py.File(f"{directory}/{dataset}"), mode="r") + return f[:] + + if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument("-list", action="store_true") parser.add_argument("-verify", action="store_true") + parser.add_argument("args", nargs="*") ns = parser.parse_args() if ns.verify: - verify_format(ns.known_args) + verify_format(*ns.args) else: generate_zarr_format(ns.list) generate_n5_format(ns.list) diff --git a/implementations/zarrita/generate_zarrita.py b/implementations/zarrita/generate_zarrita.py index f34a8494..5ed76c9c 100644 --- a/implementations/zarrita/generate_zarrita.py +++ b/implementations/zarrita/generate_zarrita.py @@ -65,9 +65,10 @@ def generate_zr3_format(list_only:bool, codecs=["gzip", "blosc", None], nested=T parser = argparse.ArgumentParser() parser.add_argument("-list", action="store_true") parser.add_argument("-verify", action="store_true") + parser.add_argument("args", nargs="*") ns = parser.parse_args() if ns.verify: - verify_format(ns.known_args) + verify_format(*ns.args) else: for nested in [False, True]: for sharded in [False, True]: diff --git a/test/test_read_all.py b/test/test_read_all.py index 67787350..8f740c93 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -34,6 +34,7 @@ """ import os +import sys import subprocess from typing import Dict, List from pathlib import Path @@ -81,83 +82,64 @@ }, } -def read_with_jzarr(fpath, ds_name, nested=None): - if ds_name == "blosc": - ds_name = "blosc/lz4" +def make_read(name, fpath, ds_name, nested=None): cmd = ( - f"implementations/jzarr/generate_data.sh " - f"-verify {str(fpath)} {ds_name}" + f"make implementations/{name}-read-fast " + f"NODEBUG=1 DIR={str(fpath)} DATASET={ds_name}" ) # will raise subprocess.CalledProcessError if return code is not 0 - subprocess.check_output(cmd, shell=True) - return None + result = subprocess.run(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if result.returncode != 0: + raise Exception( + { + "command": cmd, + "stdout": result.stdout, + "stderr": result.stderr, + } + ) + return result + +def read_with_jzarr(fpath, ds_name, nested=None): + if ds_name == "blosc": + ds_name = "blosc/lz4" + return make_read("jzarr", fpath, ds_name, nested) def read_with_zarr(fpath, ds_name, nested): - import zarr if ds_name == "blosc": ds_name = "blosc/lz4" - if str(fpath).endswith('.zr'): - if nested: - if 'FSStore' in str(fpath): - store = zarr.storage.FSStore( - os.fspath(fpath), dimension_separator='/', mode='r' - ) - else: - store = zarr.storage.NestedDirectoryStore(os.fspath(fpath)) - else: - if 'FSStore' in str(fpath): - store = zarr.storage.FSStore(os.fspath(fpath)) - else: - store = zarr.storage.DirectoryStore(fpath) - else: - store = os.fspath(fpath) - return zarr.open(store)[ds_name][:] + return make_read("zarr-python", fpath, ds_name, nested) def read_with_pyn5(fpath, ds_name, nested): - import pyn5 - return pyn5.File(fpath)[ds_name][:] + return make_read("pyn5", fpath, ds_name, nested) def read_with_z5py(fpath, ds_name, nested): - import z5py if ds_name == "blosc": ds_name = "blosc/lz4" - return z5py.File(fpath)[ds_name][:] + return make_read("z5py", fpath, ds_name, nested) def read_with_zarrita(fpath, ds_name, nested): - import zarrita if ds_name == "blosc": ds_name = "blosc/lz4" - h = zarrita.get_hierarchy(str(fpath.absolute())) - return h["/" + ds_name][:] + return make_read("zarrita", fpath, ds_name, nested) def read_with_xtensor_zarr(fpath, ds_name, nested): if ds_name == "blosc": ds_name = "blosc/lz4" - fname = "a.npz" - if os.path.exists(fname): - os.remove(fname) - subprocess.check_call(["implementations/xtensor_zarr/build/run_xtensor_zarr", fpath, ds_name]) - return np.load(fname)["a"] + return make_read("xtensor_zarr", fpath, ds_name, nested) def read_with_Rarr(fpath, ds_name, nested): if ds_name == "blosc": ds_name = "blosc/lz4" - - cmd = ( - f"Rscript implementations/Rarr/verify_data_internal.R " - f"{str(fpath)} {ds_name}" - ) - - # will raise subprocess.CalledProcessError if return code is not 0 - subprocess.check_output(cmd, shell=True) - return None + return make_read("Rarr", fpath, ds_name, nested) EXTENSIONS = {"zarr": ".zr", "N5": ".n5", "zarr-v3": ".zr3"} @@ -274,11 +256,13 @@ def test_correct_read(fmt, writing_library, reading_library, codec, nested, f"file not found: {fpath}. Make sure you have generated the data " "using 'make data'" ) - test = read_fn(fpath, codec, nested) - # Assume if None is returned, the read function has verified. - if test is not None: - assert test.shape == reference.shape - assert np.allclose(test, reference) + result = read_fn(fpath, codec, nested) + + if b"Skipping" in result.stderr: + pytest.skip(result.stderr.decode()) + else: + print(result.stdout) + print(result.stderr, file=sys.stderr) def tabulate_test_results(params, per_codec_tables=False): From 99e3837bd46e5b4cd76e4a0d8c4ad18cc32eae20 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 18:38:45 +0200 Subject: [PATCH 81/87] Attempt simplifying GH build --- .github/workflows/build.yml | 79 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5c6372f..776d754d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,50 +5,43 @@ on: [push, pull_request] jobs: - get_implementations: + build: + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - - name: List implementations - id: set-matrix - run: echo "matrix=$(cd implementations && ls -1 | jq -ncR '[inputs]')" >> $GITHUB_OUTPUT - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - - write_released: - strategy: - fail-fast: false - matrix: - implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} - platform: [ubuntu-latest] - python-version: [3.8] - zarr-python: [released] - needs: [get_implementations] - uses: ./.github/workflows/impl.yml - with: - implementation: ${{ matrix.implementation }} - platform: ${{ matrix.platform }} - python-version: ${{ matrix.python-version }} - zarr-python: ${{ matrix.zarr-python }} - action: write - - read_released: - strategy: - fail-fast: false - matrix: - implementation: ${{ fromJson(needs.get_implementations.outputs.matrix) }} - platform: [ubuntu-latest] - python-version: [3.8] - zarr-python: [pre] - # Run the read job regardless once writing is complete - # TMP needs: [write_pre, write_released] - needs: [write_released] - if: ${{ always() }} - uses: ./.github/workflows/impl.yml - with: - implementation: ${{ matrix.implementation }} - platform: ${{ matrix.platform }} - python-version: ${{ matrix.python-version }} - zarr-python: ${{ matrix.zarr-python }} - action: read + + - name: Cache conda + uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + activate-environment: ZI + channels: conda-forge,defaults + channel-priority: true + environment-file: environment.yml + mamba-version: "*" + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Run build + shell: bash -el {0} + run: | + make write; + test/test_read_all.py From 1ce56f9c3008a3dfb684896523b10fd0236aa7da Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 18:59:07 +0200 Subject: [PATCH 82/87] Debug 'import-im6.q16: unable to open X server' --- implementations/.conda_driver.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index b364b3d8..ad224b3b 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -6,6 +6,7 @@ set -e set -o pipefail +set -x ## Setup based on mamba versus conda installation if command -v mamba &> /dev/null From 7693740f268107b173e05254e7f8b5d145648864 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 19:11:20 +0200 Subject: [PATCH 83/87] Try another method to determine mamba prefix `mamba` in the github action prints additional information leading to errors: ``` + export 'CONDA_EXE= mamba version : 1.5.8 /usr/share/miniconda/bin/conda' + CONDA_EXE=' mamba version : 1.5.8 /usr/share/miniconda/bin/conda' + . mamba version : 1.5.8 /usr/share/miniconda/etc/profile.d/conda.sh ++ import re import-im6.q16: unable to open X server `' @ error/import.c/ImportImageCommand/346. ``` --- implementations/.conda_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index ad224b3b..c38694f3 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -24,7 +24,7 @@ create_or_activate(){ [[ -z "${NODEBUG}" ]] && >&2 echo "Creating $ENVNAME" $COMMAND env create -n $ENVNAME -f $IMPL/environment.yml fi - export MAMBA_ROOT_PREFIX=$(mamba info --base -q) + export MAMBA_ROOT_PREFIX=$(conda info --base -q) export MAMBA_EXE=${MAMBA_ROOT_PREFIX}/bin/mamba export CONDA_EXE=${MAMBA_ROOT_PREFIX}/bin/conda . $MAMBA_ROOT_PREFIX/etc/profile.d/conda.sh From 7e06549d4ca6c7a7b27f9006a2e17407a70c3cdd Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 19:53:06 +0200 Subject: [PATCH 84/87] Quiet built and start adding .skip files --- implementations/.conda_driver.sh | 1 - implementations/pyn5/.skip | 1 + implementations/xtensor_zarr/.skip | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 implementations/pyn5/.skip create mode 100644 implementations/xtensor_zarr/.skip diff --git a/implementations/.conda_driver.sh b/implementations/.conda_driver.sh index c38694f3..592487eb 100644 --- a/implementations/.conda_driver.sh +++ b/implementations/.conda_driver.sh @@ -6,7 +6,6 @@ set -e set -o pipefail -set -x ## Setup based on mamba versus conda installation if command -v mamba &> /dev/null diff --git a/implementations/pyn5/.skip b/implementations/pyn5/.skip new file mode 100644 index 00000000..bb855dd7 --- /dev/null +++ b/implementations/pyn5/.skip @@ -0,0 +1 @@ +pyn5 requires "n5" version attribute diff --git a/implementations/xtensor_zarr/.skip b/implementations/xtensor_zarr/.skip new file mode 100644 index 00000000..3fa6019f --- /dev/null +++ b/implementations/xtensor_zarr/.skip @@ -0,0 +1 @@ +no amd64; core dump on GH From 46f4c7718b5b58eb4042b8d86d2cdbd47ce9d750 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 20:00:52 +0200 Subject: [PATCH 85/87] Fix compile issues --- implementations/tensorstore/.skip | 1 + implementations/tensorstore/generate_tensorstore.py | 1 + implementations/z5py/generate_z5py.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/implementations/tensorstore/.skip b/implementations/tensorstore/.skip index e69de29b..5b9a6bfa 100644 --- a/implementations/tensorstore/.skip +++ b/implementations/tensorstore/.skip @@ -0,0 +1 @@ +compile issues diff --git a/implementations/tensorstore/generate_tensorstore.py b/implementations/tensorstore/generate_tensorstore.py index 9e490428..b2e00e11 100755 --- a/implementations/tensorstore/generate_tensorstore.py +++ b/implementations/tensorstore/generate_tensorstore.py @@ -28,6 +28,7 @@ def n5_metadata(compression: str): } def zr_metadata(): + if True: chunk_grid = {"name": "regular", "configuration": {"chunk_shape": chunks}} # read size sharding_codec = { diff --git a/implementations/z5py/generate_z5py.py b/implementations/z5py/generate_z5py.py index 4746ae60..3a40d61b 100644 --- a/implementations/z5py/generate_z5py.py +++ b/implementations/z5py/generate_z5py.py @@ -45,7 +45,7 @@ def generate_n5_format(list_only:bool, compressors=['gzip', 'raw']): def verify_format(directory: str, dataset: str): - f = z5py.File(f"{directory}/{dataset}"), mode="r") + f = z5py.File(f"{directory}/{dataset}", mode="r") return f[:] From c2e0f774a0294fbcb49c2ebc22743178d60d8117 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 20:31:37 +0200 Subject: [PATCH 86/87] Call test with pytest --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 776d754d..57a5b4cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,4 +44,4 @@ jobs: shell: bash -el {0} run: | make write; - test/test_read_all.py + pytest test/test_read_all.py From ab6667b82f3a611b65418037c3ac4147c71101b0 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 24 Jul 2024 21:00:07 +0200 Subject: [PATCH 87/87] more skips --- implementations/Rarr/.skip | 1 + implementations/z5py/.skip | 1 + 2 files changed, 2 insertions(+) create mode 100644 implementations/Rarr/.skip create mode 100644 implementations/z5py/.skip diff --git a/implementations/Rarr/.skip b/implementations/Rarr/.skip new file mode 100644 index 00000000..523b9b00 --- /dev/null +++ b/implementations/Rarr/.skip @@ -0,0 +1 @@ +built under R version 4.3.2 diff --git a/implementations/z5py/.skip b/implementations/z5py/.skip new file mode 100644 index 00000000..6129b3da --- /dev/null +++ b/implementations/z5py/.skip @@ -0,0 +1 @@ +read_metadata failure