ZI drivers + Rarr implementation

.dockerignore

@@ -0,0 +1,5 @@
+data/**/
+target*
+.netcdf-java
+.tensorstore
+.git

.github/workflows/build.yml

@@ -4,38 +4,44 @@ name: Build
 on: [push, pull_request]
 
 jobs:
-  test:
-    name: ${{ matrix.platform }} ${{ matrix.python-version }}
-    runs-on: ${{ matrix.platform }}
 
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [ubuntu-latest]
-        python-version: [3.7]
+  build:
 
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Cache conda
+        uses: actions/cache@v3
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
 
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v1
+        uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          channels: conda-forge,ome
+          activate-environment: ZI
+          channels: conda-forge,defaults
+          channel-priority: true
           environment-file: environment.yml
-          python-version: ${{ matrix.python-version }}
+          mamba-version: "*"
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: true
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: Run tests
-        shell: bash -l {0}
-        run: make
+      - name: Run build
+        shell: bash -el {0}
+        run: |
+          make write;
+          pytest test/test_read_all.py

.github/workflows/impl.yml

@@ -0,0 +1,90 @@
+---
+name: Implementation tests
+
+on:
+  workflow_call:
+    inputs:
+      implementation:
+        required: true
+        type: string
+      platform:
+        required: true
+        type: string
+      python-version:
+        required: true
+        type: string
+      zarr-python:
+        required: true
+        type: string
+      action:
+        required: true
+        type: string
+
+jobs:
+
+  impl:
+    name: ${{ inputs.implementation }}-${{ inputs.action }}-${{ inputs.platform }}-py${{ inputs.python-version }}-${{ inputs.zarr-python }}
+    runs-on: ${{ inputs.platform }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Cache conda
+        uses: actions/cache@v3
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles( format('{0}/{1}/{2}', 'implementations/', inputs.implementation, '/environment.yml') ) }}
+
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge,defaults
+          channel-priority: true
+          environment-file: implementations/${{ inputs.implementation }}/environment.yml
+          mamba-version: "*"
+          python-version: ${{ inputs.python-version }}
+          activate-environment: ZI_${{ inputs.implementation }}
+        env:
+          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+
+      - name: Install zarr dev (optional)
+        shell: bash -l {0}
+        if: ${{ inputs.zarr-python == 'pre' }}
+        run: |
+          python -m pip install git+https://github.com/zarr-developers/zarr-python.git
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      #
+      # Choose read or write based on {{ action }}
+      #
+
+      - name: Download previous output for testing
+        uses: actions/download-artifact@v3
+        if: ${{ inputs.action == 'read' }}
+        with:
+          name: ${{ inputs.implementation }}-${{ inputs.zarr-python }}-data
+          path: data
+
+      - name: Run build
+        shell: bash -l {0}
+        run: make implementations/${{ inputs.implementation }}-${{ inputs.action }}
+
+      - name: Save output for testing
+        uses: actions/upload-artifact@v3
+        if: ${{ inputs.action == 'write' }}
+        with:
+          name: ${{ inputs.implementation }}-${{ inputs.zarr-python }}-data
+          path: data

.gitignore

@@ -1,8 +1,9 @@
 target*
 __pycache__/
-generate_data/xtensor_zarr/build
+implmenetations/xtensor_zarr/build
 node_modules/
 report.*
 
 # ignore data subdirectories
 data/**/
+*iml

Dockerfile

@@ -2,7 +2,7 @@ FROM continuumio/miniconda3
 
 WORKDIR /src
 COPY environment.yml /src/environment.yml
-RUN apt-get update -y && apt install -y freeglut3-dev
+RUN apt-get update -y && apt install -y freeglut3-dev # Unsure why freeglut is here
 RUN conda env create -f environment.yml -n z
 SHELL ["conda", "run", "-n", "z", "/bin/bash", "-c"]
 

Makefile

@@ -1,46 +1,107 @@
+IMPLEMENTATIONS=$(wildcard implementations/*)
+CURRENT_DIR = $(shell pwd)
+
+define HELP_TEXT
+make [target]
+
+Target(s):
+  write                generate data for all implementations
+  list                 print directory\tpath pairs for all written data
+  read                 read all data written by all implementations
+  read-fast            like read, but do not re-run `write`
+
+Notes:
+  - Each implementation has a conda environment named ZI_<IMPL>
+
+endef
+
+export HELP_TEXT
+
+.PHONY: help
+help:
+	@echo "$$HELP_TEXT"
+
+.PHONY: report
+
+report: data
+	python test/test_read_all.py
+
+.PHONY: test data
+
+ifeq ($(TEST),) #################################################
+# If TEST is not set, by default build everything, generate
+# data for all implementations, and then run all pytests.
+
 test: data
-	pytest -v
+	pytest -v -k W
+
+data: $(IMPLEMENTATIONS)
+
+else
+# Otherwise, focus on a single implementation, only generating
+# its data and using the "-k W-" keyword to limit which pytests
+# get run
+
+test: implementations/$(TEST)
+	pytest -v -k W-$(TEST)
+
+data: implementations/$(TEST)
+
+endif ##########################################################
+
 
 data/reference_image.png:
 	python generate_reference_image.py
 
-.PHONY: jzarr
-jzarr: data/reference_image.png
-	bash generate_data/jzarr/generate_data.sh
+define mk-impl-target
+# For each of the items in our "implementations" directory,
+# create targets which depend on the reference data and
+# call the "driver.sh" script as necessary.
 
-.PHONY: n5java
-n5java: data/reference_image.png
-	bash generate_data/n5-java/generate_data.sh
+.PHONY: write list read-fast read $1 $1/ $1-list $1-read-fast $1-read $1-write $1-destroy clean
 
-.PHONY: pyn5
-pyn5: data/reference_image.png
-	python generate_data/generate_pyn5.py
+write: $1-write
+list: $1-list
 
-.PHONY: z5py
-z5py: data/reference_image.png
-	python generate_data/generate_z5py.py
+read-fast: $1-read-fast
+read: $1-read
 
-.PHONY: zarr
-zarr: data/reference_image.png
-	python generate_data/generate_zarr.py
+$1-write: data/reference_image.png
+	@if test -e $1/.skip; \
+	    then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \
+	else \
+	    bash $1/driver.sh write; \
+	fi
 
-.PHONY: zarrita
-zarrita: data/reference_image.png
-	python generate_data/generate_zarrita.py
+$1-list:
+	@if test -e $1/.skip; \
+	    then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \
+	else \
+	    bash $1/driver.sh list; \
+	fi
 
-.PHONY: js
-js: data/reference_image.png
-	bash generate_data/js/generate_data.sh
+$1-read-fast:
+	@if test -e $1/.skip; \
+	    then >&2 echo "Skipping $1 -- $$(shell test -e $1/.skip && cat $1/.skip)"; \
+	else \
+	    bash $1/driver.sh read $(DIR) $(DATASET); \
+	fi
 
-.PHONY: xtensor_zarr
-xtensor_zarr: data/reference_image.png
-	bash generate_data/xtensor_zarr/generate_data.sh
 
-.PHONY: data
-data: jzarr n5java pyn5 z5py zarr js xtensor_zarr zarrita
+$1-read: write $1-read-fast
 
-.PHONY: test
 
-.PHONY: report
-report: data
-	python test/test_read_all.py
+# Alias for read & write
+$1: $1-write $1-read
+
+# Alias in case the trailing slash is included
+$1/: $1
+
+# Additional target to cleanup the environment
+$1-destroy:
+	bash $1/driver.sh destroy
+
+clean: $1-destroy
+
+endef
+$(foreach impl,$(IMPLEMENTATIONS),$(eval $(call mk-impl-target,$(impl))))

README.md

@@ -5,15 +5,14 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi
 
 ## Implementations currently tested
 
-
 * https://github.com/bcdev/jzarr
 * https://github.com/saalfeldlab/n5-zarr
 * https://github.com/aschampion/rust-n5
 * https://github.com/xtensor-stack/xtensor-zarr
 * https://github.com/gzuidhof/zarr.js
 * https://github.com/zarr-developers/zarr-python
 * https://github.com/constantinpape/z5
-
+* https://gihutb.com/grimbough/Rarr
 
 ## Other implementations
 
@@ -23,3 +22,13 @@ Test for compatibility. See [doc/development_overview.md](doc/development_overvi
 * [Zarr.jl](https://github.com/meggart/Zarr.jl) #42
 * https://github.com/freeman-lab/zarr-js
 * [GDAL >= 3.4](https://gdal.org/drivers/raster/zarr.html)
+
+## Running locally
+
+* Create an environment: `conda env create -n ZI -f environment.yml`
+* Run `make`
+
+## Other features
+
+* `touch implementations/{impl}/.skip` to disable an implementation
+* Use `make NODEBUG=1 ...` to quiet output