Merge pull request #7 from bogovicj/dimSep

feat: detect dimension_separator from .zarray

src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java

@@ -248,6 +248,7 @@ public ZArrayAttributes getZArraryAttributes(final String pathName) throws IOExc
 			}
 		} else System.out.println(path.toString() + " does not exist.");
 
+		JsonElement sepElem = attributes.get("dimension_separator");
 		return new ZArrayAttributes(
 				attributes.get("zarr_format").getAsInt(),
 				gson.fromJson(attributes.get("shape"), long[].class),
@@ -256,6 +257,7 @@ public ZArrayAttributes getZArraryAttributes(final String pathName) throws IOExc
 				gson.fromJson(attributes.get("compressor"), ZarrCompressor.class),
 				attributes.get("fill_value").getAsString(),
 				attributes.get("order").getAsCharacter(),
+				sepElem != null ? sepElem.getAsString() : dimensionSeparator,
 				gson.fromJson(attributes.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType()));
 	}
 
@@ -447,12 +449,19 @@ public DataBlock<?> readBlock(
 		else
 			zarrDatasetAttributes = getZArraryAttributes(pathName).getDatasetAttributes();
 
+		final String dimSep;
+		final String attrDimensionSeparator = zarrDatasetAttributes.getDimensionSeparator();
+		if (attrDimensionSeparator != null && !attrDimensionSeparator.isEmpty())
+			dimSep = attrDimensionSeparator;
+		else
+			dimSep = dimensionSeparator;
+
 		final Path path = Paths.get(
 				basePath,
 				removeLeadingSlash(pathName),
 				getZarrDataBlockPath(
 						gridPosition,
-						dimensionSeparator,
+						dimSep,
 						zarrDatasetAttributes.isRowMajor()).toString());
 		if (!Files.exists(path))
 			return null;

src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java

@@ -245,6 +245,7 @@ public void setDatasetAttributes(
 				ZarrCompressor.fromCompression(datasetAttributes.getCompression()),
 				"0",
 				'C',
+				dimensionSeparator,
 				null);
 
 		setZArrayAttributes(pathName, zArrayAttributes);

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java

@@ -47,6 +47,7 @@ public class ZArrayAttributes {
 	protected static final String fillValueKey = "fill_value";
 	protected static final String orderKey = "order";
 	protected static final String filtersKey = "filters";
+	protected static final String dimensionSeparatorKey = "dimension_separator";
 
 	private final int zarr_format;
 	private final long[] shape;
@@ -55,6 +56,7 @@ public class ZArrayAttributes {
 	private final ZarrCompressor compressor;
 	private final String fill_value;
 	private final char order;
+	private final String dimensionSeparator;
 	private final List<Filter> filters = new ArrayList<>();
 
 	public ZArrayAttributes(
@@ -65,6 +67,7 @@ public ZArrayAttributes(
 			final ZarrCompressor compressor,
 			final String fill_value,
 			final char order,
+			final String dimensionSeparator,
 			final Collection<Filter> filters) {
 
 		this.zarr_format = zarr_format;
@@ -74,10 +77,25 @@ public ZArrayAttributes(
 		this.compressor = compressor == null ? new ZarrCompressor.Raw() : compressor;
 		this.fill_value = fill_value;
 		this.order = order;
+		this.dimensionSeparator = dimensionSeparator;
 		if (filters != null)
 			this.filters.addAll(filters);
 	}
 
+	public ZArrayAttributes(
+			final int zarr_format,
+			final long[] shape,
+			final int[] chunks,
+			final DType dtype,
+			final ZarrCompressor compressor,
+			final String fill_value,
+			final char order,
+			final Collection<Filter> filters) {
+
+		// empty dimensionSeparator so that the reader's separator is used
+		this(zarr_format, shape, chunks, dtype, compressor, fill_value, order, "", filters);
+	}
+
 	public ZarrDatasetAttributes getDatasetAttributes() {
 
 		final boolean isRowMajor = order == 'C';
@@ -95,7 +113,8 @@ public ZarrDatasetAttributes getDatasetAttributes() {
 				dtype,
 				compressor.getCompression(),
 				isRowMajor,
-				fill_value);
+				fill_value,
+				dimensionSeparator);
 	}
 
 	public long[] getShape() {
@@ -133,6 +152,10 @@ public char getOrder() {
 		return order;
 	}
 
+	public String getDimensionSeparator() {
+		return dimensionSeparator;
+	}
+
 	public String getFillValue() {
 
 		return fill_value;

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrDatasetAttributes.java

@@ -38,19 +38,33 @@ public class ZarrDatasetAttributes extends DatasetAttributes {
 	private final transient boolean isRowMajor;
 	private final transient DType dType;
 	private final transient byte[] fillBytes;
+	private final transient String dimensionSeparator;
 
 	public ZarrDatasetAttributes(
 			final long[] dimensions,
 			final int[] blockSize,
 			final DType dType,
 			final Compression compression,
 			final boolean isRowMajor,
-			final String fill_value) {
+			final String fill_value,
+			final String dimensionSeparator ) {
 
 		super(dimensions, blockSize, dType.getDataType(), compression);
 		this.dType = dType;
 		this.isRowMajor = isRowMajor;
 		this.fillBytes = dType.createFillBytes(fill_value);
+		this.dimensionSeparator = dimensionSeparator;
+	}
+
+	public ZarrDatasetAttributes(
+			final long[] dimensions,
+			final int[] blockSize,
+			final DType dType,
+			final Compression compression,
+			final boolean isRowMajor,
+			final String fill_value) {
+
+		this( dimensions, blockSize, dType, compression, isRowMajor, fill_value, "");
 	}
 
 	public boolean isRowMajor() {
@@ -68,4 +82,8 @@ public byte[] getFillBytes() {
 		// TODO Auto-generated method stub
 		return fillBytes;
 	}
+
+	public String getDimensionSeparator() {
+		return dimensionSeparator;
+	}
 }

src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java

@@ -77,6 +77,7 @@ public class N5ZarrTest extends AbstractN5Test {
 
 	static private String testDirPath = System.getProperty("user.home") + "/tmp/n5-test.zarr";
 	static private String testZarrDirPath = System.getProperty("user.home") + "/tmp/zarr-test.zarr";
+	static private String testZarrNestedDirPath = System.getProperty("user.home") + "/tmp/zarr-test-nested.zarr";
 	static private String testZarrDatasetName = "/test/data";
 
 
@@ -127,6 +128,22 @@ public void testCreateDataset() {
 		}
 	}
 
+	@Test
+	public void testCreateNestedDataset() {
+
+		final String datasetName = "/test/nested/data";
+		try {
+			N5ZarrWriter n5Nested = new N5ZarrWriter(testDirPath, "/", true );
+			n5Nested.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]);
+			assertEquals( "/", n5Nested.getZArraryAttributes(datasetName).getDimensionSeparator());
+
+			n5Nested.remove(datasetName);
+			n5Nested.close();
+		} catch (IOException e) {
+			fail(e.getMessage());
+		}
+	}
+
 	@Test
 	public void testPadCrop() throws Exception {
 
@@ -229,20 +246,20 @@ public void testMode1WriteReadByteBlock() {
 	public void testWriteReadSerializableBlock() {
 	}
 
-	private boolean runPythonTest() throws IOException, InterruptedException {
+	private boolean runPythonTest(String script) throws IOException, InterruptedException {
 
 		final boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
 		Process process;
 		if (isWindows) {
-		    process = Runtime.getRuntime().exec("cmd.exe /c python3 src\\test\\python\\zarr-test.py");
+		    process = Runtime.getRuntime().exec("cmd.exe /c python3 src\\test\\python\\" + script);
 		} else {
-		    process = Runtime.getRuntime().exec("python3 src/test/python/zarr-test.py");
+		    process = Runtime.getRuntime().exec("python3 src/test/python/" + script );
 		}
 		final int exitCode = process.waitFor();
 		new BufferedReader(new InputStreamReader(process.getErrorStream())).lines().forEach(System.out::println);
 		process.destroy();
 
-		return (exitCode == 0);
+		return (exitCode == 0 );
 	}
 
 	private static <T extends IntegerType<T>> void assertIsSequence(
@@ -276,7 +293,7 @@ private static <T extends RealType<T>> void assertIsSequence(
 	public void testReadZarrPython() throws IOException, InterruptedException {
 
 		/* create test data with python */
-		if (!runPythonTest()) {
+		if (!runPythonTest("zarr-test.py")) {
 			System.out.println("Couldn't run Python test, skipping compatibility test with Python.");
 			return;
 		}
@@ -401,6 +418,36 @@ public void testReadZarrPython() throws IOException, InterruptedException {
 		n5Zarr.remove();
 	}
 
+	@Test
+	public void testReadZarrNestedPython() throws IOException, InterruptedException {
+
+		/* create test data with python */
+		if (!runPythonTest("zarr-nested-test.py")) {
+			System.out.println("Couldn't run Python test, skipping compatibility test with Python.");
+			return;
+		}
+
+		final N5ZarrWriter n5Zarr = new N5ZarrWriter(testZarrNestedDirPath, ".", true);
+
+		/* groups */
+		System.out.println( n5Zarr.exists(testZarrDatasetName));
+		System.out.println( n5Zarr.datasetExists(testZarrDatasetName));
+		assertTrue(n5Zarr.exists(testZarrDatasetName) && !n5Zarr.datasetExists(testZarrDatasetName));
+
+		/* array parameters */
+		final DatasetAttributes datasetAttributesC = n5Zarr.getDatasetAttributes(testZarrDatasetName + "/3x2_c_|u1");
+		assertArrayEquals(datasetAttributesC.getDimensions(), new long[]{3, 2});
+		assertArrayEquals(datasetAttributesC.getBlockSize(), new int[]{3, 2});
+		assertEquals(datasetAttributesC.getDataType(), DataType.UINT8);
+		assertEquals( n5Zarr.getZArraryAttributes(testZarrDatasetName + "/3x2_c_|u1").getDimensionSeparator(), "/" );
+
+		final UnsignedByteType refUnsignedByte = new UnsignedByteType();
+		assertIsSequence(N5Utils.open(n5Zarr, testZarrDatasetName + "/3x2_c_|u1"), refUnsignedByte);
+
+		/* remove the container */
+		n5Zarr.remove();
+	}
+
 	@Test
 	public void testRawCompressorNullInZarray() throws IOException, FileNotFoundException, ParseException {
 		final N5ZarrWriter n5 = new N5ZarrWriter(testZarrDirPath);

src/test/python/zarr-nested-test.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+import numpy as np
+import zarr
+from numcodecs import Zlib, GZip, BZ2
+
+# Nested directory store
+nested_test_path = Path.home() / 'tmp' / 'zarr-test-nested.zarr'
+group_path = 'test/data'
+
+nested_store = zarr.NestedDirectoryStore(str(nested_test_path))
+nested_root = zarr.group(store=nested_store, overwrite=True)
+nested_group = nested_root.create_group(group_path)
+
+array_3x2_c = np.arange(0,3*2).reshape(2,3)
+
+nested_group.array(
+  name='3x2_c_|u1',
+  dtype='|u1',
+  data=array_3x2_c,
+  chunks=(2, 3),
+  overwrite=True)