From 7080f14d479aa6ff6f18c139b7bc5c4b510ccd19 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 7 Mar 2023 21:12:01 -0500 Subject: [PATCH 01/17] Add first version of variable length string array support TODO: * Write works but doesn't write number of serialized bytes. Hence, read does not allocate correct number of bytes. * Filter is non-functional at the moment. Adapt this such that it writes the correct json data (as seen in Anndata). --- .../janelia/saalfeldlab/n5/zarr/DType.java | 32 +++++++++++++++++-- .../janelia/saalfeldlab/n5/zarr/Filter.java | 2 +- .../saalfeldlab/n5/zarr/N5ZarrWriter.java | 5 +-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 0aba36a..85df7ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -31,7 +31,9 @@ import java.lang.reflect.Type; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; import java.util.EnumMap; +import java.util.List; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; @@ -49,6 +51,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; /** * Enumerates available zarr data types as defined at @@ -73,6 +76,7 @@ public class DType { typestrs.put(DataType.UINT64, ">u8"); typestrs.put(DataType.FLOAT32, ">f4"); typestrs.put(DataType.FLOAT64, ">f8"); + typestrs.put(DataType.VLENSTRING, "|O"); } public static enum Primitive { @@ -134,7 +138,7 @@ public DType(final String typestr) { order = typestr.charAt(0) == '<' ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN; final Primitive primitive = Primitive.fromCode(typestr.charAt(1)); - final int nB = Integer.parseInt(typestr.substring(2)); + final int nB = (primitive == Primitive.OBJECT) ? 0 : Integer.parseInt(typestr.substring(2)); switch (primitive) { case BIT: @@ -211,8 +215,15 @@ public DType(final String typestr) { byteBlockFactory = (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); break; + case OBJECT: + nBytes = 1; + nBits = 0; + dataBlockFactory = (blockSize, gridPosition, numElements) -> + new VLenStringDataBlock(blockSize, gridPosition, new String[0]); + byteBlockFactory = (blockSize, gridPosition, numElements) -> + new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); + break; // case BOOLEAN: -// case OBJECT: // not sure about this // case OTHER: // not sure about this // case STRING: // not sure about this // case UNICODE: // not sure about this @@ -333,6 +344,8 @@ protected final static DataType getDataType( default: return DataType.UINT8; // fallback } + case OBJECT: // todo: this should also depend on filters! + return DataType.VLENSTRING; default: return DataType.UINT8; // fallback } @@ -345,6 +358,21 @@ public String toString() { return typestr; } + /** + * Returns a list of {@link Filter filters} for the corresponding {@link DType}. + * + * @return list of filters + */ + public List getFilters() { + if (dataType == DataType.VLENSTRING) { + ArrayList filterSet = new ArrayList<>(); + filterSet.add(new Filter(){ public String id = "vlen-utf8"; }); + return filterSet; + } + else + return null; + } + /** * Factory for {@link DataBlock DataBlocks}. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 2b30578..cd60a6b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -36,5 +36,5 @@ * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> */ public interface Filter { - + public String id = null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java index edb43b3..5000ee0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java @@ -240,16 +240,17 @@ public void setDatasetAttributes( final int[] chunks = datasetAttributes.getBlockSize().clone(); Utils.reorder(chunks); + DType type = new DType(datasetAttributes.getDataType()); final ZArrayAttributes zArrayAttributes = new ZArrayAttributes( N5ZarrReader.VERSION.getMajor(), shape, chunks, - new DType(datasetAttributes.getDataType()), + type, ZarrCompressor.fromCompression(datasetAttributes.getCompression()), "0", 'C', dimensionSeparator, - null); + type.getFilters()); setZArrayAttributes(pathName, zArrayAttributes); } From f5b550715a6f97549acd15a5d6123cf72988f303 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 9 Mar 2023 13:22:08 -0500 Subject: [PATCH 02/17] Write and read vlen string arrays (not zarr compatible yet!) --- .../saalfeldlab/n5/zarr/N5ZarrReader.java | 32 ++++++++++++++++++- .../ZarrCompatibleVlenStringDataBlock.java | 28 ++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index 8cc82dc..4aa4c2e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -28,6 +28,8 @@ */ package org.janelia.saalfeldlab.n5.zarr; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -41,10 +43,13 @@ import java.util.HashMap; import java.util.stream.Stream; +import org.apache.commons.compress.utils.IOUtils; import org.janelia.saalfeldlab.n5.BlockReader; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.GsonAttributesParser; import org.janelia.saalfeldlab.n5.N5FSReader; @@ -57,6 +62,8 @@ import net.imglib2.type.Type; import net.imglib2.view.IntervalView; import net.imglib2.view.Views; +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; +import org.janelia.saalfeldlab.n5.blosc.BloscCompression; /** @@ -347,9 +354,12 @@ public static DataBlock readBlock( final int[] blockSize = datasetAttributes.getBlockSize(); final DType dType = datasetAttributes.getDType(); + final BlockReader reader = datasetAttributes.getCompression().getReader(); final ByteArrayDataBlock byteBlock = dType.createByteBlock(blockSize, gridPosition); - final BlockReader reader = datasetAttributes.getCompression().getReader(); + if (dType.getDataType() == DataType.VLENSTRING) { + return readVLenStringBlock(in, reader, byteBlock); + } reader.read(byteBlock, in); switch (dType.getDataType()) { @@ -423,6 +433,26 @@ public static DataBlock readBlock( return dataBlock; } + protected static VLenStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { + // read whole chunk and deserialize; this should be improved + VLenStringDataBlock dataBlock = new ZarrCompatibleVlenStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new byte[0]); + if (reader instanceof BloscCompression) { + // Blosc reader reads actual data and doesn't care about buffer size (but needs special treatment in data block) + reader.read(dataBlock, in); + } else if (reader instanceof DefaultBlockReader) { + try (final InputStream inflater = ((DefaultBlockReader) reader).getInputStream(in)) { + final DataInputStream dis = new DataInputStream(inflater); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(dis, out); + dataBlock.readData(ByteBuffer.wrap(out.toByteArray())); + } + } + else { + throw new UnsupportedOperationException("Only Blosc compression or algorithms that use DefaultBlockReader are supported.") + } + return dataBlock; + } + protected static > void copyTransposed( final RandomAccessibleInterval src, final RandomAccessibleInterval dst) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java new file mode 100644 index 0000000..1aa6c55 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java @@ -0,0 +1,28 @@ +package org.janelia.saalfeldlab.n5.zarr; + +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; + +import java.nio.ByteBuffer; + +public class ZarrCompatibleVlenStringDataBlock extends VLenStringDataBlock { + + public ZarrCompatibleVlenStringDataBlock(int[] size, long[] gridPosition, String[] data) { + super(size, gridPosition, data); + } + + public ZarrCompatibleVlenStringDataBlock(int[] size, long[] gridPosition, byte[] data) { + super(size, gridPosition, data); + } + + @Override + public void readData(final ByteBuffer buffer) { + try { + serializedData = buffer.array(); + } + catch (UnsupportedOperationException e) { // buffer is a DirectByteBuffer -> copy contents + serializedData = new byte[buffer.limit()]; + buffer.get(serializedData, 0, serializedData.length); + } + actualData = deserialize(serializedData); + } +} From 96fa009204f3e01b811d9d3adb47b56a1354358f Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 10 Mar 2023 14:38:37 -0500 Subject: [PATCH 03/17] (De-)serialize strings in zarr-compatible fashion --- .../janelia/saalfeldlab/n5/zarr/DType.java | 3 +- .../saalfeldlab/n5/zarr/N5ZarrReader.java | 7 +- .../saalfeldlab/n5/zarr/N5ZarrWriter.java | 1 + .../ZarrCompatibleVLenStringDataBlock.java | 67 +++++++++++++++++++ .../ZarrCompatibleVlenStringDataBlock.java | 28 -------- .../saalfeldlab/n5/zarr/N5ZarrTest.java | 30 +++++++++ 6 files changed, 102 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 85df7ce..027561b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -51,7 +51,6 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; /** * Enumerates available zarr data types as defined at @@ -219,7 +218,7 @@ public DType(final String typestr) { nBytes = 1; nBits = 0; dataBlockFactory = (blockSize, gridPosition, numElements) -> - new VLenStringDataBlock(blockSize, gridPosition, new String[0]); + new ZarrCompatibleVLenStringDataBlock(blockSize, gridPosition, new String[0]); byteBlockFactory = (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); break; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index 4aa4c2e..68dec90 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -62,7 +62,6 @@ import net.imglib2.type.Type; import net.imglib2.view.IntervalView; import net.imglib2.view.Views; -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; import org.janelia.saalfeldlab.n5.blosc.BloscCompression; @@ -433,9 +432,9 @@ public static DataBlock readBlock( return dataBlock; } - protected static VLenStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { + protected static ZarrCompatibleVLenStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { // read whole chunk and deserialize; this should be improved - VLenStringDataBlock dataBlock = new ZarrCompatibleVlenStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new byte[0]); + ZarrCompatibleVLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); if (reader instanceof BloscCompression) { // Blosc reader reads actual data and doesn't care about buffer size (but needs special treatment in data block) reader.read(dataBlock, in); @@ -448,7 +447,7 @@ protected static VLenStringDataBlock readVLenStringBlock(InputStream in, BlockRe } } else { - throw new UnsupportedOperationException("Only Blosc compression or algorithms that use DefaultBlockReader are supported.") + throw new UnsupportedOperationException("Only Blosc compression or algorithms that use DefaultBlockReader are supported."); } return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java index 5000ee0..9eb2c4a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java @@ -67,6 +67,7 @@ import net.imglib2.type.numeric.integer.ByteType; import net.imglib2.util.Intervals; import net.imglib2.view.Views; +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; /** * @author Stephan Saalfeld diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java new file mode 100644 index 0000000..c7a2e94 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java @@ -0,0 +1,67 @@ +package org.janelia.saalfeldlab.n5.zarr; + +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class ZarrCompatibleVLenStringDataBlock extends VLenStringDataBlock { + + public ZarrCompatibleVLenStringDataBlock(int[] size, long[] gridPosition, String[] data) { + super(size, gridPosition, data); + } + + public ZarrCompatibleVLenStringDataBlock(int[] size, long[] gridPosition, byte[] data) { + super(size, gridPosition, data); + } + + @Override + public void readData(final ByteBuffer buffer) { + try { + serializedData = buffer.array(); + } + catch (UnsupportedOperationException e) { // buffer is a DirectByteBuffer -> copy contents + serializedData = new byte[buffer.limit()]; + buffer.get(serializedData, 0, serializedData.length); + } + actualData = deserialize(serializedData); + } + + protected byte[] serialize(String[] strings) { + final int N = strings.length; + final byte[][] encodedStrings = Arrays.stream(strings).map((str) -> str.getBytes(ENCODING)).toArray(byte[][]::new); + final int[] lengths = Arrays.stream(encodedStrings).mapToInt((arr) -> arr.length).toArray(); + final int totalLength = Arrays.stream(lengths).sum(); + final ByteBuffer buf = ByteBuffer.wrap(new byte[totalLength + 4*N + 4]); + + buf.putInt(N); + for (int i = 0; i < N; ++i) { + buf.putInt(lengths[i]); + buf.put(encodedStrings[i]); + } + + return buf.array(); + } + + protected String[] deserialize(byte[] rawBytes) { + ByteBuffer buf = ByteBuffer.wrap(rawBytes); + + // sanity check to avoid out of memory errors + if (rawBytes.length < 4) + throw new RuntimeException("Corrupt buffer, data seems truncated."); + + final int N = buf.getInt(); + if (rawBytes.length < N) + throw new RuntimeException("Corrupt buffer, data seems truncated."); + + final String[] strings = new String[N]; + for (int i = 0; i < N; ++i) { + final int length = buf.getInt(); + final byte[] encodedString = new byte[length]; + buf.get(encodedString); + strings[i] = new String(encodedString, ENCODING); + } + + return strings; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java deleted file mode 100644 index 1aa6c55..0000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVlenStringDataBlock.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.janelia.saalfeldlab.n5.zarr; - -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; - -import java.nio.ByteBuffer; - -public class ZarrCompatibleVlenStringDataBlock extends VLenStringDataBlock { - - public ZarrCompatibleVlenStringDataBlock(int[] size, long[] gridPosition, String[] data) { - super(size, gridPosition, data); - } - - public ZarrCompatibleVlenStringDataBlock(int[] size, long[] gridPosition, byte[] data) { - super(size, gridPosition, data); - } - - @Override - public void readData(final ByteBuffer buffer) { - try { - serializedData = buffer.array(); - } - catch (UnsupportedOperationException e) { // buffer is a DirectByteBuffer -> copy contents - serializedData = new byte[buffer.limit()]; - buffer.get(serializedData, 0, serializedData.length); - } - actualData = deserialize(serializedData); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 4f8cd72..143ca51 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -45,16 +45,19 @@ import org.janelia.saalfeldlab.n5.AbstractN5Test; import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.VLenStringDataBlock; import org.janelia.saalfeldlab.n5.blosc.BloscCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -258,6 +261,33 @@ public void testMode1WriteReadByteBlock() { public void testWriteReadSerializableBlock() { } + @Override + @Test + public void testWriteReadStringBlock() { + DataType dataType = DataType.VLENSTRING; + int[] blockSize = new int[]{3, 2, 1}; + String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; + Compression[] compressions = this.getCompressions(); + + for (Compression compression : compressions) { + System.out.println("Testing " + compression.getType() + " " + dataType); + + try { + n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); + DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); + VLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + n5.writeBlock("/test/group/dataset", attributes, dataBlock); + DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, new long[]{0L, 0L, 0L}); + Assert.assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); + Assert.assertTrue(n5.remove("/test/group/dataset")); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail("Block cannot be written."); + } + } + + } + private boolean runPythonTest(String script) throws IOException, InterruptedException { final boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows"); From 41faeee427f7b7ccf0213f43a6f9ce0b84491896 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 10 Mar 2023 15:40:30 -0500 Subject: [PATCH 04/17] Make filter (de-)serialize to the right thing --- .../janelia/saalfeldlab/n5/zarr/DType.java | 2 +- .../janelia/saalfeldlab/n5/zarr/Filter.java | 34 ++++++++++++++++++- .../saalfeldlab/n5/zarr/N5ZarrReader.java | 1 + .../ZarrCompatibleVLenStringDataBlock.java | 4 +++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 027561b..18a18da 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -365,7 +365,7 @@ public String toString() { public List getFilters() { if (dataType == DataType.VLENSTRING) { ArrayList filterSet = new ArrayList<>(); - filterSet.add(new Filter(){ public String id = "vlen-utf8"; }); + filterSet.add(new ZarrCompatibleVLenStringDataBlock.VLenStringFilter()); return filterSet; } else diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index cd60a6b..f6ba14c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -28,6 +28,17 @@ */ package org.janelia.saalfeldlab.n5.zarr; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import java.lang.reflect.Type; + /** * Place holder interface for filters * @@ -36,5 +47,26 @@ * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> */ public interface Filter { - public String id = null; + + static public class JsonAdapter implements JsonDeserializer { + + @Override + public Filter deserialize( + final JsonElement json, + final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { + + final JsonObject jsonObject = json.getAsJsonObject(); + final JsonElement jsonId = jsonObject.get("id"); + if (jsonId == null) + return null; + final String id = jsonId.getAsString(); + switch (id) { + case "vlen-utf8": + return new ZarrCompatibleVLenStringDataBlock.VLenStringFilter(); + default: + return null; + } + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index 68dec90..9c514e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -81,6 +81,7 @@ static private GsonBuilder initGsonBuilder(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DType.class, new DType.JsonAdapter()); gsonBuilder.registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter); + gsonBuilder.registerTypeAdapter(Filter.class, new Filter.JsonAdapter()); gsonBuilder.serializeNulls(); return gsonBuilder; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java index c7a2e94..bac5fce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java @@ -64,4 +64,8 @@ protected String[] deserialize(byte[] rawBytes) { return strings; } + + public static class VLenStringFilter implements Filter { + public final String id = "vlen-utf8"; + } } From 84514a662e1375d22827381f08358c20ca3cec7c Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 10 Mar 2023 16:53:27 -0500 Subject: [PATCH 05/17] Fix endianness based on numcodecs source code In this version reading and writing arrays to/from Python works (tested manually) --- .../saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java index bac5fce..4454d8f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java @@ -3,6 +3,7 @@ import org.janelia.saalfeldlab.n5.VLenStringDataBlock; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Arrays; public class ZarrCompatibleVLenStringDataBlock extends VLenStringDataBlock { @@ -33,6 +34,7 @@ protected byte[] serialize(String[] strings) { final int[] lengths = Arrays.stream(encodedStrings).mapToInt((arr) -> arr.length).toArray(); final int totalLength = Arrays.stream(lengths).sum(); final ByteBuffer buf = ByteBuffer.wrap(new byte[totalLength + 4*N + 4]); + buf.order(ByteOrder.LITTLE_ENDIAN); buf.putInt(N); for (int i = 0; i < N; ++i) { @@ -45,6 +47,7 @@ protected byte[] serialize(String[] strings) { protected String[] deserialize(byte[] rawBytes) { ByteBuffer buf = ByteBuffer.wrap(rawBytes); + buf.order(ByteOrder.LITTLE_ENDIAN); // sanity check to avoid out of memory errors if (rawBytes.length < 4) From 5570cadb8547dfcf046fc6acf3cd6206802dc2a8 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 10 Mar 2023 20:16:01 -0500 Subject: [PATCH 06/17] Fix typo in function name --- src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java | 3 --- .../org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java | 8 ++++---- .../org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java | 7 +++---- .../java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java | 6 +++--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index f6ba14c..3566a7c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -33,9 +33,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index 9c514e2..df4834d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -241,7 +241,7 @@ public boolean groupExists(final String pathName) { return Files.exists(path) && Files.isRegularFile(path); } - public ZArrayAttributes getZArraryAttributes(final String pathName) throws IOException { + public ZArrayAttributes getZArrayAttributes(final String pathName) throws IOException { final Path path = Paths.get(basePath, removeLeadingSlash(pathName), zarrayFile); final HashMap attributes = new HashMap<>(); @@ -274,7 +274,7 @@ public ZArrayAttributes getZArraryAttributes(final String pathName) throws IOExc @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final ZArrayAttributes zArrayAttributes = getZArraryAttributes(pathName); + final ZArrayAttributes zArrayAttributes = getZArrayAttributes(pathName); return zArrayAttributes == null ? null : zArrayAttributes.getDatasetAttributes(); } @@ -326,7 +326,7 @@ public HashMap getAttributes(final String pathName) throws if (mapN5DatasetAttributes && datasetExists(pathName)) { - final DatasetAttributes datasetAttributes = getZArraryAttributes(pathName).getDatasetAttributes(); + final DatasetAttributes datasetAttributes = getZArrayAttributes(pathName).getDatasetAttributes(); attributes.put("dimensions", gson.toJsonTree(datasetAttributes.getDimensions())); attributes.put("blockSize", gson.toJsonTree(datasetAttributes.getBlockSize())); attributes.put("dataType", gson.toJsonTree(datasetAttributes.getDataType())); @@ -480,7 +480,7 @@ public DataBlock readBlock( if (datasetAttributes instanceof ZarrDatasetAttributes) zarrDatasetAttributes = (ZarrDatasetAttributes)datasetAttributes; else - zarrDatasetAttributes = getZArraryAttributes(pathName).getDatasetAttributes(); + zarrDatasetAttributes = getZArrayAttributes(pathName).getDatasetAttributes(); final String dimSep; final String attrDimensionSeparator = zarrDatasetAttributes.getDimensionSeparator(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java index 9eb2c4a..e8f6865 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java @@ -67,7 +67,6 @@ import net.imglib2.type.numeric.integer.ByteType; import net.imglib2.util.Intervals; import net.imglib2.view.Views; -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; /** * @author Stephan Saalfeld @@ -292,7 +291,7 @@ public void setAttributes( if (mapN5DatasetAttributes && datasetExists(pathName)) { attributes = new HashMap<>(attributes); - ZArrayAttributes zArrayAttributes = getZArraryAttributes(pathName); + ZArrayAttributes zArrayAttributes = getZArrayAttributes(pathName); long[] shape; int[] chunks; final DType dtype; @@ -469,7 +468,7 @@ public void writeBlock( if (datasetAttributes instanceof ZarrDatasetAttributes) zarrDatasetAttributes = (ZarrDatasetAttributes)datasetAttributes; else - zarrDatasetAttributes = getZArraryAttributes(pathName).getDatasetAttributes(); + zarrDatasetAttributes = getZArrayAttributes(pathName).getDatasetAttributes(); final Path path = Paths.get( basePath, @@ -498,7 +497,7 @@ public boolean deleteBlock(final String pathName, final long... gridPosition) th if (datasetAttributes instanceof ZarrDatasetAttributes) zarrDatasetAttributes = (ZarrDatasetAttributes)datasetAttributes; else - zarrDatasetAttributes = getZArraryAttributes(pathName).getDatasetAttributes(); + zarrDatasetAttributes = getZArrayAttributes(pathName).getDatasetAttributes(); final Path path = Paths.get( basePath, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 143ca51..8e6da01 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -136,7 +136,7 @@ public void testCreateNestedDataset() throws IOException { N5ZarrWriter n5Nested = new N5ZarrWriter(testDirPath, "/", true ); n5Nested.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]); - assertEquals( "/", n5Nested.getZArraryAttributes(datasetName).getDimensionSeparator()); + assertEquals( "/", n5Nested.getZArrayAttributes(datasetName).getDimensionSeparator()); n5Nested.remove(datasetName); n5Nested.close(); @@ -430,7 +430,7 @@ public void testReadZarrPython() throws IOException, InterruptedException { final RandomAccessibleInterval a3x2_c_bu4_f1_after = N5Utils.open(n5Zarr, datasetName); assertIsSequence(Views.interval(a3x2_c_bu4_f1_after, a3x2_c_bu4_f1), refUnsignedInt); final RandomAccess ra = a3x2_c_bu4_f1_after.randomAccess(); - final int fill_value = Integer.parseInt(n5Zarr.getZArraryAttributes(datasetName).getFillValue()); + final int fill_value = Integer.parseInt(n5Zarr.getZArrayAttributes(datasetName).getFillValue()); ra.setPosition(shape[0] - 5, 0); assertEquals(fill_value, ra.get().getInteger()); ra.setPosition(shape[1] - 5, 1); @@ -481,7 +481,7 @@ public void testReadZarrNestedPython() throws IOException, InterruptedException assertArrayEquals(datasetAttributesC.getDimensions(), new long[]{3, 2}); assertArrayEquals(datasetAttributesC.getBlockSize(), new int[]{3, 2}); assertEquals(datasetAttributesC.getDataType(), DataType.UINT8); - assertEquals( n5Zarr.getZArraryAttributes(testZarrDatasetName + "/3x2_c_|u1").getDimensionSeparator(), "/" ); + assertEquals( n5Zarr.getZArrayAttributes(testZarrDatasetName + "/3x2_c_|u1").getDimensionSeparator(), "/" ); final UnsignedByteType refUnsignedByte = new UnsignedByteType(); assertIsSequence(N5Utils.open(n5Zarr, testZarrDatasetName + "/3x2_c_|u1"), refUnsignedByte); From f7346a18787b37704cce6538a7858dc8d240de4a Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sun, 15 Oct 2023 15:58:49 -0400 Subject: [PATCH 07/17] Re-introduce special treatment of String block Without this, the data gets read incompletely --- .../n5/zarr/ZarrKeyValueReader.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index dda88dd..77e423c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5.zarr; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; @@ -34,6 +36,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.commons.compress.utils.IOUtils; import org.janelia.saalfeldlab.n5.BlockReader; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader; @@ -42,6 +45,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; @@ -50,6 +54,7 @@ import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.blosc.BloscCompression; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; import org.janelia.saalfeldlab.n5.zarr.cache.ZarrJsonCache; @@ -659,6 +664,11 @@ protected static DataBlock readBlock( final ByteArrayDataBlock byteBlock = dType.createByteBlock(blockSize, gridPosition); final BlockReader reader = datasetAttributes.getCompression().getReader(); + + if (dType.getDataType() == DataType.VLENSTRING) { + return readVLenStringBlock(in, reader, byteBlock); + } + reader.read(byteBlock, in); switch (dType.getDataType()) { @@ -676,6 +686,26 @@ protected static DataBlock readBlock( return dataBlock; } + private static ZarrCompatibleVLenStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { + // read whole chunk and deserialize; this should be improved + ZarrCompatibleVLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); + if (reader instanceof BloscCompression) { + // Blosc reader reads actual data and doesn't care about buffer size (but needs special treatment in data block) + reader.read(dataBlock, in); + } else if (reader instanceof DefaultBlockReader) { + try (final InputStream inflater = ((DefaultBlockReader) reader).getInputStream(in)) { + final DataInputStream dis = new DataInputStream(inflater); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(dis, out); + dataBlock.readData(ByteBuffer.wrap(out.toByteArray())); + } + } + else { + throw new UnsupportedOperationException("Only Blosc compression or algorithms that use DefaultBlockReader are supported."); + } + return dataBlock; + } + /** * Constructs the path for a data block in a dataset at a given grid * position. From f665d7940c9b179677fb23516b1ef4405a3b6622 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sun, 15 Oct 2023 16:04:23 -0400 Subject: [PATCH 08/17] Fix some IDE warnings --- .../janelia/saalfeldlab/n5/zarr/Filter.java | 2 +- .../saalfeldlab/n5/zarr/N5ZarrReader.java | 1 - .../saalfeldlab/n5/zarr/N5ZarrTest.java | 48 +++++++++---------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 3566a7c..b274f21 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -45,7 +45,7 @@ */ public interface Filter { - static public class JsonAdapter implements JsonDeserializer { + class JsonAdapter implements JsonDeserializer { @Override public Filter deserialize( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index 46df711..b8e400c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -59,7 +59,6 @@ public class N5ZarrReader extends ZarrKeyValueReader { * If true, fields from .zgroup, .zarray, and .zattrs will be merged * when calling getAttributes, and variants of getAttribute * @param cacheMeta cache attributes and meta data - * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the * store. This is most interesting for high latency backends. Changes diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 22aef7a..794f992 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -33,12 +33,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; @@ -96,10 +96,10 @@ */ public class N5ZarrTest extends AbstractN5Test { - static private String testZarrDatasetName = "/test/data"; + static private final String testZarrDatasetName = "/test/data"; @Override - protected String tempN5Location() throws URISyntaxException { + protected String tempN5Location() { try { return Files.createTempDirectory("n5-zarr-test").toUri().toString(); @@ -219,7 +219,7 @@ public void testCreateDatasetNameEmpty() throws IOException, URISyntaxException } @Test - public void testCreateDatasetNameSlash() throws IOException, URISyntaxException { + public void testCreateDatasetNameSlash() throws IOException { try (final N5Writer n5 = createN5Writer()) { n5.createDataset("", dimensions, blockSize, DataType.UINT64, getCompressions()[0]); @@ -236,7 +236,7 @@ public void testGetDatasetAttributesNull() throws IOException { } @Test - public void testPadCrop() throws Exception { + public void testPadCrop() { final byte[] src = new byte[]{1, 1, 1, 1}; // 2x2 final int[] srcBlockSize = new int[]{2, 2}; @@ -255,9 +255,9 @@ public void testVersion() throws NumberFormatException, IOException { try (final N5Writer writer = createN5Writer()) { - @SuppressWarnings("resource") final N5ZarrWriter zarr = (N5ZarrWriter)writer; + final N5ZarrWriter zarr = (N5ZarrWriter)writer; final Version n5Version = writer.getVersion(); - Assert.assertTrue(n5Version.equals(N5ZarrReader.VERSION)); + assertEquals(n5Version, N5ZarrReader.VERSION); final JsonObject bumpVersion = new JsonObject(); bumpVersion.add(ZarrKeyValueReader.ZARR_FORMAT_KEY, new JsonPrimitive(N5ZarrReader.VERSION.getMajor() + 1)); @@ -355,11 +355,11 @@ public void testListAttributes() throws IOException { n5.setAttribute(datasetName2, "attr5", 5.4); Map> attributesMap = n5.listAttributes(datasetName2); - assertTrue(attributesMap.get("attr1") == long[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == long.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == double.class); + assertSame(attributesMap.get("attr1"), long[].class); + assertSame(attributesMap.get("attr2"), String[].class); + assertSame(attributesMap.get("attr3"), long.class); + assertSame(attributesMap.get("attr4"), String.class); + assertSame(attributesMap.get("attr5"), double.class); n5.createGroup(groupName2); n5.setAttribute(groupName2, "attr1", new double[]{1, 2, 3}); @@ -369,11 +369,11 @@ public void testListAttributes() throws IOException { n5.setAttribute(groupName2, "attr5", 5.4); attributesMap = n5.listAttributes(datasetName2); - assertTrue(attributesMap.get("attr1") == long[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == long.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == double.class); + assertSame(attributesMap.get("attr1"), long[].class); + assertSame(attributesMap.get("attr2"), String[].class); + assertSame(attributesMap.get("attr3"), long.class); + assertSame(attributesMap.get("attr4"), String.class); + assertSame(attributesMap.get("attr5"), double.class); } } @@ -406,7 +406,7 @@ public void testWriteReadStringBlock() { DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); VLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock("/test/group/dataset", attributes, dataBlock); - DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, new long[]{0L, 0L, 0L}); + DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); Assert.assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); Assert.assertTrue(n5.remove("/test/group/dataset")); } catch (IOException e) { @@ -672,7 +672,7 @@ public void testReadZarrNestedPython() throws IOException, InterruptedException } @Test - public void testRawCompressorNullInZarray() throws IOException, FileNotFoundException, ParseException { + public void testRawCompressorNullInZarray() throws IOException, ParseException { final String testZarrDirPath = tmpPathName("n5-zarr-test-"); final N5ZarrWriter n5 = new N5ZarrWriter(testZarrDirPath); @@ -687,7 +687,7 @@ public void testRawCompressorNullInZarray() throws IOException, FileNotFoundExce try (FileReader freader = new FileReader(zArrayFile)) { final JSONObject zarray = (JSONObject)jsonParser.parse(freader); final JSONObject compressor = (JSONObject)zarray.get("compressor"); - assertTrue(compressor == null); + assertNull(compressor); } finally { n5.remove(); n5.close(); @@ -733,8 +733,8 @@ public void testAttributes() throws IOException { }.getType())); // test the case where the resulting file becomes shorter - n5.setAttribute(groupName, "key1", new Integer(1)); - n5.setAttribute(groupName, "key2", new Integer(2)); + n5.setAttribute(groupName, "key1", 1); + n5.setAttribute(groupName, "key2", 2); Assert.assertEquals(4, n5.listAttributes(groupName).size()); /* class interface */ @@ -832,7 +832,7 @@ public void testAttributeMapping() throws IOException { @Test @Override @Ignore - public void testNullAttributes() throws IOException, URISyntaxException { + public void testNullAttributes() throws IOException { // serializeNulls must be on for Zarr to be able to write datasets with raw compression @@ -882,7 +882,7 @@ public void testNullAttributes() throws IOException, URISyntaxException { @Test @Override @Ignore - public void testRootLeaves() throws IOException { + public void testRootLeaves() { // This tests serializing primitives, and arrays at the root of an n5's attributes, // since .zattrs must be a json object, this would test invalide behavior for zarr, From cddf433283bf389c8666d786ff89fb9a4896bd4d Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Mon, 16 Oct 2023 15:34:42 -0400 Subject: [PATCH 09/17] Rename Vlen string data block --- .../java/org/janelia/saalfeldlab/n5/zarr/DType.java | 10 +++++----- .../java/org/janelia/saalfeldlab/n5/zarr/Filter.java | 2 +- ...taBlock.java => ZarrCompatibleStringDataBlock.java} | 8 ++++---- .../saalfeldlab/n5/zarr/ZarrKeyValueReader.java | 6 +++--- .../org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java | 10 +++++----- 5 files changed, 18 insertions(+), 18 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/zarr/{ZarrCompatibleVLenStringDataBlock.java => ZarrCompatibleStringDataBlock.java} (87%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 18a18da..8b7a245 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -75,7 +75,7 @@ public class DType { typestrs.put(DataType.UINT64, ">u8"); typestrs.put(DataType.FLOAT32, ">f4"); typestrs.put(DataType.FLOAT64, ">f8"); - typestrs.put(DataType.VLENSTRING, "|O"); + typestrs.put(DataType.STRING, "|O"); } public static enum Primitive { @@ -218,7 +218,7 @@ public DType(final String typestr) { nBytes = 1; nBits = 0; dataBlockFactory = (blockSize, gridPosition, numElements) -> - new ZarrCompatibleVLenStringDataBlock(blockSize, gridPosition, new String[0]); + new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); byteBlockFactory = (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); break; @@ -344,7 +344,7 @@ protected final static DataType getDataType( return DataType.UINT8; // fallback } case OBJECT: // todo: this should also depend on filters! - return DataType.VLENSTRING; + return DataType.STRING; default: return DataType.UINT8; // fallback } @@ -363,9 +363,9 @@ public String toString() { * @return list of filters */ public List getFilters() { - if (dataType == DataType.VLENSTRING) { + if (dataType == DataType.STRING) { ArrayList filterSet = new ArrayList<>(); - filterSet.add(new ZarrCompatibleVLenStringDataBlock.VLenStringFilter()); + filterSet.add(new ZarrCompatibleStringDataBlock.VLenStringFilter()); return filterSet; } else diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index b274f21..4efe7b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -60,7 +60,7 @@ public Filter deserialize( final String id = jsonId.getAsString(); switch (id) { case "vlen-utf8": - return new ZarrCompatibleVLenStringDataBlock.VLenStringFilter(); + return new ZarrCompatibleStringDataBlock.VLenStringFilter(); default: return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java similarity index 87% rename from src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java rename to src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java index 4454d8f..b83c1cf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleVLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java @@ -1,18 +1,18 @@ package org.janelia.saalfeldlab.n5.zarr; -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; +import org.janelia.saalfeldlab.n5.StringDataBlock; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -public class ZarrCompatibleVLenStringDataBlock extends VLenStringDataBlock { +public class ZarrCompatibleStringDataBlock extends StringDataBlock { - public ZarrCompatibleVLenStringDataBlock(int[] size, long[] gridPosition, String[] data) { + public ZarrCompatibleStringDataBlock(int[] size, long[] gridPosition, String[] data) { super(size, gridPosition, data); } - public ZarrCompatibleVLenStringDataBlock(int[] size, long[] gridPosition, byte[] data) { + public ZarrCompatibleStringDataBlock(int[] size, long[] gridPosition, byte[] data) { super(size, gridPosition, data); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index 77e423c..a5dda8e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -665,7 +665,7 @@ protected static DataBlock readBlock( final ByteArrayDataBlock byteBlock = dType.createByteBlock(blockSize, gridPosition); final BlockReader reader = datasetAttributes.getCompression().getReader(); - if (dType.getDataType() == DataType.VLENSTRING) { + if (dType.getDataType() == DataType.STRING) { return readVLenStringBlock(in, reader, byteBlock); } @@ -686,9 +686,9 @@ protected static DataBlock readBlock( return dataBlock; } - private static ZarrCompatibleVLenStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { + private static ZarrCompatibleStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { // read whole chunk and deserialize; this should be improved - ZarrCompatibleVLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); + ZarrCompatibleStringDataBlock dataBlock = new ZarrCompatibleStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); if (reader instanceof BloscCompression) { // Blosc reader reads actual data and doesn't care about buffer size (but needs special treatment in data block) reader.read(dataBlock, in); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 794f992..370af99 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -61,8 +61,8 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.VLenStringDataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; +import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.blosc.BloscCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.json.simple.JSONObject; @@ -393,7 +393,7 @@ public void testWriteReadSerializableBlock() { @Test public void testWriteReadStringBlock() { - DataType dataType = DataType.VLENSTRING; + DataType dataType = DataType.STRING; int[] blockSize = new int[]{3, 2, 1}; String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; Compression[] compressions = this.getCompressions(); @@ -404,11 +404,11 @@ public void testWriteReadStringBlock() { try (final N5Writer n5 = createN5Writer()) { n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); - VLenStringDataBlock dataBlock = new ZarrCompatibleVLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + StringDataBlock dataBlock = new ZarrCompatibleStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock("/test/group/dataset", attributes, dataBlock); DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); - Assert.assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove("/test/group/dataset")); + assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); + assertTrue(n5.remove("/test/group/dataset")); } catch (IOException e) { e.printStackTrace(); Assert.fail("Block cannot be written."); From f4861d2937cd302c7ecc330b402495ddd42985f1 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 18 Oct 2023 03:56:37 -0400 Subject: [PATCH 10/17] Write test to check Python compatibility --- .../zarr/ZarrCompatibleStringDataBlock.java | 10 +++-- src/test/python/zarr-test.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java index b83c1cf..269db72 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java @@ -53,12 +53,14 @@ protected String[] deserialize(byte[] rawBytes) { if (rawBytes.length < 4) throw new RuntimeException("Corrupt buffer, data seems truncated."); - final int N = buf.getInt(); - if (rawBytes.length < N) + final int n = buf.getInt(); + if (rawBytes.length < n) throw new RuntimeException("Corrupt buffer, data seems truncated."); - final String[] strings = new String[N]; - for (int i = 0; i < N; ++i) { + System.out.println(" *********** ZARR: read so many bytes: " + rawBytes.length + " and n = " + n); + + final String[] strings = new String[n]; + for (int i = 0; i < n; ++i) { final int length = buf.getInt(); final byte[] encodedString = new byte[length]; buf.get(encodedString); diff --git a/src/test/python/zarr-test.py b/src/test/python/zarr-test.py index d8508ec..e89fca0 100644 --- a/src/test/python/zarr-test.py +++ b/src/test/python/zarr-test.py @@ -45,6 +45,8 @@ array_3x2_c = np.arange(0,3*2).reshape(2,3) array_3x2_f = np.asfortranarray(array_3x2_c) +array_3x2_str_c = np.array(["", "a", "bc", "de", "fgh", ":-þ"]).reshape(2,3) +array_3x2_str_f = np.asfortranarray(array_3x2_str_c) array_30x20_c = np.arange(0,30*20).reshape(20,30) array_30x20_f = np.asfortranarray(array_30x20_c) @@ -208,3 +210,38 @@ fill_value="NaN", overwrite=True) +group.array( + name='3x2_c_str', + dtype=str, + data=array_3x2_str_c, + chunks=(2, 3), + overwrite=True) +group.array( + name='3x2_f_str', + dtype=str, + data=array_3x2_str_f, + chunks=(2, 3), + order='F', + overwrite=True) +group.array( + name='3x2_str_c_u8_zlib', + dtype=str, + compressor=Zlib(level=6), + data=array_3x2_str_c, + chunks=(7, 13), + overwrite=True) +group.array( + name='3x2_str_c_u8_gzip', + dtype=str, + compressor=GZip(level=6), + data=array_3x2_str_c, + chunks=(7, 13), + overwrite=True) +group.array( + name='3x2_str_c_u8_bz2', + dtype=str, + compressor=BZ2(level=1), + data=array_3x2_str_c, + chunks=(2, 2), + overwrite=True) + From cd9d2c1d3b9954c8846e4312d01932240aac9dc6 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 18 Oct 2023 05:38:28 -0400 Subject: [PATCH 11/17] Fix missing deserialization of Filters --- src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java | 2 ++ .../org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 4efe7b1..9428d41 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -45,6 +45,8 @@ */ public interface Filter { + public static JsonAdapter jsonAdapter = new Filter.JsonAdapter(); + class JsonAdapter implements JsonDeserializer { @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index a5dda8e..4561589 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -842,6 +842,7 @@ protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.registerTypeAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.registerTypeAdapter(ZArrayAttributes.class, ZArrayAttributes.jsonAdapter); + gsonBuilder.registerTypeAdapter(Filter.class, Filter.jsonAdapter); gsonBuilder.disableHtmlEscaping(); gsonBuilder.serializeNulls(); From 9f3e5f75fdf8f55e8d5199d1e5b8e2e6c2355015 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 18 Oct 2023 06:00:51 -0400 Subject: [PATCH 12/17] Add test for verifying Python compatibility To run and debug in intellij: add PATH and CONDA_PREFIX env variables to run setting --- .../zarr/ZarrCompatibleStringDataBlock.java | 2 -- .../saalfeldlab/n5/zarr/N5ZarrTest.java | 19 +++++++++++++++++++ src/test/python/zarr-test.py | 6 +++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java index 269db72..ce72586 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java @@ -57,8 +57,6 @@ protected String[] deserialize(byte[] rawBytes) { if (rawBytes.length < n) throw new RuntimeException("Corrupt buffer, data seems truncated."); - System.out.println(" *********** ZARR: read so many bytes: " + rawBytes.length + " and n = " + n); - final String[] strings = new String[n]; for (int i = 0; i < n; ++i) { final int length = buf.getInt(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 370af99..05bc4b2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -634,6 +634,25 @@ public void testReadZarrPython() throws IOException, InterruptedException { raf.setPosition(shapef[1] - 5, 1); assertTrue(Float.isNaN(raf.get().getRealFloat())); + /* strings */ + String[] expected = {"", "a", "bc", "de", "fgh", ":-þ"}; // C-order + datasetName = testZarrDatasetName + "/3x2_c_str"; + DatasetAttributes strAttributes = n5Zarr.getDatasetAttributes(datasetName); + DataBlock loadedDataBlock = n5Zarr.readBlock(datasetName, strAttributes, 0L, 0L); + assertArrayEquals(expected, ((String[])loadedDataBlock.getData())); + + expected = new String[] {"", "de", "a", "fgh", "bc", ":-þ"}; // F-order + datasetName = testZarrDatasetName + "/3x2_f_str"; + strAttributes = n5Zarr.getDatasetAttributes(datasetName); + loadedDataBlock = n5Zarr.readBlock(datasetName, strAttributes, 0L, 0L); + assertArrayEquals(expected, ((String[])loadedDataBlock.getData())); + + expected = new String[] {"bc", "", ":-þ", ""}; // chunked and compressed (second chunk) + datasetName = testZarrDatasetName + "/3x2_c_str_bz2"; + strAttributes = n5Zarr.getDatasetAttributes(datasetName); + loadedDataBlock = n5Zarr.readBlock(datasetName, strAttributes, 1L, 0L); + assertArrayEquals(expected, ((String[])loadedDataBlock.getData())); + /* remove the container */ n5Zarr.remove(); n5Zarr.close(); diff --git a/src/test/python/zarr-test.py b/src/test/python/zarr-test.py index e89fca0..f1449ce 100644 --- a/src/test/python/zarr-test.py +++ b/src/test/python/zarr-test.py @@ -224,21 +224,21 @@ order='F', overwrite=True) group.array( - name='3x2_str_c_u8_zlib', + name='3x2_c_str_zlib', dtype=str, compressor=Zlib(level=6), data=array_3x2_str_c, chunks=(7, 13), overwrite=True) group.array( - name='3x2_str_c_u8_gzip', + name='3x2_c_str_gzip', dtype=str, compressor=GZip(level=6), data=array_3x2_str_c, chunks=(7, 13), overwrite=True) group.array( - name='3x2_str_c_u8_bz2', + name='3x2_c_str_bz2', dtype=str, compressor=BZ2(level=1), data=array_3x2_str_c, From 74b535f1807a18152d0bae9f39b0edb63171ebf6 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 18 Oct 2023 08:54:00 -0400 Subject: [PATCH 13/17] Change DType constructor call as preparation to change signature --- .../org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index 4561589..7b55229 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -35,6 +35,7 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collection; import org.apache.commons.compress.utils.IOUtils; import org.janelia.saalfeldlab.n5.BlockReader; @@ -503,8 +504,7 @@ protected JsonElement zarrToN5DatasetAttributes(final JsonElement elem ) { attrs.add(DatasetAttributes.DIMENSIONS_KEY, attrs.get(ZArrayAttributes.shapeKey)); attrs.add(DatasetAttributes.BLOCK_SIZE_KEY, attrs.get(ZArrayAttributes.chunksKey)); - attrs.addProperty(DatasetAttributes.DATA_TYPE_KEY, - new DType(attrs.get(ZArrayAttributes.dTypeKey).getAsString()).getDataType().toString()); + attrs.addProperty(DatasetAttributes.DATA_TYPE_KEY, zattrs.getDType().getDataType().toString()); JsonElement e = attrs.get(ZArrayAttributes.compressorKey); if (e == JsonNull.INSTANCE) { From edccdaa23d1885d911fa6ef6222cd057739dfdab Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 19 Oct 2023 09:54:37 -0400 Subject: [PATCH 14/17] Change construction of DType DType is constructed by a typestring and a collection of filters. In particular, it's not directly de-serialized anymore. --- .../janelia/saalfeldlab/n5/zarr/DType.java | 35 ++++--------------- .../saalfeldlab/n5/zarr/N5ZarrReader.java | 1 - .../saalfeldlab/n5/zarr/ZArrayAttributes.java | 8 +++-- .../n5/zarr/ZarrKeyValueReader.java | 1 - .../saalfeldlab/n5/zarr/N5ZarrTest.java | 9 +++-- 5 files changed, 20 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 8b7a245..13146f3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -32,6 +32,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Collection; import java.util.EnumMap; import java.util.List; @@ -76,6 +77,7 @@ public class DType { typestrs.put(DataType.FLOAT32, ">f4"); typestrs.put(DataType.FLOAT64, ">f8"); typestrs.put(DataType.STRING, "|O"); + typestrs.put(DataType.OBJECT, "|O"); } public static enum Primitive { @@ -131,7 +133,7 @@ public static Primitive fromCode(final char code) { /* the closest possible N5 DataType */ protected final DataType dataType; - public DType(final String typestr) { + public DType(final String typestr, final Collection filters) { this.typestr = typestr; @@ -217,10 +219,8 @@ public DType(final String typestr) { case OBJECT: nBytes = 1; nBits = 0; - dataBlockFactory = (blockSize, gridPosition, numElements) -> - new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); - byteBlockFactory = (blockSize, gridPosition, numElements) -> - new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); + dataBlockFactory = null; + byteBlockFactory = null; break; // case BOOLEAN: // case OTHER: // not sure about this @@ -343,8 +343,8 @@ protected final static DataType getDataType( default: return DataType.UINT8; // fallback } - case OBJECT: // todo: this should also depend on filters! - return DataType.STRING; + case OBJECT: + return DataType.OBJECT; default: return DataType.UINT8; // fallback } @@ -433,27 +433,6 @@ private static interface ByteBlockFactory { public ByteArrayDataBlock createByteBlock(final int[] blockSize, final long[] gridPosition, final int numElements); } - static public class JsonAdapter implements JsonDeserializer, JsonSerializer { - - @Override - public DType deserialize( - final JsonElement json, - final Type typeOfT, - final JsonDeserializationContext context) throws JsonParseException { - - return new DType(json.getAsString()); - } - - @Override - public JsonElement serialize( - final DType src, - final Type typeOfSrc, - final JsonSerializationContext context) { - - return new JsonPrimitive(src.toString()); - } - } - public ByteOrder getOrder() { return order; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java index b8e400c..a223601 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java @@ -80,7 +80,6 @@ public N5ZarrReader(final String basePath, new FileSystemKeyValueAccess(FileSystems.getDefault()), basePath, gsonBuilder - .registerTypeAdapter(DType.class, new DType.JsonAdapter()) .registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter), mapN5DatasetAttributes, mergeAttributes, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java index c02a92e..62ffa2c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java @@ -215,16 +215,20 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial final JsonObject obj = json.getAsJsonObject(); final JsonElement sepElem = obj.get("dimension_separator"); try { + final Collection filters = context.deserialize(obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType()); + final String typestr = context.deserialize(obj.get("dtype"), String.class); + final DType dType = new DType(typestr, filters); + return new ZArrayAttributes( obj.get("zarr_format").getAsInt(), context.deserialize( obj.get("shape"), long[].class), context.deserialize( obj.get("chunks"), int[].class), - context.deserialize( obj.get("dtype"), DType.class), // fix + dType, // fix context.deserialize( obj.get("compressor"), ZarrCompressor.class), // fix obj.get("fill_value").getAsString(), obj.get("order").getAsCharacter(), sepElem != null ? sepElem.getAsString() : ".", - context.deserialize( obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType())); + filters); } catch (Exception e) { return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index 7b55229..9ca58a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -836,7 +836,6 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) { - gsonBuilder.registerTypeAdapter(DType.class, new DType.JsonAdapter()); gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 05bc4b2..151273c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -46,6 +46,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -800,7 +801,9 @@ public void testAttributeMapping() throws IOException { int[] blkN5 = n5.getAttribute(datasetName, DatasetAttributes.BLOCK_SIZE_KEY, int[].class); assertArrayEquals(blkZarr, blkN5); - DType dtype = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, DType.class); + String typestr = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, String.class); + Collection filters = n5.getAttribute(datasetName, ZArrayAttributes.filtersKey, TypeToken.getParameterized(Collection.class, Filter.class).getType()); + DType dtype = new DType(typestr, filters); // read to a string because zarr may not have the N5 DataType deserializer DataType n5DataType = DataType.fromString(n5.getAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, String.class)); assertEquals(dtype.getDataType(), n5DataType); @@ -828,7 +831,9 @@ public void testAttributeMapping() throws IOException { n5.setAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, newDtype.toString()); - dtype = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, DType.class); + typestr = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, String.class); + filters = n5.getAttribute(datasetName, ZArrayAttributes.filtersKey, TypeToken.getParameterized(Collection.class, Filter.class).getType()); + dtype = new DType(typestr, filters); n5DataType = DataType.fromString(n5.getAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, String.class)); assertEquals(newDtype, dtype.getDataType()); assertEquals(newDtype, n5DataType); From 9dc961cbbca6741b165746037ec491118f20d417 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 19 Oct 2023 11:30:36 -0400 Subject: [PATCH 15/17] Make filters (de-)serialize to the correct thing --- .../janelia/saalfeldlab/n5/zarr/DType.java | 40 +++++++------ .../janelia/saalfeldlab/n5/zarr/Filter.java | 56 ++++++++++++++----- .../zarr/ZarrCompatibleStringDataBlock.java | 4 -- .../n5/zarr/ZarrKeyValueWriter.java | 5 +- 4 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 13146f3..3741172 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -28,13 +28,11 @@ */ package org.janelia.saalfeldlab.n5.zarr; -import java.lang.reflect.Type; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collection; import java.util.EnumMap; -import java.util.List; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; @@ -45,13 +43,7 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; +import static org.janelia.saalfeldlab.n5.zarr.Filter.VLEN_UTF8; /** * Enumerates available zarr data types as defined at @@ -219,8 +211,15 @@ public DType(final String typestr, final Collection filters) { case OBJECT: nBytes = 1; nBits = 0; - dataBlockFactory = null; - byteBlockFactory = null; + if (filters.contains(VLEN_UTF8)) { + dataBlockFactory = (blockSize, gridPosition, numElements) -> + new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + byteBlockFactory = (blockSize, gridPosition, numElements) -> + new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); + } else { + dataBlockFactory = null; + byteBlockFactory = null; + } break; // case BOOLEAN: // case OTHER: // not sure about this @@ -237,7 +236,7 @@ public DType(final String typestr, final Collection filters) { new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); } - dataType = getDataType(primitive, nBytes); + dataType = getDataType(primitive, nBytes, filters); } public DType(final DataType dataType, final int nPrimitives) { @@ -279,6 +278,11 @@ public DType(final DataType dataType, final int nPrimitives) { break; // case INT8: // case UINT8: + case STRING: + nBytes = 1; + dataBlockFactory = (blockSize, gridPosition, numElements) -> + new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + break; default: nBytes = nPrimitives; dataBlockFactory = (blockSize, gridPosition, numElements) -> @@ -300,7 +304,8 @@ public DataType getDataType() { protected final static DataType getDataType( final Primitive primitive, - final int nBytes) { + final int nBytes, + final Collection filters) { switch (primitive) { case INT: @@ -344,7 +349,10 @@ protected final static DataType getDataType( return DataType.UINT8; // fallback } case OBJECT: - return DataType.OBJECT; + if (filters.contains(VLEN_UTF8)) + return DataType.STRING; + else + return DataType.OBJECT; default: return DataType.UINT8; // fallback } @@ -362,10 +370,10 @@ public String toString() { * * @return list of filters */ - public List getFilters() { + public Collection getFilters() { if (dataType == DataType.STRING) { ArrayList filterSet = new ArrayList<>(); - filterSet.add(new ZarrCompatibleStringDataBlock.VLenStringFilter()); + filterSet.add(VLEN_UTF8); return filterSet; } else diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 9428d41..988f243 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -33,21 +33,45 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import java.lang.reflect.Type; /** - * Place holder interface for filters + * Filter types * * TODO implement some * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> + * @author Michael Innerberger */ -public interface Filter { +public enum Filter { + // Note: the JSON (de-)serializer below is very much tailored to this filter, which serializes to "{"id":"vlen-utf8"}" + // If additional filters are implemented, consider also changing the type adapter below + VLEN_UTF8("vlen-utf8"); - public static JsonAdapter jsonAdapter = new Filter.JsonAdapter(); + private final String id; - class JsonAdapter implements JsonDeserializer { + Filter(final String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public static Filter fromString(final String id) { + for (final Filter filter : values()) + if (filter.getId().equals(id)) + return filter; + return null; + } + + public static final JsonAdapter jsonAdapter = new JsonAdapter(); + + public static class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override public Filter deserialize( @@ -55,17 +79,23 @@ public Filter deserialize( final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { - final JsonObject jsonObject = json.getAsJsonObject(); - final JsonElement jsonId = jsonObject.get("id"); + final JsonElement jsonId = json.getAsJsonObject().get("id"); if (jsonId == null) return null; - final String id = jsonId.getAsString(); - switch (id) { - case "vlen-utf8": - return new ZarrCompatibleStringDataBlock.VLenStringFilter(); - default: - return null; - } + + final String stringId = jsonId.getAsString(); + return Filter.fromString(stringId); + } + + @Override + public JsonElement serialize( + final Filter filter, + final Type typeOfSrc, + final JsonSerializationContext context) { + + final JsonObject serialization = new JsonObject(); + serialization.add("id", new JsonPrimitive(filter.getId())); + return serialization; } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java index ce72586..9b166c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java @@ -67,8 +67,4 @@ protected String[] deserialize(byte[] rawBytes) { return strings; } - - public static class VLenStringFilter implements Filter { - public final String id = "vlen-utf8"; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java index 6b2bf47..b922879 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java @@ -368,17 +368,18 @@ protected ZArrayAttributes createZArrayAttributes(final DatasetAttributes datase reorder(shape); final int[] chunks = datasetAttributes.getBlockSize().clone(); reorder(chunks); + final DType dType = new DType(datasetAttributes.getDataType()); final ZArrayAttributes zArrayAttributes = new ZArrayAttributes( N5ZarrReader.VERSION.getMajor(), shape, chunks, - new DType(datasetAttributes.getDataType()), + dType, ZarrCompressor.fromCompression(datasetAttributes.getCompression()), "0", 'C', dimensionSeparator, - null); + dType.getFilters()); return zArrayAttributes; } From 970b6e357949e7a23ab09fedfc491ea63bed096a Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 19 Oct 2023 17:49:46 -0400 Subject: [PATCH 16/17] Make Filter an interface again --- .../janelia/saalfeldlab/n5/zarr/Filter.java | 34 +++++++++---------- .../n5/zarr/ZarrKeyValueReader.java | 3 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 988f243..3c756b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -47,31 +47,31 @@ * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Michael Innerberger */ -public enum Filter { - // Note: the JSON (de-)serializer below is very much tailored to this filter, which serializes to "{"id":"vlen-utf8"}" - // If additional filters are implemented, consider also changing the type adapter below - VLEN_UTF8("vlen-utf8"); +public interface Filter { - private final String id; + String getId(); - Filter(final String id) { - this.id = id; - } + // Note: the JSON (de-)serializer below is very much tailored to this filter, which serializes to "{"id":"vlen-utf8"}" + // If additional filters are implemented, consider also changing the type adapter below + Filter VLEN_UTF8 = new VLenStringFilter(); - public String getId() { - return id; - } + class VLenStringFilter implements Filter { + private static final String id = "vlen-utf8"; + @Override + public String getId() { + return id; + } + }; - public static Filter fromString(final String id) { - for (final Filter filter : values()) - if (filter.getId().equals(id)) - return filter; + static Filter fromString(final String id) { + if (VLEN_UTF8.getId().equals(id)) + return VLEN_UTF8; return null; } - public static final JsonAdapter jsonAdapter = new JsonAdapter(); + JsonAdapter jsonAdapter = new JsonAdapter(); - public static class JsonAdapter implements JsonDeserializer, JsonSerializer { + class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override public Filter deserialize( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index 9ca58a2..c059b69 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -35,7 +35,6 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Collection; import org.apache.commons.compress.utils.IOUtils; import org.janelia.saalfeldlab.n5.BlockReader; @@ -841,7 +840,7 @@ protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.registerTypeAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.registerTypeAdapter(ZArrayAttributes.class, ZArrayAttributes.jsonAdapter); - gsonBuilder.registerTypeAdapter(Filter.class, Filter.jsonAdapter); + gsonBuilder.registerTypeHierarchyAdapter(Filter.class, Filter.jsonAdapter); gsonBuilder.disableHtmlEscaping(); gsonBuilder.serializeNulls(); From 395bf3f8e5a998657dc1095fdab0f844c1a6edb9 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sat, 21 Oct 2023 07:20:16 -0400 Subject: [PATCH 17/17] Rename ZarrCompatibleStringDataBlock --- src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java | 4 ++-- .../org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java | 4 ++-- ...patibleStringDataBlock.java => ZarrStringDataBlock.java} | 6 +++--- .../java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/zarr/{ZarrCompatibleStringDataBlock.java => ZarrStringDataBlock.java} (89%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 3741172..b34a58a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -213,7 +213,7 @@ public DType(final String typestr, final Collection filters) { nBits = 0; if (filters.contains(VLEN_UTF8)) { dataBlockFactory = (blockSize, gridPosition, numElements) -> - new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + new ZarrStringDataBlock(blockSize, gridPosition, new String[0]); byteBlockFactory = (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); } else { @@ -281,7 +281,7 @@ public DType(final DataType dataType, final int nPrimitives) { case STRING: nBytes = 1; dataBlockFactory = (blockSize, gridPosition, numElements) -> - new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + new ZarrStringDataBlock(blockSize, gridPosition, new String[0]); break; default: nBytes = nPrimitives; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index c059b69..a18d542 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -685,9 +685,9 @@ protected static DataBlock readBlock( return dataBlock; } - private static ZarrCompatibleStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { + private static ZarrStringDataBlock readVLenStringBlock(InputStream in, BlockReader reader, ByteArrayDataBlock byteBlock) throws IOException { // read whole chunk and deserialize; this should be improved - ZarrCompatibleStringDataBlock dataBlock = new ZarrCompatibleStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); + ZarrStringDataBlock dataBlock = new ZarrStringDataBlock(byteBlock.getSize(), byteBlock.getGridPosition(), new String[0]); if (reader instanceof BloscCompression) { // Blosc reader reads actual data and doesn't care about buffer size (but needs special treatment in data block) reader.read(dataBlock, in); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrStringDataBlock.java similarity index 89% rename from src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java rename to src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrStringDataBlock.java index 9b166c1..5127a29 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrStringDataBlock.java @@ -6,13 +6,13 @@ import java.nio.ByteOrder; import java.util.Arrays; -public class ZarrCompatibleStringDataBlock extends StringDataBlock { +public class ZarrStringDataBlock extends StringDataBlock { - public ZarrCompatibleStringDataBlock(int[] size, long[] gridPosition, String[] data) { + public ZarrStringDataBlock(int[] size, long[] gridPosition, String[] data) { super(size, gridPosition, data); } - public ZarrCompatibleStringDataBlock(int[] size, long[] gridPosition, byte[] data) { + public ZarrStringDataBlock(int[] size, long[] gridPosition, byte[] data) { super(size, gridPosition, data); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 151273c..07bb898 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -405,7 +405,7 @@ public void testWriteReadStringBlock() { try (final N5Writer n5 = createN5Writer()) { n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); - StringDataBlock dataBlock = new ZarrCompatibleStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock("/test/group/dataset", attributes, dataBlock); DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData());