From dfc9cc105fa5cca1934c0609f787a8f532744c0c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 10 Mar 2021 21:10:06 -0500 Subject: [PATCH 01/16] FSStore: aflesh out incomplete test and add another. The first test passes after changes to FSStore. The second test fails. --- zarr/storage.py | 7 +++++-- zarr/tests/test_storage.py | 17 ++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index c332ee02f5..37dc4e9542 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1051,8 +1051,11 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def getitems(self, keys, **kwargs): - keys = [self._normalize_key(key) for key in keys] - return self.map.getitems(keys, on_error="omit") + keys_transformed = [self._normalize_key(key) for key in keys] + results = self.map.getitems(keys_transformed, on_error="omit") + # The function calling this method may not recognize the transformed keys + # So we send the values returned by self.map.getitems back into the original key space. + return {keys[keys_transformed.index(rk)]: rv for rk,rv in results.items()} def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index a6598f2781..e5d01b86c9 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -915,19 +915,20 @@ def test_create(self): path1 = tempfile.mkdtemp() path2 = tempfile.mkdtemp() g = zarr.open_group("file://" + path1, mode='w', - storage_options={"auto_mkdir": True}) - a = g.create_dataset("data", shape=(8,)) - a[:4] = [0, 1, 2, 3] + storage_options={'auto_mkdir': True}) + a = g.create_dataset("data", shape=(2,4), dtype='int') + a[0,:4] = [0, 1, 2, 3] assert "data" in os.listdir(path1) assert ".zgroup" in os.listdir(path1) g = zarr.open_group("simplecache::file://" + path1, mode='r', storage_options={"cache_storage": path2, "same_names": True}) - assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0] + assert g.data[:].tolist() == [[0, 1, 2, 3], [0, 0, 0, 0]] with pytest.raises(PermissionError): g.data[:] = 1 + def test_read_only(self): path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) @@ -1215,12 +1216,14 @@ def test_numbered_groups(self): store = self.create_store() group = zarr.group(store=store) arr = group.create_dataset('0', shape=(10, 10)) - arr[1] = 1 + values = np.arange(100).reshape((10,10)) + arr[:] = values + assert_array_equal(arr, values) # Read it back store = self.create_store(path=store.path) - zarr.open_group(store.path)["0"] - + assert_array_equal(zarr.open_group(store.path)["0"], values) + class TestTempStore(StoreTests, unittest.TestCase): From eea0172b45c5925b391be5a010e9d80d34cd4d4e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 10:44:01 -0500 Subject: [PATCH 02/16] pep8 fixes --- zarr/storage.py | 4 ++-- zarr/tests/test_storage.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 37dc4e9542..35d1360a15 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1054,8 +1054,8 @@ def getitems(self, keys, **kwargs): keys_transformed = [self._normalize_key(key) for key in keys] results = self.map.getitems(keys_transformed, on_error="omit") # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk,rv in results.items()} + # So we send the values returned by self.map.getitems back into the original key space. + return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index e5d01b86c9..6391cbd0b6 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1216,14 +1216,14 @@ def test_numbered_groups(self): store = self.create_store() group = zarr.group(store=store) arr = group.create_dataset('0', shape=(10, 10)) - values = np.arange(100).reshape((10,10)) + values = np.arange(100).reshape((10, 10)) arr[:] = values assert_array_equal(arr, values) # Read it back store = self.create_store(path=store.path) assert_array_equal(zarr.open_group(store.path)["0"], values) - + class TestTempStore(StoreTests, unittest.TestCase): From 1fddbb6339e1e2ccdba53cf111603fae0ffc2f64 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 10:57:08 -0500 Subject: [PATCH 03/16] TestNestedFSStore: tweak second assertion of test_numbered_groups --- zarr/tests/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 6391cbd0b6..b672d95d35 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1222,7 +1222,7 @@ def test_numbered_groups(self): # Read it back store = self.create_store(path=store.path) - assert_array_equal(zarr.open_group(store.path)["0"], values) + assert_array_equal(zarr.open_group(store=store)["0"], values) class TestTempStore(StoreTests, unittest.TestCase): From 03f348382d59c7ca6eac8c373ce41043c4671c62 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 16:55:33 -0500 Subject: [PATCH 04/16] FSStore: change ggetitems to return dict with input keys --- zarr/storage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index c332ee02f5..6d53a58fdf 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1051,8 +1051,11 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def getitems(self, keys, **kwargs): - keys = [self._normalize_key(key) for key in keys] - return self.map.getitems(keys, on_error="omit") + keys_transformed = [self._normalize_key(key) for key in keys] + results = self.map.getitems(keys_transformed, on_error="omit") + # The function calling this method may not recognize the transformed keys + # So we send the values returned by self.map.getitems back with the input keys. + return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} def __getitem__(self, key): key = self._normalize_key(key) From 4201043b199f626209127b000d6f587d2675611b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 16:59:18 -0500 Subject: [PATCH 05/16] TestArrayWithFSStore: add key_separator kwarg FSStore constructor --- zarr/storage.py | 7 ++----- zarr/tests/test_core.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 6d53a58fdf..c332ee02f5 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1051,11 +1051,8 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def getitems(self, keys, **kwargs): - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back with the input keys. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} + keys = [self._normalize_key(key) for key in keys] + return self.map.getitems(keys, on_error="omit") def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index eaa87800ef..5bd1a98ed7 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2435,7 +2435,8 @@ class TestArrayWithFSStore(TestArray): def create_array(read_only=False, **kwargs): path = mkdtemp() atexit.register(shutil.rmtree, path) - store = FSStore(path) + key_separator = kwargs.pop('key_separator', "/") + store = FSStore(path, key_separator=key_separator, auto_mkdir=True) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) kwargs.setdefault('compressor', Blosc()) From 1ce8c79ef435eac88085ac269b838771e84e5da0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 17:28:13 -0500 Subject: [PATCH 06/16] TestArrayWithFSStore: add key_separator arg to store constructor in create_arrray. --- zarr/tests/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 5bd1a98ed7..138fc9d58b 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2443,7 +2443,7 @@ def create_array(read_only=False, **kwargs): init_array(store, **kwargs) return Array(store, read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs) - + def test_hexdigest(self): # Check basic 1-D array z = self.create_array(shape=(1050,), chunks=100, dtype=' Date: Thu, 11 Mar 2021 17:36:09 -0500 Subject: [PATCH 07/16] revert changes. the logic I need to test actually lives in test_core, not test_storage. --- zarr/tests/test_storage.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index b672d95d35..e6b889d3dc 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -915,20 +915,19 @@ def test_create(self): path1 = tempfile.mkdtemp() path2 = tempfile.mkdtemp() g = zarr.open_group("file://" + path1, mode='w', - storage_options={'auto_mkdir': True}) - a = g.create_dataset("data", shape=(2,4), dtype='int') - a[0,:4] = [0, 1, 2, 3] + storage_options={"auto_mkdir": True}) + a = g.create_dataset("data", shape=(8,)) + a[:4] = [0, 1, 2, 3] assert "data" in os.listdir(path1) assert ".zgroup" in os.listdir(path1) g = zarr.open_group("simplecache::file://" + path1, mode='r', storage_options={"cache_storage": path2, "same_names": True}) - assert g.data[:].tolist() == [[0, 1, 2, 3], [0, 0, 0, 0]] + assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0] with pytest.raises(PermissionError): g.data[:] = 1 - def test_read_only(self): path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) @@ -1216,13 +1215,11 @@ def test_numbered_groups(self): store = self.create_store() group = zarr.group(store=store) arr = group.create_dataset('0', shape=(10, 10)) - values = np.arange(100).reshape((10, 10)) - arr[:] = values - assert_array_equal(arr, values) - + arr[1] = 1 + # Read it back store = self.create_store(path=store.path) - assert_array_equal(zarr.open_group(store=store)["0"], values) + zarr.open_group(store.path)["0"] class TestTempStore(StoreTests, unittest.TestCase): From 996808b89b7629841b6845e1e43cc05a9736a54d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 11 Mar 2021 17:45:03 -0500 Subject: [PATCH 08/16] kill some whitespace --- zarr/tests/test_core.py | 4 ++-- zarr/tests/test_storage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 138fc9d58b..5bd1a98ed7 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2443,7 +2443,7 @@ def create_array(read_only=False, **kwargs): init_array(store, **kwargs) return Array(store, read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs) - + def test_hexdigest(self): # Check basic 1-D array z = self.create_array(shape=(1050,), chunks=100, dtype=' Date: Sun, 14 Mar 2021 17:11:10 -0400 Subject: [PATCH 09/16] add nested tfsstore tests --- zarr/tests/test_core.py | 123 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 4 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 5bd1a98ed7..035c7aeb76 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -35,8 +35,9 @@ from zarr.util import buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec - # noinspection PyMethodMayBeStatic + + class TestArray(unittest.TestCase): def test_array_init(self): @@ -1066,7 +1067,7 @@ def test_structured_array_nested(self): (1, (1, ((1, 2), (2, 3), (3, 4)), 1), b'bbb'), (2, (2, ((2, 3), (3, 4), (4, 5)), 2), b'ccc')], dtype=[('foo', 'i8'), ('bar', [('foo', 'i4'), ('bar', '(3, 2)f4'), - ('baz', 'u1')]), ('baz', 'S3')]) + ('baz', 'u1')]), ('baz', 'S3')]) fill_values = None, b'', (0, (0, ((0, 0), (1, 1), (2, 2)), 0), b'zzz') self.check_structured_array(d, fill_values) @@ -1780,7 +1781,7 @@ def test_structured_array_nested(self): (1, (1, ((1, 2), (2, 3), (3, 4)), 1), b'bbb'), (2, (2, ((2, 3), (3, 4), (4, 5)), 2), b'ccc')], dtype=[('foo', 'i8'), ('bar', [('foo', 'i4'), ('bar', '(3, 2)f4'), - ('baz', 'u1')]), ('baz', 'S3')]) + ('baz', 'u1')]), ('baz', 'S3')]) fill_values = None, b'', (0, (0, ((0, 0), (1, 1), (2, 2)), 0), b'zzz') with pytest.raises(TypeError): self.check_structured_array(d, fill_values) @@ -2435,7 +2436,7 @@ class TestArrayWithFSStore(TestArray): def create_array(read_only=False, **kwargs): path = mkdtemp() atexit.register(shutil.rmtree, path) - key_separator = kwargs.pop('key_separator', "/") + key_separator = kwargs.pop('key_separator', ".") store = FSStore(path, key_separator=key_separator, auto_mkdir=True) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) @@ -2540,3 +2541,117 @@ def test_read_from_all_blocks(self): z[2:99_000] = 1 b = Array(z.store, read_only=True, partial_decompress=True) assert (b[2:99_000] == 1).all() + + +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +class TestArrayWithFSStoreNested(TestArray): + @staticmethod + def create_array(read_only=False, **kwargs): + path = mkdtemp() + atexit.register(shutil.rmtree, path) + key_separator = kwargs.pop('key_separator', "/") + store = FSStore(path, key_separator=key_separator, auto_mkdir=True) + cache_metadata = kwargs.pop('cache_metadata', True) + cache_attrs = kwargs.pop('cache_attrs', True) + kwargs.setdefault('compressor', Blosc()) + init_array(store, **kwargs) + return Array(store, read_only=read_only, cache_metadata=cache_metadata, + cache_attrs=cache_attrs) + + def test_hexdigest(self): + # Check basic 1-D array + z = self.create_array(shape=(1050,), chunks=100, dtype=' Date: Sun, 14 Mar 2021 18:17:16 -0400 Subject: [PATCH 10/16] FSStore: fsstore.listdir now handles nested keys --- zarr/storage.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 35d1360a15..a78fe1c17b 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1117,9 +1117,26 @@ def dir_path(self, path=None): def listdir(self, path=None): dir_path = self.dir_path(path) try: - out = sorted(p.rstrip('/').rsplit('/', 1)[-1] - for p in self.fs.ls(dir_path, detail=False)) - return out + children = sorted(p.rstrip('/').rsplit('/', 1)[-1] + for p in self.fs.ls(dir_path, detail=False)) + if self.key_separator == '/': + if array_meta_key in children: + # special handling of directories containing an array to map nested chunk + # keys back to standard chunk keys + new_children = [] + root_path = self.dir_path(path) + for entry in children: + entry_path = os.path.join(root_path, entry) + if _prog_number.match(entry) and self.fs.isdir(entry_path): + for dir_path, _, file_names in self.fs.walk(entry_path): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + rel_path = file_path.split(root_path)[1] + new_children.append(rel_path.replace(os.path.sep, '.')) + else: + new_children.append(entry) + return sorted(new_children) + return children except IOError: return [] @@ -2592,6 +2609,7 @@ class RedisStore(MutableMapping): Keyword arguments passed through to the `redis.Redis` function. """ + def __init__(self, prefix='zarr', **kwargs): import redis self._prefix = prefix @@ -2677,6 +2695,7 @@ class ConsolidatedMetadataStore(MutableMapping): zarr.convenience.consolidate_metadata, zarr.convenience.open_consolidated """ + def __init__(self, store, metadata_key='.zmetadata'): self.store = store From babd76a78b53277c98917051a0cfbbc8728f0452 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 15 Mar 2021 15:29:23 -0400 Subject: [PATCH 11/16] FSStore: re-order conditional evaluation in listdir --- zarr/storage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index a78fe1c17b..0e8ad196df 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1119,7 +1119,9 @@ def listdir(self, path=None): try: children = sorted(p.rstrip('/').rsplit('/', 1)[-1] for p in self.fs.ls(dir_path, detail=False)) - if self.key_separator == '/': + if self.key_separator != "/": + return children + else: if array_meta_key in children: # special handling of directories containing an array to map nested chunk # keys back to standard chunk keys @@ -1136,7 +1138,8 @@ def listdir(self, path=None): else: new_children.append(entry) return sorted(new_children) - return children + else: + return children except IOError: return [] From b679c08cc637e369a73625feddb42c7a956307e6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 15 Mar 2021 17:15:03 -0400 Subject: [PATCH 12/16] FSStore: use self.fs.find in listdir --- zarr/storage.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 0e8ad196df..aa20fc77a3 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1130,11 +1130,10 @@ def listdir(self, path=None): for entry in children: entry_path = os.path.join(root_path, entry) if _prog_number.match(entry) and self.fs.isdir(entry_path): - for dir_path, _, file_names in self.fs.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path)[1] - new_children.append(rel_path.replace(os.path.sep, '.')) + for file_name in self.fs.find(entry_path): + file_path = os.path.join(dir_path, file_name) + rel_path = file_path.split(root_path)[1] + new_children.append(rel_path.replace(os.path.sep, '.')) else: new_children.append(entry) return sorted(new_children) From 749747cdc878e0ba882bad3783878dd1fe3f93d3 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 30 Apr 2021 08:06:03 +0200 Subject: [PATCH 13/16] Add tests from #718 --- zarr/tests/test_hierarchy.py | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index afecfabdf1..ac2abfe694 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -21,13 +21,13 @@ from zarr.core import Array from zarr.creation import open_array from zarr.hierarchy import Group, group, open_group -from zarr.storage import (ABSStore, DBMStore, DirectoryStore, LMDBStore, - LRUStoreCache, MemoryStore, NestedDirectoryStore, - SQLiteStore, ZipStore, array_meta_key, atexit_rmglob, - atexit_rmtree, group_meta_key, init_array, - init_group) +from zarr.storage import (ABSStore, DBMStore, DirectoryStore, FSStore, + LMDBStore, LRUStoreCache, MemoryStore, + NestedDirectoryStore, SQLiteStore, ZipStore, + array_meta_key, atexit_rmglob, atexit_rmtree, + group_meta_key, init_array, init_group) from zarr.util import InfoReporter -from zarr.tests.util import skip_test_env_var +from zarr.tests.util import skip_test_env_var, have_fsspec # noinspection PyStatementEffect @@ -971,6 +971,51 @@ def create_store(): return store, None +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +class TestGroupWithFSStore(TestGroup): + + @staticmethod + def create_store(): + path = tempfile.mkdtemp() + atexit.register(atexit_rmtree, path) + store = FSStore(path) + return store, None + + def test_round_trip_nd(self): + data = np.arange(1000).reshape(10, 10, 10) + name = 'raw' + + store, _ = self.create_store() + f = open_group(store, mode='w') + f.create_dataset(name, data=data, chunks=(5, 5, 5), + compressor=None) + h = open_group(store, mode='r') + np.testing.assert_array_equal(h[name][:], data) + + +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +class TestGroupWithNestedFSStore(TestGroupWithFSStore): + + @staticmethod + def create_store(): + path = tempfile.mkdtemp() + atexit.register(atexit_rmtree, path) + store = FSStore(path, key_separator='/', auto_mkdir=True) + return store, None + + def test_inconsistent_dimension_separator(self): + data = np.arange(1000).reshape(10, 10, 10) + name = 'raw' + + store, _ = self.create_store() + f = open_group(store, mode='w') + + # cannot specify dimension_separator that conflicts with the store + with pytest.raises(ValueError): + f.create_dataset(name, data=data, chunks=(5, 5, 5), + compressor=None, dimension_separator='.') + + class TestGroupWithZipStore(TestGroup): @staticmethod From 7f1ba5e5bafa581dbf2a21b58f3c8aafb0cd7631 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 4 May 2021 16:11:48 +0200 Subject: [PATCH 14/16] Apply suggestion from @grlee77 --- zarr/util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zarr/util.py b/zarr/util.py index e3e9fca2f9..eb0b784ded 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -554,7 +554,6 @@ def __init__(self, store_key, chunk_store): self.map = self.chunk_store.map self.fs = self.chunk_store.fs self.store_key = store_key - self.key_path = self.map._key_to_str(store_key) self.buff = None self.nblocks = None self.start_points = None @@ -562,6 +561,12 @@ def __init__(self, store_key, chunk_store): self.start_points_max = None self.read_blocks = set() + _key_path = self.map._key_to_str(store_key) + _key_path = _key_path.split('/') + _chunk_path = [self.chunk_store._normalize_key(_key_path[-1])] + _key_path = '/'.join(_key_path[:-1] + _chunk_path) + self.key_path = _key_path + def prepare_chunk(self): assert self.buff is None header = self.fs.read_block(self.key_path, 0, 16) From e46ad5445bc76de3a78de5bdcbe43f8a774e907c Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 4 May 2021 16:15:30 +0200 Subject: [PATCH 15/16] Update PartialRead hexdigest values --- zarr/tests/test_core.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 46962be96f..5aa948bb83 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2636,14 +2636,25 @@ def create_array(read_only=False, **kwargs): partial_decompress=True, ) + def expected(self): + return [ + "94884f29b41b9beb8fc99ad7bf9c0cbf0f2ab3c9", + "077aa3bd77b8d354f8f6c15dce5ae4f545788a72", + "22be95d83c097460adb339d80b2d7fe19c513c16", + "85131cec526fa46938fd2c4a6083a58ee11037ea", + "c3167010c162c6198cb2bf3c1da2c46b047c69a1", + ] + def test_hexdigest(self): + found = [] + # Check basic 1-D array z = self.create_array(shape=(1050,), chunks=100, dtype=" Date: Tue, 4 May 2021 17:35:18 +0200 Subject: [PATCH 16/16] More hexdigest updates --- zarr/tests/test_core.py | 47 ++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 5aa948bb83..08e59acda4 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2479,28 +2479,41 @@ def create_array(read_only=False, **kwargs): return Array(store, read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs) + def expected(self): + return [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ] + def test_hexdigest(self): + found = [] + # Check basic 1-D array z = self.create_array(shape=(1050,), chunks=100, dtype='