diff --git a/zarr/storage.py b/zarr/storage.py index aa46e59e79..d698e0e0ee 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1081,8 +1081,11 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def getitems(self, keys, **kwargs): - keys = [self._normalize_key(key) for key in keys] - return self.map.getitems(keys, on_error="omit") + keys_transformed = [self._normalize_key(key) for key in keys] + results = self.map.getitems(keys_transformed, on_error="omit") + # The function calling this method may not recognize the transformed keys + # So we send the values returned by self.map.getitems back into the original key space. + return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} def __getitem__(self, key): key = self._normalize_key(key) @@ -1144,9 +1147,28 @@ def dir_path(self, path=None): def listdir(self, path=None): dir_path = self.dir_path(path) try: - out = sorted(p.rstrip('/').rsplit('/', 1)[-1] - for p in self.fs.ls(dir_path, detail=False)) - return out + children = sorted(p.rstrip('/').rsplit('/', 1)[-1] + for p in self.fs.ls(dir_path, detail=False)) + if self.key_separator != "/": + return children + else: + if array_meta_key in children: + # special handling of directories containing an array to map nested chunk + # keys back to standard chunk keys + new_children = [] + root_path = self.dir_path(path) + for entry in children: + entry_path = os.path.join(root_path, entry) + if _prog_number.match(entry) and self.fs.isdir(entry_path): + for file_name in self.fs.find(entry_path): + file_path = os.path.join(dir_path, file_name) + rel_path = file_path.split(root_path)[1] + new_children.append(rel_path.replace(os.path.sep, '.')) + else: + new_children.append(entry) + return sorted(new_children) + else: + return children except IOError: return [] @@ -2739,6 +2761,7 @@ class ConsolidatedMetadataStore(MutableMapping): zarr.convenience.consolidate_metadata, zarr.convenience.open_consolidated """ + def __init__(self, store, metadata_key='.zmetadata'): self.store = store diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 6914cff87e..08e59acda4 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -35,8 +35,9 @@ from zarr.util import buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec - # noinspection PyMethodMayBeStatic + + class TestArray(unittest.TestCase): def test_array_init(self): @@ -1079,7 +1080,7 @@ def test_structured_array_nested(self): (1, (1, ((1, 2), (2, 3), (3, 4)), 1), b'bbb'), (2, (2, ((2, 3), (3, 4), (4, 5)), 2), b'ccc')], dtype=[('foo', 'i8'), ('bar', [('foo', 'i4'), ('bar', '(3, 2)f4'), - ('baz', 'u1')]), ('baz', 'S3')]) + ('baz', 'u1')]), ('baz', 'S3')]) fill_values = None, b'', (0, (0, ((0, 0), (1, 1), (2, 2)), 0), b'zzz') self.check_structured_array(d, fill_values) @@ -1802,7 +1803,7 @@ def test_structured_array_nested(self): (1, (1, ((1, 2), (2, 3), (3, 4)), 1), b'bbb'), (2, (2, ((2, 3), (3, 4), (4, 5)), 2), b'ccc')], dtype=[('foo', 'i8'), ('bar', [('foo', 'i4'), ('bar', '(3, 2)f4'), - ('baz', 'u1')]), ('baz', 'S3')]) + ('baz', 'u1')]), ('baz', 'S3')]) fill_values = None, b'', (0, (0, ((0, 0), (1, 1), (2, 2)), 0), b'zzz') with pytest.raises(TypeError): self.check_structured_array(d, fill_values) @@ -2469,7 +2470,8 @@ class TestArrayWithFSStore(TestArray): def create_array(read_only=False, **kwargs): path = mkdtemp() atexit.register(shutil.rmtree, path) - store = FSStore(path) + key_separator = kwargs.pop('key_separator', ".") + store = FSStore(path, key_separator=key_separator, auto_mkdir=True) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) kwargs.setdefault('compressor', Blosc()) @@ -2477,28 +2479,41 @@ def create_array(read_only=False, **kwargs): return Array(store, read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs) + def expected(self): + return [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ] + def test_hexdigest(self): + found = [] + # Check basic 1-D array z = self.create_array(shape=(1050,), chunks=100, dtype='