Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixing behaviour for group parameter in open_datatree #9666

Merged
merged 26 commits into from
Oct 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
3429b2c
adding draft for fixing behaviour for group parameter
aladinor Oct 23, 2024
1507f4d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 23, 2024
e24e88b
new trial
aladinor Oct 23, 2024
34e74db
new trial
aladinor Oct 23, 2024
b6fac5b
new trial
aladinor Oct 23, 2024
ce83c89
fixing duplicate pahts and path in the root group
aladinor Oct 24, 2024
72fcee6
removing yield str(gpath)
aladinor Oct 24, 2024
bd853c8
implementing the proposed solution to hdf5 and netcdf backends
aladinor Oct 24, 2024
d6e5422
adding changes to whats-new.rst
aladinor Oct 24, 2024
12005e2
removing encoding['source_group'] line to avoid conflicts with PR #9660
aladinor Oct 24, 2024
e4384d6
adding test
aladinor Oct 24, 2024
0fab3c7
Merge branch 'main' into fix-group-param
TomNicholas Oct 24, 2024
e935e4e
adding test
aladinor Oct 24, 2024
2803f9f
Merge branch 'fix-group-param' of https://github.com/aladinor/xarray …
aladinor Oct 24, 2024
9a41b68
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 24, 2024
f5d3073
adding assert subgroup_tree.root.parent is None
aladinor Oct 24, 2024
a473778
Merge branch 'fix-group-param' of https://github.com/aladinor/xarray …
aladinor Oct 24, 2024
bb6d413
modifying tests
aladinor Oct 24, 2024
fcf3dc6
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 24, 2024
195e036
Update xarray/tests/test_backends_datatree.py
aladinor Oct 24, 2024
5ef3a56
applying suggested changes
aladinor Oct 24, 2024
90c5b4d
updating test
aladinor Oct 24, 2024
38548b0
Merge branch 'main' into fix-group-param
TomNicholas Oct 24, 2024
e78d576
adding Justus and Alfonso to the list of contributors to the DataTree…
aladinor Oct 24, 2024
762587b
adding Justus and Alfonso to the list of contributors to the DataTree…
aladinor Oct 24, 2024
0cd22c5
Merge branch 'main' into fix-group-param
TomNicholas Oct 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions doc/whats-new.rst
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@ New Features
By `Owen Littlejohns <https://github.com/owenlittlejohns>`_,
`Eni Awowale <https://github.com/eni-awowale>`_,
`Matt Savoie <https://github.com/flamingbear>`_,
`Stephan Hoyer <https://github.com/shoyer>`_ and
`Tom Nicholas <https://github.com/TomNicholas>`_.
`Stephan Hoyer <https://github.com/shoyer>`_,
`Tom Nicholas <https://github.com/TomNicholas>`_,
`Justus Magin <https://github.com/keewis>`_, and
`Alfonso Ladino <https://github.com/aladinor>`_.
- A migration guide for users of the prototype `xarray-contrib/datatree repository <https://github.com/xarray-contrib/datatree>`_ has been added, and can be found in the `DATATREE_MIGRATION_GUIDE.md` file in the repository root.
By `Tom Nicholas <https://github.com/TomNicholas>`_.
- Added zarr backends for :py:func:`open_groups` (:issue:`9430`, :pull:`9469`).
Expand Down
1 change: 0 additions & 1 deletion xarray/backends/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@ def _iter_nc_groups(root, parent="/"):
yield str(parent)
for path, group in root.groups.items():
gpath = parent / path
yield str(gpath)
yield from _iter_nc_groups(group, parent=gpath)


Expand Down
6 changes: 5 additions & 1 deletion xarray/backends/h5netcdf_.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ def open_datatree(
driver_kwds=driver_kwds,
**kwargs,
)

return datatree_from_dict_with_io_cleanup(groups_dict)

def open_groups_as_dict(
Expand Down Expand Up @@ -556,7 +557,10 @@ def open_groups_as_dict(
decode_timedelta=decode_timedelta,
)

group_name = str(NodePath(path_group))
if group:
group_name = str(NodePath(path_group).relative_to(parent))
else:
group_name = str(NodePath(path_group))
groups_dict[group_name] = group_ds

return groups_dict
Expand Down
6 changes: 5 additions & 1 deletion xarray/backends/netCDF4_.py
Original file line number Diff line number Diff line change
Expand Up @@ -729,6 +729,7 @@ def open_datatree(
autoclose=autoclose,
**kwargs,
)

return datatree_from_dict_with_io_cleanup(groups_dict)

def open_groups_as_dict(
Expand Down Expand Up @@ -789,7 +790,10 @@ def open_groups_as_dict(
use_cftime=use_cftime,
decode_timedelta=decode_timedelta,
)
group_name = str(NodePath(path_group))
if group:
group_name = str(NodePath(path_group).relative_to(parent))
else:
group_name = str(NodePath(path_group))
groups_dict[group_name] = group_ds

return groups_dict
Expand Down
8 changes: 5 additions & 3 deletions xarray/backends/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -1479,6 +1479,7 @@ def open_datatree(
zarr_format=zarr_format,
**kwargs,
)

return datatree_from_dict_with_io_cleanup(groups_dict)

def open_groups_as_dict(
Expand Down Expand Up @@ -1543,9 +1544,11 @@ def open_groups_as_dict(
use_cftime=use_cftime,
decode_timedelta=decode_timedelta,
)
group_name = str(NodePath(path_group))
if group:
group_name = str(NodePath(path_group).relative_to(parent))
else:
group_name = str(NodePath(path_group))
groups_dict[group_name] = group_ds

return groups_dict


Expand All @@ -1554,7 +1557,6 @@ def _iter_zarr_groups(root: ZarrGroup, parent: str = "/") -> Iterable[str]:
yield str(parent_nodepath)
for path, group in root.groups():
gpath = parent_nodepath / path
yield str(gpath)
yield from _iter_zarr_groups(group, parent=str(gpath))


Expand Down
24 changes: 24 additions & 0 deletions xarray/tests/test_backends_datatree.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,18 @@ def test_open_groups_to_dict(self, tmpdir) -> None:
for ds in aligned_dict_of_datasets.values():
ds.close()

def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None:
"""Test opening a specific group within a NetCDF file using `open_datatree`."""
filepath = tmpdir / "test.nc"
group = "/set1"
original_dt = simple_datatree
original_dt.to_netcdf(filepath)
expected_subtree = original_dt[group].copy()
expected_subtree.orphan()
with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree:
assert subgroup_tree.root.parent is None
assert_equal(subgroup_tree, expected_subtree)


@requires_h5netcdf
class TestH5NetCDFDatatreeIO(DatatreeIOBase):
Expand Down Expand Up @@ -502,6 +514,18 @@ def test_open_groups(self, unaligned_datatree_zarr) -> None:
for ds in unaligned_dict_of_datasets.values():
ds.close()

def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None:
"""Test opening a specific group within a Zarr store using `open_datatree`."""
filepath = tmpdir / "test.zarr"
group = "/set2"
original_dt = simple_datatree
original_dt.to_zarr(filepath)
expected_subtree = original_dt[group].copy()
expected_subtree.orphan()
with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree:
assert subgroup_tree.root.parent is None
assert_equal(subgroup_tree, expected_subtree)

@requires_dask
def test_open_groups_chunks(self, tmpdir) -> None:
"""Test `open_groups` with chunks on a zarr store."""
Expand Down
Loading