From 7f63bf9684b01fbe476d57db55d419770cd2069c Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 3 May 2024 12:58:49 -0400 Subject: [PATCH 1/4] Infer storage_options in zarr, if given mapper --- MANIFEST.in | 1 + kerchunk/combine.py | 2 +- kerchunk/zarr.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 81b1e7f4..ca08559d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include LICENSE prune tests prune examples +global-exclude *.hdf *.hdf5 *.h5 *.grib2 *.tiff *.tif *.json test*.py diff --git a/kerchunk/combine.py b/kerchunk/combine.py index 69ec3cc6..bbe48370 100644 --- a/kerchunk/combine.py +++ b/kerchunk/combine.py @@ -756,7 +756,7 @@ def _replace(l: list, i: int, v) -> list: def auto_dask( urls: List[str], - single_driver: str, + single_driver: type, single_kwargs: dict, mzz_kwargs: dict, n_batches: int, diff --git a/kerchunk/zarr.py b/kerchunk/zarr.py index 0c05af2e..07fc3881 100644 --- a/kerchunk/zarr.py +++ b/kerchunk/zarr.py @@ -33,6 +33,8 @@ def single_zarr( mapper = fsspec.get_mapper(uri_or_store, **(storage_options or {})) else: mapper = uri_or_store + if isinstance(mapper, fsspec.FSMap) and storage_options is None: + storage_options = mapper.fs.storage_options refs = out or {} for k in mapper: From dfb0dad221200e851723556657d013ef28c2e235 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 3 May 2024 13:18:39 -0400 Subject: [PATCH 2/4] later CI --- .github/workflows/tests.yml | 2 +- ci/{environment-py39.yml => environment-py312.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename ci/{environment-py39.yml => environment-py312.yml} (96%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd83bea5..cad36373 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [39, 310, 311] + python-version: [310, 311, 312] steps: - uses: actions/checkout@v4 diff --git a/ci/environment-py39.yml b/ci/environment-py312.yml similarity index 96% rename from ci/environment-py39.yml rename to ci/environment-py312.yml index e4ca09ad..c7ed630e 100644 --- a/ci/environment-py39.yml +++ b/ci/environment-py312.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.9 + - python=3.12 - dask - zarr - xarray From 58c3340115b791bf1235a6176091a2b83549f051 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 3 May 2024 13:31:37 -0400 Subject: [PATCH 3/4] back to miniconda --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cad36373..90d8bb9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,12 +13,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup conda - uses: mamba-org/setup-micromamba@v1 + uses: conda-incubator/setup-miniconda@v3 with: environment-file: ci/environment-py${{matrix.python-version}}.yml - cache-downloads: false - cache-environment: true - generate-run-shell: false - name: Install kerchunk shell: bash -l {0} run: | From 47f211654e5dff5754b6e3117e89c230e5fb6f19 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 3 May 2024 13:50:04 -0400 Subject: [PATCH 4/4] relax pin --- ci/environment-py312.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/environment-py312.yml b/ci/environment-py312.yml index c7ed630e..60b5214b 100644 --- a/ci/environment-py312.yml +++ b/ci/environment-py312.yml @@ -9,7 +9,7 @@ dependencies: - xarray - xarray-datatree - h5netcdf - - h5py<3.9 + - h5py - pandas - cfgrib - cftime