Merge branch 'main' into topk

* main: Revert "Support first, last with datetime, timedelta (#402)" (#404) Support first, last with datetime, timedelta (#402) Bump codecov/codecov-action from 4.5.0 to 4.6.0 (#401) Bump mamba-org/setup-micromamba from 1 to 2 (#400) Revert "[revert] test with Xarray PR branch" (#393) [pre-commit.ci] pre-commit autoupdate (#399) Faster subsetting for cohorts (#397) Fix default int on windows, numpy<2 (#395) Avoid rechunking when preferred_method="blockwise" (#394) Preserve dtype better when specified. (#389) Drop python 3.9, use ruff (#392) silence warning (#390) Expand groupby_reduce property tests (#385) Fix bug with NaNs in `by` and method='blockwise' (#384) Avoid explicit np.nan, np.inf (#383)
xarray-contrib · Jan 7, 2025 · 44f5f3f · 44f5f3f
2 parents 7056d18 + 3853101
commit 44f5f3f
Show file tree

Hide file tree

Showing 32 changed files with 607 additions and 346 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -22,7 +22,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up conda environment
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-name: flox-bench
           create-args: >-

diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -53,7 +53,7 @@ jobs:
           echo "TODAY=$(date  +'%Y-%m-%d')" >> $GITHUB_ENV
 
       - name: Setup micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ${{env.CONDA_ENV_FILE}}
           environment-name: flox-tests
@@ -77,7 +77,7 @@ jobs:
           --ignore flox/tests \
           --cov=./ --cov-report=xml
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v4.5.0
+        uses: codecov/codecov-action@v4.6.0
         with:
           file: ./coverage.xml
           flags: unittests
@@ -106,7 +106,7 @@ jobs:
         run: |
           echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - name: Setup micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ${{env.CONDA_ENV_FILE}}
           environment-name: flox-tests
@@ -132,7 +132,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cache-dir=.mypy_cache/ --cobertura-xml-report mypy_report
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.5.0
+        uses: codecov/codecov-action@v4.6.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -26,7 +26,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest"]
         env: ["environment"]
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.10", "3.12"]
         include:
           - os: "windows-latest"
             env: "environment"
@@ -36,7 +36,10 @@ jobs:
             python-version: "3.12"
           - os: "ubuntu-latest"
             env: "minimal-requirements"
-            python-version: "3.9"
+            python-version: "3.10"
+          - os: "windows-latest"
+            env: "env-numpy1"
+            python-version: "3.10"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -45,7 +48,7 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Set up conda environment
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ci/${{ matrix.env }}.yml
           environment-name: flox-tests
@@ -70,9 +73,10 @@ jobs:
       - name: Run Tests
         id: status
         run: |
+          python -c "import xarray; xarray.show_versions()"
           pytest --durations=20 --durations-min=0.5 -n auto --cov=./ --cov-report=xml --hypothesis-profile ci
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v4.5.0
+        uses: codecov/codecov-action@v4.6.0
         with:
           file: ./coverage.xml
           flags: unittests
@@ -101,7 +105,7 @@ jobs:
           repository: "pydata/xarray"
           fetch-depth: 0 # Fetch all history for all branches and tags.
       - name: Set up conda environment
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ci/requirements/environment.yml
           environment-name: xarray-tests

diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -43,7 +43,7 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Set up conda environment
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-name: flox-tests
           init-shell: bash

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,29 +4,25 @@ ci:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.5.0"
+    rev: "v0.6.9"
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: "v4.0.0-alpha.8"
     hooks:
       - id: prettier
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-docstring-first
 
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.4.2
-    hooks:
-      - id: black
-
   - repo: https://github.com/executablebooks/mdformat
     rev: 0.7.17
     hooks:
@@ -35,13 +31,6 @@ repos:
           - mdformat-black
           - mdformat-myst
 
-  - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.5
-    hooks:
-      - id: nbqa-black
-      - id: nbqa-ruff
-        args: [--fix]
-
   - repo: https://github.com/kynan/nbstripout
     rev: 0.7.1
     hooks:
@@ -56,12 +45,12 @@ repos:
           - tomli
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.18
+    rev: v0.20.2
     hooks:
       - id: validate-pyproject
 
   - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.1
+    rev: v1.7.3
     hooks:
       - id: actionlint
         files: ".github/workflows/"

diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py
@@ -21,7 +21,6 @@ def _skip_slow():
     >>> from . import _skip_slow
     >>> def time_something_slow():
     ...     pass
-    ...
     >>> time_something.setup = _skip_slow
     """
     if os.environ.get("ASV_SKIP_SLOW", "0") == "1":

diff --git a/asv_bench/benchmarks/cohorts.py b/asv_bench/benchmarks/cohorts.py
@@ -67,7 +67,12 @@ def track_num_layers(self):
     track_num_tasks.unit = "tasks"  # type: ignore[attr-defined] # Lazy
     track_num_tasks_optimized.unit = "tasks"  # type: ignore[attr-defined] # Lazy
     track_num_layers.unit = "layers"  # type: ignore[attr-defined] # Lazy
-    for f in [track_num_tasks, track_num_tasks_optimized, track_num_layers, track_num_cohorts]:
+    for f in [
+        track_num_tasks,
+        track_num_tasks_optimized,
+        track_num_layers,
+        track_num_cohorts,
+    ]:
         f.repeat = 1  # type: ignore[attr-defined] # Lazy
         f.rounds = 1  # type: ignore[attr-defined] # Lazy
         f.number = 1  # type: ignore[attr-defined] # Lazy
@@ -82,9 +87,7 @@ def setup(self, *args, **kwargs):
         y = np.repeat(np.arange(30), 60)
         by = x[np.newaxis, :] * y[:, np.newaxis]
 
-        self.by = flox.core._factorize_multiple((by,), expected_groups=(None,), any_by_dask=False)[
-            0
-        ][0]
+        self.by = flox.core._factorize_multiple((by,), expected_groups=(None,), any_by_dask=False)[0][0]
 
         self.array = dask.array.ones(self.by.shape, chunks=(350, 350))
         self.axis = (-2, -1)
@@ -101,7 +104,12 @@ def __init__(self, *args, **kwargs):
 
     def rechunk(self):
         self.array = flox.core.rechunk_for_cohorts(
-            self.array, -1, self.by, force_new_chunk_at=[1], chunksize=48, ignore_old_chunks=True
+            self.array,
+            -1,
+            self.by,
+            force_new_chunk_at=[1],
+            chunksize=48,
+            ignore_old_chunks=True,
         )
 
 
@@ -151,7 +159,12 @@ def setup(self, *args, **kwargs):
 
     def rechunk(self):
         self.array = flox.core.rechunk_for_cohorts(
-            self.array, -1, self.by, force_new_chunk_at=[1], chunksize=4, ignore_old_chunks=True
+            self.array,
+            -1,
+            self.by,
+            force_new_chunk_at=[1],
+            chunksize=4,
+            ignore_old_chunks=True,
         )
 
 

diff --git a/asv_bench/benchmarks/combine.py b/asv_bench/benchmarks/combine.py
@@ -65,12 +65,8 @@ def construct_member(groups) -> dict[str, Any]:
             * 2
         ]
 
-        self.x_chunk_reindexed = [
-            construct_member(groups) for groups in [np.array((1, 2, 3, 4))] * 4
-        ]
+        self.x_chunk_reindexed = [construct_member(groups) for groups in [np.array((1, 2, 3, 4))] * 4]
         self.kwargs = {
-            "agg": flox.aggregations._initialize_aggregation(
-                "sum", "float64", np.float64, 0, 0, {}
-            ),
+            "agg": flox.aggregations._initialize_aggregation("sum", "float64", np.float64, 0, 0, {}),
             "axis": (3,),
         }
diff --git a/asv_bench/benchmarks/reduce.py b/asv_bench/benchmarks/reduce.py
@@ -7,7 +7,11 @@
 
 N = 3000
 funcs = ["sum", "nansum", "mean", "nanmean", "max", "nanmax", "count"]
-engines = [None, "flox", "numpy"]  # numbagg is disabled for now since it takes ages in CI
+engines = [
+    None,
+    "flox",
+    "numpy",
+]  # numbagg is disabled for now since it takes ages in CI
 expected_groups = {
     "None": None,
     "bins": pd.IntervalIndex.from_breaks([1, 2, 4]),
@@ -17,9 +21,7 @@
 NUMBAGG_FUNCS = ["nansum", "nanmean", "nanmax", "count", "all"]
 numbagg_skip = []
 for name in expected_names:
-    numbagg_skip.extend(
-        list((func, name, "numbagg") for func in funcs if func not in NUMBAGG_FUNCS)
-    )
+    numbagg_skip.extend(list((func, name, "numbagg") for func in funcs if func not in NUMBAGG_FUNCS))
 
 
 def setup_jit():

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
@@ -6,7 +6,7 @@ dependencies:
   - build
   - cachey
   - dask-core
-  - numpy>=1.22
+  - numpy<2
   - mamba
   - pip
   - python=3.10

diff --git a/ci/docs.yml b/ci/docs.yml
@@ -16,6 +16,7 @@ dependencies:
   - myst-parser
   - myst-nb
   - sphinx
+  - sphinx-remove-toctrees
   - furo>=2024.08
   - ipykernel
   - jupyter

diff --git a/ci/env-numpy1.yml b/ci/env-numpy1.yml
@@ -0,0 +1,30 @@
+name: flox-tests
+channels:
+  - conda-forge
+dependencies:
+  - asv
+  - cachey
+  - cftime
+  - codecov
+  - cubed>=0.14.3
+  - dask-core
+  - pandas
+  - numpy<2
+  - scipy
+  - lxml # for mypy coverage report
+  - matplotlib
+  - pip
+  - pytest
+  - pytest-cov
+  - pytest-pretty
+  - pytest-xdist
+  - syrupy
+  - pre-commit
+  - numpy_groupies>=0.9.19
+  - pooch
+  - toolz
+  - numba
+  - numbagg>=0.3
+  - hypothesis
+  - pip:
+      - git+https://github.com/dcherian/xarray.git
diff --git a/ci/environment.yml b/ci/environment.yml
@@ -19,11 +19,12 @@ dependencies:
   - pytest-pretty
   - pytest-xdist
   - syrupy
-  - xarray
   - pre-commit
   - numpy_groupies>=0.9.19
   - pooch
   - toolz
   - numba
   - numbagg>=0.3
   - hypothesis
+  - pip:
+      - git+https://github.com/dcherian/xarray.git
diff --git a/ci/no-dask.yml b/ci/no-dask.yml
@@ -14,11 +14,12 @@ dependencies:
   - pytest-pretty
   - pytest-xdist
   - syrupy
-  - xarray
   - numpydoc
   - pre-commit
   - numpy_groupies>=0.9.19
   - pooch
   - toolz
   - numba
   - numbagg>=0.3
+  - pip:
+      - git+https://github.com/dcherian/xarray.git
diff --git a/ci/no-numba.yml b/ci/no-numba.yml
@@ -19,8 +19,9 @@ dependencies:
   - pytest-pretty
   - pytest-xdist
   - syrupy
-  - xarray
   - pre-commit
   - numpy_groupies>=0.9.19
   - pooch
   - toolz
+  - pip:
+      - git+https://github.com/dcherian/xarray.git
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -40,6 +40,7 @@
     "sphinx.ext.napoleon",
     "myst_nb",
     "sphinx_codeautolink",
+    "sphinx_remove_toctrees",
 ]
 
 codeautolink_concat_default = True
@@ -54,6 +55,8 @@
 master_doc = "index"
 language = "en"
 
+remove_from_toctrees = ["generated/*"]
+
 # General information about the project.
 project = "flox"
 current_year = datetime.datetime.now().year

diff --git a/docs/source/user-stories/climatology-hourly.ipynb b/docs/source/user-stories/climatology-hourly.ipynb
@@ -92,7 +92,6 @@
     "%load_ext watermark\n",
     "\n",
     "\n",
-    "\n",
     "%watermark -iv"
    ]
   },

diff --git a/flox/__init__.py b/flox/__init__.py
@@ -1,9 +1,15 @@
 #!/usr/bin/env python
 # flake8: noqa
 """Top-level module for flox ."""
+
 from . import cache
 from .aggregations import Aggregation, Scan  # noqa
-from .core import groupby_reduce, groupby_scan, rechunk_for_blockwise, rechunk_for_cohorts  # noqa
+from .core import (
+    groupby_reduce,
+    groupby_scan,
+    rechunk_for_blockwise,
+    rechunk_for_cohorts,
+)  # noqa
 
 
 def _get_version():
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,7 @@ dependencies: @@
       - build
       - cachey
       - dask-core
-      - numpy>=1.22
+      - numpy<2
       - mamba
       - pip
       - python=3.10
@@ Expand Down @@