Upstream CI Improvements (#527)

* rename action for aesthetic reasons

* use better concurrency detection

* add fail-fast: false

* switch to micromamba to build env

* fix matrix version

* remove old upstream dev env yml, add script

* switch workflow to use script

* install scipy from nightly wheels

* add statsmodel explicity to uninstall/reinstall

* use statsmodel nightly wheel instead

* will this fix the weird script issue

* conda list everywhere

* change job name to match used python version

* explicitly add scipy as dependency

* remove statsmodel from uninstall/reinstall

* make sure packages are the same everywhere

* - -> _

* allow pip installs to pull in new dependencies?

* add pandas as explicit dependency

* sort and add pandas as explicit dependency

* add to release notes

* remove env names

.github/workflows/upstream-dev-ci.yml

@@ -1,49 +1,44 @@
-name: Upstream-dev CI
+name: CI Upstream
 on:
   workflow_dispatch:
   schedule:
      - cron: '0 0 * * *' # Daily “At 00:00”
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   upstream-dev:
-    name:  upstream-dev-py310
+    name:  upstream-dev-py311
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
 
     steps:
-      - name: Cancel previous runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ github.token }}
       - name: checkout
         uses: actions/checkout@v4
         with:
           token: ${{ github.token }}
-      - name: conda_setup
-        uses: conda-incubator/setup-miniconda@v3
+      - name: set up environment
+        uses: mamba-org/setup-micromamba@v1
         with:
-          activate-environment: geocat_comp_upstream
-          channel-priority: strict
-          miniconda-version: "latest"
-          python-version: "3.10"
-          channels: conda-forge
+          environment-file: build_envs/environment.yml
+          create-args: >-
+            python=${{ matrix.python-version }}
 
-      - name: Install select upstream dependencies
+      - name: conda list
         run: |
-          pip install git+https://github.com/Unidata/MetPy.git
-          pip install git+https://github.com/numpy/numpy.git
-          pip install git+https://github.com/dask/dask.git
-          pip install git+https://github.com/dask/distributed.git
-          pip install git+https://github.com/hgrecco/pint.git
-          pip install git+https://github.com/pydata/xarray.git
-          pip install git+https://github.com/xarray-contrib/cf-xarray.git
-          pip install git+https://github.com/xarray-contrib/xskillscore.git
-
-      - name: Install conda dependencies
+          conda list
+
+      - name: Install upstream versions
         run: |
-          conda env update --file build_envs/upstream-dev-environment.yml
+          bash ci/install-upstream.sh
 
       - name: Install geocat-comp
         run: |

build_envs/environment.yml

@@ -2,7 +2,7 @@ name: geocat_comp_build
 channels:
   - conda-forge
 dependencies:
-  - python>=3.9, <3.12  # minimum support 3.9
+  - python>=3.9,<3.12  # minimum support 3.9
   - cf_xarray>=0.3.1    # min version 0.3.1 for dependencies
   - cftime
   - dask
@@ -11,6 +11,8 @@ dependencies:
   - metpy
   - numba
   - numpy
+  - scipy
+  - pandas
   - pint
   - xarray
   - xskillscore

ci/install-upstream.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# adapted from https://github.com/pydata/xarray/blob/main/ci/install-upstream-wheels.sh
+
+# forcibly remove packages to avoid artifacts
+conda remove -y --force \
+    cf_xarray \
+    dask \
+    distributed \
+    metpy \
+    numpy \
+    pandas \
+    pint \
+    scipy \
+    xarray \
+    xskillscore
+
+# conda list
+conda list
+
+# if available install from nightly wheels
+python -m pip install \
+    -i https://pypi.anaconda.org/scipy-wheels-nightly/simple \
+    --no-deps \
+    --pre \
+    --upgrade \
+    numpy \
+    pandas \
+    scipy
+
+# install rest from source
+python -m pip install \
+    git+https://github.com/xarray-contrib/cf-xarray.git \
+    git+https://github.com/dask/dask.git \
+    git+https://github.com/dask/distributed.git \
+    git+https://github.com/Unidata/MetPy.git \
+    git+https://github.com/hgrecco/pint.git \
+    git+https://github.com/pydata/xarray.git \
+    git+https://github.com/xarray-contrib/xskillscore.git