diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 38fb4e49..3a860804 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,26 +32,25 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 + - uses: prefix-dev/setup-pixi@v0.8.1 with: - environment-file: ci/environment.yml - cache-environment: true - create-args: >- - python=${{matrix.python-version}} + pixi-version: v0.39.2 + environments: ${{ matrix.environment }} + # we can freeze the environment and manually bump the dependencies to the + # latest version time to time. + frozen: true - name: Install virtualizarr run: | python -m pip install -e . --no-deps - - name: Conda list information + - name: List installed libraries run: | - micromamba env list - micromamba list + pixi list - name: Running Tests run: | - python -m pytest --run-network-tests --verbose --cov=virtualizarr --cov-report=xml + pixi run test-xml-cov - name: Upload code coverage to Codecov uses: codecov/codecov-action@v3.1.4 diff --git a/.github/workflows/min-deps.yml b/.github/workflows/min-deps.yml deleted file mode 100644 index c236a9ef..00000000 --- a/.github/workflows/min-deps.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: min-deps - -on: - push: - branches: [ "main" ] - paths-ignore: - - 'docs/**' - pull_request: - branches: [ "main" ] - paths-ignore: - - 'docs/**' - schedule: - - cron: "0 0 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - PIP_ROOT_USER_ACTION: ignore - -jobs: - test-min-deps: - name: ${{ matrix.python-version }}-build - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - strategy: - matrix: - python-version: ["3.12"] - steps: - - uses: actions/checkout@v4 - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/min-deps.yml - cache-environment: true - create-args: >- - python=${{matrix.python-version}} - - - name: Install virtualizarr - run: | - python -m pip install -e . --no-deps - - - name: Conda list information - run: | - micromamba env list - micromamba list - - - name: Running Tests - run: | - python -m pytest --verbose --cov=virtualizarr --cov-report=xml - - - name: Upload code coverage to Codecov - uses: codecov/codecov-action@v3.1.4 - with: - file: ./coverage.xml - flags: unittests - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: false diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml deleted file mode 100644 index 74867ea5..00000000 --- a/.github/workflows/upstream.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: upstream - -on: - push: - branches: [ "main" ] - paths-ignore: - - 'docs/**' - pull_request: - branches: [ "main" ] - paths-ignore: - - 'docs/**' - schedule: - - cron: "0 0 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - PIP_ROOT_USER_ACTION: ignore - -jobs: - test-upstream: - name: ${{ matrix.python-version }}-build - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - strategy: - matrix: - python-version: ["3.12"] - steps: - - uses: actions/checkout@v4 - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/upstream.yml - cache-environment: true - create-args: >- - python=${{matrix.python-version}} - - - name: Install virtualizarr - run: | - python -m pip install -e . --no-deps - - - name: Conda list information - run: | - micromamba env list - micromamba list - - - name: Running Tests - run: | - python -m pytest --verbose --cov=virtualizarr --cov-report=xml - - - name: Upload code coverage to Codecov - uses: codecov/codecov-action@v3.1.4 - with: - file: ./coverage.xml - flags: unittests - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index fc7e85e5..f46cb46e 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,10 @@ examples/ .ionide # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode + +# Pixi folder +.pixi/ + +# python virtual environment +.venv +venv diff --git a/.readthedocs.yml b/.readthedocs.yml index db97ed7c..6a64750f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,24 +1,19 @@ # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details -# Required version: 2 - -# Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "mambaforge-4.10" + # this ensures a viable `mamba` is on `$PATH`` + python: mambaforge-latest + commands: + - mamba install -c conda-forge -c nodefaults pixi + - pixi install --environment docs + - pixi run docs + - pixi run readthedocs # Build documentation in the doc/ directory with Sphinx sphinx: configuration: docs/conf.py fail_on_warning: true - -# Optionally declare the Python requirements required to build your docs -conda: - environment: ci/doc.yml -python: - install: - - method: pip - path: . diff --git a/ci/doc.yml b/ci/doc.yml deleted file mode 100644 index ccc3ded6..00000000 --- a/ci/doc.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: virtualizarr-docs -channels: - - conda-forge - - nodefaults -dependencies: - - python>=3.10 - - "sphinx" - - pip - - pip: - - "myst_nb" - - "pydata_sphinx_theme" - - "sphinx_copybutton" - - "sphinx_design" - - "sphinx_togglebutton" - - "sphinx-autodoc-typehints" diff --git a/ci/environment.yml b/ci/environment.yml deleted file mode 100644 index 08dbf246..00000000 --- a/ci/environment.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: virtualizarr-tests -channels: - - conda-forge - - nodefaults -dependencies: - - h5netcdf - - h5py - - hdf5 - - netcdf4 - - xarray>=2024.10.0 - - kerchunk>=0.2.5 - - numpy>=2.0.0 - - ujson - - packaging - - universal_pathlib - - hdf5plugin - - numcodecs - # Testing - - codecov[toml] - - pre-commit - - mypy - - ruff - - pandas-stubs - - pytest-mypy - - pytest-cov - - pytest - - pooch - - scipy - - fsspec - - s3fs - - fastparquet - - imagecodecs>=2024.6.1 - # for opening tiff files - - tifffile - # for opening FITS files - - astropy - - pip - - pip: - - imagecodecs-numcodecs==2024.6.1 diff --git a/ci/min-deps.yml b/ci/min-deps.yml deleted file mode 100644 index a0a0cf20..00000000 --- a/ci/min-deps.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: virtualizarr-min-deps -channels: - - conda-forge - - nodefaults -dependencies: - - h5py - - hdf5 - - netcdf4 - - xarray>=2024.10.0 - - numpy>=2.0.0 - - numcodecs - - packaging - - ujson - - universal_pathlib - # Testing - - codecov[toml] - - pre-commit - - mypy - - ruff - - pandas-stubs - - pytest-mypy - - pytest-cov - - pytest - - pooch - - fsspec diff --git a/ci/upstream.yml b/ci/upstream.yml deleted file mode 100644 index ed9bf6fa..00000000 --- a/ci/upstream.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: virtualizarr-upstream -channels: - - conda-forge - - nodefaults -dependencies: - - xarray>=2024.10.0,<2025.0.0 - - h5netcdf - - h5py - - hdf5 - - netcdf4 - - numpy>=2.0.0 - - packaging - - ujson - - universal_pathlib - - hdf5plugin - - numcodecs - - imagecodecs>=2024.6.1 - # Testing - - codecov[toml] - - pre-commit - - mypy - - ruff - - pandas-stubs - - pytest-asyncio - - pytest-mypy - - pytest-cov - - pytest - - pooch - - fsspec - - pip - - pip: - - icechunk==0.1.0a8 # Installs zarr v3 beta 3 as dependency - # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - - imagecodecs-numcodecs==2024.6.1 diff --git a/pyproject.toml b/pyproject.toml index 8421819e..062f6862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,34 +21,47 @@ classifiers = [ requires-python = ">=3.10" dynamic = ["version"] dependencies = [ - "xarray>=2024.10.0,<2025.0.0", + "xarray>=2024.10.0", "numpy>=2.0.0", "packaging", "universal-pathlib", "numcodecs", "ujson", + "zarr" ] [project.optional-dependencies] -hdf_reader = [ +remote = [ "fsspec", + "requests", + "aiohttp", + "s3fs", +] +hdf_reader = [ + "virtualizarr[remote]", "h5py", "hdf5plugin", "imagecodecs", "imagecodecs-numcodecs==2024.6.1", - "numcodecs" + ] icechunk = [ - "icechunk==0.1.0a8", + "icechunk==0.1.0a12", ] -test = [ - "codecov", +kerchunk = [ + "virtualizarr[remote]", "fastparquet", - "fsspec", "h5py", - "kerchunk>=0.2.5", - "mypy", + "kerchunk @ git+https://github.com/fsspec/kerchunk@main", "netcdf4", + "scipy", + "tifffile", + "astropy", + +] +test = [ + "codecov", + "mypy", "numcodecs", "pandas-stubs", "pooch", @@ -57,14 +70,24 @@ test = [ "pytest-mypy", "pytest", "ruff", - "s3fs", - "scipy", - "virtualizarr[hdf_reader]" + "virtualizarr[hdf_reader]", + "virtualizarr[icechunk]", + "virtualizarr[kerchunk]", + "virtualizarr[remote]" +] +docs = [ + "sphinx", + "myst_nb", + "pydata_sphinx_theme", + "sphinx_copybutton", + "sphinx_design", + "sphinx_togglebutton", + "sphinx-autodoc-typehints", ] [project.urls] -Home = "https://github.com/TomNicholas/VirtualiZarr" -Documentation = "https://github.com/TomNicholas/VirtualiZarr/blob/main/README.md" +Home = "https://github.com/zarr-developers/VirtualiZarr" +Documentation = "https://github.com/zarr-developers/VirtualiZarr/blob/main/README.md" [build-system] requires = [ @@ -74,6 +97,41 @@ requires = [ "check-manifest" ] +[tool.pixi.project] +channels = ["conda-forge"] +platforms = ["win-64", "linux-64", "osx-64", "osx-arm64"] + +[tool.pixi.environments] +test = { features = ["test", "remote", "hdf_reader", "kerchunk", "icechunk"], solve-group = "default" } +hdf-reader = { features = ["hdf_reader"], solve-group = "default" } +kerchunk = { features = ["kerchunk"], solve-group = "default" } +icechunk = { features = ["icechunk"], solve-group = "default" } +docs = { features = ["docs"], solve-group = "default" } + +[tool.pixi.feature.hdf_reader.dependencies] +netcdf4 = ">=1.7.2,<2" +hdf5 = ">=1.14.4,<2" +h5netcdf = ">=1.5.0,<2" + +[tool.pixi.feature.kerchunk.dependencies] +netcdf4 = ">=1.7.2,<2" +hdf5 = ">=1.14.4,<2" +h5netcdf = ">=1.5.0,<2" + +[tool.pixi.pypi-dependencies] +virtualizarr = { path = ".", editable = true } + +[tool.pixi.feature.test.tasks] +test = { cmd = "pytest --run-network-tests" } +test-no-network = { cmd = "pytest" } +test-cov = { cmd = "pytest --run-network-tests --verbose --cov=virtualizarr --cov=term-missing" } +test-xml-cov = { cmd = "pytest --run-network-tests --verbose --cov=virtualizarr --cov-report=xml" } +test-html-cov = { cmd = "pytest --run-network-tests --verbose --cov=virtualizarr --cov-report=html" } + +[tool.pixi.feature.docs.tasks] +docs = { cmd = "make html", cwd = "docs" } +readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r docs/_build/html $READTHEDOCS_OUTPUT/html" + [tool.setuptools_scm] fallback_version = "9999"