Add CircleCI to run figure comparison tests

.circleci/config.yml

@@ -0,0 +1,113 @@
+version: 2.1
+
+no-backports: &no-backports
+  name: Skip any branches called cherry-pick
+  command: |
+    if [[ "${CIRCLE_BRANCH}" == *"cherry-pick"* || "${CIRCLE_BRANCH}" == *"backport"* ]]; then
+      circleci step halt
+    fi
+
+skip-check: &skip-check
+  name: Check for [ci skip]
+  command: bash .circleci/early_exit.sh
+
+merge-check: &merge-check
+  name: Check if we need to merge upstream main
+  command: |
+    if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then
+      git fetch origin --tags
+      git fetch origin +refs/pull/$CIRCLE_PR_NUMBER/merge:pr/$CIRCLE_PR_NUMBER/merge
+      git checkout -qf pr/$CIRCLE_PR_NUMBER/merge
+    fi
+
+apt-run: &apt-install
+  name: Install apt packages
+  command: |
+    sudo apt update
+    sudo apt install -y libopenjp2-7
+
+jobs:
+  figure:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    environment:
+      TOXENV=<< parameters.jobname >>
+    steps:
+      - run: *no-backports
+      - checkout
+      - run: *skip-check
+      - run: *merge-check
+      - run: *apt-install
+      - run: pip install --user -U tox tox-pypi-filter
+      - run: tox -v
+      # - run:
+      #     name: Running codecov
+      #     command: bash -e .circleci/codecov_upload.sh -f ".tmp/${TOXENV}/coverage.xml"
+      - store_artifacts:
+          path: .tmp/<< parameters.jobname >>/figure_test_images
+
+  deploy-reference-images:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    environment:
+      TOXENV: << parameters.jobname >>
+      GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa_7b8fc81c13a3b446ec9aa50d3f626978
+    steps:
+      - checkout
+      - run: *skip-check
+      - run: *merge-check
+      - run: *apt-install
+      # Clear out all the ssh keys so that it always uses the write deploy key
+      - run: ssh-add -D
+      # Add private key for deploying to the figure tests repo
+      - add_ssh_keys:
+          fingerprints: "SHA256:vf80el6ZY/FiLKo+eXblG/DQfCuJBwrQvH1S37vjG5I"
+      - run: ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run: git config --global user.email "dkist@circleci" && git config --global user.name "DKIST Circle CI"
+      - run: git clone [email protected]:DKISTDC/dkist-figure-tests.git --depth 1 -b dkist-${CIRCLE_BRANCH} ~/dkist-figure-tests/
+      # Generate Reference images
+      - run: pip install --user -U tox tox-pypi-filter
+      - run: rm -rf /home/circleci/dkist-figure-tests/figures/$TOXENV/*
+      - run: tox -v -- --mpl-generate-path=/home/circleci/dkist-figure-tests/figures/$TOXENV | tee toxlog
+      - run: |
+          hashlib=$(grep "^figure_hashes.*\.json$" toxlog)
+          cp ./dkist/tests/$hashlib /home/circleci/dkist-figure-tests/figures/$TOXENV/
+      - run: |
+          cd ~/dkist-figure-tests/
+          git pull
+          git status
+          git add .
+          git commit -m "Update reference figures from ${CIRCLE_BRANCH}" || echo "No changes to reference images to deploy"
+          git push
+
+workflows:
+  version: 2
+
+  figure-tests:
+    jobs:
+      - figure:
+          name: << matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py312-figure"
+
+      - deploy-reference-images:
+          name: baseline-<< matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py312-figure"
+          requires:
+            - << matrix.jobname >>
+          filters:
+            branches:
+              only:
+                - main
+                - figuretests

.circleci/early_exit.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+commitmessage=$(git log --pretty=%B -n 1)
+if [[ $commitmessage = *"[ci skip]"* ]] || [[ $commitmessage = *"[skip ci]"* ]]; then
+    echo "Skipping build because [ci skip] found in commit message"
+    circleci step halt
+fi

changelog/415.feature.rst

@@ -0,0 +1 @@
+Add CircleCI config to enable figure comparison testing with pytest_mpl.

dkist/dataset/tests/test_plotting.py

@@ -4,8 +4,10 @@
 
 from astropy.visualization.wcsaxes import WCSAxes
 
+from dkist.tests.helpers import figure_test
 
-@pytest.mark.mpl_image_compare
+
+@figure_test
 @pytest.mark.parametrize("aslice", [np.s_[0, :, :], np.s_[:, 0, :], np.s_[:, :, 0]])
 def test_dataset_projection(dataset_3d, aslice):
     pytest.importorskip("ndcube", "2.0.2")  # https://github.com/sunpy/ndcube/pull/509
@@ -16,15 +18,15 @@ def test_dataset_projection(dataset_3d, aslice):
     return fig
 
 
-@pytest.mark.mpl_image_compare
+@figure_test
 @pytest.mark.parametrize("aslice", [np.s_[0, :, :], np.s_[:, 0, :], np.s_[:, :, 0]])
 def test_2d_plot(dataset_3d, aslice):
     fig = plt.figure()
     dataset_3d[aslice].plot()
     return fig
 
 
-@pytest.mark.mpl_image_compare
+@figure_test
 def test_2d_plot2(dataset_3d):
     fig = plt.figure()
     dataset_3d[:, :, 0].plot(axes_units=["Angstrom", "deg", "deg"])

dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json

@@ -0,0 +1,9 @@
+{
+  "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice0]": "f3cd5b87ef5d109090bf3183c7edbcdd8398d083417b2564991c4515bddbc8bf",
+  "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice1]": "753909f9ae5d03a03d18513783c22501784460d7f93851fd40e8ddfd493dcb6c",
+  "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice2]": "0a29ef8f286f16acf7d5ab420da6970d632ce44c1f11cfd6eb3b4c5e15c7f23a",
+  "dkist.dataset.tests.test_plotting.test_2d_plot[aslice0]": "b31423d5ec45941849564f4ec7e276f2c52a0fa5038ce0b3c8d4af1b7f848a1d",
+  "dkist.dataset.tests.test_plotting.test_2d_plot[aslice1]": "cbb84fbae51d8238803f8f0d6820c575f024fe54b1656f1b181dc4ec645e9ff9",
+  "dkist.dataset.tests.test_plotting.test_2d_plot[aslice2]": "132c5615832daff457dacb4cb770498f1fbb4460a5b90b5d4d01d224c70eeb28",
+  "dkist.dataset.tests.test_plotting.test_2d_plot2": "409b5a10ad8ccf005331261505e63ce8febdc38eb8b5a34f8863e567e3cccb9c"
+}

dkist/tests/helpers.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+from functools import wraps
+
+import matplotlib as mpl
+import pytest
+
+import astropy
+
+import ndcube
+
+
+def get_hash_library_name():
+    """
+    Generate the hash library name for this env.
+    """
+    import mpl_animators
+
+    animators_version = "dev" if (("dev" in mpl_animators.__version__) or ("rc" in mpl_animators.__version__)) else mpl_animators.__version__.replace(".", "")
+    ft2_version = f"{mpl.ft2font.__freetype_version__.replace('.', '')}"
+    mpl_version = "dev" if (("dev" in mpl.__version__) or ("rc" in mpl.__version__)) else mpl.__version__.replace(".", "")
+    astropy_version = "dev" if (("dev" in astropy.__version__) or ("rc" in astropy.__version__)) else astropy.__version__.replace(".", "")
+    ndcube_version = "dev" if (("dev" in ndcube.__version__) or ("rc" in ndcube.__version__)) else ndcube.__version__.replace(".", "")
+    return f"figure_hashes_mpl_{mpl_version}_ft_{ft2_version}_astropy_{astropy_version}_animators_{animators_version}_ndcube_{ndcube_version}.json"
+
+
+def figure_test(test_function):
+    """
+    A decorator for a test that verifies the hash of the current figure or the
+    returned figure, with the name of the test function as the hash identifier
+    in the library. A PNG is also created in the 'result_image' directory,
+    which is created on the current path.
+
+    All such decorated tests are marked with `pytest.mark.mpl_image` for convenient filtering.
+
+    Examples
+    --------
+    @figure_test
+    def test_simple_plot():
+        plt.plot([0,1])
+    """
+    hash_library_name = get_hash_library_name()
+    hash_library_file = Path(__file__).parent / hash_library_name
+
+    @pytest.mark.mpl_image_compare(hash_library=hash_library_file,
+                                   style="default")
+    @wraps(test_function)
+    def test_wrapper(*args, **kwargs):
+        return test_function(*args, **kwargs)
+    return test_wrapper

pyproject.toml

@@ -103,6 +103,14 @@ write_to = "dkist/_version.py"
   [ tool.gilesbot.milestones ]
     enabled = false
 
+  [ tool.gilesbot.circleci_artifacts]
+    enabled = true
+
+  [ tool.gilesbot.circleci_artifacts.figure_report]
+    url = ".tmp/py312-figure/figure_test_images/fig_comparison.html"
+
+  [tool.gilesbot.circleci_artifacts.figure_report_devdeps]
+
 [tool.towncrier]
     package = "dkist"
     filename = "CHANGELOG.rst"

pytest.ini

@@ -31,6 +31,8 @@ addopts =
     --doctest-rst
     -p no:unraisableexception
     -p no:threadexception
+mpl-results-path = figure_test_images
+mpl-deterministic = true
 filterwarnings =
     # Turn all warnings into errors so they do not pass silently.
     error

tox.ini

@@ -6,14 +6,15 @@ requires =
     tox-pypi-filter >= 0.14
 envlist =
     py{310,311,312}
-    py312-devdeps
+    py312-{devdeps,benchmarks,figure}
     py310-oldestdeps
-    py312-benchmarks
     codestyle
     build_docs{,-notebooks}
 
 [testenv]
 pypi_filter = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt
+allowlist_externals=
+    /bin/sh
 # Run the tests in a temporary directory to make sure that we don't import
 # the package from the source tree
 change_dir = .tmp/{envname}
@@ -47,6 +48,10 @@ deps =
     devdeps: git+https://github.com/astropy/asdf-astropy
     # Autogenerate oldest dependencies from info in setup.cfg
     oldestdeps: minimum_dependencies
+    figure: matplotlib==3.9.1
+    figure: mpl_animators==1.1.1
+    figure: astropy==6.1.1
+    figure: ndcube==2.2.2
 # The following indicates which extras_require will be installed
 extras =
     tests
@@ -55,6 +60,10 @@ commands_pre =
     oldestdeps: pip install -r requirements-min.txt cryptography<42 jsonschema==4.0.1
     pip freeze --all --no-input
 commands =
+    figure: /bin/sh -c "mkdir -p ./figure_test_images; python -c 'import matplotlib as mpl; print(mpl.ft2font.__file__, mpl.ft2font.__freetype_version__, mpl.ft2font.__freetype_build_type__)' > ./figure_test_images/figure_version_info.txt"
+    figure: /bin/sh -c "pip freeze >> ./figure_test_images/figure_version_info.txt"
+    figure: /bin/sh -c "cat ./figure_test_images/figure_version_info.txt"
+    figure: python -c "import dkist.tests.helpers as h; print(h.get_hash_library_name())"
     # To amend the pytest command for different factors you can add a line
     # which starts with a factor like `online: --remote-data=any \`
     # If you have no factors which require different commands this is all you need:
@@ -70,6 +79,11 @@ commands =
     !benchmarks: --benchmark-skip \
     benchmarks: -m benchmark \
     benchmarks: --benchmark-autosave \
+    figure: -m "mpl_image_compare" \
+    figure: --mpl \
+    figure: --remote-data=any \
+    figure: --mpl-generate-summary=html \
+    figure: --mpl-baseline-path=https://raw.githubusercontent.com/DKISTDC/dkist-figure-tests/main/figures/{envname}/ \
     oldestdeps: -o asdf_schema_tests_enabled=false \
     {posargs}