diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..1c109696 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,113 @@ +version: 2.1 + +no-backports: &no-backports + name: Skip any branches called cherry-pick + command: | + if [[ "${CIRCLE_BRANCH}" == *"cherry-pick"* || "${CIRCLE_BRANCH}" == *"backport"* ]]; then + circleci step halt + fi + +skip-check: &skip-check + name: Check for [ci skip] + command: bash .circleci/early_exit.sh + +merge-check: &merge-check + name: Check if we need to merge upstream main + command: | + if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then + git fetch origin --tags + git fetch origin +refs/pull/$CIRCLE_PR_NUMBER/merge:pr/$CIRCLE_PR_NUMBER/merge + git checkout -qf pr/$CIRCLE_PR_NUMBER/merge + fi + +apt-run: &apt-install + name: Install apt packages + command: | + sudo apt update + sudo apt install -y libopenjp2-7 + +jobs: + figure: + parameters: + jobname: + type: string + docker: + - image: cimg/python:3.12 + environment: + TOXENV=<< parameters.jobname >> + steps: + - run: *no-backports + - checkout + - run: *skip-check + - run: *merge-check + - run: *apt-install + - run: pip install --user -U tox tox-pypi-filter + - run: tox -v + # - run: + # name: Running codecov + # command: bash -e .circleci/codecov_upload.sh -f ".tmp/${TOXENV}/coverage.xml" + - store_artifacts: + path: .tmp/<< parameters.jobname >>/figure_test_images + + deploy-reference-images: + parameters: + jobname: + type: string + docker: + - image: cimg/python:3.12 + environment: + TOXENV: << parameters.jobname >> + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa_7b8fc81c13a3b446ec9aa50d3f626978 + steps: + - checkout + - run: *skip-check + - run: *merge-check + - run: *apt-install + # Clear out all the ssh keys so that it always uses the write deploy key + - run: ssh-add -D + # Add private key for deploying to the figure tests repo + - add_ssh_keys: + fingerprints: "SHA256:vf80el6ZY/FiLKo+eXblG/DQfCuJBwrQvH1S37vjG5I" + - run: ssh-keyscan github.com >> ~/.ssh/known_hosts + - run: git config --global user.email "dkist@circleci" && git config --global user.name "DKIST Circle CI" + - run: git clone git@github.com:DKISTDC/dkist-figure-tests.git --depth 1 -b dkist-${CIRCLE_BRANCH} ~/dkist-figure-tests/ + # Generate Reference images + - run: pip install --user -U tox tox-pypi-filter + - run: rm -rf /home/circleci/dkist-figure-tests/figures/$TOXENV/* + - run: tox -v -- --mpl-generate-path=/home/circleci/dkist-figure-tests/figures/$TOXENV | tee toxlog + - run: | + hashlib=$(grep "^figure_hashes.*\.json$" toxlog) + cp ./dkist/tests/$hashlib /home/circleci/dkist-figure-tests/figures/$TOXENV/ + - run: | + cd ~/dkist-figure-tests/ + git pull + git status + git add . + git commit -m "Update reference figures from ${CIRCLE_BRANCH}" || echo "No changes to reference images to deploy" + git push + +workflows: + version: 2 + + figure-tests: + jobs: + - figure: + name: << matrix.jobname >> + matrix: + parameters: + jobname: + - "py312-figure" + + - deploy-reference-images: + name: baseline-<< matrix.jobname >> + matrix: + parameters: + jobname: + - "py312-figure" + requires: + - << matrix.jobname >> + filters: + branches: + only: + - main + - figuretests diff --git a/.circleci/early_exit.sh b/.circleci/early_exit.sh new file mode 100644 index 00000000..1342039f --- /dev/null +++ b/.circleci/early_exit.sh @@ -0,0 +1,6 @@ +#!/bin/bash +commitmessage=$(git log --pretty=%B -n 1) +if [[ $commitmessage = *"[ci skip]"* ]] || [[ $commitmessage = *"[skip ci]"* ]]; then + echo "Skipping build because [ci skip] found in commit message" + circleci step halt +fi diff --git a/changelog/415.feature.rst b/changelog/415.feature.rst new file mode 100644 index 00000000..0c0596c4 --- /dev/null +++ b/changelog/415.feature.rst @@ -0,0 +1 @@ +Add CircleCI config to enable figure comparison testing with pytest_mpl. diff --git a/dkist/dataset/tests/test_plotting.py b/dkist/dataset/tests/test_plotting.py index 2965db14..df50cb4d 100644 --- a/dkist/dataset/tests/test_plotting.py +++ b/dkist/dataset/tests/test_plotting.py @@ -4,8 +4,10 @@ from astropy.visualization.wcsaxes import WCSAxes +from dkist.tests.helpers import figure_test -@pytest.mark.mpl_image_compare + +@figure_test @pytest.mark.parametrize("aslice", [np.s_[0, :, :], np.s_[:, 0, :], np.s_[:, :, 0]]) def test_dataset_projection(dataset_3d, aslice): pytest.importorskip("ndcube", "2.0.2") # https://github.com/sunpy/ndcube/pull/509 @@ -16,7 +18,7 @@ def test_dataset_projection(dataset_3d, aslice): return fig -@pytest.mark.mpl_image_compare +@figure_test @pytest.mark.parametrize("aslice", [np.s_[0, :, :], np.s_[:, 0, :], np.s_[:, :, 0]]) def test_2d_plot(dataset_3d, aslice): fig = plt.figure() @@ -24,7 +26,7 @@ def test_2d_plot(dataset_3d, aslice): return fig -@pytest.mark.mpl_image_compare +@figure_test def test_2d_plot2(dataset_3d): fig = plt.figure() dataset_3d[:, :, 0].plot(axes_units=["Angstrom", "deg", "deg"]) diff --git a/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json b/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json new file mode 100644 index 00000000..01ef2660 --- /dev/null +++ b/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json @@ -0,0 +1,9 @@ +{ + "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice0]": "f3cd5b87ef5d109090bf3183c7edbcdd8398d083417b2564991c4515bddbc8bf", + "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice1]": "753909f9ae5d03a03d18513783c22501784460d7f93851fd40e8ddfd493dcb6c", + "dkist.dataset.tests.test_plotting.test_dataset_projection[aslice2]": "0a29ef8f286f16acf7d5ab420da6970d632ce44c1f11cfd6eb3b4c5e15c7f23a", + "dkist.dataset.tests.test_plotting.test_2d_plot[aslice0]": "b31423d5ec45941849564f4ec7e276f2c52a0fa5038ce0b3c8d4af1b7f848a1d", + "dkist.dataset.tests.test_plotting.test_2d_plot[aslice1]": "cbb84fbae51d8238803f8f0d6820c575f024fe54b1656f1b181dc4ec645e9ff9", + "dkist.dataset.tests.test_plotting.test_2d_plot[aslice2]": "132c5615832daff457dacb4cb770498f1fbb4460a5b90b5d4d01d224c70eeb28", + "dkist.dataset.tests.test_plotting.test_2d_plot2": "409b5a10ad8ccf005331261505e63ce8febdc38eb8b5a34f8863e567e3cccb9c" +} \ No newline at end of file diff --git a/dkist/tests/helpers.py b/dkist/tests/helpers.py new file mode 100644 index 00000000..ec633b99 --- /dev/null +++ b/dkist/tests/helpers.py @@ -0,0 +1,49 @@ +from pathlib import Path +from functools import wraps + +import matplotlib as mpl +import pytest + +import astropy + +import ndcube + + +def get_hash_library_name(): + """ + Generate the hash library name for this env. + """ + import mpl_animators + + animators_version = "dev" if (("dev" in mpl_animators.__version__) or ("rc" in mpl_animators.__version__)) else mpl_animators.__version__.replace(".", "") + ft2_version = f"{mpl.ft2font.__freetype_version__.replace('.', '')}" + mpl_version = "dev" if (("dev" in mpl.__version__) or ("rc" in mpl.__version__)) else mpl.__version__.replace(".", "") + astropy_version = "dev" if (("dev" in astropy.__version__) or ("rc" in astropy.__version__)) else astropy.__version__.replace(".", "") + ndcube_version = "dev" if (("dev" in ndcube.__version__) or ("rc" in ndcube.__version__)) else ndcube.__version__.replace(".", "") + return f"figure_hashes_mpl_{mpl_version}_ft_{ft2_version}_astropy_{astropy_version}_animators_{animators_version}_ndcube_{ndcube_version}.json" + + +def figure_test(test_function): + """ + A decorator for a test that verifies the hash of the current figure or the + returned figure, with the name of the test function as the hash identifier + in the library. A PNG is also created in the 'result_image' directory, + which is created on the current path. + + All such decorated tests are marked with `pytest.mark.mpl_image` for convenient filtering. + + Examples + -------- + @figure_test + def test_simple_plot(): + plt.plot([0,1]) + """ + hash_library_name = get_hash_library_name() + hash_library_file = Path(__file__).parent / hash_library_name + + @pytest.mark.mpl_image_compare(hash_library=hash_library_file, + style="default") + @wraps(test_function) + def test_wrapper(*args, **kwargs): + return test_function(*args, **kwargs) + return test_wrapper diff --git a/pyproject.toml b/pyproject.toml index 86cbb5d5..d8130fb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,12 @@ write_to = "dkist/_version.py" [ tool.gilesbot.milestones ] enabled = false + [ tool.gilesbot.circleci_artifacts] + enabled = true + + [ tool.gilesbot.circleci_artifacts.figure_report] + url = ".tmp/py312-figure/figure_test_images/fig_comparison.html" + [tool.towncrier] package = "dkist" filename = "CHANGELOG.rst" diff --git a/pytest.ini b/pytest.ini index 0308ea40..36c18926 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,6 +31,8 @@ addopts = --doctest-rst -p no:unraisableexception -p no:threadexception +mpl-results-path = figure_test_images +mpl-deterministic = true filterwarnings = # Turn all warnings into errors so they do not pass silently. error diff --git a/tox.ini b/tox.ini index 1cab5d43..39133a81 100644 --- a/tox.ini +++ b/tox.ini @@ -6,14 +6,15 @@ requires = tox-pypi-filter >= 0.14 envlist = py{310,311,312} - py312-devdeps + py312-{devdeps,benchmarks,figure} py310-oldestdeps - py312-benchmarks codestyle build_docs{,-notebooks} [testenv] pypi_filter = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt +allowlist_externals= + /bin/sh # Run the tests in a temporary directory to make sure that we don't import # the package from the source tree change_dir = .tmp/{envname} @@ -47,6 +48,10 @@ deps = devdeps: git+https://github.com/astropy/asdf-astropy # Autogenerate oldest dependencies from info in setup.cfg oldestdeps: minimum_dependencies + figure: matplotlib==3.9.1 + figure: mpl_animators==1.1.1 + figure: astropy==6.1.1 + figure: ndcube==2.2.2 # The following indicates which extras_require will be installed extras = tests @@ -55,6 +60,10 @@ commands_pre = oldestdeps: pip install -r requirements-min.txt cryptography<42 jsonschema==4.0.1 pip freeze --all --no-input commands = + figure: /bin/sh -c "mkdir -p ./figure_test_images; python -c 'import matplotlib as mpl; print(mpl.ft2font.__file__, mpl.ft2font.__freetype_version__, mpl.ft2font.__freetype_build_type__)' > ./figure_test_images/figure_version_info.txt" + figure: /bin/sh -c "pip freeze >> ./figure_test_images/figure_version_info.txt" + figure: /bin/sh -c "cat ./figure_test_images/figure_version_info.txt" + figure: python -c "import dkist.tests.helpers as h; print(h.get_hash_library_name())" # To amend the pytest command for different factors you can add a line # which starts with a factor like `online: --remote-data=any \` # If you have no factors which require different commands this is all you need: @@ -70,6 +79,11 @@ commands = !benchmarks: --benchmark-skip \ benchmarks: -m benchmark \ benchmarks: --benchmark-autosave \ + figure: -m "mpl_image_compare" \ + figure: --mpl \ + figure: --remote-data=any \ + figure: --mpl-generate-summary=html \ + figure: --mpl-baseline-path=https://raw.githubusercontent.com/DKISTDC/dkist-figure-tests/main/figures/{envname}/ \ oldestdeps: -o asdf_schema_tests_enabled=false \ {posargs}