From 89abb18cfbd3f4731750c6d93a8f8cde0cd6cc83 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 8 Sep 2022 10:02:05 +0100 Subject: [PATCH 01/14] Added options to set default tolerance and style --- pytest_mpl/plugin.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index afd38f2..ef3b800 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -162,6 +162,14 @@ def pytest_addoption(parser): parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", type='bool') + style_help = "default style to use for tests, unless specified in the mpl_image_compare decorator" + group.addoption('--mpl-default-style', help=style_help, action='store') + parser.addini('mpl-default-style', help=style_help) + + tolerance_help = "default tolerance to use for tests, unless specified in the mpl_image_compare decorator" + group.addoption('--mpl-default-tolerance', help=tolerance_help, action='store') + parser.addini('mpl-default-tolerance', help=tolerance_help) + def pytest_configure(config): @@ -202,6 +210,12 @@ def pytest_configure(config): if results_dir is not None: results_dir = os.path.abspath(results_dir) + default_style = (config.getoption("--mpl-default-style") or + config.getini("mpl-default-style")) + + default_tolerance = (config.getoption("--mpl-default-tolerance") or + config.getini("mpl-default-tolerance")) + config.pluginmanager.register(ImageComparison(config, baseline_dir=baseline_dir, baseline_relative_dir=baseline_relative_dir, @@ -210,7 +224,9 @@ def pytest_configure(config): hash_library=hash_library, generate_hash_library=generate_hash_lib, generate_summary=generate_summary, - results_always=results_always)) + results_always=results_always, + default_style=default_style, + default_tolerance=default_tolerance)) else: @@ -266,7 +282,9 @@ def __init__(self, hash_library=None, generate_hash_library=None, generate_summary=None, - results_always=False + results_always=False, + default_style='classic', + default_tolerance=2 ): self.config = config self.baseline_dir = baseline_dir @@ -287,6 +305,9 @@ def __init__(self, self.generate_summary = generate_summary self.results_always = results_always + self.default_style = default_style + self.default_tolerance = default_tolerance + # Generate the containing dir for all test results if not self.results_dir: self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) @@ -467,7 +488,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary = {} compare = get_compare(item) - tolerance = compare.kwargs.get('tolerance', 2) + tolerance = compare.kwargs.get('tolerance', self.default_tolerance) ext = self._file_extension(item) @@ -683,7 +704,7 @@ def pytest_runtest_call(self, item): # noqa from matplotlib.testing.decorators import ImageComparisonTest as MplImageComparisonTest remove_ticks_and_titles = MplImageComparisonTest.remove_text - style = compare.kwargs.get('style', 'classic') + style = compare.kwargs.get('style', self.default_style) remove_text = compare.kwargs.get('remove_text', False) backend = compare.kwargs.get('backend', 'agg') From 718308a60cc631975871e1ecc117dad2afabb79d Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 20 May 2023 21:17:31 +0100 Subject: [PATCH 02/14] Apply suggestions from code review --- pytest_mpl/plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index ef3b800..588df8a 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -211,10 +211,12 @@ def pytest_configure(config): results_dir = os.path.abspath(results_dir) default_style = (config.getoption("--mpl-default-style") or - config.getini("mpl-default-style")) + config.getini("mpl-default-style") or + "classic") - default_tolerance = (config.getoption("--mpl-default-tolerance") or - config.getini("mpl-default-tolerance")) + default_tolerance = float(config.getoption("--mpl-default-tolerance") or + config.getini("mpl-default-tolerance") or + "2") config.pluginmanager.register(ImageComparison(config, baseline_dir=baseline_dir, From 232f479b75ed94c0370d8bddd535a00092497c6f Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 20 May 2023 21:50:21 +0100 Subject: [PATCH 03/14] Add `mpl-baseline-path` ini option --- docs/configuration.rst | 4 ++-- pytest_mpl/plugin.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 9e6914f..36cbc73 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -84,11 +84,11 @@ Directory containing baseline images ------------------------------------ | **kwarg**: ``baseline_dir=`` | **CLI**: ``--mpl-baseline-path=`` -| **INI**: --- +| **INI**: ``mpl-baseline-path`` | Default: ``baseline/`` *(relative to the test file)* The directory containing the baseline images that will be compared to the test figures. -The kwarg option (``baseline_dir``) is relative to the test file, while the CLI option (``--mpl-baseline-path``) is relative to where pytest was run. +The kwarg option (``baseline_dir``) is relative to the test file, while the CLI option (``--mpl-baseline-path``) and INI option (``mpl-baseline-path``) are relative to where pytest was run. Absolute paths can also be used. If the directory does not exist, it will be created along with any missing parent directories. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 588df8a..e7fd9a3 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -133,11 +133,16 @@ def pytest_addoption(parser): group.addoption('--mpl-generate-hash-library', help="filepath to save a generated hash library, relative " "to location where py.test is run", action='store') - group.addoption('--mpl-baseline-path', - help="directory containing baseline images, relative to " - "location where py.test is run unless --mpl-baseline-relative is given. " - "This can also be a URL or a set of comma-separated URLs (in case " - "mirrors are specified)", action='store') + + baseline_path_help = ( + "directory containing baseline images, relative to " + "location where py.test is run unless --mpl-baseline-relative is given. " + "This can also be a URL or a set of comma-separated URLs (in case " + "mirrors are specified)" + ) + group.addoption("--mpl-baseline-path", help=baseline_path_help, action="store") + parser.addini("mpl-baseline-path", help=baseline_path_help) + group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " "relative to the test location.", action="store_true") group.addoption('--mpl-hash-library', @@ -181,7 +186,8 @@ def pytest_configure(config): config.getoption("--mpl-generate-path") is not None or config.getoption("--mpl-generate-hash-library") is not None): - baseline_dir = config.getoption("--mpl-baseline-path") + baseline_dir = (config.getoption("--mpl-baseline-path") or + config.getini("mpl-baseline-path")) generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") From 02eec0deeebdc5b9c4c7f7258411905238b003d8 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 12:10:46 +0100 Subject: [PATCH 04/14] Add `--mpl-use-full-test-name` CLI option --- docs/configuration.rst | 2 +- pytest_mpl/plugin.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 36cbc73..aa84bdb 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -160,7 +160,7 @@ If you specify a filename that has an extension other than ``png``, you must als Whether to include the module name in the filename -------------------------------------------------- | **kwarg**: --- -| **CLI**: --- +| **CLI**: ``--mpl-use-full-test-name`` | **INI**: ``mpl-use-full-test-name`` | Default: ``False`` diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index e7fd9a3..0743625 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -164,8 +164,9 @@ def pytest_addoption(parser): help=results_always_help) parser.addini('mpl-results-always', help=results_always_help) - parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", - type='bool') + use_full_test_name_help = "use fully qualified test name as the filename." + group.addoption("--mpl-use-full-test-name", help=use_full_test_name_help, action="store_true") + parser.addini('mpl-use-full-test-name', help=use_full_test_name_help, type='bool') style_help = "default style to use for tests, unless specified in the mpl_image_compare decorator" group.addoption('--mpl-default-style', help=style_help, action='store') @@ -195,6 +196,8 @@ def pytest_configure(config): generate_summary = config.getoption("--mpl-generate-summary") results_always = (config.getoption("--mpl-results-always") or config.getini("mpl-results-always")) + use_full_test_name = (config.getoption("--mpl-use-full-test-name") or + config.getini("mpl-use-full-test-name")) if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") @@ -233,6 +236,7 @@ def pytest_configure(config): generate_hash_library=generate_hash_lib, generate_summary=generate_summary, results_always=results_always, + use_full_test_name=use_full_test_name, default_style=default_style, default_tolerance=default_tolerance)) @@ -291,6 +295,7 @@ def __init__(self, generate_hash_library=None, generate_summary=None, results_always=False, + use_full_test_name=False, default_style='classic', default_tolerance=2 ): @@ -312,6 +317,7 @@ def __init__(self, results_always = True self.generate_summary = generate_summary self.results_always = results_always + self.use_full_test_name = use_full_test_name self.default_style = default_style self.default_tolerance = default_tolerance @@ -360,7 +366,7 @@ def generate_filename(self, item): Given a pytest item, generate the figure filename. """ ext = self._file_extension(item) - if self.config.getini('mpl-use-full-test-name'): + if self.use_full_test_name: filename = generate_test_name(item) + f'.{ext}' else: compare = get_compare(item) From 2864ecf022a3a65df45ae9938ad3fb96c523ccf8 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 13:10:08 +0100 Subject: [PATCH 05/14] Add `mpl-hash-library` INI option --- docs/configuration.rst | 10 +++++++--- pytest_mpl/plugin.py | 25 ++++++++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index aa84bdb..19545d4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -191,15 +191,19 @@ File containing baseline hashes ------------------------------- | **kwarg**: ``hash_library=`` | **CLI**: ``--mpl-hash-library=`` -| **INI**: --- +| **INI**: ``mpl-hash-library = `` | Default: *no hash comparison* The file containing the baseline hashes that will be compared to the test figures. -Both the kwarg option (``hash_library``) and the CLI option (``--mpl-hash-library``) are relative to the test file. -In this case, the CLI option takes precedence over the kwarg option. +The kwarg option (``hash_library``) is relative to the test file, while the INI option (``mpl-hash-library``) is relative to where pytest was run. The file must be a JSON file in the same format as one generated by ``--mpl-generate-hash-library``. If its directory does not exist, it will be created along with any missing parent directories. +.. attention:: + + For backwards compatibility, the CLI option (``--mpl-hash-library``) is relative to the test file. + Also, the CLI option takes precedence over the kwarg option, but the kwarg option takes precedence over the INI option as usual. + Configuring this option disables baseline image comparison. If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" `, you must explicitly set the :ref:`baseline directory configuration option `. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 0743625..c252420 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -145,9 +145,9 @@ def pytest_addoption(parser): group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " "relative to the test location.", action="store_true") - group.addoption('--mpl-hash-library', - help="json library of image hashes, relative to " - "location where py.test is run", action='store') + mpl_hash_library_help = "json library of image hashes, relative to location where py.test is run" + group.addoption('--mpl-hash-library', help=mpl_hash_library_help, action='store') + parser.addini("mpl-hash-library", help=mpl_hash_library_help) group.addoption('--mpl-generate-summary', action='store', help="Generate a summary report of any failed tests" ", in --mpl-results-path. The type of the report should be " @@ -192,7 +192,8 @@ def pytest_configure(config): generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") - hash_library = config.getoption("--mpl-hash-library") + hash_library = config.getoption("--mpl-hash-library") or config.getini("mpl-hash-library") + _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility generate_summary = config.getoption("--mpl-generate-summary") results_always = (config.getoption("--mpl-results-always") or config.getini("mpl-results-always")) @@ -218,6 +219,10 @@ def pytest_configure(config): baseline_dir = os.path.abspath(generate_dir) if results_dir is not None: results_dir = os.path.abspath(results_dir) + if hash_library is not None: + # For backwards compatibility, don't make absolute if set via CLI option + if not _hash_library_from_cli: + hash_library = os.path.abspath(hash_library) default_style = (config.getoption("--mpl-default-style") or config.getini("mpl-default-style") or @@ -238,7 +243,8 @@ def pytest_configure(config): results_always=results_always, use_full_test_name=use_full_test_name, default_style=default_style, - default_tolerance=default_tolerance)) + default_tolerance=default_tolerance, + _hash_library_from_cli=_hash_library_from_cli)) else: @@ -297,7 +303,8 @@ def __init__(self, results_always=False, use_full_test_name=False, default_style='classic', - default_tolerance=2 + default_tolerance=2, + _hash_library_from_cli=False, # for backwards compatibility ): self.config = config self.baseline_dir = baseline_dir @@ -305,6 +312,7 @@ def __init__(self, self.generate_dir = path_is_not_none(generate_dir) self.results_dir = path_is_not_none(results_dir) self.hash_library = path_is_not_none(hash_library) + self._hash_library_from_cli = _hash_library_from_cli # for backwards compatibility self.generate_hash_library = path_is_not_none(generate_hash_library) if generate_summary: generate_summary = {i.lower() for i in generate_summary.split(',')} @@ -637,7 +645,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): # Use hash library name of current test as results hash library name self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name - hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) + # Order of precedence for hash library: CLI, kwargs, INI (for backwards compatibility) + hash_library_filename = compare.kwargs.get("hash_library", None) or self.hash_library + if self._hash_library_from_cli: # for backwards compatibility + hash_library_filename = self.hash_library hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() if not Path(hash_library_filename).exists(): From 3a414700aca7282c0687e84a654cb6bcb15c3916 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 15:12:26 +0100 Subject: [PATCH 06/14] Document tolerance and style CLI and INI options --- docs/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 19545d4..9828476 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -226,8 +226,8 @@ Adjusting these options *may* allow tests to pass across a range of Matplotlib a RMS tolerance ------------- | **kwarg**: ``tolerance=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-tolerance=`` +| **INI**: ``mpl-default-tolerance = `` | Default: ``2`` The maximum RMS difference between the result image and the baseline image before the test fails. @@ -310,8 +310,8 @@ A dictionary of keyword arguments to pass to :func:`matplotlib.pyplot.savefig`. Matplotlib style ---------------- | **kwarg**: ``style=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-style=`` +| **INI**: ``mpl-default-style = `` | Default: ``"classic"`` The Matplotlib style to use when saving the figure. From 562b765a8981f52453b442791c87e345d0a64b7b Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 19:44:52 +0100 Subject: [PATCH 07/14] Add backend CLI and INI options --- docs/configuration.rst | 4 ++-- pytest_mpl/plugin.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 9828476..f0f79ab 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -340,8 +340,8 @@ See the :func:`matplotlib.style.context` ``style`` documentation for the options Matplotlib backend ------------------ | **kwarg**: ``backend=`` -| **CLI**: --- -| **INI**: --- +| **CLI**: ``--mpl-default-backend=`` +| **INI**: ``mpl-default-backend = `` | Default: ``"agg"`` The Matplotlib backend to use when saving the figure. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index c252420..121aa13 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -176,6 +176,11 @@ def pytest_addoption(parser): group.addoption('--mpl-default-tolerance', help=tolerance_help, action='store') parser.addini('mpl-default-tolerance', help=tolerance_help) + msg = "default backend to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-backend" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + def pytest_configure(config): @@ -232,6 +237,10 @@ def pytest_configure(config): config.getini("mpl-default-tolerance") or "2") + default_backend = (config.getoption("--mpl-default-backend") or + config.getini("mpl-default-backend") or + "agg") + config.pluginmanager.register(ImageComparison(config, baseline_dir=baseline_dir, baseline_relative_dir=baseline_relative_dir, @@ -244,6 +253,7 @@ def pytest_configure(config): use_full_test_name=use_full_test_name, default_style=default_style, default_tolerance=default_tolerance, + default_backend=default_backend, _hash_library_from_cli=_hash_library_from_cli)) else: @@ -304,6 +314,7 @@ def __init__(self, use_full_test_name=False, default_style='classic', default_tolerance=2, + default_backend='agg', _hash_library_from_cli=False, # for backwards compatibility ): self.config = config @@ -329,6 +340,7 @@ def __init__(self, self.default_style = default_style self.default_tolerance = default_tolerance + self.default_backend = default_backend # Generate the containing dir for all test results if not self.results_dir: @@ -731,7 +743,7 @@ def pytest_runtest_call(self, item): # noqa style = compare.kwargs.get('style', self.default_style) remove_text = compare.kwargs.get('remove_text', False) - backend = compare.kwargs.get('backend', 'agg') + backend = compare.kwargs.get('backend', self.default_backend) ext = self._file_extension(item) From d71296fc806fd60f468b8d0e4b5aa8cee92d5dd4 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 15:33:39 +0100 Subject: [PATCH 08/14] Add `mpl-generate-summary` INI option --- docs/configuration.rst | 2 +- pytest_mpl/plugin.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index f0f79ab..52c4272 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -399,7 +399,7 @@ Generate test summaries ----------------------- | **kwarg**: --- | **CLI**: ``--mpl-generate-summary={html,json,basic-html}`` -| **INI**: --- +| **INI**: ``mpl-generate-summary = {html,json,basic-html}`` | Default: ``None`` This option specifies the format of the test summary report to generate, if any. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 121aa13..53c6195 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -148,11 +148,15 @@ def pytest_addoption(parser): mpl_hash_library_help = "json library of image hashes, relative to location where py.test is run" group.addoption('--mpl-hash-library', help=mpl_hash_library_help, action='store') parser.addini("mpl-hash-library", help=mpl_hash_library_help) - group.addoption('--mpl-generate-summary', action='store', - help="Generate a summary report of any failed tests" - ", in --mpl-results-path. The type of the report should be " - "specified. Supported types are `html`, `json` and `basic-html`. " - "Multiple types can be specified separated by commas.") + + mpl_generate_summary_help = ( + "Generate a summary report of any failed tests" + ", in --mpl-results-path. The type of the report should be " + "specified. Supported types are `html`, `json` and `basic-html`. " + "Multiple types can be specified separated by commas." + ) + group.addoption("--mpl-generate-summary", help=mpl_generate_summary_help, action="store") + parser.addini("mpl-generate-summary", help=mpl_generate_summary_help) results_path_help = "directory for test results, relative to location where py.test is run" group.addoption('--mpl-results-path', help=results_path_help, action='store') @@ -199,7 +203,8 @@ def pytest_configure(config): results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") hash_library = config.getoption("--mpl-hash-library") or config.getini("mpl-hash-library") _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility - generate_summary = config.getoption("--mpl-generate-summary") + generate_summary = (config.getoption("--mpl-generate-summary") or + config.getini("mpl-generate-summary")) results_always = (config.getoption("--mpl-results-always") or config.getini("mpl-results-always")) use_full_test_name = (config.getoption("--mpl-use-full-test-name") or From b066651cabfc8c6a7f1aa4d3f3c6b865fd78cdf8 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 19:42:09 +0100 Subject: [PATCH 09/14] Tidy formatting of options code using patterns from #150 --- pytest_mpl/plugin.py | 213 ++++++++++++++++++++++--------------------- 1 file changed, 111 insertions(+), 102 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 53c6195..62a9ea0 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -44,7 +44,11 @@ from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html -SUPPORTED_FORMATS = {'html', 'json', 'basic-html'} +DEFAULT_STYLE = "classic" +DEFAULT_TOLERANCE: float = 2 +DEFAULT_BACKEND = "agg" + +SUPPORTED_FORMATS = {"html", "json", "basic-html"} SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match. Expected shape: {expected_shape} @@ -125,60 +129,71 @@ def pytest_report_header(config, startdir): def pytest_addoption(parser): group = parser.getgroup("matplotlib image comparison") - group.addoption('--mpl', action='store_true', - help="Enable comparison of matplotlib figures to reference files") - group.addoption('--mpl-generate-path', - help="directory to generate reference images in, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-generate-hash-library', - help="filepath to save a generated hash library, relative " - "to location where py.test is run", action='store') - - baseline_path_help = ( + + msg = "Enable comparison of matplotlib figures to reference files" + group.addoption("--mpl", help=msg, action="store_true") + + msg = "directory to generate reference images in, relative to location where py.test is run" + group.addoption("--mpl-generate-path", help=msg, action="store") + + msg = "filepath to save a generated hash library, relative to location where py.test is run" + group.addoption("--mpl-generate-hash-library", help=msg, action="store") + + msg = ( "directory containing baseline images, relative to " "location where py.test is run unless --mpl-baseline-relative is given. " "This can also be a URL or a set of comma-separated URLs (in case " "mirrors are specified)" ) - group.addoption("--mpl-baseline-path", help=baseline_path_help, action="store") - parser.addini("mpl-baseline-path", help=baseline_path_help) + option = "mpl-baseline-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) - group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " - "relative to the test location.", action="store_true") - mpl_hash_library_help = "json library of image hashes, relative to location where py.test is run" - group.addoption('--mpl-hash-library', help=mpl_hash_library_help, action='store') - parser.addini("mpl-hash-library", help=mpl_hash_library_help) + msg = "interpret the baseline directory as relative to the test location." + group.addoption("--mpl-baseline-relative", help=msg, action="store_true") - mpl_generate_summary_help = ( + msg = "json library of image hashes, relative to location where py.test is run" + option = "mpl-hash-library" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = ( "Generate a summary report of any failed tests" ", in --mpl-results-path. The type of the report should be " "specified. Supported types are `html`, `json` and `basic-html`. " "Multiple types can be specified separated by commas." ) - group.addoption("--mpl-generate-summary", help=mpl_generate_summary_help, action="store") - parser.addini("mpl-generate-summary", help=mpl_generate_summary_help) + option = "mpl-generate-summary" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) - results_path_help = "directory for test results, relative to location where py.test is run" - group.addoption('--mpl-results-path', help=results_path_help, action='store') - parser.addini('mpl-results-path', help=results_path_help) + msg = "directory for test results, relative to location where py.test is run" + option = "mpl-results-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) - results_always_help = ("Always compare to baseline images and save result images, even for passing tests. " - "This option is automatically applied when generating a HTML summary.") - group.addoption('--mpl-results-always', action='store_true', - help=results_always_help) - parser.addini('mpl-results-always', help=results_always_help) + msg = ( + "Always compare to baseline images and save result images, even for passing tests. " + "This option is automatically applied when generating a HTML summary." + ) + option = "mpl-results-always" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg) - use_full_test_name_help = "use fully qualified test name as the filename." - group.addoption("--mpl-use-full-test-name", help=use_full_test_name_help, action="store_true") - parser.addini('mpl-use-full-test-name', help=use_full_test_name_help, type='bool') + msg = "use fully qualified test name as the filename." + option = "mpl-use-full-test-name" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg, type="bool") - style_help = "default style to use for tests, unless specified in the mpl_image_compare decorator" - group.addoption('--mpl-default-style', help=style_help, action='store') - parser.addini('mpl-default-style', help=style_help) + msg = "default style to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-style" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) - tolerance_help = "default tolerance to use for tests, unless specified in the mpl_image_compare decorator" - group.addoption('--mpl-default-tolerance', help=tolerance_help, action='store') - parser.addini('mpl-default-tolerance', help=tolerance_help) + msg = "default tolerance to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-tolerance" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) msg = "default backend to use for tests, unless specified in the mpl_image_compare decorator" option = "mpl-default-backend" @@ -188,36 +203,40 @@ def pytest_addoption(parser): def pytest_configure(config): - config.addinivalue_line('markers', - "mpl_image_compare: Compares matplotlib figures " - "against a baseline image") + config.addinivalue_line( + "markers", + "mpl_image_compare: Compares matplotlib figures against a baseline image", + ) + + if ( + config.getoption("--mpl") + or config.getoption("--mpl-generate-path") is not None + or config.getoption("--mpl-generate-hash-library") is not None + ): - if (config.getoption("--mpl") or - config.getoption("--mpl-generate-path") is not None or - config.getoption("--mpl-generate-hash-library") is not None): + def get_cli_or_ini(name, default=None): + return config.getoption(f"--{name}") or config.getini(name) or default - baseline_dir = (config.getoption("--mpl-baseline-path") or - config.getini("mpl-baseline-path")) generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") - results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") - hash_library = config.getoption("--mpl-hash-library") or config.getini("mpl-hash-library") - _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility - generate_summary = (config.getoption("--mpl-generate-summary") or - config.getini("mpl-generate-summary")) - results_always = (config.getoption("--mpl-results-always") or - config.getini("mpl-results-always")) - use_full_test_name = (config.getoption("--mpl-use-full-test-name") or - config.getini("mpl-use-full-test-name")) + baseline_dir = get_cli_or_ini("mpl-baseline-path") if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") else: baseline_relative_dir = None + use_full_test_name = get_cli_or_ini("mpl-use-full-test-name") + + hash_library = get_cli_or_ini("mpl-hash-library") + _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility - # Note that results_dir is an empty string if not specified - if not results_dir: - results_dir = None + default_tolerance = float(get_cli_or_ini("mpl-default-tolerance", DEFAULT_TOLERANCE)) + default_style = get_cli_or_ini("mpl-default-style", DEFAULT_STYLE) + default_backend = get_cli_or_ini("mpl-default-backend", DEFAULT_BACKEND) + + results_dir = get_cli_or_ini("mpl-results-path") + results_always = get_cli_or_ini("mpl-results-always") + generate_summary = get_cli_or_ini("mpl-generate-summary") if generate_dir is not None: if baseline_dir is not None: @@ -234,35 +253,25 @@ def pytest_configure(config): if not _hash_library_from_cli: hash_library = os.path.abspath(hash_library) - default_style = (config.getoption("--mpl-default-style") or - config.getini("mpl-default-style") or - "classic") - - default_tolerance = float(config.getoption("--mpl-default-tolerance") or - config.getini("mpl-default-tolerance") or - "2") - - default_backend = (config.getoption("--mpl-default-backend") or - config.getini("mpl-default-backend") or - "agg") - - config.pluginmanager.register(ImageComparison(config, - baseline_dir=baseline_dir, - baseline_relative_dir=baseline_relative_dir, - generate_dir=generate_dir, - results_dir=results_dir, - hash_library=hash_library, - generate_hash_library=generate_hash_lib, - generate_summary=generate_summary, - results_always=results_always, - use_full_test_name=use_full_test_name, - default_style=default_style, - default_tolerance=default_tolerance, - default_backend=default_backend, - _hash_library_from_cli=_hash_library_from_cli)) + plugin = ImageComparison( + config, + baseline_dir=baseline_dir, + baseline_relative_dir=baseline_relative_dir, + generate_dir=generate_dir, + results_dir=results_dir, + hash_library=hash_library, + generate_hash_library=generate_hash_lib, + generate_summary=generate_summary, + results_always=results_always, + use_full_test_name=use_full_test_name, + default_style=default_style, + default_tolerance=default_tolerance, + default_backend=default_backend, + _hash_library_from_cli=_hash_library_from_cli, + ) + config.pluginmanager.register(plugin) else: - config.pluginmanager.register(FigureCloser(config)) @@ -305,23 +314,23 @@ def path_is_not_none(apath): class ImageComparison: - - def __init__(self, - config, - baseline_dir=None, - baseline_relative_dir=None, - generate_dir=None, - results_dir=None, - hash_library=None, - generate_hash_library=None, - generate_summary=None, - results_always=False, - use_full_test_name=False, - default_style='classic', - default_tolerance=2, - default_backend='agg', - _hash_library_from_cli=False, # for backwards compatibility - ): + def __init__( + self, + config, + baseline_dir=None, + baseline_relative_dir=None, + generate_dir=None, + results_dir=None, + hash_library=None, + generate_hash_library=None, + generate_summary=None, + results_always=False, + use_full_test_name=False, + default_style=DEFAULT_STYLE, + default_tolerance=DEFAULT_TOLERANCE, + default_backend=DEFAULT_BACKEND, + _hash_library_from_cli=False, # for backwards compatibility + ): self.config = config self.baseline_dir = baseline_dir self.baseline_relative_dir = path_is_not_none(baseline_relative_dir) From 8c12b1dc66d9f45b009b7c0e55fb8874a4023e28 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 29 May 2023 20:11:48 +0100 Subject: [PATCH 10/14] Prefer parsing tolerance as int instead of float --- pytest_mpl/plugin.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 62a9ea0..8a96be0 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -45,7 +45,7 @@ from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html DEFAULT_STYLE = "classic" -DEFAULT_TOLERANCE: float = 2 +DEFAULT_TOLERANCE = 2 DEFAULT_BACKEND = "agg" SUPPORTED_FORMATS = {"html", "json", "basic-html"} @@ -230,7 +230,12 @@ def get_cli_or_ini(name, default=None): hash_library = get_cli_or_ini("mpl-hash-library") _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility - default_tolerance = float(get_cli_or_ini("mpl-default-tolerance", DEFAULT_TOLERANCE)) + default_tolerance = get_cli_or_ini("mpl-default-tolerance", DEFAULT_TOLERANCE) + if isinstance(default_tolerance, str): + if default_tolerance.isdigit(): # prefer int if possible + default_tolerance = int(default_tolerance) + else: + default_tolerance = float(default_tolerance) default_style = get_cli_or_ini("mpl-default-style", DEFAULT_STYLE) default_backend = get_cli_or_ini("mpl-default-backend", DEFAULT_BACKEND) From cac5e00ff8bb41f83e42343d85c8f9d1560360b2 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 28 Sep 2023 23:32:32 +0100 Subject: [PATCH 11/14] Test new config options Adds tests for the new config options across ini, CLI and kwarg where relevant. Asserts that they work and have the expected order of precedence. Still need to add dedicated tests for all the other options. --- setup.cfg | 1 + tests/test_baseline_path.py | 48 ++++++++++++++++++++++++ tests/test_default_backend.py | 35 ++++++++++++++++++ tests/test_default_style.py | 37 +++++++++++++++++++ tests/test_default_tolerance.py | 57 +++++++++++++++++++++++++++++ tests/test_generate_summary.py | 63 ++++++++++++++++++++++++++++++++ tests/test_hash_library.py | 48 ++++++++++++++++++++++++ tests/test_use_full_test_name.py | 52 ++++++++++++++++++++++++++ 8 files changed, 341 insertions(+) create mode 100644 tests/test_baseline_path.py create mode 100644 tests/test_default_backend.py create mode 100644 tests/test_default_style.py create mode 100644 tests/test_default_tolerance.py create mode 100644 tests/test_generate_summary.py create mode 100644 tests/test_hash_library.py create mode 100644 tests/test_use_full_test_name.py diff --git a/setup.cfg b/setup.cfg index 7f9a3e3..f8026f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ filterwarnings = error ignore:distutils Version classes are deprecated ignore:the imp module is deprecated in favour of importlib + ignore:The NumPy module was reloaded [flake8] max-line-length = 100 diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py new file mode 100644 index 0000000..767baea --- /dev/null +++ b/tests/test_baseline_path.py @@ -0,0 +1,48 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected_baseline_path, success_expected", + [ + ("dir1", None, None, "dir1", True), + ("dir1", "dir2", None, "dir2", True), + ("dir1", "dir2", "dir3", "dir3", True), + ("dir1", "dir2", "dir3", "dir2", False), + (None, None, "dir3", "dir3", True), + ], +) +def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): + (pytester.path / expected_baseline_path).mkdir() + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / expected_baseline_path / "test_mpl.png", + ) + ini = f"mpl-baseline-path = {pytester.path / ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + {ini} + """ + ) + kwarg = f"baseline_dir=r'{pytester.path / kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-baseline-path={pytester.path / cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_default_backend.py b/tests/test_default_backend.py new file mode 100644 index 0000000..192d3cd --- /dev/null +++ b/tests/test_default_backend.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("backend1", None, None, "backend1"), + ("backend1", "backend2", None, "backend2"), + ("backend1", "backend2", "backend3", "backend3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = f"mpl-default-backend = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"backend='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-backend={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.re_match_lines([f".*(ModuleNotFound|Value)Error: .*{expected}.*"]) diff --git a/tests/test_default_style.py b/tests/test_default_style.py new file mode 100644 index 0000000..a193d54 --- /dev/null +++ b/tests/test_default_style.py @@ -0,0 +1,37 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("sty1", None, None, "sty1"), + ("sty1", "sty2", None, "sty2"), + ("sty1", "sty2", "sty3", "sty3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = "mpl-default-style = " + ini if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + kwarg = f"style='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + cli = "--mpl-default-style=" + cli if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*OSError: *'{expected}'*"]) diff --git a/tests/test_default_tolerance.py b/tests/test_default_tolerance.py new file mode 100644 index 0000000..93cb21b --- /dev/null +++ b/tests/test_default_tolerance.py @@ -0,0 +1,57 @@ +from pathlib import Path + +import pytest +from PIL import Image, ImageDraw + +TEST_NAME = "test_base_style" + + +@pytest.fixture(scope="module") +def baseline_image(tmpdir_factory): + path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png" + image = Image.open(path) + draw = ImageDraw.Draw(image) + draw.rectangle(((0, 0), (100, 100)), fill="red") + output = Path(tmpdir_factory.mktemp("data").join(f"{TEST_NAME}.png")) + image.save(output) + return output + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + (40, None, None, True), + (30, 40, None, True), + (30, 30, 40, True), + (30, 40, 30, False), + (40, 30, 30, False), + ], +) +def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected): + ini = f"mpl-default-tolerance = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {baseline_image.parent} + {ini} + """ + ) + kwarg = f"tolerance={kwarg}" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def {TEST_NAME}(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-tolerance={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py new file mode 100644 index 0000000..3632207 --- /dev/null +++ b/tests/test_generate_summary.py @@ -0,0 +1,63 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("json", None, {"json"}), + ("json", "html", {"html"}), + ("basic-html", "json", {"json"}), + (None, "json,basic-html,html", {"json", "basic-html", "html"}), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-generate-summary = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-results-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-generate-summary={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + + json_summary = pytester.path / "results.json" + if "json" in expected: + with open(json_summary) as fp: + results = json.load(fp) + assert "test_config.test_mpl" in results + else: + assert not json_summary.exists() + + html_summary = pytester.path / "fig_comparison.html" + if "html" in expected: + with open(html_summary) as fp: + raw = fp.read() + assert "bootstrap" in raw + assert "test_config.test_mpl" in raw + else: + assert not html_summary.exists() + + basic_html_summary = pytester.path / "fig_comparison_basic.html" + if "basic-html" in expected: + with open(basic_html_summary) as fp: + raw = fp.read() + assert "bootstrap" not in raw + assert "test_config.test_mpl" in raw + else: + assert not basic_html_summary.exists() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py new file mode 100644 index 0000000..893f270 --- /dev/null +++ b/tests/test_hash_library.py @@ -0,0 +1,48 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("bad", None, None, False), + ("good", None, None, True), + ("bad", "good", None, True), + ("bad", "bad", "good", False), # Note: CLI overrides kwarg + ("bad", "good", "bad", True), + ], +) +def test_config(pytester, ini, cli, kwarg, success_expected): + hash_libraries = { + "good": pytester.path / "good_hash_library.json", + "bad": pytester.path / "bad_hash_library.json", + } + ini = f"mpl-hash-library = {hash_libraries[ini]}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"hash_library=r'{hash_libraries[kwarg]}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + pytester.runpytest(f"--mpl-generate-hash-library={hash_libraries['good']}") + with open(hash_libraries["bad"], "w") as fp: + json.dump({"test_config.test_mpl": "bad-value"}, fp) + cli = f"--mpl-hash-library={hash_libraries[cli]}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py new file mode 100644 index 0000000..f78bd48 --- /dev/null +++ b/tests/test_use_full_test_name.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path + +import pytest + +FULL_TEST_NAME = "test_config.TestClass.test_mpl" +SHORT_TEST_NAME = "test_mpl" + + +@pytest.mark.parametrize( + "ini, cli, expected_baseline_name, success_expected", + [ + (None, None, SHORT_TEST_NAME, True), + (False, None, SHORT_TEST_NAME, True), + (True, None, FULL_TEST_NAME, True), + (False, True, FULL_TEST_NAME, True), + (None, True, FULL_TEST_NAME, True), + (True, True, "bad_name", False), + ], +) +def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / f"{expected_baseline_name}.png", + ) + ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + class TestClass: + @pytest.mark.mpl_image_compare + def test_mpl(self): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-use-full-test-name" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) From 47a83c8a36fa683cf7565e93b0e7b2d763aee83d Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Fri, 29 Sep 2023 08:45:35 +0100 Subject: [PATCH 12/14] Test other config options --- tests/test_generate.py | 44 ++++++++++++++++++++++++++++++++++++ tests/test_results_always.py | 41 +++++++++++++++++++++++++++++++++ tests/test_results_path.py | 34 ++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 tests/test_generate.py create mode 100644 tests/test_results_always.py create mode 100644 tests/test_results_path.py diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..c87c8d6 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,44 @@ +PYFILE = ( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ +) + + +def test_generate_baseline_images(pytester): + pytester.makepyfile(PYFILE) + baseline_dir = pytester.path / "alternative_baseline" + result = pytester.runpytest(f"--mpl-generate-path={baseline_dir}") + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + + +def test_generate_baseline_hashes(pytester): + pytester.makepyfile(PYFILE) + hash_library = pytester.path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-hash-library={hash_library}", + f"--mpl-results-path={pytester.path}", + ) + result.assert_outcomes(failed=1) # this option enables --mpl + assert hash_library.exists() + assert (pytester.path / "test_generate_baseline_hashes.test_mpl" / "result.png").exists() + + +def test_generate_baseline_images_and_hashes(pytester): + pytester.makepyfile(PYFILE) + baseline_dir = pytester.path / "alternative_baseline" + hash_library = pytester.path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-path={baseline_dir}", + f"--mpl-generate-hash-library={hash_library}", + ) + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + assert hash_library.exists() diff --git a/tests/test_results_always.py b/tests/test_results_always.py new file mode 100644 index 0000000..c39f5b3 --- /dev/null +++ b/tests/test_results_always.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, enabled_expected", + [ + (None, None, False), + (True, None, True), + (False, None, False), + (False, True, True), + (True, True, True), + ], +) +def test_config(pytester, ini, cli, enabled_expected): + ini = f"mpl-results-always = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {Path(__file__).parent / "baseline" / "2.0.x"} + mpl-results-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_base_style(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-results-always" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(passed=1) + assert (pytester.path / "test_config.test_base_style" / "result.png").exists() == enabled_expected diff --git a/tests/test_results_path.py b/tests/test_results_path.py new file mode 100644 index 0000000..962a210 --- /dev/null +++ b/tests/test_results_path.py @@ -0,0 +1,34 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("dir1", None, "dir1"), + ("dir1", "dir2", "dir2"), + (None, "dir2", "dir2"), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-results-path = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-results-path={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + assert (pytester.path / expected / "test_config.test_mpl" / "result.png").exists() From d5913f4321aebb483fbbf4da0bdb57640f16d1a1 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 7 Oct 2023 20:21:41 +0100 Subject: [PATCH 13/14] Upgrade readthedocs config https://blog.readthedocs.com/drop-support-system-packages/ --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e265082..02fe0e9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,6 @@ build: python: "3.11" python: - system_packages: false install: - method: pip path: . From a36ac38bf1c0c81cd22245334412a0dcdd03a95a Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 7 Oct 2023 22:26:43 +0100 Subject: [PATCH 14/14] Test new config options Adds tests for the new config options across ini, CLI and kwarg where relevant. Asserts that they work and have the expected order of precedence. Still need to add dedicated tests for all the other options. --- tests/helpers.py | 7 +++++++ tests/test_baseline_path.py | 12 +++++++----- tests/test_default_style.py | 3 ++- tests/test_generate.py | 16 ++++++++++------ tests/test_generate_summary.py | 10 ++++++---- tests/test_hash_library.py | 6 ++++-- tests/test_results_always.py | 6 ++++-- tests/test_results_path.py | 3 ++- tests/test_use_full_test_name.py | 6 ++++-- 9 files changed, 46 insertions(+), 23 deletions(-) create mode 100644 tests/helpers.py diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..037343d --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,7 @@ +from pathlib import Path + + +def pytester_path(pytester): + if hasattr(pytester, "path"): + return pytester.path + return Path(pytester.tmpdir) # pytest v5 diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py index 767baea..745006a 100644 --- a/tests/test_baseline_path.py +++ b/tests/test_baseline_path.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -15,12 +16,13 @@ ], ) def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): - (pytester.path / expected_baseline_path).mkdir() + path = pytester_path(pytester) + (path / expected_baseline_path).mkdir() shutil.copyfile( # Test will only pass if baseline is at expected path Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", - pytester.path / expected_baseline_path / "test_mpl.png", + path / expected_baseline_path / "test_mpl.png", ) - ini = f"mpl-baseline-path = {pytester.path / ini}" if ini is not None else "" + ini = f"mpl-baseline-path = {path / ini}" if ini is not None else "" pytester.makeini( f""" [pytest] @@ -28,7 +30,7 @@ def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expec {ini} """ ) - kwarg = f"baseline_dir=r'{pytester.path / kwarg}'" if kwarg else "" + kwarg = f"baseline_dir=r'{path / kwarg}'" if kwarg else "" pytester.makepyfile( f""" import matplotlib.pyplot as plt @@ -40,7 +42,7 @@ def test_mpl(): return fig """ ) - cli = f"--mpl-baseline-path={pytester.path / cli}" if cli else "" + cli = f"--mpl-baseline-path={path / cli}" if cli else "" result = pytester.runpytest("--mpl", cli) if success_expected: result.assert_outcomes(passed=1) diff --git a/tests/test_default_style.py b/tests/test_default_style.py index a193d54..9088008 100644 --- a/tests/test_default_style.py +++ b/tests/test_default_style.py @@ -1,4 +1,5 @@ import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -14,7 +15,7 @@ def test_config(pytester, ini, cli, kwarg, expected): pytester.makeini( f""" [pytest] - mpl-baseline-path = {pytester.path} + mpl-baseline-path = {pytester_path(pytester)} {ini} """ ) diff --git a/tests/test_generate.py b/tests/test_generate.py index c87c8d6..61b1447 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,3 +1,5 @@ +from helpers import pytester_path + PYFILE = ( """ import matplotlib.pyplot as plt @@ -13,7 +15,7 @@ def test_mpl(): def test_generate_baseline_images(pytester): pytester.makepyfile(PYFILE) - baseline_dir = pytester.path / "alternative_baseline" + baseline_dir = pytester_path(pytester) / "alternative_baseline" result = pytester.runpytest(f"--mpl-generate-path={baseline_dir}") result.assert_outcomes(skipped=1) assert (baseline_dir / "test_mpl.png").exists() @@ -21,20 +23,22 @@ def test_generate_baseline_images(pytester): def test_generate_baseline_hashes(pytester): pytester.makepyfile(PYFILE) - hash_library = pytester.path / "alternative_baseline" / "hash_library_1.json" + path = pytester_path(pytester) + hash_library = path / "alternative_baseline" / "hash_library_1.json" result = pytester.runpytest( f"--mpl-generate-hash-library={hash_library}", - f"--mpl-results-path={pytester.path}", + f"--mpl-results-path={path}", ) result.assert_outcomes(failed=1) # this option enables --mpl assert hash_library.exists() - assert (pytester.path / "test_generate_baseline_hashes.test_mpl" / "result.png").exists() + assert (path / "test_generate_baseline_hashes.test_mpl" / "result.png").exists() def test_generate_baseline_images_and_hashes(pytester): pytester.makepyfile(PYFILE) - baseline_dir = pytester.path / "alternative_baseline" - hash_library = pytester.path / "alternative_baseline" / "hash_library_1.json" + path = pytester_path(pytester) + baseline_dir = path / "alternative_baseline" + hash_library = path / "alternative_baseline" / "hash_library_1.json" result = pytester.runpytest( f"--mpl-generate-path={baseline_dir}", f"--mpl-generate-hash-library={hash_library}", diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py index 3632207..6350975 100644 --- a/tests/test_generate_summary.py +++ b/tests/test_generate_summary.py @@ -1,6 +1,7 @@ import json import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -13,11 +14,12 @@ ], ) def test_config(pytester, ini, cli, expected): + path = pytester_path(pytester) ini = f"mpl-generate-summary = {ini}" if ini else "" pytester.makeini( f""" [pytest] - mpl-results-path = {pytester.path} + mpl-results-path = {path} {ini} """ ) @@ -36,7 +38,7 @@ def test_mpl(): result = pytester.runpytest("--mpl", cli) result.assert_outcomes(failed=1) - json_summary = pytester.path / "results.json" + json_summary = path / "results.json" if "json" in expected: with open(json_summary) as fp: results = json.load(fp) @@ -44,7 +46,7 @@ def test_mpl(): else: assert not json_summary.exists() - html_summary = pytester.path / "fig_comparison.html" + html_summary = path / "fig_comparison.html" if "html" in expected: with open(html_summary) as fp: raw = fp.read() @@ -53,7 +55,7 @@ def test_mpl(): else: assert not html_summary.exists() - basic_html_summary = pytester.path / "fig_comparison_basic.html" + basic_html_summary = path / "fig_comparison_basic.html" if "basic-html" in expected: with open(basic_html_summary) as fp: raw = fp.read() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py index 893f270..58c457b 100644 --- a/tests/test_hash_library.py +++ b/tests/test_hash_library.py @@ -1,6 +1,7 @@ import json import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -14,9 +15,10 @@ ], ) def test_config(pytester, ini, cli, kwarg, success_expected): + path = pytester_path(pytester) hash_libraries = { - "good": pytester.path / "good_hash_library.json", - "bad": pytester.path / "bad_hash_library.json", + "good": path / "good_hash_library.json", + "bad": path / "bad_hash_library.json", } ini = f"mpl-hash-library = {hash_libraries[ini]}" if ini else "" pytester.makeini( diff --git a/tests/test_results_always.py b/tests/test_results_always.py index c39f5b3..ff2cc84 100644 --- a/tests/test_results_always.py +++ b/tests/test_results_always.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -14,13 +15,14 @@ ], ) def test_config(pytester, ini, cli, enabled_expected): + path = pytester_path(pytester) ini = f"mpl-results-always = {ini}" if ini else "" pytester.makeini( f""" [pytest] mpl-default-style = fivethirtyeight mpl-baseline-path = {Path(__file__).parent / "baseline" / "2.0.x"} - mpl-results-path = {pytester.path} + mpl-results-path = {path} {ini} """ ) @@ -38,4 +40,4 @@ def test_base_style(): cli = "--mpl-results-always" if cli else "" result = pytester.runpytest("--mpl", cli) result.assert_outcomes(passed=1) - assert (pytester.path / "test_config.test_base_style" / "result.png").exists() == enabled_expected + assert (path / "test_config.test_base_style" / "result.png").exists() == enabled_expected diff --git a/tests/test_results_path.py b/tests/test_results_path.py index 962a210..a727c2a 100644 --- a/tests/test_results_path.py +++ b/tests/test_results_path.py @@ -1,4 +1,5 @@ import pytest +from helpers import pytester_path @pytest.mark.parametrize( @@ -31,4 +32,4 @@ def test_mpl(): cli = f"--mpl-results-path={cli}" if cli else "" result = pytester.runpytest("--mpl", cli) result.assert_outcomes(failed=1) - assert (pytester.path / expected / "test_config.test_mpl" / "result.png").exists() + assert (pytester_path(pytester) / expected / "test_config.test_mpl" / "result.png").exists() diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py index f78bd48..56aaca5 100644 --- a/tests/test_use_full_test_name.py +++ b/tests/test_use_full_test_name.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from helpers import pytester_path FULL_TEST_NAME = "test_config.TestClass.test_mpl" SHORT_TEST_NAME = "test_mpl" @@ -19,16 +20,17 @@ ], ) def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + path = pytester_path(pytester) shutil.copyfile( # Test will only pass if baseline is at expected path Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", - pytester.path / f"{expected_baseline_name}.png", + path / f"{expected_baseline_name}.png", ) ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" pytester.makeini( f""" [pytest] mpl-default-style = fivethirtyeight - mpl-baseline-path = {pytester.path} + mpl-baseline-path = {path} {ini} """ )