Merge pull request #377 from Lnaden/pytest5

PyTest Inner Workings Overhaul

docs/qcfractal/source/install.rst

@@ -90,7 +90,12 @@ Then, run the following command:
 
 .. code-block::
 
-   >>> pytest -p qcfractal.testing --pyargs qcfractal
+   >>> pytest --pyargs qcfractal
+
+QCFractal ships with a small testing plugin which should be automatically detected and gives you access to the
+``--runslow`` and ``--runexamples`` PyTest CLI flags. The ``--runslow`` flag tells the testing suite to run any test
+the developers think are a bit more time consuming than the others. Without this flag, you will see many tests (such
+as those for the CLI) skipped.
 
 
 Developing from Source

examples/test_examples.py

@@ -4,6 +4,9 @@
 
 import os
 import time
+
+import pytest
+
 from qcfractal import testing
 
 _pwd = os.path.abspath(os.path.dirname(__file__))
@@ -20,10 +23,9 @@ def wait_true(wait_time, func, *args, **kwargs):
 
     return False
 
-
+@pytest.mark.example
 @testing.using_psi4
 @testing.using_unix
-@testing.mark_example
 def test_local_server_example():
     """Make sure the Fireworks example works as intended"""
 

qcfractal/cli/tests/test_cli.py

@@ -34,20 +34,20 @@ def qcfractal_base_init():
     yield f"--base-folder={tmpdir.name}"
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_server_boot(qcfractal_base_init):
     port = "--port=" + str(testing.find_open_port())
     args = ["qcfractal-server", "start", qcfractal_base_init, port]
     assert testing.run_process(args, interupt_after=10, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_upgrade(qcfractal_base_init):
     args = ["qcfractal-server", "upgrade", qcfractal_base_init]
     assert testing.run_process(args, interupt_after=10, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_user_add(qcfractal_base_init):
     args = ["qcfractal-server", "user", qcfractal_base_init, "add", "test_user_add_1", "--permissions", "admin"]
     assert testing.run_process(args, **_options)
@@ -63,7 +63,7 @@ def test_cli_user_add(qcfractal_base_init):
     assert testing.run_process(args, **_options) is False
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_user_show(qcfractal_base_init):
     args = ["qcfractal-server", "user", qcfractal_base_init, "add", "test_user_show", "--permissions", "admin"]
     assert testing.run_process(args, **_options)
@@ -75,7 +75,7 @@ def test_cli_user_show(qcfractal_base_init):
     assert testing.run_process(args, **_options) is False
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_user_modify(qcfractal_base_init):
     args = ["qcfractal-server", "user", qcfractal_base_init, "add", "test_user_modify", "--permissions", "read"]
     assert testing.run_process(args, **_options)
@@ -96,7 +96,7 @@ def test_cli_user_modify(qcfractal_base_init):
     assert testing.run_process(args, **_options) is False
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_user_remove(qcfractal_base_init):
     args = ["qcfractal-server", "user", qcfractal_base_init, "add", "test_user_remove", "--permissions", "admin"]
     assert testing.run_process(args, **_options)
@@ -109,7 +109,7 @@ def test_cli_user_remove(qcfractal_base_init):
 
 
 @pytest.mark.skip(reason="Failing on Travis for unknown reasons.")
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_server_local_boot(qcfractal_base_init):
     port = "--port=" + str(testing.find_open_port())
     args = ["qcfractal-server", "start", "--local-manager=1", port, qcfractal_base_init]
@@ -128,7 +128,7 @@ def active_server(request, qcfractal_base_init):
         yield server
 
 
-@testing.mark_slow
+@pytest.mark.slow
 @pytest.mark.parametrize("log_apis", [0, 1])
 def test_with_api_logging(postgres_server, log_apis):
 
@@ -147,20 +147,20 @@ def test_with_api_logging(postgres_server, log_apis):
     assert testing.run_process(args, interupt_after=10, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_manager_local_testing_process():
     assert testing.run_process(["qcfractal-manager", "--adapter=pool", "--test", "--tasks-per-worker=2"], **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_manager_executor_manager_boot(active_server):
     args = [
         "qcfractal-manager", active_server.test_uri_cli, "--adapter=pool", "--tasks-per-worker=2", "--verify=False"
     ]
     assert testing.run_process(args, interupt_after=7, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_manager_executor_manager_boot_from_file(active_server, tmp_path):
 
     yaml_file = """
@@ -180,23 +180,23 @@ def test_manager_executor_manager_boot_from_file(active_server, tmp_path):
     assert testing.run_process(args, interupt_after=7, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def cli_manager_runs(config_data, tmp_path):
     temp_config = tmp_path / "temp_config.yaml"
     temp_config.write_text(yaml.dump(config_data))
     args = ["qcfractal-manager", f"--config-file={temp_config}", "--test"]
     assert testing.run_process(args, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def load_manager_config(adapter, scheduler):
     config = read_config_file(os.path.join(_pwd, "manager_boot_template.yaml"))
     config["common"]["adapter"] = adapter
     config["cluster"]["scheduler"] = scheduler
     return config
 
 
-@testing.mark_slow
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "adapter,scheduler",
     [
@@ -220,15 +220,15 @@ def test_cli_managers(adapter, scheduler, tmp_path):
     cli_manager_runs(config, tmp_path)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 @testing.using_parsl
 def test_cli_manager_parsl_launchers(tmp_path):
     config = load_manager_config("parsl", "slurm")
     config["parsl"]["provider"].update({"launcher": {"launcher_class": "singleNODELauncher"}})
     cli_manager_runs(config, tmp_path)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 @pytest.mark.parametrize("adapter", [
     pytest.param("dask", marks=testing.using_dask_jobqueue),
     pytest.param("parsl", marks=testing.using_parsl),
@@ -240,7 +240,7 @@ def test_cli_managers_missing(adapter, tmp_path):
     cli_manager_runs(config, tmp_path)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 @pytest.mark.parametrize("adapter", [
     pytest.param("dask", marks=testing.using_dask_jobqueue),
     pytest.param("parsl", marks=testing.using_parsl),
@@ -252,21 +252,21 @@ def test_cli_managers_none(adapter, tmp_path):
     cli_manager_runs(config, tmp_path)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_managers_help():
     """Test that qcfractal_manager --help works"""
     args = ["qcfractal-manager", "--help"]
     testing.run_process(args, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_managers_schema():
     """Test that qcfractal_manager --schema works"""
     args = ["qcfractal-manager", "--schema"]
     testing.run_process(args, **_options)
 
 
-@testing.mark_slow
+@pytest.mark.slow
 def test_cli_managers_skel(tmp_path):
     """Test that qcfractal_manager --skeleton works"""
     config = tmp_path / "config.yaml"

qcfractal/interface/tests/test_visualization.py

@@ -8,6 +8,7 @@
 from . import portal
 
 try:
+    import plotly
     _has_ploty = True
 except ModuleNotFoundError:
     _has_ploty = False

qcfractal/testing.py

@@ -27,6 +27,48 @@
 ### Addon testing capabilities
 
 
+def pytest_addoption(parser):
+    """
+    Additional PyTest CLI flags to add
+
+    See `pytest_collection_modifyitems` for handling and `pytest_configure` for adding known in-line marks.
+    """
+    parser.addoption("--runslow", action="store_true", default=False, help="run slow tests")
+    parser.addoption("--runexamples", action="store_true", default=False, help="run example tests")
+
+
+def pytest_collection_modifyitems(config, items):
+    """
+    Handle test triggers based on the CLI flags
+
+    Use decorators:
+    @pytest.mark.slow
+    @pyrest.mark.example
+    """
+    runslow = config.getoption("--runslow")
+    runexamples = config.getoption("--runexamples")
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    skip_example = pytest.mark.skip(reason="need --runexamples option to run")
+    for item in items:
+        if "slow" in item.keywords and not runslow:
+            item.add_marker(skip_slow)
+        if "example" in item.keywords and not runexamples:
+            item.add_marker(skip_example)
+
+
+def pytest_configure(config):
+    import sys
+    sys._called_from_test = True
+    config.addinivalue_line("markers", "example: Mark a given test as an example which can be run")
+    config.addinivalue_line("markers", "slow: Mark a given test as slower than most other tests, needing a special "
+                                       "flag to run.")
+
+
+def pytest_unconfigure(config):
+    import sys
+    del sys._called_from_test
+
+
 def _plugin_import(plug):
     plug_spec = pkgutil.find_loader(plug)
     if plug_spec is None:
@@ -82,35 +124,13 @@ def _build_pytest_skip(program):
                                 reason='Not on Unix operating system, '
                                 'assuming Bash is not present')
 
-### Generic helpers
-
-
-def mark_slow(func):
-    try:
-        if not pytest.config.getoption("--runslow"):
-            func = pytest.mark.skip("need --runslow option to run")(func)
-    except (AttributeError, ValueError):
-        # AttributeError: module 'pytest' has no attribute 'config'
-        pass
-
-    return func
-
-
-def mark_example(func):
-    try:
-        if not pytest.config.getoption("--runexamples"):
-            func = pytest.mark.skip("need --runexample option to run")(func)
-    except AttributeError:
-        # AttributeError: module 'pytest' has no attribute 'config'
-        pass
-
-    return func
 
+### Generic helpers
 
 def recursive_dict_merge(base_dict, dict_to_merge_in):
     """Recursive merge for more complex than a simple top-level merge {**x, **y} which does not handle nested dict."""
     for k, v in dict_to_merge_in.items():
-        if (k in base_dict and isinstance(base_dict[k], dict) and isinstance(dict_to_merge_in[k], Mapping)):
+        if k in base_dict and isinstance(base_dict[k], dict) and isinstance(dict_to_merge_in[k], Mapping):
             recursive_dict_merge(base_dict[k], dict_to_merge_in[k])
         else:
             base_dict[k] = dict_to_merge_in[k]

qcfractal/tests/test_collections.py

@@ -7,7 +7,7 @@
 
 import qcfractal.interface as ptl
 from qcfractal import testing
-from qcfractal.testing import fractal_compute_server, mark_slow
+from qcfractal.testing import fractal_compute_server
 
 
 def test_collection_query(fractal_compute_server):
@@ -292,7 +292,7 @@ def test_missing_collection(fractal_compute_server):
         client.get_collection("reactiondataset", "_waffles_")
 
 
-@mark_slow
+@pytest.mark.slow
 @testing.using_torsiondrive
 @testing.using_geometric
 @testing.using_rdkit

qcfractal/tests/test_managers.py

@@ -12,7 +12,7 @@
 
 import qcfractal.interface as ptl
 from qcfractal import FractalServer, queue, testing
-from qcfractal.testing import reset_server_database, test_server, mark_slow
+from qcfractal.testing import reset_server_database, test_server
 
 CLIENT_USERNAME = "test_compute_adapter"
 
@@ -76,7 +76,7 @@ def test_queue_manager_single_tags(compute_adapter_fixture):
         assert manager["username"] == CLIENT_USERNAME
 
 
-@mark_slow
+@pytest.mark.slow
 @testing.using_rdkit
 def test_queue_manager_statistics(compute_adapter_fixture, caplog):
     """Test statistics are correctly generated"""

qcfractal/tests/test_server.py

@@ -11,7 +11,7 @@
 
 import qcfractal.interface as ptl
 from qcfractal import FractalServer, FractalSnowflake, FractalSnowflakeHandler
-from qcfractal.testing import (await_true, find_open_port, mark_slow, pristine_loop,
+from qcfractal.testing import (await_true, find_open_port, pristine_loop,
                                test_server, using_geometric, using_rdkit, using_torsiondrive)
 
 meta_set = {'errors', 'n_inserted', 'success', 'duplicates', 'error_description', 'validation_errors'}
@@ -114,7 +114,7 @@ def test_storage_socket(test_server):
     assert pdata["data"][0] == storage
 
 
-@mark_slow
+@pytest.mark.slow
 def test_snowflakehandler_restart():
 
     with FractalSnowflakeHandler() as server:
@@ -142,7 +142,7 @@ def test_snowflakehandler_log():
     assert proc.poll() is not None
 
 
-@mark_slow
+@pytest.mark.slow
 @using_geometric
 @using_torsiondrive
 @using_rdkit