Move QiskitTestCase to qiskit.test (#1616)

* Move JobTestCase to test.python.ibmq

* Move common testing functionality to qiskit.test

Temporary commit for moving the files to qiskit.test.

* Split qiskit.test.common into separate modules

* Style and docstring adjustments

* Add new Path.QASMS, revise existing ones

* Update CHANGELOG

.pylintrc

@@ -115,7 +115,8 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme
 # pi = the PI constant
 # op = operation iterator
 # b = basis iterator
-good-names=i,j,k,n,m,ex,v,w,x,y,z,Run,_,logger,q,r,qr,cr,qc,pi,op,b,ar,br
+good-names=i,j,k,n,m,ex,v,w,x,y,z,Run,_,logger,q,r,qr,cr,qc,pi,op,b,ar,br,
+    __unittest
 
 # Bad variable names which should always be refused, separated by a comma
 bad-names=foo,bar,toto,tutu,tata

CHANGELOG.rst

@@ -25,6 +25,8 @@ Changed
 
 - The ``Exception`` subclasses have been moved to an ``.exceptions`` module
   within each package (for example, ``qiskit.exceptions.QiskitError``). (#1600).
+- The ``QiskitTestCase`` and testing utilities are now included as part of
+  ``qiskit.test`` and thus available for third-party implementations. (#1616).
 
 Removed
 -------

qiskit/test/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018, IBM.
+#
+# This source code is licensed under the Apache License, Version 2.0 found in
+# the LICENSE.txt file in the root directory of this source tree.
+
+"""Functionality and helpers for testing Qiskit."""
+
+from .base import QiskitTestCase
+from .decorators import requires_cpp_simulator, requires_qe_access, slow_test
+from .utils import Path

qiskit/test/base.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018, IBM.
+#
+# This source code is licensed under the Apache License, Version 2.0 found in
+# the LICENSE.txt file in the root directory of this source tree.
+
+"""Base TestCases for the unit tests.
+
+Implementors of unit tests for Terra are encouraged to subclass
+``QiskitTestCase`` in order to take advantage of utility functions (for example,
+the environment variables for customizing different options), and the
+decorators in the ``decorators`` package.
+"""
+
+import inspect
+import logging
+import os
+import unittest
+from unittest.util import safe_repr
+
+from .utils import Path, _AssertNoLogsContext, setup_test_logging
+
+
+__unittest = True  # Allows shorter stack trace for .assertDictAlmostEqual
+
+
+class QiskitTestCase(unittest.TestCase):
+    """Helper class that contains common functionality."""
+
+    @classmethod
+    def setUpClass(cls):
+        # Determines if the TestCase is using IBMQ credentials.
+        cls.using_ibmq_credentials = False
+
+        # Set logging to file and stdout if the LOG_LEVEL envar is set.
+        cls.log = logging.getLogger(cls.__name__)
+        if os.getenv('LOG_LEVEL'):
+            filename = '%s.log' % os.path.splitext(inspect.getfile(cls))[0]
+            setup_test_logging(cls.log, os.getenv('LOG_LEVEL'), filename)
+
+    def tearDown(self):
+        # Reset the default providers, as in practice they acts as a singleton
+        # due to importing the wrapper from qiskit.
+        from qiskit.providers.ibmq import IBMQ
+        from qiskit.providers.builtinsimulators import BasicAer
+
+        IBMQ._accounts.clear()
+        BasicAer._backends = BasicAer._verify_backends()
+
+    @staticmethod
+    def _get_resource_path(filename, path=Path.TEST):
+        """Get the absolute path to a resource.
+
+        Args:
+            filename (string): filename or relative path to the resource.
+            path (Path): path used as relative to the filename.
+        Returns:
+            str: the absolute path to the resource.
+        """
+        return os.path.normpath(os.path.join(path.value, filename))
+
+    def assertNoLogs(self, logger=None, level=None):
+        """Assert that no message is sent to the specified logger and level.
+
+        Context manager to test that no message is sent to the specified
+        logger and level (the opposite of TestCase.assertLogs()).
+        """
+        return _AssertNoLogsContext(self, logger, level)
+
+    def assertDictAlmostEqual(self, dict1, dict2, delta=None, msg=None,
+                              places=None, default_value=0):
+        """Assert two dictionaries with numeric values are almost equal.
+
+        Fail if the two dictionaries are unequal as determined by
+        comparing that the difference between values with the same key are
+        not greater than delta (default 1e-8), or that difference rounded
+        to the given number of decimal places is not zero. If a key in one
+        dictionary is not in the other the default_value keyword argument
+        will be used for the missing value (default 0). If the two objects
+        compare equal then they will automatically compare almost equal.
+
+        Args:
+            dict1 (dict): a dictionary.
+            dict2 (dict): a dictionary.
+            delta (number): threshold for comparison (defaults to 1e-8).
+            msg (str): return a custom message on failure.
+            places (int): number of decimal places for comparison.
+            default_value (number): default value for missing keys.
+
+        Raises:
+            TypeError: raises TestCase failureException if the test fails.
+        """
+        def valid_comparison(value):
+            if places is not None:
+                return round(value, places) == 0
+            else:
+                return value < delta
+
+        # Check arguments.
+        if dict1 == dict2:
+            return
+        if places is not None:
+            if delta is not None:
+                raise TypeError("specify delta or places not both")
+            msg_suffix = ' within %s places' % places
+        else:
+            delta = delta or 1e-8
+            msg_suffix = ' within %s delta' % delta
+
+        # Compare all keys in both dicts, populating error_msg.
+        error_msg = ''
+        for key in set(dict1.keys()) | set(dict2.keys()):
+            val1 = dict1.get(key, default_value)
+            val2 = dict2.get(key, default_value)
+            if not valid_comparison(abs(val1 - val2)):
+                error_msg += '(%s: %s != %s), ' % (safe_repr(key),
+                                                   safe_repr(val1),
+                                                   safe_repr(val2))
+
+        if error_msg:
+            msg = self._formatMessage(msg, error_msg[:-2] + msg_suffix)
+            raise self.failureException(msg)

qiskit/test/decorators.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018, IBM.
+#
+# This source code is licensed under the Apache License, Version 2.0 found in
+# the LICENSE.txt file in the root directory of this source tree.
+
+"""Decorator for using with Qiskit unit tests."""
+
+import functools
+import os
+import unittest
+
+from qiskit.providers.ibmq.credentials import Credentials, discover_credentials
+from qiskit.providers.legacysimulators import QasmSimulator
+
+from .utils import Path
+from .http_recorder import http_recorder
+from .testing_options import get_test_options
+
+
+def is_cpp_simulator_available():
+    """Check if the C++ simulator can be instantiated.
+
+    Returns:
+        bool: True if simulator executable is available
+    """
+    try:
+        QasmSimulator()
+    except FileNotFoundError:
+        return False
+    return True
+
+
+def requires_cpp_simulator(test_item):
+    """Decorator that skips test if C++ simulator is not available
+
+    Args:
+        test_item (callable): function or class to be decorated.
+
+    Returns:
+        callable: the decorated function.
+    """
+    reason = 'C++ simulator not found, skipping test'
+    return unittest.skipIf(not is_cpp_simulator_available(), reason)(test_item)
+
+
+def slow_test(func):
+    """Decorator that signals that the test takes minutes to run.
+
+    Args:
+        func (callable): test function to be decorated.
+
+    Returns:
+        callable: the decorated function.
+    """
+
+    @functools.wraps(func)
+    def _wrapper(*args, **kwargs):
+        skip_slow = not TEST_OPTIONS['run_slow']
+        if skip_slow:
+            raise unittest.SkipTest('Skipping slow tests')
+
+        return func(*args, **kwargs)
+
+    return _wrapper
+
+
+def _get_credentials(test_object, test_options):
+    """Finds the credentials for a specific test and options.
+
+    Args:
+        test_object (QiskitTestCase): The test object asking for credentials
+        test_options (dict): Options after QISKIT_TESTS was parsed by get_test_options.
+
+    Returns:
+        Credentials: set of credentials
+
+    Raises:
+        Exception: When the credential could not be set and they are needed for that set of options
+    """
+
+    dummy_credentials = Credentials('dummyapiusersloginWithTokenid01',
+                                    'https://quantumexperience.ng.bluemix.net/api')
+
+    if test_options['mock_online']:
+        return dummy_credentials
+
+    if os.getenv('USE_ALTERNATE_ENV_CREDENTIALS', ''):
+        # Special case: instead of using the standard credentials mechanism,
+        # load them from different environment variables. This assumes they
+        # will always be in place, as is used by the Travis setup.
+        return Credentials(os.getenv('IBMQ_TOKEN'), os.getenv('IBMQ_URL'))
+    else:
+        # Attempt to read the standard credentials.
+        discovered_credentials = discover_credentials()
+
+        if discovered_credentials:
+            # Decide which credentials to use for testing.
+            if len(discovered_credentials) > 1:
+                try:
+                    # Attempt to use QE credentials.
+                    return discovered_credentials[dummy_credentials.unique_id()]
+                except KeyError:
+                    pass
+
+            # Use the first available credentials.
+            return list(discovered_credentials.values())[0]
+
+    # No user credentials were found.
+    if test_options['rec']:
+        raise Exception('Could not locate valid credentials. You need them for recording '
+                        'tests against the remote API.')
+
+    test_object.log.warning("No user credentials were detected. Running with mocked data.")
+    test_options['mock_online'] = True
+    return dummy_credentials
+
+
+def requires_qe_access(func):
+    """Decorator that signals that the test uses the online API:
+
+    It involves:
+        * determines if the test should be skipped by checking environment
+            variables.
+        * if the `USE_ALTERNATE_ENV_CREDENTIALS` environment variable is
+          set, it reads the credentials from an alternative set of environment
+          variables.
+        * if the test is not skipped, it reads `qe_token` and `qe_url` from
+            `Qconfig.py`, environment variables or qiskitrc.
+        * if the test is not skipped, it appends `qe_token` and `qe_url` as
+            arguments to the test function.
+
+    Args:
+        func (callable): test function to be decorated.
+
+    Returns:
+        callable: the decorated function.
+    """
+
+    @functools.wraps(func)
+    def _wrapper(self, *args, **kwargs):
+        if TEST_OPTIONS['skip_online']:
+            raise unittest.SkipTest('Skipping online tests')
+
+        credentials = _get_credentials(self, TEST_OPTIONS)
+        self.using_ibmq_credentials = credentials.is_ibmq()
+        kwargs.update({'qe_token': credentials.token,
+                       'qe_url': credentials.url})
+
+        decorated_func = func
+        if TEST_OPTIONS['rec'] or TEST_OPTIONS['mock_online']:
+            # For recording or for replaying existing cassettes, the test
+            # should be decorated with @use_cassette.
+            vcr_mode = 'new_episodes' if TEST_OPTIONS['rec'] else 'none'
+            decorated_func = http_recorder(
+                vcr_mode, Path.CASSETTES.value).use_cassette()(decorated_func)
+
+        return decorated_func(self, *args, **kwargs)
+
+    return _wrapper
+
+
+TEST_OPTIONS = get_test_options()