Remove Hypothesis internals from tracebacks

hypothesis-python/RELEASE.rst

@@ -0,0 +1,12 @@
+RELEASE_TYPE: patch
+
+This patch stops Hypothesis from leaking implementation details - each
+traceback now has only a single entry for Hypothesis, so you can focus
+on debugging your code without having to parse out dozens of lines of
+irrelevant output (:issue:`848`).
+
+If you *do* want to see the internal details, you can disable frame
+elision by setting :obj:`~hypothesis.settings.verbosity` to ``debug``.
+
+*The traceback handling this feature requires is only possible on Python 3.
+Under Python 2, Hypothesis retains the old behaviour of showing internals.*

hypothesis-python/src/hypothesis/core.py

@@ -49,12 +49,12 @@
 from hypothesis.executors import new_style_executor
 from hypothesis.reporting import report, verbose_report, current_verbosity
 from hypothesis.statistics import note_engine_for_statistics
-from hypothesis.internal.compat import ceil, hbytes, qualname, \
+from hypothesis.internal.compat import PY2, ceil, hbytes, qualname, \
     str_to_bytes, benchmark_time, get_type_hints, getfullargspec, \
     int_from_bytes, bad_django_TestCase
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.utils.conventions import infer, not_set
-from hypothesis.internal.escalation import \
+from hypothesis.internal.escalation import clip_traceback, \
     escalate_hypothesis_internal_error
 from hypothesis.internal.reflection import is_mock, proxies, nicerepr, \
     arg_string, impersonate, function_digest, fully_qualified_name, \
@@ -564,16 +564,18 @@ def evaluate_test_data(self, data):
             raise
         except Exception as e:
             escalate_hypothesis_internal_error()
-            data.__expected_traceback = traceback.format_exc()
+            _, _, tb = sys.exc_info()
+            if not PY2:  # pragma: no branch
+                tb = clip_traceback(tb)
+            trace_text = ''.join(traceback.format_exception(type(e), e, tb))
+            data.__expected_traceback = trace_text
             data.__expected_exception = e
-            verbose_report(data.__expected_traceback)
-
-            error_class, _, tb = sys.exc_info()
+            verbose_report(trace_text)
 
             origin = traceback.extract_tb(tb)[-1]
             filename = origin[0]
             lineno = origin[1]
-            data.mark_interesting((error_class, filename, lineno))
+            data.mark_interesting((type(e), filename, lineno))
 
     def run(self):
         # Tell pytest to omit the body of this function from tracebacks
@@ -665,10 +667,13 @@ def run(self):
                     'Unreliable assumption: An example which satisfied '
                     'assumptions on the first run now fails it.'
                 )
-            except BaseException:
+            except BaseException as e:
                 if len(self.falsifying_examples) <= 1:
                     raise
-                report(traceback.format_exc())
+                _, _, tb = sys.exc_info()
+                if not PY2:  # pragma: no branch
+                    tb = clip_traceback(tb)
+                report(''.join(traceback.format_exception(type(e), e, tb)))
             finally:  # pragma: no cover
                 # This section is in fact entirely covered by the tests in
                 # test_reproduce_failure, but it seems to trigger a lovely set
@@ -915,7 +920,7 @@ def wrapped_test(*arguments, **kwargs):
                         setattr(runner, 'subTest', subTest)
                 else:
                     state.run()
-            except BaseException:
+            except BaseException as error:
                 generated_seed = \
                     wrapped_test._hypothesis_internal_use_generated_seed
                 if generated_seed is not None and not state.failed_normally:
@@ -930,7 +935,13 @@ def wrapped_test(*arguments, **kwargs):
                             report(
                                 'You can add @seed(%d) to this test to '
                                 'reproduce this failure.' % (generated_seed,))
-                raise
+                if PY2:
+                    raise
+                with local_settings(settings):
+                    the_error_hypothesis_found = clip_traceback(
+                        error.__traceback__, error
+                    )
+                raise the_error_hypothesis_found
 
         for attrib in dir(test):
             if not (attrib.startswith('_') or hasattr(wrapped_test, attrib)):

hypothesis-python/src/hypothesis/internal/escalation.py

@@ -19,10 +19,12 @@
 
 import os
 import sys
+import traceback
+from inspect import getframeinfo
 
 import hypothesis
 from hypothesis.errors import StopTest, DeadlineExceeded, \
-    HypothesisException, UnsatisfiedAssumption
+    MultipleFailures, HypothesisException, UnsatisfiedAssumption
 from hypothesis.internal.compat import text_type, binary_type, \
     encoded_filepath
 
@@ -72,9 +74,28 @@ def escalate_hypothesis_internal_error():
     error_type, e, tb = sys.exc_info()
     if getattr(e, 'hypothesis_internal_always_escalate', False):
         raise
-    import traceback
     filepath = traceback.extract_tb(tb)[-1][0]
     if is_hypothesis_file(filepath) and not isinstance(
         e, (HypothesisException,) + HYPOTHESIS_CONTROL_EXCEPTIONS,
     ):
         raise
+
+
+def is_hypothesis_frame(frame):
+    return is_hypothesis_file(getframeinfo(frame)[0]) or \
+        frame.f_globals.get('__hypothesistracebackhide__') is True
+
+
+def clip_traceback(tb, error=None):
+    """Remove Hypothesis internal frames from the error traceback."""
+    # Don't mess with the traceback in debug mode, or for internal errors
+    if all([
+        hypothesis.settings.default.verbosity < hypothesis.Verbosity.debug,
+        (isinstance(error, MultipleFailures) or not
+         is_hypothesis_file(traceback.extract_tb(tb)[-1][0]))
+    ]):
+        while tb is not None and is_hypothesis_frame(tb.tb_frame):
+            tb = tb.tb_next
+    if error is not None:
+        return error.with_traceback(tb)
+    return tb

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -544,6 +544,7 @@ def accept(f):
         f.__name__ = target.__name__
         f.__module__ = target.__module__
         f.__doc__ = target.__doc__
+        f.__globals__['__hypothesistracebackhide__'] = True
         return f
     return accept
 

hypothesis-python/tests/cover/test_escalation.py

@@ -17,11 +17,14 @@
 
 from __future__ import division, print_function, absolute_import
 
+import traceback
+
 import pytest
 
 import hypothesis.strategies as st
 import hypothesis.internal.escalation as esc
-from hypothesis import given
+from hypothesis import Verbosity, given, settings
+from hypothesis.internal.compat import PY2
 
 
 def test_does_not_escalate_errors_in_non_hypothesis_file():
@@ -66,3 +69,23 @@ def test(i):
         test()
 
     assert count == [1]
+
+
+@pytest.mark.skipif(PY2, reason='no support for traceback manipulation')
+@pytest.mark.parametrize('verbosity', [Verbosity.normal, Verbosity.debug])
+def test_tracebacks_omit_hypothesis_internals(verbosity):
+    @settings(verbosity=verbosity)
+    @given(st.none())
+    def simplest_failure(x):
+        assert x
+
+    try:
+        simplest_failure()
+    except AssertionError as e:
+        tb = traceback.extract_tb(e.__traceback__)
+        # Unless in debug mode, Hypothesis adds 1 frame - the least possible!
+        # (4 frames: this one, simplest_failure, internal frame, assert False)
+        if verbosity < Verbosity.debug:
+            assert len(tb) == 4
+        else:
+            assert len(tb) >= 5