Remove Hypothesis internals from tracebacks

hypothesis-python/RELEASE.rst

@@ -0,0 +1,12 @@
+RELEASE_TYPE: patch
+
+This patch shortens tracebacks from Hypothesis, so you can see exactly
+happened in your code without having to skip over irrelevant details
+about our internals (:issue:`848`).
+
+In the example test (see :pull:`1582`), this reduces tracebacks from
+nine frames to just three - and for a test with multiple errors, from
+seven frames per error to just one!
+
+If you *do* want to see the internal details, you can disable frame
+elision by setting :obj:`~hypothesis.settings.verbosity` to ``debug``.

hypothesis-python/src/hypothesis/core.py

@@ -22,7 +22,6 @@
 
 import os
 import ast
-import sys
 import zlib
 import base64
 import random as rnd_module
@@ -49,12 +48,12 @@
 from hypothesis.executors import new_style_executor
 from hypothesis.reporting import report, verbose_report, current_verbosity
 from hypothesis.statistics import note_engine_for_statistics
-from hypothesis.internal.compat import ceil, hbytes, qualname, \
+from hypothesis.internal.compat import PY2, ceil, hbytes, qualname, \
     binary_type, str_to_bytes, benchmark_time, get_type_hints, \
     getfullargspec, int_from_bytes, bad_django_TestCase
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.utils.conventions import infer, not_set
-from hypothesis.internal.escalation import \
+from hypothesis.internal.escalation import get_trimmed_traceback, \
     escalate_hypothesis_internal_error
 from hypothesis.internal.reflection import is_mock, proxies, nicerepr, \
     arg_string, impersonate, function_digest, fully_qualified_name, \
@@ -570,16 +569,17 @@ def evaluate_test_data(self, data):
             raise
         except Exception as e:
             escalate_hypothesis_internal_error()
-            data.__expected_traceback = traceback.format_exc()
+            tb = get_trimmed_traceback()
+            data.__expected_traceback = ''.join(
+                traceback.format_exception(type(e), e, tb)
+            )
             data.__expected_exception = e
             verbose_report(data.__expected_traceback)
 
-            error_class, _, tb = sys.exc_info()
-
             origin = traceback.extract_tb(tb)[-1]
             filename = origin[0]
             lineno = origin[1]
-            data.mark_interesting((error_class, filename, lineno))
+            data.mark_interesting((type(e), filename, lineno))
 
     def run(self):
         # Tell pytest to omit the body of this function from tracebacks
@@ -671,10 +671,11 @@ def run(self):
                     'Unreliable assumption: An example which satisfied '
                     'assumptions on the first run now fails it.'
                 )
-            except BaseException:
+            except BaseException as e:
                 if len(self.falsifying_examples) <= 1:
                     raise
-                report(traceback.format_exc())
+                tb = get_trimmed_traceback()
+                report(''.join(traceback.format_exception(type(e), e, tb)))
             finally:  # pragma: no cover
                 # This section is in fact entirely covered by the tests in
                 # test_reproduce_failure, but it seems to trigger a lovely set
@@ -921,11 +922,11 @@ def wrapped_test(*arguments, **kwargs):
                         setattr(runner, 'subTest', subTest)
                 else:
                     state.run()
-            except BaseException:
+            except BaseException as e:
                 generated_seed = \
                     wrapped_test._hypothesis_internal_use_generated_seed
-                if generated_seed is not None and not state.failed_normally:
-                    with local_settings(settings):
+                with local_settings(settings):
+                    if not (state.failed_normally or generated_seed is None):
                         if running_under_pytest:
                             report(
                                 'You can add @seed(%(seed)d) to this test or '
@@ -936,7 +937,25 @@ def wrapped_test(*arguments, **kwargs):
                             report(
                                 'You can add @seed(%d) to this test to '
                                 'reproduce this failure.' % (generated_seed,))
-                raise
+                    # The dance here is to avoid showing users long tracebacks
+                    # full of Hypothesis internals they don't care about.
+                    # We have to do this inline, to avoid adding another
+                    # internal stack frame just when we've removed the rest.
+                    if PY2:
+                        # Python 2 doesn't have Exception.with_traceback(...);
+                        # instead it has a three-argument form of the `raise`
+                        # statement.  Which is a SyntaxError on Python 3.
+                        exec(
+                            'raise type(e), e, get_trimmed_traceback()',
+                            globals(), locals()
+                        )
+                    # On Python 3, we swap out the real traceback for our
+                    # trimmed version.  Using a variable ensures that the line
+                    # which will actually appear in trackbacks is as clear as
+                    # possible - "raise the_error_hypothesis_found".
+                    the_error_hypothesis_found = \
+                        e.with_traceback(get_trimmed_traceback())
+                    raise the_error_hypothesis_found
 
         for attrib in dir(test):
             if not (attrib.startswith('_') or hasattr(wrapped_test, attrib)):

hypothesis-python/src/hypothesis/internal/escalation.py

@@ -19,10 +19,12 @@
 
 import os
 import sys
+import traceback
+from inspect import getframeinfo
 
 import hypothesis
 from hypothesis.errors import StopTest, DeadlineExceeded, \
-    HypothesisException, UnsatisfiedAssumption
+    MultipleFailures, HypothesisException, UnsatisfiedAssumption
 from hypothesis.internal.compat import text_type, binary_type, \
     encoded_filepath
 
@@ -72,9 +74,30 @@ def escalate_hypothesis_internal_error():
     error_type, e, tb = sys.exc_info()
     if getattr(e, 'hypothesis_internal_always_escalate', False):
         raise
-    import traceback
     filepath = traceback.extract_tb(tb)[-1][0]
     if is_hypothesis_file(filepath) and not isinstance(
         e, (HypothesisException,) + HYPOTHESIS_CONTROL_EXCEPTIONS,
     ):
         raise
+
+
+def get_trimmed_traceback():
+    """Return the current traceback, minus any frames added by Hypothesis."""
+    error_type, _, tb = sys.exc_info()
+    if all([
+        # If verbosity is debug, leave the full traceback as-is
+        hypothesis.settings.default.verbosity < hypothesis.Verbosity.debug,
+        # If it's raised from inside Hypothesis and *not* MultipleFailures,
+        # it's probably an internal bug - so don't destroy the evidence!
+        (issubclass(error_type, MultipleFailures) or not
+         is_hypothesis_file(traceback.extract_tb(tb)[-1][0]))
+    ]):
+        while tb is not None and (
+            # If the frame is from one of our files, it's ours.
+            is_hypothesis_file(getframeinfo(tb.tb_frame)[0]) or
+            # But our `@proxies` decorator overrides the source location,
+            # so we check for an attribute it injects into the frame too.
+            tb.tb_frame.f_globals.get('__hypothesistracebackhide__') is True
+        ):
+            tb = tb.tb_next
+    return tb

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -544,6 +544,7 @@ def accept(f):
         f.__name__ = target.__name__
         f.__module__ = target.__module__
         f.__doc__ = target.__doc__
+        f.__globals__['__hypothesistracebackhide__'] = True
         return f
     return accept
 

hypothesis-python/tests/cover/test_escalation.py

@@ -17,11 +17,15 @@
 
 from __future__ import division, print_function, absolute_import
 
+import sys
+import traceback
+
 import pytest
 
 import hypothesis.strategies as st
 import hypothesis.internal.escalation as esc
-from hypothesis import given
+from hypothesis import Verbosity, given, settings
+from hypothesis.errors import MultipleFailures
 
 
 def test_does_not_escalate_errors_in_non_hypothesis_file():
@@ -66,3 +70,41 @@ def test(i):
         test()
 
     assert count == [1]
+
+
+@pytest.mark.parametrize('verbosity', [Verbosity.normal, Verbosity.debug])
+def test_tracebacks_omit_hypothesis_internals(verbosity):
+    @settings(verbosity=verbosity, database=None)
+    @given(st.none())
+    def simplest_failure(x):
+        assert x
+
+    try:
+        simplest_failure()
+    except AssertionError:
+        tb = traceback.extract_tb(sys.exc_info()[2])
+        # Unless in debug mode, Hypothesis adds 1 frame - the least possible!
+        # (4 frames: this one, simplest_failure, internal frame, assert False)
+        if verbosity < Verbosity.debug:
+            assert len(tb) == 4
+        else:
+            assert len(tb) >= 5
+
+
+@pytest.mark.parametrize('verbosity', [Verbosity.normal, Verbosity.debug])
+def test_tracebacks_omit_hypothesis_internals_for_multierrors(verbosity):
+    @settings(verbosity=verbosity, database=None)
+    @given(st.integers())
+    def multiple_failures(x):
+        assert x >= -999
+        assert x <= 100
+
+    try:
+        multiple_failures()
+    except MultipleFailures:
+        tb = traceback.extract_tb(sys.exc_info()[2])
+        # (4 frames: this one, multiple_failures, internal frame)
+        if verbosity < Verbosity.debug:
+            assert len(tb) == 3
+        else:
+            assert len(tb) >= 4