From 23f24ddffe65fdf6f2c5b0edff40bbbc2aa4a958 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Sat, 15 Sep 2018 15:42:56 +1000 Subject: [PATCH] Report statistics when using pytest-xdist --- hypothesis-python/RELEASE.rst | 5 ++ .../src/hypothesis/extra/pytestplugin.py | 62 ++++++++++--------- .../src/hypothesis/statistics.py | 19 ++++++ .../tests/pytest/test_statistics.py | 17 +++++ 4 files changed, 74 insertions(+), 29 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..01d875fef2 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,5 @@ +RELEASE_TYPE: patch + +Hypothesis can now :ref:`show statistics ` when running +under :pypi:`pytest-xdist`. Previously, statistics were only reported +when all tests were run in a single process (:issue:`700`). diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py index 4bb3972009..daf60eae95 100644 --- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py +++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py @@ -17,6 +17,8 @@ from __future__ import division, print_function, absolute_import +from distutils.version import LooseVersion + import pytest from hypothesis import core, settings @@ -103,7 +105,9 @@ def pytest_runtest_call(item): store = StoringReporter(item.config) def note_statistics(stats): - gathered_statistics[item.nodeid] = stats + lines = [item.nodeid + ':', ''] + stats.get_description() + [''] + gathered_statistics[item.nodeid] = lines + item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): @@ -120,42 +124,42 @@ def pytest_runtest_makereport(item, call): 'Hypothesis', '\n'.join(item.hypothesis_report_information) )) + if hasattr(item, 'hypothesis_statistics') and report.when == 'teardown': + # Running on pytest < 3.5 where user_properties doesn't exist, fall + # back on the global gathered_statistics (which breaks under xdist) + if hasattr(report, 'user_properties'): # pragma: no branch + val = ('hypothesis-stats', item.hypothesis_statistics) + # Workaround for https://github.com/pytest-dev/pytest/issues/4034 + if isinstance(report.user_properties, tuple): + report.user_properties += (val,) + else: + report.user_properties.append(val) def pytest_terminal_summary(terminalreporter): if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION): return terminalreporter.section('Hypothesis Statistics') - for name, statistics in gathered_statistics.items(): - terminalreporter.write_line(name + ':') - terminalreporter.write_line('') - if not statistics.has_runs: - terminalreporter.write_line(' - Test was never run') - continue + if LooseVersion(pytest.__version__) < '3.5': # pragma: no cover + if not gathered_statistics: + terminalreporter.write_line( + 'Reporting Hypothesis statistics with pytest-xdist enabled ' + 'requires pytest >= 3.5' + ) + for lines in gathered_statistics.values(): + for li in lines: + terminalreporter.write_line(li) + return - terminalreporter.write_line(( - ' - %d passing examples, %d failing examples,' - ' %d invalid examples') % ( - statistics.passing_examples, statistics.failing_examples, - statistics.invalid_examples, - )) - terminalreporter.write_line( - ' - Typical runtimes: %s' % (statistics.runtimes,) - ) - terminalreporter.write_line( - ' - Fraction of time spent in data generation: %s' % ( - statistics.draw_time_percentage,)) - terminalreporter.write_line( - ' - Stopped because %s' % (statistics.exit_reason,) - ) - if statistics.events: - terminalreporter.write_line(' - Events:') - for event in statistics.events: - terminalreporter.write_line( - ' * %s' % (event,) - ) - terminalreporter.write_line('') + # terminalreporter.stats is a dict, where the empty string appears to + # always be the key for a list of _pytest.reports.TestReport objects + # (where we stored the statistics data in pytest_runtest_makereport above) + for test_report in terminalreporter.stats.get('', []): + for name, lines in test_report.user_properties: + if name == 'hypothesis-stats' and test_report.when == 'teardown': + for li in lines: + terminalreporter.write_line(li) def pytest_collection_modifyitems(items): diff --git a/hypothesis-python/src/hypothesis/statistics.py b/hypothesis-python/src/hypothesis/statistics.py index e5bc88478e..8d1e8266a3 100644 --- a/hypothesis-python/src/hypothesis/statistics.py +++ b/hypothesis-python/src/hypothesis/statistics.py @@ -103,6 +103,25 @@ def __init__(self, engine): self.draw_time_percentage = '~ %d%%' % ( round(draw_time_percentage),) + def get_description(self): + """Return a list of lines describing the statistics, to be printed.""" + if not self.has_runs: + return [' - Test was never run'] + lines = [ + ' - %d passing examples, %d failing examples, %d invalid examples' + % (self.passing_examples, self.failing_examples, + self.invalid_examples), + ' - Typical runtimes: %s' % (self.runtimes,), + ' - Fraction of time spent in data generation: %s' % ( + self.draw_time_percentage, + ), + ' - Stopped because %s' % (self.exit_reason,) + ] + if self.events: + lines.append(' - Events:') + lines += [' * %s' % (event,) for event in self.events] + return lines + def note_engine_for_statistics(engine): callback = collector.value diff --git a/hypothesis-python/tests/pytest/test_statistics.py b/hypothesis-python/tests/pytest/test_statistics.py index bf5dc151ec..ff060ca4a0 100644 --- a/hypothesis-python/tests/pytest/test_statistics.py +++ b/hypothesis-python/tests/pytest/test_statistics.py @@ -17,6 +17,10 @@ from __future__ import division, print_function, absolute_import +from distutils.version import LooseVersion + +import pytest + from hypothesis.extra.pytestplugin import PRINT_STATISTICS_OPTION pytest_plugins = 'pytester' @@ -67,6 +71,19 @@ def test_prints_statistics_given_option(testdir): assert '< 10% of examples satisfied assumptions' in out +@pytest.mark.skipif(LooseVersion(pytest.__version__) < '3.5', reason='too old') +def test_prints_statistics_given_option_under_xdist(testdir): + script = testdir.makepyfile(TESTSUITE) + result = testdir.runpytest(script, PRINT_STATISTICS_OPTION, '-n', '2') + out = '\n'.join(result.stdout.lines) + assert 'Hypothesis Statistics' in out + assert 'timeout=0.2' in out + assert 'max_examples=100' in out + assert '< 10% of examples satisfied assumptions' in out + # Check that xdist doesn't have us report the same thing twice + assert out.count('Stopped because settings.timeout=0.2') == 1 + + UNITTEST_TESTSUITE = """ from hypothesis import given