From f08da6663158eeb61c9db288bd78783e14933c5f Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 13 May 2023 22:39:05 -0700 Subject: [PATCH 1/4] gh-86275: Don't fail PBTs for running slowly --- Lib/test/support/hypothesis_helper.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index 76bd2490fe6e3d..a614923e81402b 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -2,3 +2,12 @@ import hypothesis except ImportError: from . import _hypothesis_stubs as hypothesis +else: + # When using the real Hypothesis, we'll configure it to ignore occasional + # slow tests (avoiding flakiness from random VM slowness in CI). + hypothesis.settings.register_profile( + "slow-is-ok", + deadline=None, + suppress_health_check=[hypothesis.HealthCheck.too_slow], + ) + hypothesis.settings.load_profile("slow-is-ok") From 59a2793f4b082522f6693c74824bc081237dc7ad Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 13 May 2023 23:05:40 -0700 Subject: [PATCH 2/4] gh-86275: Cache Hypothesis' db in CI --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69b78e5567adb1..063772ed3a9587 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -368,6 +368,14 @@ jobs: echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + - name: 'Restore Hypothesis database' + id: cache-hypothesis-database + uses: actions/cache@v3 + with: + path: ./hypothesis + key: hypothesis-database-${{ github.head_ref || github.run_id }} + restore-keys: + - hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | From 2d3b8119c1da2d494f3c4a61f0cc0d37d5c2fc3a Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 13 May 2023 23:15:34 -0700 Subject: [PATCH 3/4] gh-86275: use pull-through cache --- .github/workflows/build.yml | 5 +++++ Lib/test/support/hypothesis_helper.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 063772ed3a9587..cc3a12833a4f5f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -396,6 +396,11 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig + - uses: actions/upload-artifact@v3 + if: always() + with: + name: hypothesis-example-db + path: .hypothesis/examples/ build_asan: diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index a614923e81402b..da16eb50c25958 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -1,3 +1,5 @@ +import os + try: import hypothesis except ImportError: @@ -11,3 +13,23 @@ suppress_health_check=[hypothesis.HealthCheck.too_slow], ) hypothesis.settings.load_profile("slow-is-ok") + + # For local development, we'll write to the default on-local-disk database + # of failing examples, and also use a pull-through cache to automatically + # replay any failing examples discovered in CI. For details on how this + # works, see https://hypothesis.readthedocs.io/en/latest/database.html + if "CI" not in os.environ: + from hypothesis.database import ( + GitHubArtifactDatabase, + MultiplexedDatabase, + ReadOnlyDatabase, + ) + + hypothesis.settings.register_profile( + "cpython-local-dev", + database=MultiplexedDatabase( + hypothesis.settings.default.database, + ReadOnlyDatabase(GitHubArtifactDatabase("python", "cpython")), + ), + ) + hypothesis.settings.load_profile("cpython-local-dev") From ccbb010c8bc2fd44802063b0884337f387105e34 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Mon, 15 May 2023 18:57:29 -0700 Subject: [PATCH 4/4] Make restore-keys a string --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc3a12833a4f5f..41abddffa5d648 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -374,7 +374,7 @@ jobs: with: path: ./hypothesis key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: + restore-keys: | - hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }}