From e4710783da54c44573aa7f1312fb8158e8cb3f15 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 7 Jul 2022 15:44:24 +0100 Subject: [PATCH 1/8] Add TEST_ONLY_IGNORE_POETRY_LOCKFILE build argument --- docker/Dockerfile | 15 +++++++++++++-- scripts-dev/complement.sh | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f4d8e6c92575..164abf7295d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -68,7 +68,18 @@ COPY pyproject.toml poetry.lock /synapse/ # reason, such as when a git repository is used directly as a dependency. ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION -RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes} +# If specified, we won't use the Poetry lockfile. +# Instead, we'll just install what a regular `pip install` would from PyPI. +ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE + +# Export the dependencies, but only if we're actually going to use the Poetry lockfile. +# Otherwise, just create an empty requirements file so that the Dockerfile can +# proceed. +RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ + /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ + else \ + touch /synapse/requirements.txt; \ + fi ### ### Stage 1: builder @@ -109,7 +120,7 @@ COPY synapse /synapse/synapse/ COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. -RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse +RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all] ### ### Stage 2: runtime diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 6381f7092e67..eab23f18f1e2 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -101,6 +101,7 @@ if [ -z "$skip_docker_build" ]; then echo_if_github "::group::Build Docker image: matrixdotorg/synapse" docker build -t matrixdotorg/synapse \ --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ + --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ -f "docker/Dockerfile" . echo_if_github "::endgroup::" From ce0c1f197f0f6e06028858980266cd069c6827b4 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 7 Jul 2022 15:44:46 +0100 Subject: [PATCH 2/8] Enable Complement tests against latest deps --- .github/workflows/latest_deps.yml | 32 ++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index c537a5a60f9f..df54d01e115d 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -135,7 +135,37 @@ jobs: /logs/**/*.log* - # TODO: run complement (as with twisted trunk, see #12473). + complement: + if: "${{ !failure() && !cancelled() }}" + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - arrangement: monolith + database: SQLite + + - arrangement: monolith + database: Postgres + + - arrangement: workers + database: Postgres + + steps: + - name: Run actions/checkout@v2 for synapse + uses: actions/checkout@v2 + with: + path: synapse + + - name: Prepare Complement's Prerequisites + run: synapse/.ci/scripts/setup_complement_prerequisites.sh + + - run: | + set -o pipefail + TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + shell: bash + name: Run Complement Tests # open an issue if the build fails, so we know about it. open-issue: From 03d5b0a19799a8aaeb8059a83cb0e16d3af86f59 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 7 Jul 2022 15:44:53 +0100 Subject: [PATCH 3/8] TMP enable latest deps in PR --- .github/workflows/latest_deps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index df54d01e115d..2585cab0790b 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -16,6 +16,7 @@ on: schedule: - cron: 0 7 * * * workflow_dispatch: + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From ffdffae73bf62478eb6207f6da6da8f06aca6c04 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 7 Jul 2022 15:56:31 +0100 Subject: [PATCH 4/8] Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/13213.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13213.misc diff --git a/changelog.d/13213.misc b/changelog.d/13213.misc new file mode 100644 index 000000000000..b50d26ac0cdf --- /dev/null +++ b/changelog.d/13213.misc @@ -0,0 +1 @@ +Enable Complement CI tests in the 'latest deps' test run. \ No newline at end of file From 909fdef967bfe1c26b510e62607e573920ff3450 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Fri, 22 Jul 2022 10:46:53 +0100 Subject: [PATCH 5/8] Install deps using pip if we're skipping the lockfile --- docker/Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 164abf7295d2..e50f6dea627e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -119,8 +119,17 @@ COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. COPY pyproject.toml README.rst /synapse/ +# Repeat of earlier build argument declaration, as this is a new build stage. +ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE + # Install the synapse package itself. -RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all] +# Unless we haven't populated requirements.txt, we don't install any dependencies +# as we should already have those from the previous `pip install` step. +RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ + pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \ + else \ + pip install --prefix="/install" --no-warn-script-location /synapse[all]; \ + fi ### ### Stage 2: runtime From 97785ee10961e5cee8429848776e42e71a2e7753 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Mon, 25 Jul 2022 14:14:04 +0100 Subject: [PATCH 6/8] Don't open issues if we're in a PR --- .github/workflows/latest_deps.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 2585cab0790b..23bdf151c345 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -168,9 +168,10 @@ jobs: shell: bash name: Run Complement Tests - # open an issue if the build fails, so we know about it. + # Open an issue if the build fails, so we know about it. + # Only do this if we're not experimenting with this action in a PR. open-issue: - if: failure() + if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'" needs: # TODO: should mypy be included here? It feels more brittle than the other two. - mypy From 4f2e0803f53d2551d4e437f0462b09748996c3c6 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 1 Aug 2022 11:23:13 +0100 Subject: [PATCH 7/8] Update docker/Dockerfile Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e50f6dea627e..97bb03b08fc1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -123,7 +123,7 @@ COPY pyproject.toml README.rst /synapse/ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE # Install the synapse package itself. -# Unless we haven't populated requirements.txt, we don't install any dependencies +# If we have populated requirements.txt, we don't install any dependencies # as we should already have those from the previous `pip install` step. RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \ From 87fcda471adb175032805c744821820f7698d4e5 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Mon, 1 Aug 2022 11:23:38 +0100 Subject: [PATCH 8/8] Revert "TMP enable latest deps in PR" This reverts commit 03d5b0a19799a8aaeb8059a83cb0e16d3af86f59. --- .github/workflows/latest_deps.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 23bdf151c345..f263cf612dc5 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -16,7 +16,6 @@ on: schedule: - cron: 0 7 * * * workflow_dispatch: - pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref }}