test: deflaker CI for testnet branch (#9820)

Run a subset of e2e and kind-network-tests 5 times.

.github/workflows/ci-deflake.yml

@@ -0,0 +1,178 @@
+# A testnet release-oriented set of tests to run 5 times to deflake them.
+name: CI (deploy branch)
+on:
+  push:
+    branches:
+      - release/testnet*
+      - '*/*deflake'
+  workflow_dispatch:
+    inputs: {}
+
+concurrency:
+  # force parallelism in master
+  group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}"
+  RUN_ID: ${{ github.run_id }}
+  RUN_ATTEMPT: ${{ github.run_attempt }}
+  GITHUB_TOKEN: ${{ github.token }}
+  GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+  GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+  # kludge until we move away from runners
+  WAIT_FOR_RUNNERS: false
+
+jobs:
+  configure:
+    runs-on: ubuntu-20.04
+    # Required permissions.
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step.
+    # Set username to a value derived from our github username.
+    outputs:
+      username: ${{ steps.compute_username.outputs.username }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - name: Compute Username
+        id: compute_username
+        shell: bash
+        env:
+          REPO: "${{ github.repository }}"
+          BRANCH: "${{ github.ref_name }}"
+        run: |
+          GIT_HASH="${{ github.sha }}"
+          GIT_HASH_LAST8=${GIT_HASH: -8}
+          GIT_HASH_LAST8_DEC=$(printf "%d" 0x$GIT_HASH_LAST8)
+          GIT_HASH_MODULO_8=$((GIT_HASH_LAST8_DEC % 8))
+          echo "username=master-${GIT_HASH_MODULO_8}"
+          echo "username=master-${GIT_HASH_MODULO_8}" >> $GITHUB_OUTPUT
+
+  setup:
+    needs: [configure]
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      username: ${{ needs.configure.outputs.username }}
+      runner_type: builder-x86
+    secrets: inherit
+
+  build:
+    needs: [setup, configure]
+    runs-on: ${{ needs.configure.outputs.username }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-x86
+      # prepare images locally, tagged by commit hash
+      - name: "Build E2E Image"
+        timeout-minutes: 40
+        run: |
+          earthly-ci ./yarn-project+export-e2e-test-images
+
+  # run specific end to end tests to deflake
+  e2e:
+    needs: [build, configure]
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - e2e_2_pxes
+          - e2e_block_building
+          - e2e_prover_coordination
+          - e2e_prover_full
+          - e2e_p2p_gossip
+          - e2e_p2p_rediscovery
+          - e2e_p2p_reqresp
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Determine runner type
+        id: runner_type
+        run: |
+          runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }})
+          echo "type=$runner" >> $GITHUB_OUTPUT
+      - name: Setup and Test
+        timeout-minutes: 200
+        uses: ./.github/ensure-tester-with-images
+        env:
+          USERNAME: ${{ needs.configure.outputs.username }}
+        with:
+          runner_type: ${{ steps.runner_type.outputs.type }}
+          builder_type: builder-x86
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 200
+          run: |
+            set -eux
+            cd ./yarn-project/end-to-end/
+            export FORCE_COLOR=1
+            for i in {1..5} ; do
+              echo "START OF RUN $i"
+              ./scripts/e2e_test.sh ${{ matrix.test }}
+              echo "END OF RUN $i"
+            done
+
+  # note: proving disabled
+  kind-network-test:
+    needs: [build, configure]
+    runs-on: ${{ needs.configure.outputs.username }}-x86
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - test: reorg.test.ts
+            values: 16-validators
+            runner_type: 16core-tester-x86-high-memory
+            timeout: 360
+          - test: 4epochs.test.ts
+            values: 48-validators
+            runner_type: 32core-tester-x86
+            timeout: 300
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: kind-network-${{ matrix.config.test }}
+      - name: Setup and KIND Network Test
+        timeout-minutes: ${{ matrix.config.timeout }}
+        uses: ./.github/ensure-tester-with-images
+        env:
+          USERNAME: ${{ needs.configure.outputs.username }}
+        with:
+          runner_type: ${{ matrix.config.runner_type }}
+          spot_strategy: None # use on-demand machines
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: ${{ matrix.config.timeout }}
+          run: |
+            cd yarn-project/end-to-end
+            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
+            test=${{ matrix.config.test }}
+            for i in {1..5} ; do
+              echo "START OF RUN $i"
+              NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${{ matrix.config.values }}.yaml ./scripts/network_test.sh ./src/spartan/$test
+              echo "END OF RUN $i"
+            done
+      - name: Copy Network Logs
+        if: always() # Run on failure too
+        run: copy_from_tester /home/ubuntu/run-${{ env.RUN_ID }}/yarn-project/end-to-end/scripts/network-test.log network-test.log
+      - name: Upload Network Logs
+        if: always() # Run on failure too
+        uses: actions/upload-artifact@v4
+        with:
+          name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}.log
+          path: network-test.log

.github/workflows/publish-aztec-packages.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - release/testnet*
       - "*/release-master*"
   workflow_dispatch:
     inputs: