diff --git a/.github/workflows/ci-deflake.yml b/.github/workflows/ci-deflake.yml new file mode 100644 index 00000000000..338051a12c9 --- /dev/null +++ b/.github/workflows/ci-deflake.yml @@ -0,0 +1,178 @@ +# A testnet release-oriented set of tests to run 5 times to deflake them. +name: CI (deploy branch) +on: + push: + branches: + - release/testnet* + - '*/*deflake' + workflow_dispatch: + inputs: {} + +concurrency: + # force parallelism in master + group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} + cancel-in-progress: true + +env: + DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_TOKEN: ${{ github.token }} + GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + # kludge until we move away from runners + WAIT_FOR_RUNNERS: false + +jobs: + configure: + runs-on: ubuntu-20.04 + # Required permissions. + permissions: + pull-requests: read + # Set job outputs to values from filter step. + # Set username to a value derived from our github username. + outputs: + username: ${{ steps.compute_username.outputs.username }} + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - name: Compute Username + id: compute_username + shell: bash + env: + REPO: "${{ github.repository }}" + BRANCH: "${{ github.ref_name }}" + run: | + GIT_HASH="${{ github.sha }}" + GIT_HASH_LAST8=${GIT_HASH: -8} + GIT_HASH_LAST8_DEC=$(printf "%d" 0x$GIT_HASH_LAST8) + GIT_HASH_MODULO_8=$((GIT_HASH_LAST8_DEC % 8)) + echo "username=master-${GIT_HASH_MODULO_8}" + echo "username=master-${GIT_HASH_MODULO_8}" >> $GITHUB_OUTPUT + + setup: + needs: [configure] + uses: ./.github/workflows/setup-runner.yml + with: + username: ${{ needs.configure.outputs.username }} + runner_type: builder-x86 + secrets: inherit + + build: + needs: [setup, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + with: + concurrency_key: build-x86 + # prepare images locally, tagged by commit hash + - name: "Build E2E Image" + timeout-minutes: 40 + run: | + earthly-ci ./yarn-project+export-e2e-test-images + + # run specific end to end tests to deflake + e2e: + needs: [build, configure] + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + test: + - e2e_2_pxes + - e2e_block_building + - e2e_prover_coordination + - e2e_prover_full + - e2e_p2p_gossip + - e2e_p2p_rediscovery + - e2e_p2p_reqresp + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + - name: Determine runner type + id: runner_type + run: | + runner=$(./scripts/ci/select_runner.sh ${{ matrix.test }}) + echo "type=$runner" >> $GITHUB_OUTPUT + - name: Setup and Test + timeout-minutes: 200 + uses: ./.github/ensure-tester-with-images + env: + USERNAME: ${{ needs.configure.outputs.username }} + with: + runner_type: ${{ steps.runner_type.outputs.type }} + builder_type: builder-x86 + builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} + builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images + tester_ttl: 200 + run: | + set -eux + cd ./yarn-project/end-to-end/ + export FORCE_COLOR=1 + for i in {1..5} ; do + echo "START OF RUN $i" + ./scripts/e2e_test.sh ${{ matrix.test }} + echo "END OF RUN $i" + done + + # note: proving disabled + kind-network-test: + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + strategy: + fail-fast: false + matrix: + config: + - test: reorg.test.ts + values: 16-validators + runner_type: 16core-tester-x86-high-memory + timeout: 360 + - test: 4epochs.test.ts + values: 48-validators + runner_type: 32core-tester-x86 + timeout: 300 + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + with: + concurrency_key: kind-network-${{ matrix.config.test }} + - name: Setup and KIND Network Test + timeout-minutes: ${{ matrix.config.timeout }} + uses: ./.github/ensure-tester-with-images + env: + USERNAME: ${{ needs.configure.outputs.username }} + with: + runner_type: ${{ matrix.config.runner_type }} + spot_strategy: None # use on-demand machines + builder_type: builder-x86 + # these are copied to the tester and expected by the earthly command below + # if they fail to copy, it will try to build them on the tester and fail + builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} + # command to produce the images in case they don't exist + builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images + tester_ttl: ${{ matrix.config.timeout }} + run: | + cd yarn-project/end-to-end + echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin + test=${{ matrix.config.test }} + for i in {1..5} ; do + echo "START OF RUN $i" + NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${{ matrix.config.values }}.yaml ./scripts/network_test.sh ./src/spartan/$test + echo "END OF RUN $i" + done + - name: Copy Network Logs + if: always() # Run on failure too + run: copy_from_tester /home/ubuntu/run-${{ env.RUN_ID }}/yarn-project/end-to-end/scripts/network-test.log network-test.log + - name: Upload Network Logs + if: always() # Run on failure too + uses: actions/upload-artifact@v4 + with: + name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}.log + path: network-test.log diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml index 1fb170f4f99..7369207ab0d 100644 --- a/.github/workflows/publish-aztec-packages.yml +++ b/.github/workflows/publish-aztec-packages.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - release/testnet* - "*/release-master*" workflow_dispatch: inputs: