feat: earthly split runners, structure reverts (#5524)

- use published multiplatform base for bb builds, add ci command to
publish (ran manually in github actions UI)
- move bench compilation to `build` runner, split out bench binaries
from running
- move bb-native-tests to `test` runner (along with
preset-release-assert-all), add parallelism lock for native tests,
hopefully won't degrade builds anymore
- roll back non-determinism in builds (remove cache mounts)
- roll back a bit to old /usr/src format inside image builds

.github/workflows/ci.yml

@@ -81,7 +81,7 @@ jobs:
         run:
           # Flags:
           # - e2e_build=true to get images from a local build, not a dockerhub registry strategy
-          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_build=true
+          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build
 
   # all the end-to-end integration tests for aztec
   e2e-x86:
@@ -123,7 +123,7 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         run: |
-          earthly-cloud build x86 --no-output +${{ matrix.test }}
+          earthly-cloud build x86 --no-output +${{ matrix.test }} --e2e_mode=cache
 
   # barretenberg (prover) native tests
   bb-native-tests:
@@ -158,7 +158,7 @@ jobs:
       - name: Test
         working-directory: ./barretenberg/cpp/
         run: |
-          earthly-cloud build ${{ matrix.environment }} --no-output +test
+          earthly-cloud test ${{ matrix.environment }} --no-output +test
 
   # All benchmarks, purposefully ran sequential on a machine
   # they should use parallelism within the benchmark, but only one thing should run at a time
@@ -170,9 +170,9 @@ jobs:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      # TODO no longer exclusive.. revisit this
-      group: ${{ github.actor }}-x86
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      # TODO this allows NO concurrency of bench jobs
+      # TODO if this takes too long, we need to divide up into different bench machines for each pipeline step
+      group: bench-x86
     steps:
       - uses: earthly/actions-setup@v1
         with:
@@ -188,10 +188,15 @@ jobs:
         working-directory: ./scripts
         run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
 
+      - name: Build and Push Binaries
+        working-directory: ./barretenberg/cpp/
+        run: earthly-cloud build x86 --push +bench-base
+
+      # Use bench_mode=cache to read the pushed build above
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
-        run: earthly-cloud bench x86 --no-output +bench-client-ivc
+        run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
-        run: earthly-cloud bench x86 --no-output +bench-ultra-honk
+        run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache

.github/workflows/publish_base_images.yml

@@ -0,0 +1,39 @@
+# Publishes our base images with custom installs or builds etc
+# These publish a multi-arch image by first publishing with x86, and then with arm
+# This is a bit of a hack, but earthly needs to see both image types to make a multiplatform image
+# and its easiest for arm to just pull the x86 image after.
+name: Publish Base Images
+on:
+  workflow_dispatch: {}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      group: publish-base-images-${{ github.event.pull_request.number || github.ref_name }}
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+
+      - name: Publish Barretenberg Base Images
+        working-directory: ./barretenberg/cpp
+        run: |
+          # see note above about how this creates a multiplatform image
+          earthly-cloud build x86 --push +build-base
+          earthly-cloud build arm --push +build-base
+

avm-transpiler/Earthfile

@@ -3,10 +3,10 @@ IMPORT ../noir AS noir
 # we rely on noir source, which this image has
 FROM noir+nargo
 
-# move noir contents to /build/noir
-RUN mv /build /noir && mkdir /build && mv /noir /build
+# move noir contents to /usr/src/noir
+RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src
 # work in avm-transpiler
-WORKDIR /build/avm-transpiler
+WORKDIR /usr/src/avm-transpiler
 
 RUN apt-get update && apt-get install -y git
 
@@ -24,5 +24,5 @@ build:
 run: 
     #TODO needed?
     FROM ubuntu:focal
-    COPY +build/avm-transpiler /build/avm-transpiler
+    COPY +build/avm-transpiler /usr/src/avm-transpiler
     ENTRYPOINT ["sh", "-c"]