Merge branch 'master' into 04-24-feat_configure_prover_as_separate_pr…

…ocess

.github/workflows/ci.yml

@@ -48,7 +48,7 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        run: earthly ./yarn-project+export-end-to-end
+        run: earthly-ci ./yarn-project+export-end-to-end
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of end-to-end jobs
@@ -76,7 +76,7 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache
+        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
       # TODO
       # - name: Upload logs
       #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
@@ -101,7 +101,7 @@ jobs:
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 25
         # limit our parallelism to half our cores
-        run: earthly --no-output +test --hardware_concurrency=64
+        run: earthly-ci --no-output +test --hardware_concurrency=64
 
   noir-projects:
     needs: setup
@@ -117,7 +117,7 @@ jobs:
           concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86
       - name: "Noir Projects"
         timeout-minutes: 25
-        run: earthly --no-output ./noir-projects/+test
+        run: earthly-ci --no-output ./noir-projects/+test
 
   yarn-project-formatting:
     needs: setup
@@ -134,7 +134,7 @@ jobs:
           concurrency_key: yarn-project-formatting-${{ github.actor }}-x86
       - name: "Yarn Project Tests"
         timeout-minutes: 25
-        run: earthly --no-output ./yarn-project/+format-check
+        run: earthly-ci --no-output ./yarn-project/+format-check
 
   yarn-project-test:
     needs: noir-projects
@@ -151,7 +151,7 @@ jobs:
           concurrency_key: yarn-project-test-${{ github.actor }}-x86
       - name: "Yarn Project Tests"
         timeout-minutes: 25
-        run: earthly --no-output ./yarn-project/+test
+        run: earthly-ci --no-output ./yarn-project/+test
 
   # push benchmarking binaries to dockerhub registry
   bb-bench-binaries:
@@ -169,7 +169,7 @@ jobs:
       - name: Build and Push Binaries
         timeout-minutes: 15
         working-directory: ./barretenberg/cpp/
-        run: earthly --push +bench-binaries
+        run: earthly-ci --push +bench-binaries
 
   setup-bench:
     uses: ./.github/workflows/setup-runner.yml
@@ -200,25 +200,34 @@ jobs:
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-client-ivc --bench_mode=cache
+        run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-ultra-honk --bench_mode=cache
+        run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
 
   merge-check:
     runs-on: ubuntu-latest
     needs:
-      [
-        e2e,
-        bb-native-tests,
-        bb-bench,
-        yarn-project-formatting,
-        yarn-project-test,
-      ]
+      - e2e
+      - bb-native-tests
+      - bb-bench
+      - yarn-project-formatting
+      - yarn-project-test
+    if: always()
     steps:
-      - run: echo Pull request merging now allowed.
+      - run: |
+          echo "e2e status: ${{ needs.e2e.result }}"
+          echo "bb-native-tests status: ${{ needs.bb-native-tests.result }}"
+          echo "bb-bench status: ${{ needs.bb-bench.result }}"
+          echo "yarn-project-formatting status: ${{ needs.yarn-project-formatting.result }}"
+          echo "yarn-project-test status: ${{ needs.yarn-project-test.result }}"
+          if [[ "${{ needs.e2e.result }}" != 'success' || "${{ needs.bb-native-tests.result }}" != 'success' || "${{ needs.bb-bench.result }}" != 'success' || "${{ needs.yarn-project-formatting.result }}" != 'success' || "${{ needs.yarn-project-test.result }}" != 'success' ]]; then
+            echo "Pull request merging not allowed due to failures."
+            exit 1
+          fi
+          echo "Pull request merging now allowed."
 
   notify:
     needs:

.github/workflows/setup-runner.yml

@@ -62,6 +62,30 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Checkout Merge Pipeline Files
+        uses: actions/checkout@v4
+        # Only check PRs for consistency (not master)
+        if: ${{ github.event.pull_request.head.sha != '' }}
+        with:
+          path: merge-commit-pipeline-files
+          sparse-checkout: |
+            .github/workflows/ci.yml
+            .github/workflows/setup-runner.yml
+
+      - name: Ensure CI Consistency
+        # Only check PRs for consistency (not master)
+        if: ${{ github.event.pull_request.head.sha != '' }}
+        run: |
+          # Compare the checked-out CI configuration files with the reference files
+          if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then
+            echo "Error: ci.yml changes in master (or PR base). Please merge these changes."
+            exit 1
+          fi
+          if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then
+            echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes."
+            exit 1
+          fi
+
       - name: Start EC2 runner
         uses: ./.github/spot-runner-action
         with:

.noir-sync-commit

@@ -1 +1 @@
-0cf2e2a1b8d247bed03ba5b7b1be5cd30f0d51b2
+1ec9cdc7013e867db3672d27e3a6104e4b7e7eef

barretenberg/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 2edbb89f3954bccf7cff3d338abca9ac22216bed
-	parent = 99e12b1abd7e66e871b41572a54cee63b5300d96
+	commit = 795c999a3b7fe8d85af05ffc09ddfd349c00e5a4
+	parent = 0a64279ba1b2b3bb6627c675b8a0b116be17f579
 	method = merge
 	cmdver = 0.4.6

barretenberg/acir_tests/Dockerfile.bb

@@ -10,8 +10,12 @@ COPY . .
 # Run every acir test through native bb build prove_then_verify flow for UltraPlonk.
 # This ensures we test independent pk construction through real/garbage witness data paths.
 RUN FLOW=prove_then_verify ./run_acir_tests.sh
-# Construct and verify a UltraHonk proof for all acir programs
-RUN FLOW=prove_and_verify_ultra_honk ./run_acir_tests.sh
+# Construct and separately verify a UltraHonk proof for a single program
+RUN FLOW=prove_then_verify_ultra_honk ./run_acir_tests.sh double_verify_nested_proof
+# Construct and separately verify a GoblinUltraHonk proof for all acir programs
+RUN FLOW=prove_then_verify_goblin_ultra_honk ./run_acir_tests.sh
+# Construct and verify a UltraHonk proof for a single program
+RUN FLOW=prove_and_verify_ultra_honk ./run_acir_tests.sh double_verify_nested_proof
 # Construct and verify a Goblin UltraHonk (GUH) proof for a single arbitrary program
 RUN FLOW=prove_and_verify_goblin_ultra_honk ./run_acir_tests.sh 6_array
 # Construct and verify a UltraHonk proof for all ACIR programs using the new witness stack workflow

barretenberg/acir_tests/flows/prove_then_verify_goblin_ultra_honk.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/acir.gz"
+FLAGS="-c $CRS_PATH $VFLAG"
+
+# Test we can perform the proof/verify flow.
+# This ensures we test independent pk construction through real/garbage witness data paths.
+$BIN prove_goblin_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_goblin_ultra_honk -o vk $FLAGS $BFLAG
+$BIN verify_goblin_ultra_honk -k vk -p proof $FLAGS

barretenberg/acir_tests/flows/prove_then_verify_ultra_honk.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/acir.gz"
+FLAGS="-c $CRS_PATH $VFLAG"
+
+# Test we can perform the proof/verify flow.
+# This ensures we test independent pk construction through real/garbage witness data paths.
+$BIN prove_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_ultra_honk -o vk $FLAGS $BFLAG
+$BIN verify_ultra_honk -k vk -p proof $FLAGS

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -579,6 +579,116 @@ bool avm_verify(const std::filesystem::path& proof_path)
     return true;
 }
 
+/**
+ * @brief Creates a proof for an ACIR circuit
+ *
+ * Communication:
+ * - stdout: The proof is written to stdout as a byte array
+ * - Filesystem: The proof is written to the path specified by outputPath
+ *
+ * @param bytecodePath Path to the file containing the serialized circuit
+ * @param witnessPath Path to the file containing the serialized witness
+ * @param outputPath Path to write the proof to
+ */
+template <IsUltraFlavor Flavor>
+void prove_honk(const std::string& bytecodePath, const std::string& witnessPath, const std::string& outputPath)
+{
+    using Builder = Flavor::CircuitBuilder;
+    using Prover = UltraProver_<Flavor>;
+
+    auto constraint_system = get_constraint_system(bytecodePath);
+    auto witness = get_witness(witnessPath);
+
+    auto builder = acir_format::create_circuit<Builder>(constraint_system, 0, witness);
+
+    const size_t additional_gates_buffer = 15; // conservatively large to be safe
+    size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + additional_gates_buffer);
+    init_bn254_crs(srs_size);
+
+    // Construct Honk proof
+    Prover prover{ builder };
+    auto proof = prover.construct_proof();
+
+    if (outputPath == "-") {
+        writeRawBytesToStdout(to_buffer</*include_size=*/true>(proof));
+        vinfo("proof written to stdout");
+    } else {
+        write_file(outputPath, to_buffer</*include_size=*/true>(proof));
+        vinfo("proof written to: ", outputPath);
+    }
+}
+
+/**
+ * @brief Verifies a proof for an ACIR circuit
+ *
+ * Note: The fact that the proof was computed originally by parsing an ACIR circuit is not of importance
+ * because this method uses the verification key to verify the proof.
+ *
+ * Communication:
+ * - proc_exit: A boolean value is returned indicating whether the proof is valid.
+ *   an exit code of 0 will be returned for success and 1 for failure.
+ *
+ * @param proof_path Path to the file containing the serialized proof
+ * @param vk_path Path to the file containing the serialized verification key
+ * @return true If the proof is valid
+ * @return false If the proof is invalid
+ */
+template <IsUltraFlavor Flavor> bool verify_honk(const std::string& proof_path, const std::string& vk_path)
+{
+    using VerificationKey = Flavor::VerificationKey;
+    using Verifier = UltraVerifier_<Flavor>;
+    using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
+
+    auto g2_data = get_bn254_g2_data(CRS_PATH);
+    srs::init_crs_factory({}, g2_data);
+    auto proof = from_buffer<std::vector<bb::fr>>(read_file(proof_path));
+    auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(read_file(vk_path)));
+    verification_key->pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
+
+    Verifier verifier{ verification_key };
+
+    bool verified = verifier.verify_proof(proof);
+
+    vinfo("verified: ", verified);
+    return verified;
+}
+
+/**
+ * @brief Writes a verification key for an ACIR circuit to a file
+ *
+ * Communication:
+ * - stdout: The verification key is written to stdout as a byte array
+ * - Filesystem: The verification key is written to the path specified by outputPath
+ *
+ * @param bytecodePath Path to the file containing the serialized circuit
+ * @param outputPath Path to write the verification key to
+ */
+template <IsUltraFlavor Flavor> void write_vk_honk(const std::string& bytecodePath, const std::string& outputPath)
+{
+    using Builder = Flavor::CircuitBuilder;
+    using ProverInstance = ProverInstance_<Flavor>;
+    using VerificationKey = Flavor::VerificationKey;
+
+    auto constraint_system = get_constraint_system(bytecodePath);
+    auto builder = acir_format::create_circuit<Builder>(constraint_system, 0, {});
+
+    const size_t additional_gates_buffer = 15; // conservatively large to be safe
+    size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + additional_gates_buffer);
+    init_bn254_crs(srs_size);
+
+    ProverInstance prover_inst(builder);
+    VerificationKey vk(
+        prover_inst.proving_key); // uses a partial form of the proving key which only has precomputed entities
+
+    auto serialized_vk = to_buffer(vk);
+    if (outputPath == "-") {
+        writeRawBytesToStdout(serialized_vk);
+        vinfo("vk written to stdout");
+    } else {
+        write_file(outputPath, serialized_vk);
+        vinfo("vk written to: ", outputPath);
+    }
+}
 /**
  * @brief Creates a proof for an ACIR circuit, outputs the proof and verification key in binary and 'field' format
  *
@@ -721,6 +831,22 @@ int main(int argc, char* argv[])
         } else if (command == "avm_verify") {
             std::filesystem::path proof_path = get_option(args, "-p", "./proofs/avm_proof");
             return avm_verify(proof_path) ? 0 : 1;
+        } else if (command == "prove_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./proofs/proof");
+            prove_honk<UltraFlavor>(bytecode_path, witness_path, output_path);
+        } else if (command == "verify_ultra_honk") {
+            return verify_honk<UltraFlavor>(proof_path, vk_path) ? 0 : 1;
+        } else if (command == "write_vk_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<UltraFlavor>(bytecode_path, output_path);
+        } else if (command == "prove_goblin_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./proofs/proof");
+            prove_honk<GoblinUltraFlavor>(bytecode_path, witness_path, output_path);
+        } else if (command == "verify_goblin_ultra_honk") {
+            return verify_honk<GoblinUltraFlavor>(proof_path, vk_path) ? 0 : 1;
+        } else if (command == "write_vk_goblin_ultra_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<GoblinUltraFlavor>(bytecode_path, output_path);
         } else {
             std::cerr << "Unknown command: " << command << "\n";
             return 1;

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk_rounds.bench.cpp

@@ -55,7 +55,6 @@ BB_PROFILE static void test_round_inner(State& state, GoblinUltraProver& prover,
     // we need to get the relation_parameters and prover_polynomials from the oink_prover
     prover.instance->proving_key = std::move(oink_prover.proving_key);
     prover.instance->relation_parameters = oink_prover.relation_parameters;
-    prover.instance->prover_polynomials = GoblinUltraFlavor::ProverPolynomials(prover.instance->proving_key);
     time_if_index(RELATION_CHECK, [&] { prover.execute_relation_check_rounds(); });
     time_if_index(ZEROMORPH, [&] { prover.execute_zeromorph_rounds(); });
 }

barretenberg/cpp/src/barretenberg/common/serialize.hpp

@@ -371,7 +371,8 @@ template <typename B, typename T> inline void read(B& it, std::optional<T>& opt_
 }
 
 template <typename T>
-concept HasGetAll = requires(T t) { t.get_all(); };
+concept HasGetAll = requires(T t) { t.get_all(); } && !
+msgpack_concepts::HasMsgPack<T>;
 
 // Write out a struct that defines get_all()
 template <typename B, HasGetAll T> inline void write(B& buf, T const& value)