diff --git a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py index 94c66435f46..532079f847e 100644 --- a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py +++ b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py @@ -1,9 +1,17 @@ import json +import argparse from pathlib import Path -PREFIX = Path("build-op-count-time") -IVC_BENCH_JSON = Path("client_ivc_bench.json") -BENCHMARK = "ClientIVCBench/Full/6" +# Define command-line arguments with defaults +parser = argparse.ArgumentParser(description="Analyze benchmark JSON data.") +parser.add_argument("--json", type=Path, default=Path("client_ivc_bench.json"), help="Benchmark JSON file name.") +parser.add_argument("--benchmark", type=str, default="ClientIVCBench/Full/6", help="Benchmark name to analyze.") +parser.add_argument("--prefix", type=Path, default=Path("build-op-count-time"), help="Prefix path for benchmark files.") +args = parser.parse_args() + +IVC_BENCH_JSON = args.json +BENCHMARK = args.benchmark +PREFIX = args.prefix # Single out an independent set of functions accounting for most of BENCHMARK's real_time to_keep = [ @@ -16,25 +24,27 @@ "TranslatorProver::construct_proof(t)", "Goblin::merge(t)" ] -with open(PREFIX/IVC_BENCH_JSON, "r") as read_file: + +with open(PREFIX / IVC_BENCH_JSON, "r") as read_file: read_result = json.load(read_file) for _bench in read_result["benchmarks"]: if _bench["name"] == BENCHMARK: bench = _bench + bench_components = dict(filter(lambda x: x[0] in to_keep, bench.items())) # For each kept time, get the proportion over all kept times. -sum_of_kept_times_ms = sum(float(time) - for _, time in bench_components.items())/1e6 +sum_of_kept_times_ms = sum(float(time) for _, time in bench_components.items()) / 1e6 + max_label_length = max(len(label) for label in to_keep) column = {"function": "function", "ms": "ms", "%": "% sum"} -print( - f"{column['function']:<{max_label_length}}{column['ms']:>8} {column['%']:>8}") +print(f"{column['function']:<{max_label_length}}{column['ms']:>8} {column['%']:>8}") + for key in to_keep: if key not in bench: time_ms = 0 else: - time_ms = bench[key]/1e6 + time_ms = bench[key] / 1e6 print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/sum_of_kept_times_ms:>8.2%}") # Validate that kept times account for most of the total measured time. @@ -70,7 +80,6 @@ total_time_ms = bench["ProtogalaxyProver::fold_instances(t)"]/1e6 print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/total_time_ms:>8.2%}") - # Extract a set of components from the benchmark data and display timings and relative percentages def print_contributions(prefix, ivc_bench_json, bench_name, components): diff --git a/barretenberg/cpp/scripts/benchmark_client_ivc.sh b/barretenberg/cpp/scripts/benchmark_client_ivc.sh index 7991ef87940..0841f4429ba 100755 --- a/barretenberg/cpp/scripts/benchmark_client_ivc.sh +++ b/barretenberg/cpp/scripts/benchmark_client_ivc.sh @@ -1,21 +1,30 @@ #!/usr/bin/env bash set -eu -TARGET="client_ivc_bench" -# Note: to run structured trace version, change "Full" to "FullStructured" here and in analyze script -FILTER="ClientIVCBench/Full/6$" -BUILD_DIR=build-op-count-time +TARGET=${1:-"client_ivc_bench"} + +if [ "$TARGET" = "client_ivc_bench" ]; then + BENCHMARK="ClientIVCBench/Full/6" +elif [ "$TARGET" = "aztec_ivc_bench" ]; then + BENCHMARK="AztecIVCBench/FullStructured/6" +else + echo "Error: Unrecognized TARGET '$TARGET'." + exit 1 +fi + +BUILD_DIR="build-op-count-time" +FILTER="${BENCHMARK}$" # '$' to ensure only specified bench is run # Move above script dir. cd $(dirname $0)/.. # Measure the benchmarks with ops time counting -./scripts/benchmark_remote.sh client_ivc_bench\ - "./client_ivc_bench --benchmark_filter=$FILTER\ - --benchmark_out=$TARGET.json\ - --benchmark_out_format=json"\ +./scripts/benchmark_remote.sh "$TARGET"\ + "./$TARGET --benchmark_filter=$FILTER\ + --benchmark_out=$TARGET.json\ + --benchmark_out_format=json"\ op-count-time\ - build-op-count-time + "$BUILD_DIR" # Retrieve output from benching instance cd $BUILD_DIR @@ -23,4 +32,4 @@ scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json . # Analyze the results cd ../ -python3 ./scripts/analyze_client_ivc_bench.py +python3 ./scripts/analyze_client_ivc_bench.py --json "$TARGET.json" --benchmark "$BENCHMARK" --prefix "$BUILD_DIR" diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index e76758ca571..ab990b18d9f 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(basics_bench) add_subdirectory(decrypt_bench) add_subdirectory(goblin_bench) add_subdirectory(ipa_bench) +add_subdirectory(aztec_ivc_bench) add_subdirectory(client_ivc_bench) add_subdirectory(pippenger_bench) add_subdirectory(plonk_bench) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/CMakeLists.txt new file mode 100644 index 00000000000..e720a99f87e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(aztec_ivc_bench aztec_ivc stdlib_honk_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/aztec_ivc.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/aztec_ivc.bench.cpp new file mode 100644 index 00000000000..a24535ea22b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/aztec_ivc_bench/aztec_ivc.bench.cpp @@ -0,0 +1,153 @@ + +#include + +#include "barretenberg/aztec_ivc/aztec_ivc.hpp" +#include "barretenberg/common/op_count.hpp" +#include "barretenberg/common/op_count_google_bench.hpp" +#include "barretenberg/goblin/mock_circuits.hpp" +#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp" +#include "barretenberg/ultra_honk/ultra_verifier.hpp" + +using namespace benchmark; +using namespace bb; + +namespace { + +/** + * @brief Benchmark suite for the aztec client PG-Goblin IVC scheme + * + */ +class AztecIVCBench : public benchmark::Fixture { + public: + using Builder = MegaCircuitBuilder; + using VerifierInstance = VerifierInstance_; + using Proof = AztecIVC::Proof; + + // Number of function circuits to accumulate(based on Zacs target numbers) + static constexpr size_t NUM_ITERATIONS_MEDIUM_COMPLEXITY = 6; + + void SetUp([[maybe_unused]] const ::benchmark::State& state) override + { + bb::srs::init_crs_factory("../srs_db/ignition"); + bb::srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); + } + + /** + * @brief Verify an IVC proof + * + */ + static bool verify_ivc(Proof& proof, AztecIVC& ivc) + { + auto verifier_inst = std::make_shared(ivc.verification_queue[0].instance_vk); + bool verified = ivc.verify(proof, { ivc.verifier_accumulator, verifier_inst }); + + // This is a benchmark, not a test, so just print success or failure to the log + if (verified) { + info("IVC successfully verified!"); + } else { + info("IVC failed to verify."); + } + return verified; + } + + /** + * @brief Precompute the verification keys for the bench given the number of circuits in the IVC + * + * @param ivc + * @param num_function_circuits + * @return auto + */ + static auto precompute_verification_keys(AztecIVC& ivc, const size_t num_circuits) + { + // Produce the set of mocked circuits to be accumulated + MockCircuitMaker mock_circuit_maker; + std::vector circuits; + for (size_t circuit_idx = 0; circuit_idx < num_circuits; ++circuit_idx) { + circuits.emplace_back(mock_circuit_maker.create_next_circuit(ivc)); + } + + // Compute and return the corresponding set of verfication keys + return ivc.precompute_folding_verification_keys(circuits); + } + + /** + * @brief Manage the construction of mock app/kernel circuits + * @details Per the medium complexity benchmark spec, the first app circuit is size 2^19. Subsequent app and kernel + * circuits are size 2^17. Circuits produced are alternatingly app and kernel. + */ + class MockCircuitMaker { + size_t circuit_counter = 0; + + public: + Builder create_next_circuit(AztecIVC& ivc) + { + circuit_counter++; + + bool is_kernel = (circuit_counter % 2 == 0); // Every other circuit is a kernel, starting from the second + + Builder circuit{ ivc.goblin.op_queue }; + if (is_kernel) { // construct mock kernel + GoblinMockCircuits::construct_mock_folding_kernel(circuit); + } else { // construct mock app + bool use_large_circuit = (circuit_counter == 1); + GoblinMockCircuits::construct_mock_app_circuit(circuit, use_large_circuit); + } + return circuit; + } + }; + + /** + * @brief Perform a specified number of circuit accumulation rounds + * + * @param NUM_CIRCUITS Number of circuits to accumulate (apps + kernels) + */ + static void perform_ivc_accumulation_rounds(size_t NUM_CIRCUITS, AztecIVC& ivc, auto& precomputed_vks) + { + ASSERT(precomputed_vks.size() == NUM_CIRCUITS); // ensure presence of a precomputed VK for each circuit + + MockCircuitMaker mock_circuit_maker; + + for (size_t circuit_idx = 0; circuit_idx < NUM_CIRCUITS; ++circuit_idx) { + Builder circuit; + { + BB_OP_COUNT_TIME_NAME("construct_circuits"); + circuit = mock_circuit_maker.create_next_circuit(ivc); + } + + ivc.accumulate(circuit, precomputed_vks[circuit_idx]); + } + } +}; + +/** + * @brief Benchmark the prover work for the full PG-Goblin IVC protocol + * + */ +BENCHMARK_DEFINE_F(AztecIVCBench, FullStructured)(benchmark::State& state) +{ + AztecIVC ivc; + ivc.trace_structure = TraceStructure::AZTEC_IVC_BENCH; + + auto total_num_circuits = 2 * static_cast(state.range(0)); // 2x accounts for kernel circuits + + // Precompute the verification keys for the benchmark circuits + auto precomputed_vkeys = precompute_verification_keys(ivc, total_num_circuits); + + Proof proof; + for (auto _ : state) { + BB_REPORT_OP_COUNT_IN_BENCH(state); + perform_ivc_accumulation_rounds(total_num_circuits, ivc, precomputed_vkeys); + proof = ivc.prove(); + } + + // For good measure, ensure the IVC verifies + verify_ivc(proof, ivc); +} + +#define ARGS Arg(AztecIVCBench::NUM_ITERATIONS_MEDIUM_COMPLEXITY) + +BENCHMARK_REGISTER_F(AztecIVCBench, FullStructured)->Unit(benchmark::kMillisecond)->ARGS; + +} // namespace + +BENCHMARK_MAIN(); diff --git a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp index 9e696ae3580..e64847a7a0e 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp @@ -41,6 +41,35 @@ class GoblinMockCircuits { std::shared_ptr verification_key; }; + /** + * @brief Populate a builder with some arbitrary but nontrivial constraints + * @details Although the details of the circuit constructed here are arbitrary, the intent is to mock something a + * bit more realistic than a circuit comprised entirely of arithmetic gates. E.g. the circuit should respond + * realistically to efforts to parallelize circuit construction. + * + * @param builder + * @param large If true, construct a "large" circuit (2^19), else a medium circuit (2^17) + */ + static void construct_mock_app_circuit(MegaBuilder& builder, bool large = false) + { + if (large) { // Results in circuit size 2^19 + stdlib::generate_sha256_test_circuit(builder, 12); + stdlib::generate_ecdsa_verification_test_circuit(builder, 11); + stdlib::generate_merkle_membership_test_circuit(builder, 12); + } else { // Results in circuit size 2^17 + stdlib::generate_sha256_test_circuit(builder, 9); + stdlib::generate_ecdsa_verification_test_circuit(builder, 2); + stdlib::generate_merkle_membership_test_circuit(builder, 10); + } + + // TODO(https://github.com/AztecProtocol/barretenberg/issues/911): We require goblin ops to be added to the + // function circuit because we cannot support zero commtiments. While the builder handles this at + // ProverInstance creation stage via the add_gates_to_ensure_all_polys_are_non_zero function for other MegaHonk + // circuits (where we don't explicitly need to add goblin ops), in IVC merge proving happens prior to folding + // where the absense of goblin ecc ops will result in zero commitments. + MockCircuits::construct_goblin_ecc_op_circuit(builder); + } + /** * @brief Populate a builder with some arbitrary but nontrivial constraints * @details Although the details of the circuit constructed here are arbitrary, the intent is to mock something a diff --git a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp index b37cdd16206..38c14d4b5cb 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp +++ b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp @@ -32,4 +32,21 @@ TEST_F(MegaMockCircuitsPinning, FunctionSizes) }; run_test(true); run_test(false); +} + +TEST_F(MegaMockCircuitsPinning, AppCircuitSizes) +{ + const auto run_test = [](bool large) { + GoblinProver goblin; + MegaCircuitBuilder app_circuit{ goblin.op_queue }; + GoblinMockCircuits::construct_mock_app_circuit(app_circuit, large); + auto instance = std::make_shared(app_circuit); + if (large) { + EXPECT_EQ(instance->proving_key.log_circuit_size, 19); + } else { + EXPECT_EQ(instance->proving_key.log_circuit_size, 17); + }; + }; + run_test(true); + run_test(false); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index a32d7d83f09..14789bfa263 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -28,7 +28,7 @@ struct StackTraces { // A set of fixed block size conigurations to be used with the structured execution trace. The actual block sizes // corresponding to these settings are defined in the corresponding arithmetization classes (Ultra/Mega). For efficiency // it is best to use the smallest possible block sizes to accommodate a given situation. -enum class TraceStructure { NONE, SMALL_TEST, CLIENT_IVC_BENCH, E2E_FULL_TEST }; +enum class TraceStructure { NONE, SMALL_TEST, CLIENT_IVC_BENCH, AZTEC_IVC_BENCH, E2E_FULL_TEST }; /** * @brief Basic structure for storing gate data in a builder diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index d5348abd956..8f0e9e35721 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -77,6 +77,23 @@ template class MegaArith { } }; + // A minimal structuring specifically tailored to the medium complexity transaction for the AztecIvc benchmark + struct AztecIvcBenchStructuredBlockSizes : public MegaTraceBlocks { + AztecIvcBenchStructuredBlockSizes() + { + this->ecc_op = 1 << 10; + this->pub_inputs = 1 << 7; + this->arithmetic = 187000; + this->delta_range = 90000; + this->elliptic = 9000; + this->aux = 137000; + this->lookup = 72000; + this->busread = 1 << 7; + this->poseidon_external = 3000; + this->poseidon_internal = 17000; + } + }; + // Structuring tailored to the full e2e TS test (TO BE UPDATED ACCORDINGLY) struct E2eStructuredBlockSizes : public MegaTraceBlocks { E2eStructuredBlockSizes() @@ -178,6 +195,9 @@ template class MegaArith { case TraceStructure::CLIENT_IVC_BENCH: fixed_block_sizes = ClientIvcBenchStructuredBlockSizes(); break; + case TraceStructure::AZTEC_IVC_BENCH: + fixed_block_sizes = AztecIvcBenchStructuredBlockSizes(); + break; case TraceStructure::E2E_FULL_TEST: fixed_block_sizes = E2eStructuredBlockSizes(); break; diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index 402ce07f9e0..38d3028300d 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -90,6 +90,7 @@ template class UltraArith { // We don't use Ultra in ClientIvc so no need for anything other than sizing for simple unit tests case TraceStructure::SMALL_TEST: case TraceStructure::CLIENT_IVC_BENCH: + case TraceStructure::AZTEC_IVC_BENCH: case TraceStructure::E2E_FULL_TEST: fixed_block_sizes = SmallTestStructuredBlockSizes(); break;