From 2c03259653c45d7f17086320a9ea76225d1595ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Rodr=C3=ADguez?= Date: Thu, 1 Aug 2024 17:09:15 +0200 Subject: [PATCH] feat: Report gates and VKs of private protocol circuits with megahonk (#7722) --- barretenberg/cpp/src/barretenberg/bb/main.cpp | 12 ++-- noir-projects/gates_report.sh | 19 ++++++- .../mega_honk_circuits.json | 4 ++ .../scripts/flamegraph.sh | 20 ++++++- .../scripts/generate_vk_json.js | 57 ++++++++++++++++--- .../profiler/src/cli/gates_flamegraph_cmd.rs | 5 ++ .../tooling/profiler/src/gates_provider.rs | 3 +- 7 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 noir-projects/noir-protocol-circuits/mega_honk_circuits.json diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index a7a010f0ef2..2bed3bc75c2 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -678,16 +678,16 @@ void prove(const std::string& bytecodePath, const std::string& witnessPath, cons * * @param bytecodePath Path to the file containing the serialized circuit */ -void gateCount(const std::string& bytecodePath, bool honk_recursion) +template void gateCount(const std::string& bytecodePath, bool honk_recursion) { // All circuit reports will be built into the string below std::string functions_string = "{\"functions\": [\n "; auto constraint_systems = get_constraint_systems(bytecodePath, honk_recursion); size_t i = 0; for (auto constraint_system : constraint_systems) { - acir_proofs::AcirComposer acir_composer(0, verbose_logging); - acir_composer.create_circuit(constraint_system, {}, true); - auto circuit_size = acir_composer.get_total_circuit_size(); + auto builder = acir_format::create_circuit( + constraint_system, 0, {}, honk_recursion, std::make_shared(), true); + auto circuit_size = builder.get_total_circuit_size(); // Build individual circuit report std::string gates_per_opcode_str; @@ -1422,7 +1422,9 @@ int main(int argc, char* argv[]) auto tube_vk_path = output_path + "/vk"; return verify_honk(tube_proof_path, tube_vk_path) ? 0 : 1; } else if (command == "gates") { - gateCount(bytecode_path, honk_recursion); + gateCount(bytecode_path, honk_recursion); + } else if (command == "gates_mega_honk") { + gateCount(bytecode_path, honk_recursion); } else if (command == "verify") { return verify(proof_path, vk_path) ? 0 : 1; } else if (command == "contract") { diff --git a/noir-projects/gates_report.sh b/noir-projects/gates_report.sh index ff638e59ac6..8df0443aae4 100755 --- a/noir-projects/gates_report.sh +++ b/noir-projects/gates_report.sh @@ -15,11 +15,28 @@ echo "{\"programs\": [" > gates_report.json # Bound for checking where to place last parentheses NUM_ARTIFACTS=$(ls -1q "$PROTOCOL_CIRCUITS_DIR/target"/*.json | wc -l) +MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' mega_honk_circuits.json) + ITER="1" for pathname in "$PROTOCOL_CIRCUITS_DIR/target"/*.json; do ARTIFACT_NAME=$(basename -s .json "$pathname") - GATES_INFO=$($BB_BIN gates -h -b "./target/$ARTIFACT_NAME.json") + # Check if the current artifact is a mega honk circuit + IS_MEGA_HONK_CIRCUIT="false" + for pattern in $MEGA_HONK_CIRCUIT_PATTERNS; do + if echo "$ARTIFACT_NAME" | grep -qE "$pattern"; then + IS_MEGA_HONK_CIRCUIT="true" + break + fi + done + + # If it's mega honk, we need to use the gates_mega_honk command + if [ "$IS_MEGA_HONK_CIRCUIT" = "true" ]; then + GATES_INFO=$($BB_BIN gates_mega_honk -h -b "$pathname") + else + GATES_INFO=$($BB_BIN gates -h -b "$pathname") + fi + MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"') echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json diff --git a/noir-projects/noir-protocol-circuits/mega_honk_circuits.json b/noir-projects/noir-protocol-circuits/mega_honk_circuits.json new file mode 100644 index 00000000000..812e1aba8b5 --- /dev/null +++ b/noir-projects/noir-protocol-circuits/mega_honk_circuits.json @@ -0,0 +1,4 @@ +[ + "private_kernel.*", + "empty_nested" +] \ No newline at end of file diff --git a/noir-projects/noir-protocol-circuits/scripts/flamegraph.sh b/noir-projects/noir-protocol-circuits/scripts/flamegraph.sh index 55bbcd38504..131fea270ca 100755 --- a/noir-projects/noir-protocol-circuits/scripts/flamegraph.sh +++ b/noir-projects/noir-protocol-circuits/scripts/flamegraph.sh @@ -71,8 +71,26 @@ fi DEST="$SCRIPT_DIR/../dest" mkdir -p $DEST +MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' "$SCRIPT_DIR/../mega_honk_circuits.json") + +# Check if the target circuit is a mega honk circuit. +ARTIFACT_FILE_NAME=$(basename -s .json "$ARTIFACT") + +IS_MEGA_HONK_CIRCUIT="false" +for pattern in $MEGA_HONK_CIRCUIT_PATTERNS; do + if echo "$ARTIFACT_FILE_NAME" | grep -qE "$pattern"; then + IS_MEGA_HONK_CIRCUIT="true" + break + fi +done + # At last, generate the flamegraph. -$PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb" --output "$DEST" -- -h +# If it's a mega honk circuit, we need to set the backend_gates_command argument to "gates_mega_honk". +if [ "$IS_MEGA_HONK_CIRCUIT" = "true" ]; then + $PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb" --output "$DEST" --backend-gates-command "gates_mega_honk" -- -h +else + $PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb" --output "$DEST" -- -h +fi # Serve the file over http if -s is set. if $SERVE; then diff --git a/noir-projects/noir-protocol-circuits/scripts/generate_vk_json.js b/noir-projects/noir-protocol-circuits/scripts/generate_vk_json.js index 8071d782d5b..a4ac5dc4b5b 100644 --- a/noir-projects/noir-protocol-circuits/scripts/generate_vk_json.js +++ b/noir-projects/noir-protocol-circuits/scripts/generate_vk_json.js @@ -8,7 +8,11 @@ const { S3 } = require("@aws-sdk/client-s3"); const crypto = require("crypto"); -const BB_BIN_PATH = process.env.BB_BIN || "../../barretenberg/cpp/build/bin/bb"; +const megaHonkPatterns = require("../mega_honk_circuits.json"); + +const BB_BIN_PATH = + process.env.BB_BIN || + path.join(__dirname, "../../../barretenberg/cpp/build/bin/bb"); const BUCKET_NAME = "aztec-ci-artifacts"; const PREFIX = "protocol"; @@ -49,10 +53,25 @@ function getBarretenbergHash() { }); } -async function getNewArtifactHash(artifactPath, outputFolder, artifactName) { +function generateArtifactHash(barretenbergHash, bytecodeHash, isMegaHonk) { + return `${barretenbergHash}-${bytecodeHash}-${ + isMegaHonk ? "mega-honk" : "ultra-honk" + }`; +} + +async function getNewArtifactHash( + artifactPath, + outputFolder, + artifactName, + isMegaHonk +) { const bytecodeHash = await getBytecodeHash(artifactPath); const barretenbergHash = await getBarretenbergHash(); - const artifactHash = `${barretenbergHash}-${bytecodeHash}`; + const artifactHash = generateArtifactHash( + barretenbergHash, + bytecodeHash, + isMegaHonk + ); const vkDataPath = vkDataFileNameForArtifactName(outputFolder, artifactName); try { @@ -72,13 +91,23 @@ async function getNewArtifactHash(artifactPath, outputFolder, artifactName) { return artifactHash; } +function isMegaHonkCircuit(artifactName) { + // TODO Uncomment when mega honk vks are supported in the protocol + // return megaHonkPatterns.some((pattern) => + // artifactName.match(new RegExp(pattern)) + // ); + return false; +} + async function processArtifact(artifactPath, outputFolder) { const artifactName = path.basename(artifactPath, ".json"); + const isMegaHonk = isMegaHonkCircuit(artifactName); const artifactHash = await getNewArtifactHash( artifactPath, outputFolder, - artifactName + artifactName, + isMegaHonk ); if (!artifactHash) { console.log("Reusing on disk vk for", artifactName); @@ -92,7 +121,8 @@ async function processArtifact(artifactPath, outputFolder) { artifactName, outputFolder, artifactPath, - artifactHash + artifactHash, + isMegaHonk ); await writeVKToS3(artifactName, artifactHash, JSON.stringify(vkData)); } else { @@ -109,9 +139,14 @@ async function generateVKData( artifactName, outputFolder, artifactPath, - artifactHash + artifactHash, + isMegaHonk ) { - console.log("Generating new vk for", artifactName); + if (isMegaHonk) { + console.log("Generating new mega honk vk for", artifactName); + } else { + console.log("Generating new vk for", artifactName); + } const binaryVkPath = vkBinaryFileNameForArtifactName( outputFolder, @@ -119,8 +154,12 @@ async function generateVKData( ); const jsonVkPath = vkJsonFileNameForArtifactName(outputFolder, artifactName); - const writeVkCommand = `${BB_BIN_PATH} write_vk_ultra_honk -h -b "${artifactPath}" -o "${binaryVkPath}"`; - const vkAsFieldsCommand = `${BB_BIN_PATH} vk_as_fields_ultra_honk -k "${binaryVkPath}" -o "${jsonVkPath}"`; + const writeVkCommand = `${BB_BIN_PATH} ${ + isMegaHonk ? "write_vk_mega_honk" : "write_vk_ultra_honk" + } -h -b "${artifactPath}" -o "${binaryVkPath}"`; + const vkAsFieldsCommand = `${BB_BIN_PATH} ${ + isMegaHonk ? "vk_as_fields_mega_honk" : "vk_as_fields_ultra_honk" + } -k "${binaryVkPath}" -o "${jsonVkPath}"`; await new Promise((resolve, reject) => { child_process.exec(`${writeVkCommand} && ${vkAsFieldsCommand}`, (err) => { diff --git a/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs b/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs index 98e89e42015..f465408a8d8 100644 --- a/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs +++ b/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs @@ -19,6 +19,10 @@ pub(crate) struct GatesFlamegraphCommand { #[clap(long, short)] backend_path: String, + /// Command to get a gates report from the backend. Defaults to "gates" + #[clap(long, short, default_value = "gates")] + backend_gates_command: String, + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] backend_extra_args: Vec, @@ -32,6 +36,7 @@ pub(crate) fn run(args: GatesFlamegraphCommand) -> eyre::Result<()> { &PathBuf::from(args.artifact_path), &BackendGatesProvider { backend_path: PathBuf::from(args.backend_path), + gates_command: args.backend_gates_command, extra_args: args.backend_extra_args, }, &InfernoFlamegraphGenerator { count_name: "gates".to_string() }, diff --git a/noir/noir-repo/tooling/profiler/src/gates_provider.rs b/noir/noir-repo/tooling/profiler/src/gates_provider.rs index f96b1292987..3f07f3e4be6 100644 --- a/noir/noir-repo/tooling/profiler/src/gates_provider.rs +++ b/noir/noir-repo/tooling/profiler/src/gates_provider.rs @@ -10,6 +10,7 @@ pub(crate) trait GatesProvider { pub(crate) struct BackendGatesProvider { pub(crate) backend_path: PathBuf, + pub(crate) gates_command: String, pub(crate) extra_args: Vec, } @@ -17,7 +18,7 @@ impl GatesProvider for BackendGatesProvider { fn get_gates(&self, artifact_path: &Path) -> eyre::Result { let mut backend_gates_cmd = Command::new(&self.backend_path); - backend_gates_cmd.arg("gates").arg("-b").arg(artifact_path); + backend_gates_cmd.arg(self.gates_command.clone()).arg("-b").arg(artifact_path); for arg in &self.extra_args { backend_gates_cmd.arg(arg);