feat: Report gates and VKs of private protocol circuits with megahonk (…

…#7722)

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -678,16 +678,16 @@ void prove(const std::string& bytecodePath, const std::string& witnessPath, cons
  *
  * @param bytecodePath Path to the file containing the serialized circuit
  */
-void gateCount(const std::string& bytecodePath, bool honk_recursion)
+template <typename Builder = UltraCircuitBuilder> void gateCount(const std::string& bytecodePath, bool honk_recursion)
 {
     // All circuit reports will be built into the string below
     std::string functions_string = "{\"functions\": [\n  ";
     auto constraint_systems = get_constraint_systems(bytecodePath, honk_recursion);
     size_t i = 0;
     for (auto constraint_system : constraint_systems) {
-        acir_proofs::AcirComposer acir_composer(0, verbose_logging);
-        acir_composer.create_circuit(constraint_system, {}, true);
-        auto circuit_size = acir_composer.get_total_circuit_size();
+        auto builder = acir_format::create_circuit<Builder>(
+            constraint_system, 0, {}, honk_recursion, std::make_shared<bb::ECCOpQueue>(), true);
+        auto circuit_size = builder.get_total_circuit_size();
 
         // Build individual circuit report
         std::string gates_per_opcode_str;
@@ -1422,7 +1422,9 @@ int main(int argc, char* argv[])
             auto tube_vk_path = output_path + "/vk";
             return verify_honk<UltraFlavor>(tube_proof_path, tube_vk_path) ? 0 : 1;
         } else if (command == "gates") {
-            gateCount(bytecode_path, honk_recursion);
+            gateCount<UltraCircuitBuilder>(bytecode_path, honk_recursion);
+        } else if (command == "gates_mega_honk") {
+            gateCount<MegaCircuitBuilder>(bytecode_path, honk_recursion);
         } else if (command == "verify") {
             return verify(proof_path, vk_path) ? 0 : 1;
         } else if (command == "contract") {

noir-projects/gates_report.sh

@@ -15,11 +15,28 @@ echo "{\"programs\": [" > gates_report.json
 # Bound for checking where to place last parentheses
 NUM_ARTIFACTS=$(ls -1q "$PROTOCOL_CIRCUITS_DIR/target"/*.json | wc -l)
 
+MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' mega_honk_circuits.json)
+
 ITER="1"
 for pathname in "$PROTOCOL_CIRCUITS_DIR/target"/*.json; do
     ARTIFACT_NAME=$(basename -s .json "$pathname")
 
-    GATES_INFO=$($BB_BIN gates -h -b "./target/$ARTIFACT_NAME.json")
+    # Check if the current artifact is a mega honk circuit
+    IS_MEGA_HONK_CIRCUIT="false"
+    for pattern in $MEGA_HONK_CIRCUIT_PATTERNS; do
+        if echo "$ARTIFACT_NAME" | grep -qE "$pattern"; then
+            IS_MEGA_HONK_CIRCUIT="true"
+            break
+        fi
+    done
+
+    # If it's mega honk, we need to use the gates_mega_honk command    
+    if [ "$IS_MEGA_HONK_CIRCUIT" = "true" ]; then
+        GATES_INFO=$($BB_BIN gates_mega_honk -h -b "$pathname")
+    else
+        GATES_INFO=$($BB_BIN gates -h -b "$pathname")
+    fi
+
     MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"')
     echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json
 

noir-projects/noir-protocol-circuits/mega_honk_circuits.json

@@ -0,0 +1,4 @@
+[
+  "private_kernel.*",
+  "empty_nested"
+]

noir-projects/noir-protocol-circuits/scripts/flamegraph.sh

@@ -71,8 +71,26 @@ fi
 DEST="$SCRIPT_DIR/../dest"
 mkdir -p $DEST
 
+MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' "$SCRIPT_DIR/../mega_honk_circuits.json")
+
+# Check if the target circuit is a mega honk circuit.
+ARTIFACT_FILE_NAME=$(basename -s .json "$ARTIFACT")
+
+IS_MEGA_HONK_CIRCUIT="false"
+for pattern in $MEGA_HONK_CIRCUIT_PATTERNS; do
+    if echo "$ARTIFACT_FILE_NAME" | grep -qE "$pattern"; then
+        IS_MEGA_HONK_CIRCUIT="true"
+        break
+    fi
+done
+
 # At last, generate the flamegraph.
-$PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb"  --output "$DEST" -- -h
+# If it's a mega honk circuit, we need to set the backend_gates_command argument to "gates_mega_honk".
+if [ "$IS_MEGA_HONK_CIRCUIT" = "true" ]; then
+    $PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb"  --output "$DEST" --backend-gates-command "gates_mega_honk" -- -h
+else
+    $PROFILER gates-flamegraph --artifact-path "${ARTIFACT}" --backend-path "$SCRIPT_DIR/../../../barretenberg/cpp/build/bin/bb"  --output "$DEST" -- -h
+fi
 
 # Serve the file over http if -s is set.
 if $SERVE; then

noir-projects/noir-protocol-circuits/scripts/generate_vk_json.js

@@ -8,7 +8,11 @@ const { S3 } = require("@aws-sdk/client-s3");
 
 const crypto = require("crypto");
 
-const BB_BIN_PATH = process.env.BB_BIN || "../../barretenberg/cpp/build/bin/bb";
+const megaHonkPatterns = require("../mega_honk_circuits.json");
+
+const BB_BIN_PATH =
+  process.env.BB_BIN ||
+  path.join(__dirname, "../../../barretenberg/cpp/build/bin/bb");
 const BUCKET_NAME = "aztec-ci-artifacts";
 const PREFIX = "protocol";
 
@@ -49,10 +53,25 @@ function getBarretenbergHash() {
   });
 }
 
-async function getNewArtifactHash(artifactPath, outputFolder, artifactName) {
+function generateArtifactHash(barretenbergHash, bytecodeHash, isMegaHonk) {
+  return `${barretenbergHash}-${bytecodeHash}-${
+    isMegaHonk ? "mega-honk" : "ultra-honk"
+  }`;
+}
+
+async function getNewArtifactHash(
+  artifactPath,
+  outputFolder,
+  artifactName,
+  isMegaHonk
+) {
   const bytecodeHash = await getBytecodeHash(artifactPath);
   const barretenbergHash = await getBarretenbergHash();
-  const artifactHash = `${barretenbergHash}-${bytecodeHash}`;
+  const artifactHash = generateArtifactHash(
+    barretenbergHash,
+    bytecodeHash,
+    isMegaHonk
+  );
 
   const vkDataPath = vkDataFileNameForArtifactName(outputFolder, artifactName);
   try {
@@ -72,13 +91,23 @@ async function getNewArtifactHash(artifactPath, outputFolder, artifactName) {
   return artifactHash;
 }
 
+function isMegaHonkCircuit(artifactName) {
+  // TODO Uncomment when mega honk vks are supported in the protocol
+  // return megaHonkPatterns.some((pattern) =>
+  //   artifactName.match(new RegExp(pattern))
+  // );
+  return false;
+}
+
 async function processArtifact(artifactPath, outputFolder) {
   const artifactName = path.basename(artifactPath, ".json");
+  const isMegaHonk = isMegaHonkCircuit(artifactName);
 
   const artifactHash = await getNewArtifactHash(
     artifactPath,
     outputFolder,
-    artifactName
+    artifactName,
+    isMegaHonk
   );
   if (!artifactHash) {
     console.log("Reusing on disk vk for", artifactName);
@@ -92,7 +121,8 @@ async function processArtifact(artifactPath, outputFolder) {
       artifactName,
       outputFolder,
       artifactPath,
-      artifactHash
+      artifactHash,
+      isMegaHonk
     );
     await writeVKToS3(artifactName, artifactHash, JSON.stringify(vkData));
   } else {
@@ -109,18 +139,27 @@ async function generateVKData(
   artifactName,
   outputFolder,
   artifactPath,
-  artifactHash
+  artifactHash,
+  isMegaHonk
 ) {
-  console.log("Generating new vk for", artifactName);
+  if (isMegaHonk) {
+    console.log("Generating new mega honk vk for", artifactName);
+  } else {
+    console.log("Generating new vk for", artifactName);
+  }
 
   const binaryVkPath = vkBinaryFileNameForArtifactName(
     outputFolder,
     artifactName
   );
   const jsonVkPath = vkJsonFileNameForArtifactName(outputFolder, artifactName);
 
-  const writeVkCommand = `${BB_BIN_PATH} write_vk_ultra_honk -h -b "${artifactPath}" -o "${binaryVkPath}"`;
-  const vkAsFieldsCommand = `${BB_BIN_PATH} vk_as_fields_ultra_honk -k "${binaryVkPath}" -o "${jsonVkPath}"`;
+  const writeVkCommand = `${BB_BIN_PATH} ${
+    isMegaHonk ? "write_vk_mega_honk" : "write_vk_ultra_honk"
+  } -h -b "${artifactPath}" -o "${binaryVkPath}"`;
+  const vkAsFieldsCommand = `${BB_BIN_PATH} ${
+    isMegaHonk ? "vk_as_fields_mega_honk" : "vk_as_fields_ultra_honk"
+  } -k "${binaryVkPath}" -o "${jsonVkPath}"`;
 
   await new Promise((resolve, reject) => {
     child_process.exec(`${writeVkCommand} && ${vkAsFieldsCommand}`, (err) => {

noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs

@@ -19,6 +19,10 @@ pub(crate) struct GatesFlamegraphCommand {
     #[clap(long, short)]
     backend_path: String,
 
+    /// Command to get a gates report from the backend. Defaults to "gates"
+    #[clap(long, short, default_value = "gates")]
+    backend_gates_command: String,
+
     #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
     backend_extra_args: Vec<String>,
 
@@ -32,6 +36,7 @@ pub(crate) fn run(args: GatesFlamegraphCommand) -> eyre::Result<()> {
         &PathBuf::from(args.artifact_path),
         &BackendGatesProvider {
             backend_path: PathBuf::from(args.backend_path),
+            gates_command: args.backend_gates_command,
             extra_args: args.backend_extra_args,
         },
         &InfernoFlamegraphGenerator { count_name: "gates".to_string() },

noir/noir-repo/tooling/profiler/src/gates_provider.rs

@@ -10,14 +10,15 @@ pub(crate) trait GatesProvider {
 
 pub(crate) struct BackendGatesProvider {
     pub(crate) backend_path: PathBuf,
+    pub(crate) gates_command: String,
     pub(crate) extra_args: Vec<String>,
 }
 
 impl GatesProvider for BackendGatesProvider {
     fn get_gates(&self, artifact_path: &Path) -> eyre::Result<BackendGatesResponse> {
         let mut backend_gates_cmd = Command::new(&self.backend_path);
 
-        backend_gates_cmd.arg("gates").arg("-b").arg(artifact_path);
+        backend_gates_cmd.arg(self.gates_command.clone()).arg("-b").arg(artifact_path);
 
         for arg in &self.extra_args {
             backend_gates_cmd.arg(arg);