feat: configurable external check failures

.github/scripts/check_test_results.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -eu
+
+# Usage: ./check_test_results.sh <expected.json> <actual.jsonl>
+# Compares the output of two test results of the same repository.
+# If any of the files doesn't exist or is empty, the script will consider that the test suite
+# couldn't be compiled.
+
+function process_json_lines() {
+  cat $1 | jq -c 'select(.type == "test" and .event == "failed") | {suite: .suite, name: .name, status: .event}' | jq -s -c 'sort_by(.suite, .name) | .[]' > $1.jq
+}
+
+if [ -f $1 ] && [ -f $2 ]; then
+  # Both files exist, let's compare them
+  $(process_json_lines $1)
+  $(process_json_lines $2)
+  diff $1.jq $2.jq
+elif [ -f $1 ]; then
+  # Only the expected file exists, which means the actual test couldn't be compiled.
+  echo "Error: external library tests couldn't be compiled."
+  echo "You could rename '$1' to '$1.does_not_compile' if it's expected that the external library can't be compiled."
+  exit -1
+elif [ -f $2 ]; then
+  # Only the actual file exists, which means we are expecting the external library
+  # not to compile but it did.
+  echo "Error: expected external library not to compile, but it did."
+  echo "You could create '$1' with these contents:"
+  $(process_json_lines $2)
+  cat $2.jq
+  exit -1
+else
+  # Both files don't exists, which means we are expecting the external library not
+  # to compile, and it didn't, so all is good.
+  exit 0
+fi

.github/workflows/test-js-packages.yml

@@ -557,11 +557,17 @@ jobs:
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-lib }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-lib }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/reset-kernel-lib }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-lib }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/types }
+          # This one currently doesn't compile (it runs out of memory or it takes too long)
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-lib, nargo_args: "--test-threads 1" }
 
     name: Check external repo - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: noir-repo
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -592,9 +598,14 @@ jobs:
 
       - name: Run nargo test
         working-directory: ./test-repo/${{ matrix.project.path }}
-        run: nargo test -q --silence-warnings
+        run: |
+          nargo test --silence-warnings --format json ${{ matrix.project.nargo_args }} > ${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl
         env:
           NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true
+
+      - name: Compare test results
+        working-directory: ./noir-repo
+        run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.expected.jsonl .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl
 
   # This is a job which depends on all test jobs and reports the overall status.
   # This allows us to add/remove test jobs without having to update the required workflows.

scripts/check-critical-libraries.sh

@@ -31,7 +31,7 @@ for REPO in ${REPOS_TO_CHECK[@]}; do
     TAG=$(getLatestReleaseTagForRepo $REPO)
     git clone $REPO -c advice.detachedHead=false --depth 1 --branch $TAG $TMP_DIR
 
-    nargo test -q --program-dir $TMP_DIR
+    nargo test --program-dir $TMP_DIR --format json
 
     rm -rf $TMP_DIR
 done

tooling/nargo_cli/src/cli/test_cmd/formatters.rs

@@ -417,6 +417,7 @@ impl Formatter for JsonFormatter {
         let mut json = Map::new();
         json.insert("type".to_string(), json!("test"));
         json.insert("name".to_string(), json!(&test_result.name));
+        json.insert("suite".to_string(), json!(&test_result.package_name));
         json.insert("exec_time".to_string(), json!(test_result.time_to_run.as_secs_f64()));
 
         let mut stdout = String::new();