Add x509-limbo patch and reporting tool

.github/workflows/actions-ci.yml

@@ -591,3 +591,21 @@ jobs:
           run: |
             sudo pkg install -y git gmake cmake go ninja
             tests/ci/run_bsd_tests.sh
+  # Temporary to test the x509-limbo patch and building of the reporting tool.
+  # This will move into a separate project in the next PR. But doing this for now to
+  # cutdown the review size.
+  x509-limbo-tooling:
+    if: github.repository_owner == 'aws'
+    needs: [sanity-test-run]
+    name: x509-limbo tooling
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13' 
+      - name: Verify x509-limbo patch and reporting tool
+        run: |
+          ./tests/ci/run_x509_limbo.sh

tests/ci/run_x509_limbo.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+set -euxo pipefail
+
+source tests/ci/common_posix_setup.sh
+
+# For now we will just verify that the patch applies and our reporting to builds and tests successfully
+# Subsequent follow-up PRs will wire this up into a new CodeBuild project and handle producing and tracking
+# the reports.
+
+SCRATCH_DIR="${SYS_ROOT}/scratch"
+X509_CI_DIR="${SRC_ROOT}/tests/ci/x509"
+X509_LIMBO_SRC="${SCRATCH_DIR}/x509-limbo"
+
+function build_reporting_tool() {
+    pushd "${X509_CI_DIR}/limbo-report"
+    make
+    mv ./limbo-report "${SCRATCH_DIR}/"
+    popd # "${X509_CI_DIR}/limbo-report"
+}
+
+function clone_and_patch_x509_limbo() {
+    git clone https://github.com/C2SP/x509-limbo.git "${X509_LIMBO_SRC}"
+    pushd "${X509_LIMBO_SRC}"
+    patch -p1 -i "${X509_CI_DIR}/x509-limbo.patch"
+    popd # "${X509_LIMBO_SRC}"
+}
+
+function run_aws_lc_harness() {
+    pushd "${X509_LIMBO_SRC}"
+    AWS_LC_SRC_DIR="${SRC_ROOT}" make test-aws-lc
+    popd # "${X509_LIMBO_SRC}"
+}
+
+mkdir -p "${SCRATCH_DIR}"
+rm -rf "${SCRATCH_DIR:?}"/*
+pushd "${SCRATCH_DIR}"
+
+build_reporting_tool
+clone_and_patch_x509_limbo
+run_aws_lc_harness
+
+popd # "${SCRATCH_DIR}"
+# rm -rf "${SCRATCH_DIR:?}"

tests/ci/x509/limbo-report/.gitignore

@@ -0,0 +1 @@
+limbo-report

tests/ci/x509/limbo-report/Makefile

@@ -0,0 +1,15 @@
+.PHONY: all build test fmt vet
+
+all: build vet test
+
+build:
+	go build ./...
+
+test:
+	go test ./...
+
+fmt:
+	go fmt ./...
+
+vet:
+	go vet ./...

tests/ci/x509/limbo-report/annotate.go

@@ -0,0 +1,122 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sort"
+)
+
+var annotateHelpDoc string = `report annotate [-csv] <limboFile> <resultFile>
+
+Annotates a standard x509-limbo results file with information from the provided <limboFile> test descriptors.
+By default this will write the result back to the process standard output.
+
+Options:
+-csv Write the results to standard output in csv format rather then the default json format
+`
+
+var annotateCommand struct {
+	formatAsCsv bool
+}
+
+var annotateFlagSet = func() *flag.FlagSet {
+	fs := flag.NewFlagSet("annotate", flag.ExitOnError)
+	fs.BoolVar(&annotateCommand.formatAsCsv, "csv", false, "format output as csv rather then the default")
+	fs.Usage = func() {
+		fmt.Fprint(fs.Output(), annotateHelpDoc)
+	}
+
+	return fs
+}()
+
+func runAnnotateCommand(args []string) error {
+	if err := annotateFlagSet.Parse(args); err != nil {
+		return err
+	}
+
+	if len(annotateFlagSet.Args()) != 2 {
+		return fmt.Errorf("expect two positional arguments")
+	}
+
+	limbo, err := parseLimboFile(annotateFlagSet.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	harnessFilePath := annotateFlagSet.Arg(1)
+	harnessBytes, err := os.ReadFile(harnessFilePath)
+	if err != nil {
+		log.Fatalf("failed to read harnessFile(%v): %v", harnessFilePath, err)
+	}
+
+	var hr HarnessResults
+	if err := json.Unmarshal(harnessBytes, &hr); err != nil {
+		log.Fatalf("failed to parse json: %v", err)
+	}
+	if err := hr.Annotate(limbo); err != nil {
+		return err
+	}
+
+	if annotateCommand.formatAsCsv {
+		return writeCsvJudgementReport(&hr, os.Stdout)
+	}
+
+	return writeJsonJudgementReport(&hr, os.Stdout)
+}
+
+func parseLimboFile(filePath string) (limbo *Limbo, err error) {
+	testCasesBytes, err := os.ReadFile(filePath)
+	if err != nil {
+		return limbo, fmt.Errorf("failed to read limbo test cases: %w", err)
+	}
+
+	var unmarshaled Limbo
+
+	if err := json.Unmarshal(testCasesBytes, &unmarshaled); err != nil {
+		return limbo, fmt.Errorf("failed to parse json: %w", err)
+	}
+
+	limbo = &unmarshaled
+
+	return limbo, nil
+}
+
+func writeJsonJudgementReport(hr *HarnessResults, out io.Writer) error {
+	jsonWriter := json.NewEncoder(out)
+	jsonWriter.SetIndent("", " ")
+
+	if err := jsonWriter.Encode(hr); err != nil {
+		return fmt.Errorf("failed to encode harness judgement: %w", err)
+	}
+
+	return nil
+}
+
+func writeCsvJudgementReport(hr *HarnessResults, out io.Writer) error {
+	csvWriter := csv.NewWriter(out)
+	defer csvWriter.Flush()
+
+	csvWriter.Write([]string{"id", "expected", "result", "judgement", "context"})
+
+	// Normalize The Report for Diffing
+	var sortedIds []string
+	for id := range hr.Results {
+		sortedIds = append(sortedIds, id)
+	}
+	sort.Strings(sortedIds)
+
+	for _, id := range sortedIds {
+		tr := hr.Results[id]
+		csvWriter.Write([]string{id, string(tr.ExpectedResult), string(tr.ActualResult), string(tr.Judgement), tr.Context})
+	}
+
+	return nil
+}