feat: track spans

cspell.json

@@ -222,6 +222,7 @@
     "rushstack",
     "schnorr",
     "secp",
+    "SEMRESATTRS",
     "sigchld",
     "Signerless",
     "siloes",

docker-compose.yml

@@ -127,6 +127,13 @@ services:
       - source: grafana-sources
         target: /etc/grafana/provisioning/datasources/default.yml
 
+  jaeger:
+    image: jaegertracing/all-in-one
+    ports:
+      - 16686:16686
+    profiles:
+      - metrics
+
 volumes:
   aztec:
   grafana:
@@ -171,9 +178,17 @@ configs:
         prometheus:
           endpoint: 0.0.0.0:8889
           metric_expiration: 5m
+        otlp/jaeger:
+            endpoint: "jaeger:4317"
+            tls:
+              insecure: true
 
       service:
         pipelines:
+          traces:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [otlp/jaeger]
           metrics:
             receivers: [otlp]
             processors: [batch]

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -203,6 +203,7 @@ export class AztecNodeService implements AztecNode {
           archiver,
           prover!,
           simulationProvider,
+          telemetry,
         );
 
     return new AztecNodeService(
@@ -760,6 +761,7 @@ export class AztecNodeService implements AztecNode {
       merkleTrees.asLatest(),
       this.contractDataSource,
       new WASMSimulator(),
+      this.telemetry,
     );
     const processor = await publicProcessorFactory.create(prevHeader, newGlobalVariables);
     // REFACTOR: Consider merging ProcessReturnValues into ProcessedTx

yarn-project/bb-prover/src/instrumentation.ts

@@ -6,6 +6,7 @@ import {
   type Histogram,
   Metrics,
   type TelemetryClient,
+  type Tracer,
   ValueType,
 } from '@aztec/telemetry-client';
 
@@ -21,7 +22,10 @@ export class ProverInstrumentation {
   private circuitSize: Gauge;
   private circuitPublicInputCount: Gauge;
 
+  public readonly tracer: Tracer;
+
   constructor(telemetry: TelemetryClient, name: string = 'bb-prover') {
+    this.tracer = telemetry.getTracer(name);
     const meter = telemetry.getMeter(name);
     const msBuckets = [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 90_000];
     this.simulationDuration = meter.createHistogram(Metrics.CIRCUIT_SIMULATION_DURATION, {

yarn-project/bb-prover/src/prover/bb_prover.ts

@@ -57,7 +57,7 @@ import {
   convertRootRollupOutputsFromWitnessMap,
 } from '@aztec/noir-protocol-circuits-types';
 import { NativeACVMSimulator } from '@aztec/simulator';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { trackSpan, type TelemetryClient, Attributes } from '@aztec/telemetry-client';
 
 import { abiEncode } from '@noir-lang/noirc_abi';
 import { type Abi, type WitnessMap } from '@noir-lang/types';
@@ -110,6 +110,10 @@ export class BBNativeRollupProver implements ServerCircuitProver {
     this.instrumentation = new ProverInstrumentation(telemetry);
   }
 
+  get tracer() {
+    return this.instrumentation.tracer;
+  }
+
   static async new(config: BBProverConfig, telemetry: TelemetryClient) {
     await fs.access(config.acvmBinaryPath, fs.constants.R_OK);
     await fs.mkdir(config.acvmWorkingDirectory, { recursive: true });
@@ -126,6 +130,7 @@ export class BBNativeRollupProver implements ServerCircuitProver {
    * @param inputs - Inputs to the circuit.
    * @returns The public inputs of the parity circuit.
    */
+  @trackSpan('getBaseParityProof', { [Attributes.PROTOCOL_CIRCUIT_NAME]: 'base-parity' })
   public async getBaseParityProof(inputs: BaseParityInputs): Promise<RootParityInput<typeof RECURSIVE_PROOF_LENGTH>> {
     const { circuitOutput, proof } = await this.createRecursiveProof(
       inputs,
@@ -147,6 +152,7 @@ export class BBNativeRollupProver implements ServerCircuitProver {
    * @param inputs - Inputs to the circuit.
    * @returns The public inputs of the parity circuit.
    */
+  @trackSpan('getBaseParityProof', { [Attributes.PROTOCOL_CIRCUIT_NAME]: 'root-parity' })
   public async getRootParityProof(
     inputs: RootParityInputs,
   ): Promise<RootParityInput<typeof NESTED_RECURSIVE_PROOF_LENGTH>> {
@@ -170,6 +176,9 @@ export class BBNativeRollupProver implements ServerCircuitProver {
    * @param inputs - The inputs to the AVM circuit.
    * @returns The proof.
    */
+  @trackSpan('getAvmProof', (inputs) => ({
+    [Attributes.APP_CIRCUIT_NAME]: inputs.functionName,
+  }))
   public async getAvmProof(inputs: AvmCircuitInputs): Promise<ProofAndVerificationKey> {
     const proofAndVk = await this.createAvmProof(inputs);
     await this.verifyAvmProof(proofAndVk.proof, proofAndVk.verificationKey);
@@ -181,6 +190,9 @@ export class BBNativeRollupProver implements ServerCircuitProver {
    * @param kernelRequest - The object encapsulating the request for a proof
    * @returns The requested circuit's public inputs and proof
    */
+  @trackSpan('getPublicKernelProof', (kernelReq) => ({
+    [Attributes.PROTOCOL_CIRCUIT_NAME]: mapProtocolArtifactNameToCircuitName(PublicKernelArtifactMapping[kernelReq.type]!.artifact),
+  }))
   public async getPublicKernelProof(
     kernelRequest: PublicKernelNonTailRequest,
   ): Promise<PublicInputsAndRecursiveProof<PublicKernelCircuitPublicInputs>> {

yarn-project/bb-prover/src/test/test_circuit_prover.ts

@@ -57,7 +57,7 @@ import {
   convertSimulatedPublicTailOutputFromWitnessMap,
 } from '@aztec/noir-protocol-circuits-types';
 import { type SimulationProvider, WASMSimulator, emitCircuitSimulationStats } from '@aztec/simulator';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { type TelemetryClient, trackSpan } from '@aztec/telemetry-client';
 
 import { ProverInstrumentation } from '../instrumentation.js';
 import { SimulatedPublicKernelArtifactMapping } from '../mappings/mappings.js';
@@ -93,6 +93,10 @@ export class TestCircuitProver implements ServerCircuitProver {
     this.instrumentation = new ProverInstrumentation(telemetry);
   }
 
+  get tracer() {
+    return this.instrumentation.tracer;
+  }
+
   public async getEmptyPrivateKernelProof(
     inputs: PrivateKernelEmptyInputData,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {
@@ -117,6 +121,7 @@ export class TestCircuitProver implements ServerCircuitProver {
    * @param inputs - Inputs to the circuit.
    * @returns The public inputs of the parity circuit.
    */
+  @trackSpan('getBaseParityProof')
   public async getBaseParityProof(inputs: BaseParityInputs): Promise<RootParityInput<typeof RECURSIVE_PROOF_LENGTH>> {
     const timer = new Timer();
     const witnessMap = convertBaseParityInputsToWitnessMap(inputs);
@@ -149,6 +154,7 @@ export class TestCircuitProver implements ServerCircuitProver {
    * @param inputs - Inputs to the circuit.
    * @returns The public inputs of the parity circuit.
    */
+  @trackSpan('getRootParityProof')
   public async getRootParityProof(
     inputs: RootParityInputs,
   ): Promise<RootParityInput<typeof NESTED_RECURSIVE_PROOF_LENGTH>> {
@@ -183,6 +189,7 @@ export class TestCircuitProver implements ServerCircuitProver {
    * @param input - Inputs to the circuit.
    * @returns The public inputs as outputs of the simulation.
    */
+  @trackSpan('getBaseRollupProof')
   public async getBaseRollupProof(
     input: BaseRollupInputs,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
@@ -213,6 +220,7 @@ export class TestCircuitProver implements ServerCircuitProver {
    * @param input - Inputs to the circuit.
    * @returns The public inputs as outputs of the simulation.
    */
+  @trackSpan('getMergeRollupProof')
   public async getMergeRollupProof(
     input: MergeRollupInputs,
   ): Promise<PublicInputsAndRecursiveProof<BaseOrMergeRollupPublicInputs>> {
@@ -244,6 +252,7 @@ export class TestCircuitProver implements ServerCircuitProver {
    * @param input - Inputs to the circuit.
    * @returns The public inputs as outputs of the simulation.
    */
+  @trackSpan('getRootRollupProof')
   public async getRootRollupProof(
     input: RootRollupInputs,
   ): Promise<PublicInputsAndRecursiveProof<RootRollupPublicInputs>> {
@@ -270,6 +279,7 @@ export class TestCircuitProver implements ServerCircuitProver {
     );
   }
 
+  @trackSpan('getPublicKernelProof')
   public async getPublicKernelProof(
     kernelRequest: PublicKernelNonTailRequest,
   ): Promise<PublicInputsAndRecursiveProof<PublicKernelCircuitPublicInputs>> {
@@ -303,6 +313,7 @@ export class TestCircuitProver implements ServerCircuitProver {
     );
   }
 
+  @trackSpan('getPublicTailProof')
   public async getPublicTailProof(
     kernelRequest: PublicKernelTailRequest,
   ): Promise<PublicInputsAndRecursiveProof<KernelCircuitPublicInputs>> {

yarn-project/foundation/.prettierrc.json

@@ -6,5 +6,5 @@
   "importOrder": ["^@aztec/(.*)$", "<THIRD_PARTY_MODULES>", "^\\./|\\.\\./"],
   "importOrderSeparation": true,
   "importOrderSortSpecifiers": true,
-  "importOrderParserPlugins": ["importAssertions", "typescript"]
+  "importOrderParserPlugins": ["importAssertions", "typescript", "decorators"]
 }

yarn-project/prover-client/src/mocks/test_context.ts

@@ -96,6 +96,7 @@ export class TestContext {
     const publicWorldStateDB = mock<WorldStatePublicDB>();
     const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator());
     const actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest());
+    const telemetry = new NoopTelemetryClient();
     const processor = new PublicProcessor(
       actualDb,
       publicExecutor,
@@ -104,6 +105,7 @@ export class TestContext {
       Header.empty(),
       publicContractsDB,
       publicWorldStateDB,
+      telemetry,
     );
 
     let localProver: ServerCircuitProver;
@@ -125,7 +127,7 @@ export class TestContext {
     }
 
     const queue = new MemoryProvingQueue();
-    const orchestrator = new ProvingOrchestrator(actualDb, queue);
+    const orchestrator = new ProvingOrchestrator(actualDb, queue, telemetry);
     const agent = new ProverAgent(localProver, proverCount);
 
     queue.start();

yarn-project/prover-client/src/orchestrator/orchestrator.ts

@@ -53,6 +53,7 @@ import { createDebugLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { BufferReader, type Tuple } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
+import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client';
 import { type MerkleTreeOperations } from '@aztec/world-state';
 
 import { inspect } from 'util';
@@ -91,7 +92,16 @@ export class ProvingOrchestrator {
   private pendingProvingJobs: AbortController[] = [];
   private paddingTx: PaddingProcessedTx | undefined = undefined;
 
-  constructor(private db: MerkleTreeOperations, private prover: ServerCircuitProver, private initialHeader?: Header) {}
+  public readonly tracer: Tracer;
+
+  constructor(
+    private db: MerkleTreeOperations,
+    private prover: ServerCircuitProver,
+    telemetryClient: TelemetryClient,
+    private initialHeader?: Header,
+  ) {
+    this.tracer = telemetryClient.getTracer('ProvingOrchestrator');
+  }
 
   /**
    * Resets the orchestrator's cached padding tx.
@@ -108,6 +118,10 @@ export class ProvingOrchestrator {
    * @param verificationKeys - The private kernel verification keys
    * @returns A proving ticket, containing a promise notifying of proving completion
    */
+  @trackSpan('startNewBlock', (numTxs, globalVariables) => ({
+    [Attributes.BLOCK_SIZE]: numTxs,
+    [Attributes.BLOCK_NUMBER]: globalVariables.blockNumber.toNumber(),
+  }))
   public async startNewBlock(
     numTxs: number,
     globalVariables: GlobalVariables,
@@ -193,6 +207,9 @@ export class ProvingOrchestrator {
    * The interface to add a simulated transaction to the scheduler
    * @param tx - The transaction to be proven
    */
+  @trackSpan('addNewTx', tx => ({
+    [Attributes.TX_HASH]: tx.hash.toString(),
+  }))
   public async addNewTx(tx: ProcessedTx): Promise<void> {
     if (!this.provingState) {
       throw new Error(`Invalid proving state, call startNewBlock before adding transactions`);
@@ -213,6 +230,13 @@ export class ProvingOrchestrator {
   /**
    * Marks the block as full and pads it to the full power of 2 block size, no more transactions will be accepted.
    */
+  @trackSpan('setBlockCompleted', function () {
+    return {
+      [Attributes.BLOCK_NUMBER]: this.provingState!.globalVariables.blockNumber.toNumber(),
+      [Attributes.BLOCK_SIZE]: this.provingState!.totalNumTxs,
+      [Attributes.BLOCK_TXS_COUNT]: this.provingState!.transactionsReceived,
+    };
+  })
   public async setBlockCompleted() {
     if (!this.provingState) {
       throw new Error(`Invalid proving state, call startNewBlock before adding transactions or completing the block`);
@@ -319,6 +343,13 @@ export class ProvingOrchestrator {
    * Performs the final tree update for the block and returns the fully proven block.
    * @returns The fully proven block and proof.
    */
+  @trackSpan('finaliseBlock', function () {
+    return {
+      [Attributes.BLOCK_NUMBER]: this.provingState!.globalVariables.blockNumber.toNumber(),
+      [Attributes.BLOCK_TXS_COUNT]: this.provingState!.transactionsReceived,
+      [Attributes.BLOCK_SIZE]: this.provingState!.totalNumTxs,
+    };
+  })
   public async finaliseBlock() {
     try {
       if (
@@ -496,6 +527,9 @@ export class ProvingOrchestrator {
   }
 
   // Updates the merkle trees for a transaction. The first enqueued job for a transaction
+  @trackSpan('prepareBaseRollupInputs', (_, tx) => ({
+    [Attributes.TX_HASH]: tx.hash.toString(),
+  }))
   private async prepareBaseRollupInputs(
     provingState: ProvingState | undefined,
     tx: ProcessedTx,

yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts

@@ -30,7 +30,7 @@ describe('prover/orchestrator/failures', () => {
 
     beforeEach(() => {
       mockProver = new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator());
-      orchestrator = new ProvingOrchestrator(context.actualDb, mockProver);
+      orchestrator = new ProvingOrchestrator(context.actualDb, mockProver, new NoopTelemetryClient());
     });
 
     it.each([

yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts

@@ -143,7 +143,7 @@ describe('prover/orchestrator/lifecycle', () => {
 
     it('cancels proving requests', async () => {
       const prover: ServerCircuitProver = new TestCircuitProver(new NoopTelemetryClient());
-      const orchestrator = new ProvingOrchestrator(context.actualDb, prover);
+      const orchestrator = new ProvingOrchestrator(context.actualDb, prover, new NoopTelemetryClient());
 
       const spy = jest.spyOn(prover, 'getBaseParityProof');
       const deferredPromises: PromiseWithResolvers<any>[] = [];

yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts

@@ -11,6 +11,7 @@ import { makeGlobalVariables, makeRootParityInput } from '@aztec/circuits.js/tes
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
 import { openTmpStore } from '@aztec/kv-store/utils';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state';
 
 import { type MockProxy, mock } from 'jest-mock-extended';
@@ -25,7 +26,7 @@ describe('prover/orchestrator', () => {
     beforeEach(async () => {
       actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest());
       mockProver = mock<ServerCircuitProver>();
-      orchestrator = new ProvingOrchestrator(actualDb, mockProver);
+      orchestrator = new ProvingOrchestrator(actualDb, mockProver, new NoopTelemetryClient());
     });
 
     it('calls root parity circuit only when ready', async () => {

yarn-project/prover-client/src/tx-prover/tx-prover.ts

@@ -34,7 +34,12 @@ export class TxProver implements ProverClient {
     initialHeader?: Header,
   ) {
     this.queue = new MemoryProvingQueue(config.proverJobTimeoutMs, config.proverJobPollIntervalMs);
-    this.orchestrator = new ProvingOrchestrator(worldStateSynchronizer.getLatest(), this.queue, initialHeader);
+    this.orchestrator = new ProvingOrchestrator(
+      worldStateSynchronizer.getLatest(),
+      this.queue,
+      telemetry,
+      initialHeader,
+    );
   }
 
   async updateProverConfig(config: Partial<ProverClientConfig & { vks: VerificationKeys }>): Promise<void> {

yarn-project/sequencer-client/package.json

@@ -35,6 +35,7 @@
     "@aztec/p2p": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
     "@aztec/simulator": "workspace:^",
+    "@aztec/telemetry-client": "workspace:^",
     "@aztec/types": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "@noir-lang/acvm_js": "portal:../../noir/packages/acvm_js",