diff --git a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts index 60ffada5411..c93bb135395 100644 --- a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts +++ b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts @@ -2,12 +2,18 @@ import { type DebugLogger } from '@aztec/aztec.js'; import { type LogFn } from '@aztec/foundation/log'; import { type BootnodeConfig, bootnodeConfigMappings } from '@aztec/p2p'; import runBootstrapNode from '@aztec/p2p-bootstrap'; +import { + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryClientConfig, +} from '@aztec/telemetry-client/start'; import { extractRelevantOptions } from '../util.js'; export const startP2PBootstrap = async (options: any, userLog: LogFn, debugLogger: DebugLogger) => { // Start a P2P bootstrap node. const config = extractRelevantOptions(options, bootnodeConfigMappings, 'p2p'); - await runBootstrapNode(config, debugLogger); + const telemetryClient = await createAndStartTelemetryClient(getTelemetryClientConfig()); + + await runBootstrapNode(config, telemetryClient, debugLogger); userLog(`P2P bootstrap node started on ${config.udpListenAddress}`); }; diff --git a/yarn-project/circuit-types/src/p2p/topic_type.ts b/yarn-project/circuit-types/src/p2p/topic_type.ts index 88aed5b8bc1..8094905276c 100644 --- a/yarn-project/circuit-types/src/p2p/topic_type.ts +++ b/yarn-project/circuit-types/src/p2p/topic_type.ts @@ -17,3 +17,21 @@ export enum TopicType { block_attestation = 'block_attestation', epoch_proof_quote = 'epoch_proof_quote', } + +/** + * Convert the topic string into a set of labels + * + * In the form: + * { + * "/aztec/tx/0.1.0": "tx", + * ... + * } + */ +export function metricsTopicStrToLabels() { + const topicStrToLabel = new Map(); + for (const topic in TopicType) { + topicStrToLabel.set(createTopicString(TopicType[topic as keyof typeof TopicType]), topic); + } + + return topicStrToLabel; +} diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index 2520cd225b4..5e8e5d50e94 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -3,6 +3,7 @@ import { sleep } from '@aztec/aztec.js'; import fs from 'fs'; +import { METRICS_PORT } from '../fixtures/fixtures.js'; import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; import { createPXEServiceAndSubmitTransactions } from './shared.js'; @@ -19,7 +20,13 @@ describe('e2e_p2p_network', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_network', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + // Uncomment to collect metrics - run in aztec-packages `docker compose --profile metrics up` + // metricsPort: METRICS_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); @@ -50,6 +57,7 @@ describe('e2e_p2p_network', () => { NUM_NODES, BOOT_NODE_UDP_PORT, DATA_DIR, + METRICS_PORT, ); // wait a bit for peers to discover each other diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 040c7f6cb15..58eeec87c18 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -17,6 +17,7 @@ import { } from '../fixtures/setup_p2p_test.js'; import { type ISnapshotManager, type SubsystemsContext, createSnapshotManager } from '../fixtures/snapshot_manager.js'; import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { getEndToEndTestTelemetryClient } from '../fixtures/with_telemetry_utils.js'; // Use a fixed bootstrap node private key so that we can re-use the same snapshot and the nodes can find each other const BOOTSTRAP_NODE_PRIVATE_KEY = '080212208f988fc0899e4a73a5aee4d271a5f20670603a756ad8d84f2c94263a6427c591'; @@ -41,6 +42,8 @@ export class P2PNetworkTest { private numberOfNodes: number, initialValidatorAddress: string, initialValidatorConfig: AztecNodeConfig, + // If set enable metrics collection + metricsPort?: number, ) { this.logger = createDebugLogger(`aztec:e2e_p2p:${testName}`); @@ -58,13 +61,25 @@ export class P2PNetworkTest { l1BlockTime: ETHEREUM_SLOT_DURATION, salt: 420, initialValidators, + metricsPort: metricsPort, }); } - static async create(testName: string, numberOfNodes: number, basePort?: number) { + static async create({ + testName, + numberOfNodes, + basePort, + metricsPort, + }: { + testName: string; + numberOfNodes: number; + basePort?: number; + metricsPort?: number; + }) { const port = basePort || (await getPort()); - const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port); + const telemetry = await getEndToEndTestTelemetryClient(metricsPort, /*service name*/ `bootstrapnode`); + const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry); const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); const initialValidatorConfig = await createValidatorConfig({} as AztecNodeConfig, bootstrapNodeEnr); diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts index 51c222e238a..b1596b24dcb 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts @@ -19,7 +19,11 @@ describe('e2e_p2p_rediscovery', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_rediscovery', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_rediscovery', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts index 8000c72bc09..9de73adca98 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts @@ -20,7 +20,11 @@ describe('e2e_p2p_reqresp_tx', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_reqresp_tx', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_reqresp_tx', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts index 665bc08c3cf..20ebfba62fc 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts @@ -32,7 +32,11 @@ describe('e2e_p2p_governance_proposer', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_gerousia', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index 6470f015229..c9ee5dd9601 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -1,3 +1,5 @@ +export const METRICS_PORT = 4318; + export const MNEMONIC = 'test test test test test test test test test test test junk'; export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex'); export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex'); diff --git a/yarn-project/end-to-end/src/fixtures/index.ts b/yarn-project/end-to-end/src/fixtures/index.ts index c2a32f7e035..af43dc38012 100644 --- a/yarn-project/end-to-end/src/fixtures/index.ts +++ b/yarn-project/end-to-end/src/fixtures/index.ts @@ -2,3 +2,4 @@ export * from './fixtures.js'; export * from './logging.js'; export * from './utils.js'; export * from './token_utils.js'; +export * from './with_telemetry_utils.js'; diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 48e89c6512d..46994fbfd7f 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -5,12 +5,12 @@ import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; import { type SentTx, createDebugLogger } from '@aztec/aztec.js'; import { type AztecAddress } from '@aztec/circuits.js'; import { type PXEService } from '@aztec/pxe'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import getPort from 'get-port'; import { generatePrivateKey } from 'viem/accounts'; import { getPrivateKeyFromIndex } from './utils.js'; +import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; export interface NodeContext { node: AztecNodeService; @@ -48,6 +48,7 @@ export function createNodes( numNodes: number, bootNodePort: number, dataDirectory?: string, + metricsPort?: number, ): Promise { const nodePromises = []; for (let i = 0; i < numNodes; i++) { @@ -55,7 +56,7 @@ export function createNodes( const port = bootNodePort + i + 1; const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; - const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir); + const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort); nodePromises.push(nodePromise); } return Promise.all(nodePromises); @@ -69,6 +70,7 @@ export async function createNode( bootstrapNode: string | undefined, publisherAddressIndex: number, dataDirectory?: string, + metricsPort?: number, ) { const validatorConfig = await createValidatorConfig( config, @@ -78,9 +80,12 @@ export async function createNode( publisherAddressIndex, dataDirectory, ); + + const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort, /*serviceName*/ `node:${tcpPort}`); + return await AztecNodeService.createAndSync( validatorConfig, - new NoopTelemetryClient(), + telemetryClient, createDebugLogger(`aztec:node-${tcpPort}`), ); } diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 2caf027e280..6809dcab9e6 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -39,6 +39,7 @@ import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { setupL1Contracts } from './setup_l1_contracts.js'; import { type SetupOptions, getPrivateKeyFromIndex, startAnvil } from './utils.js'; +import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; export type SubsystemsContext = { anvil: Anvil; @@ -386,7 +387,8 @@ async function setupFromFresh( aztecNodeConfig.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } - const telemetry = await createAndStartTelemetryClient(getTelemetryConfig()); + const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ 'basenode'); + logger.verbose('Creating and synching an aztec node...'); const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, telemetry); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 3df1c510368..6818c4a8628 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -230,6 +230,8 @@ async function setupWithRemoteEnvironment( export type SetupOptions = { /** State load */ stateLoad?: string; + /** Whether to enable metrics collection, if undefined, metrics collection is disabled */ + metricsPort?: number | undefined; /** Previously deployed contracts on L1 */ deployL1ContractsValues?: DeployL1Contracts; /** Whether to skip deployment of protocol contracts (auth registry, etc) */ diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts new file mode 100644 index 00000000000..43f9b2f9b46 --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts @@ -0,0 +1,31 @@ +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { + type TelemetryClientConfig, + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryConfig, +} from '@aztec/telemetry-client/start'; + +export function getEndToEndTestTelemetryClient(metricsPort?: number, serviceName?: string): Promise { + return !metricsPort + ? Promise.resolve(new NoopTelemetryClient()) + : createAndStartTelemetryClient(getEndToEndTestTelemetryConfig(metricsPort, serviceName)); +} + +/** + * Utility functions for setting up end-to-end tests with telemetry. + * + * Read from env vars, override if metricsPort is set + */ +export function getEndToEndTestTelemetryConfig(metricsPort?: number, serviceName?: string) { + const telemetryConfig: TelemetryClientConfig = getTelemetryConfig(); + if (metricsPort) { + telemetryConfig.metricsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/metrics`); + telemetryConfig.tracesCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/traces`); + telemetryConfig.logsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/logs`); + } + if (serviceName) { + telemetryConfig.serviceName = serviceName; + } + return telemetryConfig; +} diff --git a/yarn-project/p2p-bootstrap/package.json b/yarn-project/p2p-bootstrap/package.json index 57225e75e38..8c6280c74fe 100644 --- a/yarn-project/p2p-bootstrap/package.json +++ b/yarn-project/p2p-bootstrap/package.json @@ -26,6 +26,7 @@ "dependencies": { "@aztec/foundation": "workspace:^", "@aztec/p2p": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "dotenv": "^16.0.3", "koa": "^2.15.3", "koa-router": "^12.0.1", diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index b22492d3cac..2704f519de3 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -1,5 +1,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import Koa from 'koa'; import Router from 'koa-router'; @@ -11,8 +13,12 @@ const { HTTP_PORT } = process.env; /** * The application entry point. */ -async function main(config: BootnodeConfig, logger = debugLogger) { - const bootstrapNode = new BootstrapNode(logger); +async function main( + config: BootnodeConfig, + telemetryClient: TelemetryClient = new NoopTelemetryClient(), + logger = debugLogger, +) { + const bootstrapNode = new BootstrapNode(telemetryClient, logger); await bootstrapNode.start(config); logger.info('DiscV5 Bootnode started'); diff --git a/yarn-project/p2p-bootstrap/tsconfig.json b/yarn-project/p2p-bootstrap/tsconfig.json index d4c2d4767aa..ac997674d5c 100644 --- a/yarn-project/p2p-bootstrap/tsconfig.json +++ b/yarn-project/p2p-bootstrap/tsconfig.json @@ -11,6 +11,9 @@ }, { "path": "../p2p" + }, + { + "path": "../telemetry-client" } ], "include": ["src"] diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json index 284973b6c85..ba5f7fd21a4 100644 --- a/yarn-project/p2p/package.json +++ b/yarn-project/p2p/package.json @@ -82,6 +82,7 @@ "@libp2p/peer-id": "4.0.7", "@libp2p/peer-id-factory": "4.1.1", "@libp2p/peer-store": "10.0.16", + "@libp2p/prometheus-metrics": "^4.2.4", "@libp2p/tcp": "9.0.24", "@multiformats/multiaddr": "12.1.14", "interface-datastore": "^8.2.11", diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index d3840ba5129..c9587195bdf 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -1,4 +1,5 @@ import { createDebugLogger } from '@aztec/foundation/log'; +import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5'; import { SignableENR } from '@chainsafe/enr'; @@ -17,7 +18,7 @@ export class BootstrapNode { private node?: Discv5 = undefined; private peerId?: PeerId; - constructor(private logger = createDebugLogger('aztec:p2p_bootstrap')) {} + constructor(private telemetry: TelemetryClient, private logger = createDebugLogger('aztec:p2p_bootstrap')) {} /** * Starts the bootstrap node. @@ -41,7 +42,7 @@ export class BootstrapNode { enr.set(AZTEC_ENR_KEY, Uint8Array.from([AZTEC_NET])); this.logger.info(`Starting bootstrap node ${peerId}, listening on ${listenAddrUdp.toString()}`); - + const metricsRegistry = new OtelMetricsAdapter(this.telemetry); this.node = Discv5.create({ enr, peerId, @@ -50,6 +51,7 @@ export class BootstrapNode { lookupTimeout: 2000, allowUnverifiedSessions: true, }, + metricsRegistry, }); (this.node as Discv5EventEmitter).on('multiaddrUpdated', (addr: Multiaddr) => { diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts index e6a4d273f0e..2b4c498ca59 100644 --- a/yarn-project/p2p/src/client/index.ts +++ b/yarn-project/p2p/src/client/index.ts @@ -49,7 +49,7 @@ export const createP2PClient = async ( // Create peer discovery service const peerId = await createLibP2PPeerId(config.peerIdPrivateKey); - const discoveryService = new DiscV5Service(peerId, config); + const discoveryService = new DiscV5Service(peerId, config, telemetry); p2pService = await LibP2PService.new( config, diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 5127a8c6238..27c37a475fa 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -6,6 +6,7 @@ import { } from '@aztec/circuit-types'; import { type DataStoreConfig } from '@aztec/kv-store/utils'; import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { gossipsub } from '@chainsafe/libp2p-gossipsub'; import { noise } from '@chainsafe/libp2p-noise'; @@ -90,7 +91,7 @@ export async function createLibp2pNode( * Test Libp2p service * P2P functionality is operational, however everything else is default * - * WORKTODO: more description + * */ export async function createTestLibP2PService( boostrapAddrs: string[] = [], @@ -114,7 +115,7 @@ export async function createTestLibP2PService( p2pEnabled: true, peerIdPrivateKey: Buffer.from(peerId.privateKey!).toString('hex'), } as P2PConfig & DataStoreConfig; - const discoveryService = new DiscV5Service(peerId, config); + const discoveryService = new DiscV5Service(peerId, config, telemetry); const proofVerifier = new AlwaysTrueCircuitVerifier(); // No bootstrap nodes provided as the libp2p service will register them in the constructor @@ -233,20 +234,27 @@ export function createBootstrapNodeConfig(privateKey: string, port: number): Boo }; } -export function createBootstrapNodeFromPrivateKey(privateKey: string, port: number): Promise { +export function createBootstrapNodeFromPrivateKey( + privateKey: string, + port: number, + telemetry: TelemetryClient = new NoopTelemetryClient(), +): Promise { const config = createBootstrapNodeConfig(privateKey, port); - return startBootstrapNode(config); + return startBootstrapNode(config, telemetry); } -export async function createBootstrapNode(port: number): Promise { +export async function createBootstrapNode( + port: number, + telemetry: TelemetryClient = new NoopTelemetryClient(), +): Promise { const peerId = await createLibP2PPeerId(); const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port); - return startBootstrapNode(config); + return startBootstrapNode(config, telemetry); } -async function startBootstrapNode(config: BootnodeConfig) { - const bootstrapNode = new BootstrapNode(); +async function startBootstrapNode(config: BootnodeConfig, telemetry: TelemetryClient) { + const bootstrapNode = new BootstrapNode(telemetry); await bootstrapNode.start(config); return bootstrapNode; } diff --git a/yarn-project/p2p/src/service/discV5_service.ts b/yarn-project/p2p/src/service/discV5_service.ts index 5e62268fe5a..06c3b740e01 100644 --- a/yarn-project/p2p/src/service/discV5_service.ts +++ b/yarn-project/p2p/src/service/discV5_service.ts @@ -1,5 +1,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; +import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5'; import { ENR, SignableENR } from '@chainsafe/enr'; @@ -41,7 +42,12 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private startTime = 0; - constructor(private peerId: PeerId, config: P2PConfig, private logger = createDebugLogger('aztec:discv5_service')) { + constructor( + private peerId: PeerId, + config: P2PConfig, + telemetry: TelemetryClient, + private logger = createDebugLogger('aztec:discv5_service'), + ) { super(); const { tcpAnnounceAddress, udpAnnounceAddress, udpListenAddress, bootstrapNodes } = config; this.bootstrapNodes = bootstrapNodes; @@ -66,6 +72,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService this.enr.setLocationMultiaddr(multiAddrUdp); this.enr.setLocationMultiaddr(multiAddrTcp); + const metricsRegistry = new OtelMetricsAdapter(telemetry); this.discv5 = Discv5.create({ enr: this.enr, peerId, @@ -75,6 +82,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService requestTimeout: 2000, allowUnverifiedSessions: true, }, + metricsRegistry, }); this.logger.info(`ENR NodeId: ${this.enr.nodeId}`); diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 8a4106d4f4f..75ae683676f 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -1,4 +1,5 @@ import { sleep } from '@aztec/foundation/sleep'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; import type { PeerId } from '@libp2p/interface'; @@ -42,7 +43,8 @@ describe('Discv5Service', () => { }; beforeEach(async () => { - bootNode = new BootstrapNode(); + const telemetryClient = new NoopTelemetryClient(); + bootNode = new BootstrapNode(telemetryClient); await bootNode.start(baseConfig); bootNodePeerId = bootNode.getPeerId(); }); @@ -138,6 +140,6 @@ describe('Discv5Service', () => { keepProvenTxsInPoolFor: 0, l1ChainId: 31337, }; - return new DiscV5Service(peerId, config); + return new DiscV5Service(peerId, config, new NoopTelemetryClient()); }; }); diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 463471041bb..b6a793645d2 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -12,13 +12,14 @@ import { Tx, TxHash, type WorldStateSynchronizer, + metricsTopicStrToLabels, } from '@aztec/circuit-types'; import { Fr } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { RunningPromise } from '@aztec/foundation/running-promise'; import type { AztecKVStore } from '@aztec/kv-store'; -import { Attributes, type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client'; +import { Attributes, OtelMetricsAdapter, type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client'; import { type ENR } from '@chainsafe/enr'; import { type GossipSub, type GossipSubComponents, gossipsub } from '@chainsafe/libp2p-gossipsub'; @@ -218,6 +219,8 @@ export class LibP2PService extends WithTracer implements P2PService { const datastore = new AztecDatastore(store); + const otelMetricsAdapter = new OtelMetricsAdapter(telemetry); + const node = await createLibp2p({ start: false, peerId, @@ -257,6 +260,8 @@ export class LibP2PService extends WithTracer implements P2PService { heartbeatInterval: config.gossipsubInterval, mcacheLength: config.gossipsubMcacheLength, mcacheGossip: config.gossipsubMcacheGossip, + metricsRegister: otelMetricsAdapter, + metricsTopicStrToLabel: metricsTopicStrToLabels(), scoreParams: createPeerScoreParams({ topics: { [Tx.p2pTopic]: createTopicScoreParams({ diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json index 15cfc0c839d..fdd9d252faf 100644 --- a/yarn-project/telemetry-client/package.json +++ b/yarn-project/telemetry-client/package.json @@ -41,6 +41,7 @@ "@opentelemetry/sdk-trace-node": "^1.25.0", "@opentelemetry/semantic-conventions": "^1.25.0", "@opentelemetry/winston-transport": "^0.7.0", + "prom-client": "^15.1.3", "winston": "^3.10.0" }, "devDependencies": { diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts index c48a9be6bc4..099fff27a0e 100644 --- a/yarn-project/telemetry-client/src/config.ts +++ b/yarn-project/telemetry-client/src/config.ts @@ -26,7 +26,7 @@ export const telemetryClientConfigMappings: ConfigMappingsType { + let mockLogger: Logger; + let mockMeter: Meter; + let mockObservableGauge: ObservableGauge; + let otelGauge: OtelGauge>; + + beforeEach(() => { + // Mocking Logger, Meter, and ObservableGauge + mockLogger = { error: jest.fn(), warn: jest.fn() } as unknown as Logger; + mockMeter = { createObservableGauge: jest.fn(() => mockObservableGauge) } as unknown as Meter; + mockObservableGauge = { addCallback: jest.fn() } as unknown as ObservableGauge; + + otelGauge = new OtelGauge(mockLogger, mockMeter, 'test_gauge', 'Test gauge'); + }); + + test('increments value without labels', () => { + otelGauge.inc(); + expect(otelGauge['currentValue']).toBe(1); + + otelGauge.inc(5); + expect(otelGauge['currentValue']).toBe(6); + }); + + test('sets value without labels', () => { + otelGauge.set(10); + expect(otelGauge['currentValue']).toBe(10); + }); + + test('decrements value without labels', () => { + otelGauge.set(5); + otelGauge.dec(); + expect(otelGauge['currentValue']).toBe(4); + + otelGauge.dec(); + expect(otelGauge['currentValue']).toBe(3); + }); + + test('resets value without labels', () => { + otelGauge.set(5); + otelGauge.reset(); + expect(otelGauge['currentValue']).toBe(0); + }); + + test('increments and sets value with labels', () => { + const labelConfig = { region: 'us-east-1', instance: 'test-instance' }; + const otelGaugeWithLabels = new OtelGauge( + mockLogger, + mockMeter, + 'test_gauge_with_labels', + 'Test gauge with labels', + ['region', 'instance'], + ); + + otelGaugeWithLabels.inc(labelConfig, 3); + expect(otelGaugeWithLabels['labeledValues'].get(JSON.stringify(labelConfig))).toBe(3); + + otelGaugeWithLabels.set(labelConfig, 10); + expect(otelGaugeWithLabels['labeledValues'].get(JSON.stringify(labelConfig))).toBe(10); + }); + + test('decrements value with labels', () => { + const labelConfig = { region: 'us-east-1', instance: 'test-instance' }; + const otelGaugeWithLabels = new OtelGauge( + mockLogger, + mockMeter, + 'test_gauge_with_labels', + 'Test gauge with labels', + ['region', 'instance'], + ); + + otelGaugeWithLabels.set(labelConfig, 5); + otelGaugeWithLabels.dec(labelConfig); + expect(otelGaugeWithLabels['labeledValues'].get(JSON.stringify(labelConfig))).toBe(4); + }); + + test('handles invalid labels with error', () => { + const invalidLabelConfig = { invalid: 'label' }; + const otelGaugeWithLabels = new OtelGauge( + mockLogger, + mockMeter, + 'test_gauge_with_labels', + 'Test gauge with labels', + ['region', 'instance'], + ); + + expect(() => otelGaugeWithLabels.inc(invalidLabelConfig as any, 1)).toThrowError('Invalid label key: invalid'); + expect(mockLogger.error).not.toHaveBeenCalled(); + }); + + test('executes addCollect functions', () => { + const collectFn = jest.fn(); + otelGauge.addCollect(collectFn); + otelGauge['handleObservation']({ observe: jest.fn() }); + + expect(collectFn).toHaveBeenCalledWith(otelGauge); + }); + + test('parses and observes labeled values safely', () => { + otelGauge = new OtelGauge(mockLogger, mockMeter, 'test_gauge', 'Test gauge', ['type']); + + const labelConfig = { type: 'ping' }; + const labelStr = JSON.stringify(labelConfig); + otelGauge['labeledValues'].set(labelStr, 5); + + const mockResult = { observe: jest.fn() }; + otelGauge['handleObservation'](mockResult); + + expect(mockResult.observe).toHaveBeenCalledWith(5, labelConfig); + }); +}); diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts new file mode 100644 index 00000000000..23e8e610bac --- /dev/null +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -0,0 +1,319 @@ +import { type Logger, createDebugLogger } from '@aztec/foundation/log'; + +import { Registry } from 'prom-client'; + +import { type Meter, type Metrics, type ObservableGauge, type TelemetryClient } from './telemetry.js'; + +/** + * Types matching the gossipsub and libp2p services + */ +type TopicStr = string; +export type TopicLabel = string; +export type TopicStrToLabel = Map; + +export enum MessageSource { + forward = 'forward', + publish = 'publish', +} + +type NoLabels = Record; +type LabelsGeneric = Record; +type LabelKeys = Extract; +interface CollectFn { + (metric: IGauge): void; +} + +interface IGauge { + inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void; + set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void; + + collect?(): void; + addCollect(fn: CollectFn): void; +} + +interface IHistogram { + startTimer(): () => void; + + observe: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void; + + reset(): void; +} + +interface IAvgMinMax { + set: NoLabels extends Labels ? (values: number[]) => void : (labels: Labels, values: number[]) => void; +} + +export type GaugeConfig = { + name: string; + help: string; +} & (NoLabels extends Labels + ? { labelNames?: never } + : { labelNames: [LabelKeys, ...Array>] }); + +export type HistogramConfig = GaugeConfig & { + buckets?: number[]; +}; + +export type AvgMinMaxConfig = GaugeConfig; + +export interface MetricsRegister { + gauge(config: GaugeConfig): IGauge; + histogram(config: HistogramConfig): IHistogram; + avgMinMax(config: AvgMinMaxConfig): IAvgMinMax; +} + +/**Otel Metrics Adapters + * + * Some dependencies we use export metrics directly in a Prometheus format + * This adapter is used to convert those metrics to a format that we can use with OpenTelemetry + * + * Affected services include: + * - chainsafe/gossipsub + * - libp2p + */ + +export class OtelGauge implements IGauge { + private gauge: ObservableGauge; + private currentValue: number = 0; + private labeledValues: Map = new Map(); + private collectFns: CollectFn[] = []; + + private _collect: () => void = () => {}; + get collect(): () => void { + return this._collect; + } + set collect(fn: () => void) { + this._collect = fn; + } + + constructor( + private logger: Logger, + meter: Meter, + name: string, + help: string, + private labelNames: Array = [], + ) { + this.gauge = meter.createObservableGauge(name as Metrics, { + description: help, + }); + + // Only observe in the callback when collect() is called + this.gauge.addCallback(this.handleObservation.bind(this)); + } + + addCollect(fn: CollectFn): void { + this.collectFns.push(fn); + } + + handleObservation(result: any): void { + // Execute the main collect function if assigned + this._collect(); + + // Execute all the collect functions + for (const fn of this.collectFns) { + fn(this); + } + + // Report the current values + if (this.labelNames.length === 0) { + result.observe(this.currentValue); + return; + } + + for (const [labelStr, value] of this.labeledValues.entries()) { + const labels = this.parseLabelsSafely(labelStr); + if (labels) { + result.observe(value, labels); + } + } + } + + /** + * Increments the gauge value + * @param labelsOrValue - Labels object or numeric value + * @param value - Value to increment by (defaults to 1) + */ + inc(value?: number): void; + inc(labels: Labels, value?: number): void; + inc(labelsOrValue?: Labels | number, value?: number): void { + if (typeof labelsOrValue === 'number') { + this.currentValue += labelsOrValue; + return; + } + + if (labelsOrValue) { + this.validateLabels(labelsOrValue); + const labelKey = JSON.stringify(labelsOrValue); + const currentValue = this.labeledValues.get(labelKey) ?? 0; + this.labeledValues.set(labelKey, currentValue + (value ?? 1)); + return; + } + + this.currentValue += value ?? 1; + } + + /** + * Sets the gauge value + * @param labelsOrValue - Labels object or numeric value + * @param value - Value to set + */ + set(value: number): void; + set(labels: Labels, value: number): void; + set(labelsOrValue: Labels | number, value?: number): void { + if (typeof labelsOrValue === 'number') { + this.currentValue = labelsOrValue; + return; + } + + this.validateLabels(labelsOrValue); + const labelKey = JSON.stringify(labelsOrValue); + this.labeledValues.set(labelKey, value!); + } + + /** + * Decrements the gauge value + * @param labels - Optional labels object + */ + dec(labels?: Labels): void { + if (labels) { + this.validateLabels(labels); + const labelKey = JSON.stringify(labels); + const currentValue = this.labeledValues.get(labelKey) ?? 0; + this.labeledValues.set(labelKey, currentValue - 1); + return; + } + + this.currentValue -= 1; + } + + /** + * Resets the gauge to initial state + */ + reset(): void { + this.currentValue = 0; + this.labeledValues.clear(); + } + + /** + * Validates that provided labels match the expected schema + * @param labels - Labels object to validate + * @throws Error if invalid labels are provided + */ + private validateLabels(labels: Labels): void { + if (this.labelNames.length === 0) { + throw new Error('Gauge was initialized without labels support'); + } + + for (const key of Object.keys(labels)) { + if (!this.labelNames.includes(key as keyof Labels)) { + throw new Error(`Invalid label key: ${key}`); + } + } + } + + /** + * Safely parses label string back to object + * @param labelStr - Stringified labels object + * @returns Labels object or null if parsing fails + */ + private parseLabelsSafely(labelStr: string): Labels | null { + try { + return JSON.parse(labelStr) as Labels; + } catch { + this.logger.error(`Failed to parse label string: ${labelStr}`); + return null; + } + } +} + +/** + * Noop implementation of a Historgram collec + */ +class NoopOtelHistogram implements IHistogram { + constructor( + private logger: Logger, + _meter: Meter, + _name: string, // Metrics must be registered in the aztec labels registry + _help: string, + _buckets: number[] = [], + _labelNames: Array = [], + ) {} + + // Overload signatures + observe(_value: number): void; + observe(_labels: Labels, _value: number): void; + observe(_valueOrLabels: number | Labels, _value?: number): void {} + + startTimer(_labels?: Labels): (_labels?: Labels) => number { + return () => 0; + } + + reset(): void { + // OpenTelemetry histograms cannot be reset, but we implement the interface + this.logger.warn('OpenTelemetry histograms cannot be reset'); + } +} + +/** + * Noop implementation of an AvgMinMax collector + */ +class NoopOtelAvgMinMax implements IAvgMinMax { + constructor( + private _logger: Logger, + _meter: Meter, + _name: string, // Metrics must be registered in the aztec labels registry + _help: string, + _labelNames: Array = [], + ) {} + + set(_values: number[]): void; + set(_labels: Labels, _values: number[]): void; + set(_valueOrLabels: number[] | Labels, _values?: number[]): void {} + + reset(): void {} +} + +/** + * Otel metrics Adapter + * + * Maps the PromClient based MetricsRegister from gossipsub and discv5 services to the Otel MetricsRegister + */ +export class OtelMetricsAdapter extends Registry implements MetricsRegister { + private readonly meter: Meter; + + constructor(telemetryClient: TelemetryClient, private logger: Logger = createDebugLogger('otel-metrics-adapter')) { + super(); + this.meter = telemetryClient.getMeter('metrics-adapter'); + } + + gauge(configuration: GaugeConfig): IGauge { + return new OtelGauge( + this.logger, + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.labelNames, + ); + } + + histogram(configuration: HistogramConfig): IHistogram { + return new NoopOtelHistogram( + this.logger, + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.buckets, + configuration.labelNames, + ); + } + + avgMinMax(configuration: AvgMinMaxConfig): IAvgMinMax { + return new NoopOtelAvgMinMax( + this.logger, + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.labelNames, + ); + } +} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 256524913e3..3398aa8aaac 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -828,6 +828,7 @@ __metadata: dependencies: "@aztec/foundation": "workspace:^" "@aztec/p2p": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 "@types/koa": ^2.15.0 @@ -868,6 +869,7 @@ __metadata: "@libp2p/peer-id": 4.0.7 "@libp2p/peer-id-factory": 4.1.1 "@libp2p/peer-store": 10.0.16 + "@libp2p/prometheus-metrics": ^4.2.4 "@libp2p/tcp": 9.0.24 "@multiformats/multiaddr": 12.1.14 "@types/jest": ^29.5.0 @@ -1172,6 +1174,7 @@ __metadata: "@opentelemetry/winston-transport": ^0.7.0 "@types/jest": ^29.5.0 jest: ^29.5.0 + prom-client: ^15.1.3 ts-node: ^10.9.1 typescript: ^5.0.4 winston: ^3.10.0 @@ -2794,6 +2797,20 @@ __metadata: languageName: node linkType: hard +"@libp2p/interface@npm:^2.2.0": + version: 2.2.0 + resolution: "@libp2p/interface@npm:2.2.0" + dependencies: + "@multiformats/multiaddr": ^12.2.3 + it-pushable: ^3.2.3 + it-stream-types: ^2.0.1 + multiformats: ^13.1.0 + progress-events: ^1.0.0 + uint8arraylist: ^2.4.8 + checksum: 277634721147384af134232fe68c6d904eb6794f48b29eb070a85f414a4a3d362151f0d7c1d76ad013f09f9efaf9010ca6d7fe4ab99b5f5c98dc1c22e184d40f + languageName: node + linkType: hard + "@libp2p/kad-dht@npm:10.0.4": version: 10.0.4 resolution: "@libp2p/kad-dht@npm:10.0.4" @@ -3036,6 +3053,19 @@ __metadata: languageName: node linkType: hard +"@libp2p/prometheus-metrics@npm:^4.2.4": + version: 4.2.4 + resolution: "@libp2p/prometheus-metrics@npm:4.2.4" + dependencies: + "@libp2p/interface": ^2.2.0 + it-foreach: ^2.1.0 + it-stream-types: ^2.0.1 + prom-client: ^15.1.2 + uint8arraylist: ^2.4.8 + checksum: f5aef58ede8371e7f6d451b3d3cc1e3136fd26acc12609030dc03218fdaeada0a03c5a2ee404c16e15f9b0191f0017a31d8b909012f9094c4c95b12cf6b978ee + languageName: node + linkType: hard + "@libp2p/pubsub@npm:^9.0.8": version: 9.0.17 resolution: "@libp2p/pubsub@npm:9.0.17" @@ -3301,6 +3331,20 @@ __metadata: languageName: node linkType: hard +"@multiformats/multiaddr@npm:^12.2.3": + version: 12.3.1 + resolution: "@multiformats/multiaddr@npm:12.3.1" + dependencies: + "@chainsafe/is-ip": ^2.0.1 + "@chainsafe/netmask": ^2.0.0 + "@multiformats/dns": ^1.0.3 + multiformats: ^13.0.0 + uint8-varint: ^2.0.1 + uint8arrays: ^5.0.0 + checksum: 086a71f86caeb441ea8895e497d7ce1e46d8bda881bc275812054b02a94d08323f100ca78d1ff9979347488145563b492d1d4ffc8d800baf5ce5c726d02af451 + languageName: node + linkType: hard + "@noble/ciphers@npm:^0.4.0": version: 0.4.1 resolution: "@noble/ciphers@npm:0.4.1" @@ -3466,7 +3510,7 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0": +"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.4.0, @opentelemetry/api@npm:^1.9.0": version: 1.9.0 resolution: "@opentelemetry/api@npm:1.9.0" checksum: 9e88e59d53ced668f3daaecfd721071c5b85a67dd386f1c6f051d1be54375d850016c881f656ffbe9a03bedae85f7e89c2f2b635313f9c9b195ad033cdc31020 @@ -6132,6 +6176,13 @@ __metadata: languageName: node linkType: hard +"bintrees@npm:1.0.2": + version: 1.0.2 + resolution: "bintrees@npm:1.0.2" + checksum: 56a52b7d3634e30002b1eda740d2517a22fa8e9e2eb088e919f37c030a0ed86e364ab59e472fc770fc8751308054bb1c892979d150e11d9e11ac33bcc1b5d16e + languageName: node + linkType: hard + "bl@npm:^4.1.0": version: 4.1.0 resolution: "bl@npm:4.1.0" @@ -10605,6 +10656,15 @@ __metadata: languageName: node linkType: hard +"it-foreach@npm:^2.1.0": + version: 2.1.1 + resolution: "it-foreach@npm:2.1.1" + dependencies: + it-peekable: ^3.0.0 + checksum: 17cb7917fdecd7a8e2106a26be13d7edea5f02ce1163f866afd9bb3dab2acf3b919064bd383c89fcb2843073474257ecf21dc3d086f695ed3e2d45bae6e03f0c + languageName: node + linkType: hard + "it-length-prefixed-stream@npm:^1.0.0, it-length-prefixed-stream@npm:^1.1.6": version: 1.1.7 resolution: "it-length-prefixed-stream@npm:1.1.7" @@ -13650,6 +13710,16 @@ __metadata: languageName: node linkType: hard +"prom-client@npm:^15.1.2, prom-client@npm:^15.1.3": + version: 15.1.3 + resolution: "prom-client@npm:15.1.3" + dependencies: + "@opentelemetry/api": ^1.4.0 + tdigest: ^0.1.1 + checksum: 9a57f3c16f39aa9a03da021883a4231c0bb56fc9d02f6ef9c28f913379f275640a5a33b98d9946ebf53c71011a29b580e9d2d6e3806cb1c229a3f59c65993968 + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -15341,6 +15411,15 @@ __metadata: languageName: node linkType: hard +"tdigest@npm:^0.1.1": + version: 0.1.2 + resolution: "tdigest@npm:0.1.2" + dependencies: + bintrees: 1.0.2 + checksum: 44de8246752b6f8c2924685f969fd3d94c36949f22b0907e99bef2b2220726dd8467f4730ea96b06040b9aa2587c0866049640039d1b956952dfa962bc2075a3 + languageName: node + linkType: hard + "terser-webpack-plugin@npm:^5.3.10": version: 5.3.10 resolution: "terser-webpack-plugin@npm:5.3.10"