feat: extract gossipsub / discv5 dependency gauge prometheus metrics (#…

…9710) Get access to all of the p2p metrics counts to help investigate issues fixes: #9691
AztecProtocol · Nov 7, 2024 · 58e75cd · 58e75cd
1 parent d77e473
commit 58e75cd
Show file tree

Hide file tree

Showing 29 changed files with 685 additions and 30 deletions.
diff --git a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts
@@ -2,12 +2,18 @@ import { type DebugLogger } from '@aztec/aztec.js';
 import { type LogFn } from '@aztec/foundation/log';
 import { type BootnodeConfig, bootnodeConfigMappings } from '@aztec/p2p';
 import runBootstrapNode from '@aztec/p2p-bootstrap';
+import {
+  createAndStartTelemetryClient,
+  getConfigEnvVars as getTelemetryClientConfig,
+} from '@aztec/telemetry-client/start';
 
 import { extractRelevantOptions } from '../util.js';
 
 export const startP2PBootstrap = async (options: any, userLog: LogFn, debugLogger: DebugLogger) => {
   // Start a P2P bootstrap node.
   const config = extractRelevantOptions<BootnodeConfig>(options, bootnodeConfigMappings, 'p2p');
-  await runBootstrapNode(config, debugLogger);
+  const telemetryClient = await createAndStartTelemetryClient(getTelemetryClientConfig());
+
+  await runBootstrapNode(config, telemetryClient, debugLogger);
   userLog(`P2P bootstrap node started on ${config.udpListenAddress}`);
 };
diff --git a/yarn-project/circuit-types/src/p2p/topic_type.ts b/yarn-project/circuit-types/src/p2p/topic_type.ts
@@ -17,3 +17,21 @@ export enum TopicType {
   block_attestation = 'block_attestation',
   epoch_proof_quote = 'epoch_proof_quote',
 }
+
+/**
+ * Convert the topic string into a set of labels
+ *
+ * In the form:
+ * {
+ *  "/aztec/tx/0.1.0": "tx",
+ *  ...
+ * }
+ */
+export function metricsTopicStrToLabels() {
+  const topicStrToLabel = new Map<string, string>();
+  for (const topic in TopicType) {
+    topicStrToLabel.set(createTopicString(TopicType[topic as keyof typeof TopicType]), topic);
+  }
+
+  return topicStrToLabel;
+}
diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts
@@ -3,6 +3,7 @@ import { sleep } from '@aztec/aztec.js';
 
 import fs from 'fs';
 
+import { METRICS_PORT } from '../fixtures/fixtures.js';
 import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js';
 import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js';
 import { createPXEServiceAndSubmitTransactions } from './shared.js';
@@ -19,7 +20,13 @@ describe('e2e_p2p_network', () => {
   let nodes: AztecNodeService[];
 
   beforeEach(async () => {
-    t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT);
+    t = await P2PNetworkTest.create({
+      testName: 'e2e_p2p_network',
+      numberOfNodes: NUM_NODES,
+      basePort: BOOT_NODE_UDP_PORT,
+      // Uncomment to collect metrics - run in aztec-packages `docker compose --profile metrics up`
+      // metricsPort: METRICS_PORT,
+    });
     await t.applyBaseSnapshots();
     await t.setup();
   });
@@ -50,6 +57,7 @@ describe('e2e_p2p_network', () => {
       NUM_NODES,
       BOOT_NODE_UDP_PORT,
       DATA_DIR,
+      METRICS_PORT,
     );
 
     // wait a bit for peers to discover each other

diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts
@@ -17,6 +17,7 @@ import {
 } from '../fixtures/setup_p2p_test.js';
 import { type ISnapshotManager, type SubsystemsContext, createSnapshotManager } from '../fixtures/snapshot_manager.js';
 import { getPrivateKeyFromIndex } from '../fixtures/utils.js';
+import { getEndToEndTestTelemetryClient } from '../fixtures/with_telemetry_utils.js';
 
 // Use a fixed bootstrap node private key so that we can re-use the same snapshot and the nodes can find each other
 const BOOTSTRAP_NODE_PRIVATE_KEY = '080212208f988fc0899e4a73a5aee4d271a5f20670603a756ad8d84f2c94263a6427c591';
@@ -41,6 +42,8 @@ export class P2PNetworkTest {
     private numberOfNodes: number,
     initialValidatorAddress: string,
     initialValidatorConfig: AztecNodeConfig,
+    // If set enable metrics collection
+    metricsPort?: number,
   ) {
     this.logger = createDebugLogger(`aztec:e2e_p2p:${testName}`);
 
@@ -58,13 +61,25 @@ export class P2PNetworkTest {
       l1BlockTime: ETHEREUM_SLOT_DURATION,
       salt: 420,
       initialValidators,
+      metricsPort: metricsPort,
     });
   }
 
-  static async create(testName: string, numberOfNodes: number, basePort?: number) {
+  static async create({
+    testName,
+    numberOfNodes,
+    basePort,
+    metricsPort,
+  }: {
+    testName: string;
+    numberOfNodes: number;
+    basePort?: number;
+    metricsPort?: number;
+  }) {
     const port = basePort || (await getPort());
 
-    const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port);
+    const telemetry = await getEndToEndTestTelemetryClient(metricsPort, /*service name*/ `bootstrapnode`);
+    const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry);
     const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt();
 
     const initialValidatorConfig = await createValidatorConfig({} as AztecNodeConfig, bootstrapNodeEnr);

diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts
@@ -19,7 +19,11 @@ describe('e2e_p2p_rediscovery', () => {
   let nodes: AztecNodeService[];
 
   beforeEach(async () => {
-    t = await P2PNetworkTest.create('e2e_p2p_rediscovery', NUM_NODES, BOOT_NODE_UDP_PORT);
+    t = await P2PNetworkTest.create({
+      testName: 'e2e_p2p_rediscovery',
+      numberOfNodes: NUM_NODES,
+      basePort: BOOT_NODE_UDP_PORT,
+    });
     await t.applyBaseSnapshots();
     await t.setup();
   });

diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts
@@ -20,7 +20,11 @@ describe('e2e_p2p_reqresp_tx', () => {
   let nodes: AztecNodeService[];
 
   beforeEach(async () => {
-    t = await P2PNetworkTest.create('e2e_p2p_reqresp_tx', NUM_NODES, BOOT_NODE_UDP_PORT);
+    t = await P2PNetworkTest.create({
+      testName: 'e2e_p2p_reqresp_tx',
+      numberOfNodes: NUM_NODES,
+      basePort: BOOT_NODE_UDP_PORT,
+    });
     await t.applyBaseSnapshots();
     await t.setup();
   });

diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts
@@ -32,7 +32,11 @@ describe('e2e_p2p_governance_proposer', () => {
   let nodes: AztecNodeService[];
 
   beforeEach(async () => {
-    t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT);
+    t = await P2PNetworkTest.create({
+      testName: 'e2e_p2p_gerousia',
+      numberOfNodes: NUM_NODES,
+      basePort: BOOT_NODE_UDP_PORT,
+    });
     await t.applyBaseSnapshots();
     await t.setup();
   });

diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts
@@ -1,3 +1,5 @@
+export const METRICS_PORT = 4318;
+
 export const MNEMONIC = 'test test test test test test test test test test test junk';
 export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex');
 export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex');

diff --git a/yarn-project/end-to-end/src/fixtures/index.ts b/yarn-project/end-to-end/src/fixtures/index.ts
@@ -2,3 +2,4 @@ export * from './fixtures.js';
 export * from './logging.js';
 export * from './utils.js';
 export * from './token_utils.js';
+export * from './with_telemetry_utils.js';
diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts
@@ -5,12 +5,12 @@ import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node';
 import { type SentTx, createDebugLogger } from '@aztec/aztec.js';
 import { type AztecAddress } from '@aztec/circuits.js';
 import { type PXEService } from '@aztec/pxe';
-import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import getPort from 'get-port';
 import { generatePrivateKey } from 'viem/accounts';
 
 import { getPrivateKeyFromIndex } from './utils.js';
+import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js';
 
 export interface NodeContext {
   node: AztecNodeService;
@@ -48,14 +48,15 @@ export function createNodes(
   numNodes: number,
   bootNodePort: number,
   dataDirectory?: string,
+  metricsPort?: number,
 ): Promise<AztecNodeService[]> {
   const nodePromises = [];
   for (let i = 0; i < numNodes; i++) {
     // We run on ports from the bootnode upwards if a port if provided, otherwise we get a random port
     const port = bootNodePort + i + 1;
 
     const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined;
-    const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir);
+    const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort);
     nodePromises.push(nodePromise);
   }
   return Promise.all(nodePromises);
@@ -69,6 +70,7 @@ export async function createNode(
   bootstrapNode: string | undefined,
   publisherAddressIndex: number,
   dataDirectory?: string,
+  metricsPort?: number,
 ) {
   const validatorConfig = await createValidatorConfig(
     config,
@@ -78,9 +80,12 @@ export async function createNode(
     publisherAddressIndex,
     dataDirectory,
   );
+
+  const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort, /*serviceName*/ `node:${tcpPort}`);
+
   return await AztecNodeService.createAndSync(
     validatorConfig,
-    new NoopTelemetryClient(),
+    telemetryClient,
     createDebugLogger(`aztec:node-${tcpPort}`),
   );
 }

diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts
@@ -39,6 +39,7 @@ import { getACVMConfig } from './get_acvm_config.js';
 import { getBBConfig } from './get_bb_config.js';
 import { setupL1Contracts } from './setup_l1_contracts.js';
 import { type SetupOptions, getPrivateKeyFromIndex, startAnvil } from './utils.js';
+import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js';
 
 export type SubsystemsContext = {
   anvil: Anvil;
@@ -386,7 +387,8 @@ async function setupFromFresh(
     aztecNodeConfig.bbWorkingDirectory = bbConfig.bbWorkingDirectory;
   }
 
-  const telemetry = await createAndStartTelemetryClient(getTelemetryConfig());
+  const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ 'basenode');
+
   logger.verbose('Creating and synching an aztec node...');
   const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, telemetry);
 

diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts
@@ -230,6 +230,8 @@ async function setupWithRemoteEnvironment(
 export type SetupOptions = {
   /** State load */
   stateLoad?: string;
+  /** Whether to enable metrics collection, if undefined, metrics collection is disabled */
+  metricsPort?: number | undefined;
   /** Previously deployed contracts on L1 */
   deployL1ContractsValues?: DeployL1Contracts;
   /** Whether to skip deployment of protocol contracts (auth registry, etc) */

diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts
@@ -0,0 +1,31 @@
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
+import {
+  type TelemetryClientConfig,
+  createAndStartTelemetryClient,
+  getConfigEnvVars as getTelemetryConfig,
+} from '@aztec/telemetry-client/start';
+
+export function getEndToEndTestTelemetryClient(metricsPort?: number, serviceName?: string): Promise<TelemetryClient> {
+  return !metricsPort
+    ? Promise.resolve(new NoopTelemetryClient())
+    : createAndStartTelemetryClient(getEndToEndTestTelemetryConfig(metricsPort, serviceName));
+}
+
+/**
+ * Utility functions for setting up end-to-end tests with telemetry.
+ *
+ * Read from env vars, override if metricsPort is set
+ */
+export function getEndToEndTestTelemetryConfig(metricsPort?: number, serviceName?: string) {
+  const telemetryConfig: TelemetryClientConfig = getTelemetryConfig();
+  if (metricsPort) {
+    telemetryConfig.metricsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/metrics`);
+    telemetryConfig.tracesCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/traces`);
+    telemetryConfig.logsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/logs`);
+  }
+  if (serviceName) {
+    telemetryConfig.serviceName = serviceName;
+  }
+  return telemetryConfig;
+}
diff --git a/yarn-project/p2p-bootstrap/package.json b/yarn-project/p2p-bootstrap/package.json
@@ -26,6 +26,7 @@
   "dependencies": {
     "@aztec/foundation": "workspace:^",
     "@aztec/p2p": "workspace:^",
+    "@aztec/telemetry-client": "workspace:^",
     "dotenv": "^16.0.3",
     "koa": "^2.15.3",
     "koa-router": "^12.0.1",

diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts
@@ -1,5 +1,7 @@
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import Koa from 'koa';
 import Router from 'koa-router';
@@ -11,8 +13,12 @@ const { HTTP_PORT } = process.env;
 /**
  * The application entry point.
  */
-async function main(config: BootnodeConfig, logger = debugLogger) {
-  const bootstrapNode = new BootstrapNode(logger);
+async function main(
+  config: BootnodeConfig,
+  telemetryClient: TelemetryClient = new NoopTelemetryClient(),
+  logger = debugLogger,
+) {
+  const bootstrapNode = new BootstrapNode(telemetryClient, logger);
   await bootstrapNode.start(config);
   logger.info('DiscV5 Bootnode started');
 

diff --git a/yarn-project/p2p-bootstrap/tsconfig.json b/yarn-project/p2p-bootstrap/tsconfig.json
@@ -11,6 +11,9 @@
     },
     {
       "path": "../p2p"
+    },
+    {
+      "path": "../telemetry-client"
     }
   ],
   "include": ["src"]

diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json
@@ -82,6 +82,7 @@
     "@libp2p/peer-id": "4.0.7",
     "@libp2p/peer-id-factory": "4.1.1",
     "@libp2p/peer-store": "10.0.16",
+    "@libp2p/prometheus-metrics": "^4.2.4",
     "@libp2p/tcp": "9.0.24",
     "@multiformats/multiaddr": "12.1.14",
     "interface-datastore": "^8.2.11",

diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts
@@ -1,4 +1,5 @@
 import { createDebugLogger } from '@aztec/foundation/log';
+import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client';
 
 import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5';
 import { SignableENR } from '@chainsafe/enr';
@@ -17,7 +18,7 @@ export class BootstrapNode {
   private node?: Discv5 = undefined;
   private peerId?: PeerId;
 
-  constructor(private logger = createDebugLogger('aztec:p2p_bootstrap')) {}
+  constructor(private telemetry: TelemetryClient, private logger = createDebugLogger('aztec:p2p_bootstrap')) {}
 
   /**
    * Starts the bootstrap node.
@@ -41,7 +42,7 @@ export class BootstrapNode {
     enr.set(AZTEC_ENR_KEY, Uint8Array.from([AZTEC_NET]));
 
     this.logger.info(`Starting bootstrap node ${peerId}, listening on ${listenAddrUdp.toString()}`);
-
+    const metricsRegistry = new OtelMetricsAdapter(this.telemetry);
     this.node = Discv5.create({
       enr,
       peerId,
@@ -50,6 +51,7 @@ export class BootstrapNode {
         lookupTimeout: 2000,
         allowUnverifiedSessions: true,
       },
+      metricsRegistry,
     });
 
     (this.node as Discv5EventEmitter).on('multiaddrUpdated', (addr: Multiaddr) => {

diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts
@@ -49,7 +49,7 @@ export const createP2PClient = async (
 
     // Create peer discovery service
     const peerId = await createLibP2PPeerId(config.peerIdPrivateKey);
-    const discoveryService = new DiscV5Service(peerId, config);
+    const discoveryService = new DiscV5Service(peerId, config, telemetry);
 
     p2pService = await LibP2PService.new(
       config,
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,9 @@ @@
         },
         {
           "path": "../p2p"
+        },
+        {
+          "path": "../telemetry-client"
         }
       ],
       "include": ["src"]
@@ Expand Down @@