diff --git a/yarn-project/archiver/src/rpc/index.ts b/yarn-project/archiver/src/rpc/index.ts index 5845eacb4b8..3535b22b32a 100644 --- a/yarn-project/archiver/src/rpc/index.ts +++ b/yarn-project/archiver/src/rpc/index.ts @@ -1,11 +1,11 @@ import { type ArchiverApi, ArchiverApiSchema } from '@aztec/circuit-types'; -import { createSafeJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; -import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client'; +import { createTracedJsonRpcServer, makeTracedFetch } from '@aztec/telemetry-client'; -export function createArchiverClient(url: string, fetch = makeFetch([1, 2, 3], true)): ArchiverApi { +export function createArchiverClient(url: string, fetch = makeTracedFetch([1, 2, 3], true)): ArchiverApi { return createSafeJsonRpcClient(url, ArchiverApiSchema, false, 'archiver', fetch); } export function createArchiverRpcServer(handler: ArchiverApi) { - return createSafeJsonRpcServer(handler, ArchiverApiSchema); + return createTracedJsonRpcServer(handler, ArchiverApiSchema); } diff --git a/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts b/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts index 3ceaf4c6c69..04b21424e61 100644 --- a/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts +++ b/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts @@ -1,5 +1,5 @@ import { type AztecNode, AztecNodeApiSchema } from '@aztec/circuit-types'; -import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createTracedJsonRpcServer } from '@aztec/telemetry-client'; /** * Wrap an AztecNode instance with a JSON RPC HTTP server. @@ -7,5 +7,5 @@ import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; * @returns An JSON-RPC HTTP server */ export function createAztecNodeRpcServer(node: AztecNode) { - return createSafeJsonRpcServer(node, AztecNodeApiSchema); + return createTracedJsonRpcServer(node, AztecNodeApiSchema); } diff --git a/yarn-project/aztec/src/cli/aztec_start_action.ts b/yarn-project/aztec/src/cli/aztec_start_action.ts index 7e8e81b142e..f97db8d053d 100644 --- a/yarn-project/aztec/src/cli/aztec_start_action.ts +++ b/yarn-project/aztec/src/cli/aztec_start_action.ts @@ -7,6 +7,7 @@ import { } from '@aztec/foundation/json-rpc/server'; import { type LogFn, type Logger } from '@aztec/foundation/log'; import { fileURLToPath } from '@aztec/foundation/url'; +import { getOtelJsonRpcPropagationMiddleware } from '@aztec/telemetry-client'; import { readFileSync } from 'fs'; import { dirname, resolve } from 'path'; @@ -102,7 +103,11 @@ export async function aztecStart(options: any, userLog: LogFn, debugLogger: Logg installSignalHandlers(debugLogger.info, signalHandlers); if (Object.entries(services).length > 0) { - const rpcServer = createNamespacedSafeJsonRpcServer(services, { http200OnError: false, log: debugLogger }); + const rpcServer = createNamespacedSafeJsonRpcServer(services, { + http200OnError: false, + log: debugLogger, + middlewares: [getOtelJsonRpcPropagationMiddleware()], + }); const { port } = await startHttpRpcServer(rpcServer, { port: options.port }); debugLogger.info(`Aztec Server listening on port ${port}`); } diff --git a/yarn-project/aztec/src/cli/cmds/start_pxe.ts b/yarn-project/aztec/src/cli/cmds/start_pxe.ts index 949995d70b7..278892571a1 100644 --- a/yarn-project/aztec/src/cli/cmds/start_pxe.ts +++ b/yarn-project/aztec/src/cli/cmds/start_pxe.ts @@ -16,6 +16,7 @@ import { allPxeConfigMappings, createPXEService, } from '@aztec/pxe'; +import { makeTracedFetch } from '@aztec/telemetry-client'; import { L2BasicContractsMap, Network } from '@aztec/types/network'; import { extractRelevantOptions } from '../util.js'; @@ -76,7 +77,7 @@ export async function addPXE( process.exit(1); } - const node = deps.node ?? createAztecNodeClient(nodeUrl!); + const node = deps.node ?? createAztecNodeClient(nodeUrl!, makeTracedFetch([1, 2, 3], true)); const pxe = await createPXEService(node, pxeConfig as PXEServiceConfig); // register basic contracts diff --git a/yarn-project/bot/src/factory.ts b/yarn-project/bot/src/factory.ts index 440b219a3f5..d5f4eb73fe3 100644 --- a/yarn-project/bot/src/factory.ts +++ b/yarn-project/bot/src/factory.ts @@ -11,6 +11,7 @@ import { type AztecNode, type FunctionCall, type PXE } from '@aztec/circuit-type import { Fr, deriveSigningKey } from '@aztec/circuits.js'; import { EasyPrivateTokenContract } from '@aztec/noir-contracts.js/EasyPrivateToken'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; +import { makeTracedFetch } from '@aztec/telemetry-client'; import { type BotConfig, SupportedTokenContracts } from './config.js'; import { getBalances, getPrivateBalance, isStandardTokenContract } from './utils.js'; @@ -39,7 +40,7 @@ export class BotFactory { return; } this.log.info(`Using remote PXE at ${config.pxeUrl!}`); - this.pxe = createPXEClient(config.pxeUrl!); + this.pxe = createPXEClient(config.pxeUrl!, makeTracedFetch([1, 2, 3], false)); } /** diff --git a/yarn-project/bot/src/rpc.ts b/yarn-project/bot/src/rpc.ts index db21442eaec..d565919b239 100644 --- a/yarn-project/bot/src/rpc.ts +++ b/yarn-project/bot/src/rpc.ts @@ -1,4 +1,5 @@ -import { type ApiHandler, createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { type ApiHandler } from '@aztec/foundation/json-rpc/server'; +import { createTracedJsonRpcServer } from '@aztec/telemetry-client'; import { BotRunnerApiSchema } from './interface.js'; import { type BotRunner } from './runner.js'; @@ -9,7 +10,7 @@ import { type BotRunner } from './runner.js'; * @returns An JSON-RPC HTTP server */ export function createBotRunnerRpcServer(botRunner: BotRunner) { - createSafeJsonRpcServer(botRunner, BotRunnerApiSchema, { + createTracedJsonRpcServer(botRunner, BotRunnerApiSchema, { http200OnError: false, healthCheck: botRunner.isHealthy.bind(botRunner), }); diff --git a/yarn-project/bot/src/runner.ts b/yarn-project/bot/src/runner.ts index 626f8f6582c..e2731439d7b 100644 --- a/yarn-project/bot/src/runner.ts +++ b/yarn-project/bot/src/runner.ts @@ -1,6 +1,6 @@ import { type AztecNode, type PXE, createAztecNodeClient, createLogger } from '@aztec/aztec.js'; import { RunningPromise } from '@aztec/foundation/running-promise'; -import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client'; +import { type TelemetryClient, type Traceable, type Tracer, makeTracedFetch, trackSpan } from '@aztec/telemetry-client'; import { Bot } from './bot.js'; import { type BotConfig } from './config.js'; @@ -26,19 +26,24 @@ export class BotRunner implements BotRunnerApi, Traceable { if (!dependencies.node && !config.nodeUrl) { throw new Error(`Missing node URL in config or dependencies`); } - this.node = dependencies.node ?? createAztecNodeClient(config.nodeUrl!); + this.node = dependencies.node ?? createAztecNodeClient(config.nodeUrl!, makeTracedFetch([1, 2, 3], true)); this.runningPromise = new RunningPromise(() => this.#work(), this.log, config.txIntervalSeconds * 1000); } /** Initializes the bot if needed. Blocks until the bot setup is finished. */ public async setup() { if (!this.bot) { - this.log.verbose(`Setting up bot`); - await this.#createBot(); - this.log.info(`Bot set up completed`); + await this.doSetup(); } } + @trackSpan('Bot.setup') + private async doSetup() { + this.log.verbose(`Setting up bot`); + await this.#createBot(); + this.log.info(`Bot set up completed`); + } + /** * Initializes the bot if needed and starts sending txs at regular intervals. * Blocks until the bot setup is finished. diff --git a/yarn-project/foundation/src/json-rpc/client/fetch.ts b/yarn-project/foundation/src/json-rpc/client/fetch.ts index 70ecc8ec363..fa02103a121 100644 --- a/yarn-project/foundation/src/json-rpc/client/fetch.ts +++ b/yarn-project/foundation/src/json-rpc/client/fetch.ts @@ -21,6 +21,7 @@ export async function defaultFetch( rpcMethod: string, body: any, useApiEndpoints: boolean, + extraHeaders: Record = {}, noRetry = false, ) { log.debug(format(`JsonRpcClient.fetch`, host, rpcMethod, '->', body)); @@ -30,13 +31,13 @@ export async function defaultFetch( resp = await fetch(`${host}/${rpcMethod}`, { method: 'POST', body: jsonStringify(body), - headers: { 'content-type': 'application/json' }, + headers: { 'content-type': 'application/json', ...extraHeaders }, }); } else { resp = await fetch(host, { method: 'POST', body: jsonStringify({ ...body, method: rpcMethod }), - headers: { 'content-type': 'application/json' }, + headers: { 'content-type': 'application/json', ...extraHeaders }, }); } } catch (err) { @@ -55,7 +56,7 @@ export async function defaultFetch( } if (!resp.ok) { - const errorMessage = `(JSON-RPC PROPAGATED) (host ${host}) (method ${rpcMethod}) (code ${resp.status}) ${responseJson.error.message}`; + const errorMessage = `Error ${resp.status} from server ${host} on ${rpcMethod}: ${responseJson.error.message}`; if (noRetry || (resp.status >= 400 && resp.status < 500)) { throw new NoRetryError(errorMessage); } else { @@ -73,10 +74,17 @@ export async function defaultFetch( * @param log - Optional logger for logging attempts. * @returns A fetch function. */ -export function makeFetch(retries: number[], defaultNoRetry: boolean, log?: Logger) { - return async (host: string, rpcMethod: string, body: any, useApiEndpoints: boolean, noRetry?: boolean) => { +export function makeFetch(retries: number[], defaultNoRetry: boolean, log?: Logger): typeof defaultFetch { + return async ( + host: string, + rpcMethod: string, + body: any, + useApiEndpoints: boolean, + extraHeaders: Record = {}, + noRetry?: boolean, + ) => { return await retry( - () => defaultFetch(host, rpcMethod, body, useApiEndpoints, noRetry ?? defaultNoRetry), + () => defaultFetch(host, rpcMethod, body, useApiEndpoints, extraHeaders, noRetry ?? defaultNoRetry), `JsonRpcClient request ${rpcMethod} to ${host}`, makeBackoff(retries), log, diff --git a/yarn-project/foundation/src/json-rpc/server/safe_json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/safe_json_rpc_server.ts index 796e8482a63..723c0a00eb8 100644 --- a/yarn-project/foundation/src/json-rpc/server/safe_json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/safe_json_rpc_server.ts @@ -1,6 +1,6 @@ import cors from '@koa/cors'; import http from 'http'; -import Koa from 'koa'; +import { type default as Application, default as Koa } from 'koa'; import bodyParser from 'koa-bodyparser'; import compress from 'koa-compress'; import Router from 'koa-router'; @@ -14,6 +14,15 @@ import { type ApiSchema, type ApiSchemaFor, parseWithOptionals, schemaHasMethod import { jsonStringify } from '../convert.js'; import { assert } from '../js_utils.js'; +export type DiagnosticsData = { + id: number | string | null; + method: string; + params: any[]; + headers: http.IncomingHttpHeaders; +}; + +export type DiagnosticsMiddleware = (ctx: DiagnosticsData, next: () => Promise) => Promise; + export class SafeJsonRpcServer { /** * The HTTP server accepting remote requests. @@ -31,6 +40,8 @@ export class SafeJsonRpcServer { private http200OnError = false, /** Health check function */ private readonly healthCheck: StatusCheckFn = () => true, + /** Additional middlewares */ + private extraMiddlewares: Application.Middleware[] = [], /** Logger */ private log = createLogger('json-rpc:server'), ) {} @@ -90,8 +101,11 @@ export class SafeJsonRpcServer { this.log.error(`Error on API handler: ${error}`); }); - app.use(compress({ br: false } as any)); + app.use(compress({ br: false })); app.use(jsonResponse); + for (const middleware of this.extraMiddlewares) { + app.use(middleware); + } app.use(exceptionHandler); app.use(bodyParser({ jsonLimit: '50mb', enableTypes: ['json'], detectJSON: () => true })); app.use(cors()); @@ -114,7 +128,9 @@ export class SafeJsonRpcServer { // Fail if not a registered function in the proxy if (typeof method !== 'string' || method === 'constructor' || !this.proxy.hasMethod(method)) { ctx.status = 400; - ctx.body = { jsonrpc, id, error: { code: -32601, message: `Method not found: ${method}` } }; + const code = -32601; + const message = `Method not found: ${method}`; + ctx.body = { jsonrpc, id, error: { code, message } }; } else { ctx.status = 200; const result = await this.proxy.call(method, params); @@ -263,10 +279,11 @@ function makeAggregateHealthcheck(namedHandlers: NamespacedApiHandlers, log?: Lo }; } -type SafeJsonRpcServerOptions = { +export type SafeJsonRpcServerOptions = { http200OnError: boolean; healthCheck?: StatusCheckFn; log?: Logger; + middlewares?: Application.Middleware[]; }; /** @@ -276,25 +293,24 @@ type SafeJsonRpcServerOptions = { */ export function createNamespacedSafeJsonRpcServer( handlers: NamespacedApiHandlers, - options: Omit = { - http200OnError: false, + options: Partial> = { log: createLogger('json-rpc:server'), }, ): SafeJsonRpcServer { - const { http200OnError, log } = options; + const { middlewares, http200OnError, log } = options; const proxy = new NamespacedSafeJsonProxy(handlers); const healthCheck = makeAggregateHealthcheck(handlers, log); - return new SafeJsonRpcServer(proxy, http200OnError, healthCheck, log); + return new SafeJsonRpcServer(proxy, http200OnError, healthCheck, middlewares, log); } export function createSafeJsonRpcServer( handler: T, schema: ApiSchemaFor, - options: SafeJsonRpcServerOptions = { http200OnError: false }, + options: Partial = {}, ) { - const { http200OnError, log, healthCheck } = options; + const { http200OnError, log, healthCheck, middlewares: extraMiddlewares } = options; const proxy = new SafeJsonProxy(handler, schema); - return new SafeJsonRpcServer(proxy, http200OnError, healthCheck, log); + return new SafeJsonRpcServer(proxy, http200OnError, healthCheck, extraMiddlewares, log); } /** diff --git a/yarn-project/foundation/src/json-rpc/server/telemetry.ts b/yarn-project/foundation/src/json-rpc/server/telemetry.ts new file mode 100644 index 00000000000..e69de29bb2d diff --git a/yarn-project/prover-client/src/prover-agent/rpc.ts b/yarn-project/prover-client/src/prover-agent/rpc.ts index 0e1cdc91fd0..14d8d3c90a6 100644 --- a/yarn-project/prover-client/src/prover-agent/rpc.ts +++ b/yarn-project/prover-client/src/prover-agent/rpc.ts @@ -1,14 +1,14 @@ import { ProverAgentApiSchema, type ProvingJobSource, ProvingJobSourceSchema } from '@aztec/circuit-types'; -import { createSafeJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; -import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client'; +import { createTracedJsonRpcServer, makeTracedFetch } from '@aztec/telemetry-client'; import { type ProverAgent } from './prover-agent.js'; export function createProvingJobSourceServer(queue: ProvingJobSource) { - return createSafeJsonRpcServer(queue, ProvingJobSourceSchema); + return createTracedJsonRpcServer(queue, ProvingJobSourceSchema); } -export function createProvingJobSourceClient(url: string, fetch = makeFetch([1, 2, 3], false)): ProvingJobSource { +export function createProvingJobSourceClient(url: string, fetch = makeTracedFetch([1, 2, 3], false)): ProvingJobSource { return createSafeJsonRpcClient(url, ProvingJobSourceSchema, false, 'provingJobSource', fetch); } @@ -18,5 +18,5 @@ export function createProvingJobSourceClient(url: string, fetch = makeFetch([1, * @returns An JSON-RPC HTTP server */ export function createProverAgentRpcServer(agent: ProverAgent) { - return createSafeJsonRpcServer(agent, ProverAgentApiSchema); + return createTracedJsonRpcServer(agent, ProverAgentApiSchema); } diff --git a/yarn-project/prover-client/src/proving_broker/rpc.ts b/yarn-project/prover-client/src/proving_broker/rpc.ts index cabc716c988..74a66580357 100644 --- a/yarn-project/prover-client/src/proving_broker/rpc.ts +++ b/yarn-project/prover-client/src/proving_broker/rpc.ts @@ -9,9 +9,10 @@ import { ProvingJobStatus, ProvingRequestType, } from '@aztec/circuit-types'; -import { createSafeJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; -import { type SafeJsonRpcServer, createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client'; +import { type SafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { type ApiSchemaFor, optional } from '@aztec/foundation/schemas'; +import { createTracedJsonRpcServer, makeTracedFetch } from '@aztec/telemetry-client'; import { z } from 'zod'; @@ -47,17 +48,23 @@ export const ProvingJobBrokerSchema: ApiSchemaFor = { }; export function createProvingBrokerServer(broker: ProvingJobBroker): SafeJsonRpcServer { - return createSafeJsonRpcServer(broker, ProvingJobBrokerSchema); + return createTracedJsonRpcServer(broker, ProvingJobBrokerSchema); } -export function createProvingJobBrokerClient(url: string, fetch = makeFetch([1, 2, 3], false)): ProvingJobBroker { +export function createProvingJobBrokerClient(url: string, fetch = makeTracedFetch([1, 2, 3], false)): ProvingJobBroker { return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, false, 'proverBroker', fetch); } -export function createProvingJobProducerClient(url: string, fetch = makeFetch([1, 2, 3], false)): ProvingJobProducer { +export function createProvingJobProducerClient( + url: string, + fetch = makeTracedFetch([1, 2, 3], false), +): ProvingJobProducer { return createSafeJsonRpcClient(url, ProvingJobProducerSchema, false, 'provingJobProducer', fetch); } -export function createProvingJobConsumerClient(url: string, fetch = makeFetch([1, 2, 3], false)): ProvingJobConsumer { +export function createProvingJobConsumerClient( + url: string, + fetch = makeTracedFetch([1, 2, 3], false), +): ProvingJobConsumer { return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, false, 'provingJobConsumer', fetch); } diff --git a/yarn-project/prover-node/src/http.ts b/yarn-project/prover-node/src/http.ts index 0178e4be79d..dc15d4c3b31 100644 --- a/yarn-project/prover-node/src/http.ts +++ b/yarn-project/prover-node/src/http.ts @@ -1,5 +1,5 @@ import { ProverNodeApiSchema } from '@aztec/circuit-types'; -import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createTracedJsonRpcServer } from '@aztec/telemetry-client'; import { type ProverNode } from './prover-node.js'; @@ -9,5 +9,5 @@ import { type ProverNode } from './prover-node.js'; * @returns An JSON-RPC HTTP server */ export function createProverNodeRpcServer(node: ProverNode) { - return createSafeJsonRpcServer(node, ProverNodeApiSchema); + return createTracedJsonRpcServer(node, ProverNodeApiSchema); } diff --git a/yarn-project/prover-node/src/prover-coordination/factory.ts b/yarn-project/prover-node/src/prover-coordination/factory.ts index 88731deec2a..6a308f0830e 100644 --- a/yarn-project/prover-node/src/prover-coordination/factory.ts +++ b/yarn-project/prover-node/src/prover-coordination/factory.ts @@ -10,7 +10,7 @@ import { type EpochCache } from '@aztec/epoch-cache'; import { createLogger } from '@aztec/foundation/log'; import { type DataStoreConfig } from '@aztec/kv-store/config'; import { createP2PClient } from '@aztec/p2p'; -import { type TelemetryClient } from '@aztec/telemetry-client'; +import { type TelemetryClient, makeTracedFetch } from '@aztec/telemetry-client'; import { type ProverNodeConfig } from '../config.js'; @@ -64,7 +64,7 @@ export async function createProverCoordination( if (config.proverCoordinationNodeUrl) { log.info('Using prover coordination via node url'); - return createAztecNodeClient(config.proverCoordinationNodeUrl); + return createAztecNodeClient(config.proverCoordinationNodeUrl, makeTracedFetch([1, 2, 3], false)); } else { throw new Error(`Aztec Node URL for Tx Provider is not set.`); } diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json index 48cbd579392..887088da917 100644 --- a/yarn-project/telemetry-client/package.json +++ b/yarn-project/telemetry-client/package.json @@ -50,6 +50,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", + "@types/koa": "^2.15.0", "jest": "^29.5.0", "ts-node": "^10.9.1", "typescript": "^5.0.4" diff --git a/yarn-project/telemetry-client/src/index.ts b/yarn-project/telemetry-client/src/index.ts index e593b4575f4..c8617bdc5ef 100644 --- a/yarn-project/telemetry-client/src/index.ts +++ b/yarn-project/telemetry-client/src/index.ts @@ -5,3 +5,4 @@ export * from './prom_otel_adapter.js'; export * from './lmdb_metrics.js'; export * from './wrappers/index.js'; export * from './start.js'; +export * from './otel_propagation.js'; diff --git a/yarn-project/telemetry-client/src/otel_propagation.ts b/yarn-project/telemetry-client/src/otel_propagation.ts new file mode 100644 index 00000000000..99f766769c7 --- /dev/null +++ b/yarn-project/telemetry-client/src/otel_propagation.ts @@ -0,0 +1,50 @@ +import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api'; +import type Koa from 'koa'; + +import { getTelemetryClient } from './start.js'; +import { + ATTR_JSONRPC_ERROR_CODE, + ATTR_JSONRPC_ERROR_MSG, + ATTR_JSONRPC_METHOD, + ATTR_JSONRPC_REQUEST_ID, +} from './vendor/attributes.js'; + +export function getOtelJsonRpcPropagationMiddleware( + scope = 'JsonRpcServer', +): (ctx: Koa.Context, next: () => Promise) => void { + return function otelJsonRpcPropagation(ctx: Koa.Context, next: () => Promise) { + const tracer = getTelemetryClient().getTracer(scope); + const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers); + const method = (ctx.request.body as any)?.method; + return tracer.startActiveSpan( + `JsonRpcServer.${method ?? 'unknown'}`, + { kind: SpanKind.SERVER }, + context, + async (span: Span): Promise => { + if (ctx.id) { + span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id); + } + if (method) { + span.setAttribute(ATTR_JSONRPC_METHOD, method); + } + + try { + await next(); + const err = (ctx.body as any).error?.message; + const code = (ctx.body as any).error?.code; + if (err) { + span.setStatus({ code: SpanStatusCode.ERROR, message: err }); + span.setAttribute(ATTR_JSONRPC_ERROR_CODE, code); + span.setAttribute(ATTR_JSONRPC_ERROR_MSG, err); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + } catch (err) { + span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) }); + } finally { + span.end(); + } + }, + ); + }; +} diff --git a/yarn-project/telemetry-client/src/telemetry.ts b/yarn-project/telemetry-client/src/telemetry.ts index 6672ce17659..3e4a2a1e887 100644 --- a/yarn-project/telemetry-client/src/telemetry.ts +++ b/yarn-project/telemetry-client/src/telemetry.ts @@ -20,7 +20,7 @@ import * as Attributes from './attributes.js'; import * as Metrics from './metrics.js'; import { getTelemetryClient } from './start.js'; -export { Span, ValueType } from '@opentelemetry/api'; +export { Span, SpanStatusCode, ValueType } from '@opentelemetry/api'; type ValuesOf = T extends Record ? U : never; diff --git a/yarn-project/telemetry-client/src/vendor/attributes.ts b/yarn-project/telemetry-client/src/vendor/attributes.ts new file mode 100644 index 00000000000..9603a5254a0 --- /dev/null +++ b/yarn-project/telemetry-client/src/vendor/attributes.ts @@ -0,0 +1,5 @@ +// See https://opentelemetry.io/docs/specs/semconv/rpc/json-rpc/ +export const ATTR_JSONRPC_METHOD = 'rpc.method'; +export const ATTR_JSONRPC_REQUEST_ID = 'rpc.jsonrpc.request_id'; +export const ATTR_JSONRPC_ERROR_CODE = 'rpc.jsonrpc.error_code'; +export const ATTR_JSONRPC_ERROR_MSG = 'rpc.jsonrpc.error_message'; diff --git a/yarn-project/telemetry-client/src/wrappers/fetch.ts b/yarn-project/telemetry-client/src/wrappers/fetch.ts new file mode 100644 index 00000000000..d4506deb824 --- /dev/null +++ b/yarn-project/telemetry-client/src/wrappers/fetch.ts @@ -0,0 +1,52 @@ +import { defaultFetch } from '@aztec/foundation/json-rpc/client'; +import { type Logger } from '@aztec/foundation/log'; +import { makeBackoff, retry } from '@aztec/foundation/retry'; + +import { SpanKind, SpanStatusCode, context, propagation } from '@opentelemetry/api'; + +import { getTelemetryClient } from '../start.js'; +import { ATTR_JSONRPC_METHOD, ATTR_JSONRPC_REQUEST_ID } from '../vendor/attributes.js'; + +/** + * Makes a fetch function that retries based on the given attempts and propagates trace information. + * @param retries - Sequence of intervals (in seconds) to retry. + * @param noRetry - Whether to stop retries on server errors. + * @param log - Optional logger for logging attempts. + * @returns A fetch function. + */ +export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, log?: Logger) { + return ( + host: string, + rpcMethod: string, + body: any, + useApiEndpoints: boolean, + extraHeaders: Record = {}, + noRetry?: boolean, + ) => { + const telemetry = getTelemetryClient(); + return telemetry + .getTracer('fetch') + .startActiveSpan(`JsonRpcClient.${rpcMethod}`, { kind: SpanKind.CLIENT }, async span => { + try { + if (body && typeof body.id === 'number') { + span.setAttribute(ATTR_JSONRPC_REQUEST_ID, body.id); + } + span.setAttribute(ATTR_JSONRPC_METHOD, rpcMethod); + const headers = { ...extraHeaders }; + propagation.inject(context.active(), headers); + return await retry( + () => defaultFetch(host, rpcMethod, body, useApiEndpoints, headers, noRetry ?? defaultNoRetry), + `JsonRpcClient request ${rpcMethod} to ${host}`, + makeBackoff(retries), + log, + false, + ); + } catch (err: any) { + span.setStatus({ code: SpanStatusCode.ERROR, message: err?.message ?? String(err) }); + throw err; + } finally { + span.end(); + } + }); + }; +} diff --git a/yarn-project/telemetry-client/src/wrappers/index.ts b/yarn-project/telemetry-client/src/wrappers/index.ts index 3a6efb1f42e..ea9c9632436 100644 --- a/yarn-project/telemetry-client/src/wrappers/index.ts +++ b/yarn-project/telemetry-client/src/wrappers/index.ts @@ -1 +1,3 @@ export * from './l2_block_stream.js'; +export * from './fetch.js'; +export * from './json_rpc_server.js'; diff --git a/yarn-project/telemetry-client/src/wrappers/json_rpc_server.ts b/yarn-project/telemetry-client/src/wrappers/json_rpc_server.ts new file mode 100644 index 00000000000..f600fedd4a7 --- /dev/null +++ b/yarn-project/telemetry-client/src/wrappers/json_rpc_server.ts @@ -0,0 +1,15 @@ +import { type SafeJsonRpcServerOptions, createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { type ApiSchemaFor } from '@aztec/foundation/schemas'; + +import { getOtelJsonRpcPropagationMiddleware } from '../otel_propagation.js'; + +export function createTracedJsonRpcServer( + handler: T, + schema: ApiSchemaFor, + options: Partial = {}, +) { + return createSafeJsonRpcServer(handler, schema, { + ...options, + middlewares: [...(options.middlewares ?? []), getOtelJsonRpcPropagationMiddleware()], + }); +} diff --git a/yarn-project/txe/src/index.ts b/yarn-project/txe/src/index.ts index a2d48d610b7..915eb46e09d 100644 --- a/yarn-project/txe/src/index.ts +++ b/yarn-project/txe/src/index.ts @@ -1,7 +1,7 @@ import { loadContractArtifact } from '@aztec/aztec.js'; -import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { type Logger } from '@aztec/foundation/log'; import { type ApiSchemaFor, type ZodFor } from '@aztec/foundation/schemas'; +import { createTracedJsonRpcServer } from '@aztec/telemetry-client'; import { readFile, readdir } from 'fs/promises'; import { join } from 'path'; @@ -120,5 +120,7 @@ const TXEDispatcherApiSchema: ApiSchemaFor = { * @returns A TXE RPC server. */ export function createTXERpcServer(logger: Logger) { - return createSafeJsonRpcServer(new TXEDispatcher(logger), TXEDispatcherApiSchema, { http200OnError: true }); + return createTracedJsonRpcServer(new TXEDispatcher(logger), TXEDispatcherApiSchema, { + http200OnError: true, + }); } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index cf5d80eacad..96530873d7b 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -1296,6 +1296,7 @@ __metadata: "@opentelemetry/sdk-trace-node": "npm:^1.28.0" "@opentelemetry/semantic-conventions": "npm:^1.28.0" "@types/jest": "npm:^29.5.0" + "@types/koa": "npm:^2.15.0" jest: "npm:^29.5.0" prom-client: "npm:^15.1.3" ts-node: "npm:^10.9.1"