From 6ba2bf208c08191b6d9ebbc2fe54a956ba29f7df Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Wed, 30 Aug 2023 21:59:56 -0300 Subject: [PATCH 01/15] Refactor combiner --- .../combiner/src/common/error.ts | 19 +++ .../combiner/src/common/handlers.ts | 155 ++++++++++++++++-- .../src/domain/endpoints/disable/action.ts | 29 ++-- .../src/domain/endpoints/quota/action.ts | 28 ++-- .../src/domain/endpoints/sign/action.ts | 30 ++-- .../combiner/src/index.ts | 4 +- .../src/pnp/endpoints/quota/action.ts | 33 ++-- .../combiner/src/pnp/endpoints/sign/action.ts | 34 ++-- .../combiner/src/server.ts | 87 +++++++--- 9 files changed, 289 insertions(+), 130 deletions(-) create mode 100644 packages/phone-number-privacy/combiner/src/common/error.ts diff --git a/packages/phone-number-privacy/combiner/src/common/error.ts b/packages/phone-number-privacy/combiner/src/common/error.ts new file mode 100644 index 00000000000..91060f23691 --- /dev/null +++ b/packages/phone-number-privacy/combiner/src/common/error.ts @@ -0,0 +1,19 @@ +import { ErrorType } from '@celo/phone-number-privacy-common' + +export class OdisError extends Error { + constructor(readonly code: ErrorType, readonly parent?: Error, readonly status: number = 500) { + // This is necessary when extending Error Classes + super(code) // 'Error' breaks prototype chain here + Object.setPrototypeOf(this, new.target.prototype) // restore prototype chain + } +} + +export function wrapError( + valueOrError: Promise, + code: ErrorType, + status: number = 500 +): Promise { + return valueOrError.catch((parentErr) => { + throw new OdisError(code, parentErr, status) + }) +} diff --git a/packages/phone-number-privacy/combiner/src/common/handlers.ts b/packages/phone-number-privacy/combiner/src/common/handlers.ts index ac0872993bf..f6681450598 100644 --- a/packages/phone-number-privacy/combiner/src/common/handlers.ts +++ b/packages/phone-number-privacy/combiner/src/common/handlers.ts @@ -1,13 +1,22 @@ import { ErrorMessage, + ErrorType, OdisRequest, OdisResponse, + PnpQuotaStatus, + SequentialDelayDomainState, WarningMessage, + send, } from '@celo/phone-number-privacy-common' +import opentelemetry, { SpanStatusCode } from '@opentelemetry/api' +import { SemanticAttributes } from '@opentelemetry/semantic-conventions' import Logger from 'bunyan' import { Request, Response } from 'express' -import { performance, PerformanceObserver } from 'perf_hooks' -import { sendFailure } from './io' +import { PerformanceObserver, performance } from 'perf_hooks' +import { getCombinerVersion } from '../config' +import { OdisError } from './error' + +const tracer = opentelemetry.trace.getTracer('combiner-tracer') export interface Locals { logger: Logger @@ -18,24 +27,22 @@ export type PromiseHandler = ( res: Response, Locals> ) => Promise -type ParentHandler = (req: Request<{}, {}, any>, res: Response) => Promise - export function catchErrorHandler( handler: PromiseHandler -): ParentHandler { +): PromiseHandler { return async (req, res) => { - const logger: Logger = res.locals.logger try { await handler(req, res) } catch (err) { + const logger: Logger = res.locals.logger logger.error(ErrorMessage.CAUGHT_ERROR_IN_ENDPOINT_HANDLER) logger.error(err) if (!res.headersSent) { - logger.info('Responding with error in outer endpoint handler') - res.status(500).json({ - success: false, - error: ErrorMessage.UNKNOWN_ERROR, - }) + if (err instanceof OdisError) { + sendFailure(err.code, err.status, res, req.url) + } else { + sendFailure(ErrorMessage.UNKNOWN_ERROR, 500, res, req.url) + } } else { logger.error(ErrorMessage.ERROR_AFTER_RESPONSE_SENT) } @@ -43,6 +50,39 @@ export function catchErrorHandler( } } +export function tracingHandler( + handler: PromiseHandler +): PromiseHandler { + return async (req, res) => { + return tracer.startActiveSpan( + req.url, + { + attributes: { + [SemanticAttributes.HTTP_ROUTE]: req.path, + [SemanticAttributes.HTTP_METHOD]: req.method, + [SemanticAttributes.HTTP_CLIENT_IP]: req.ip, + }, + }, + async (span) => { + try { + await handler(req, res) + span.setStatus({ + code: SpanStatusCode.OK, + }) + } catch (err: any) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : 'Fail', + }) + throw err + } finally { + span.end() + } + } + ) + } +} + export function meteringHandler( handler: PromiseHandler ): PromiseHandler { @@ -86,9 +126,98 @@ export function meteringHandler( } } +export function timeoutHandler( + timeoutMs: number, + handler: PromiseHandler +): PromiseHandler { + return async (req, res) => { + const timeoutSignal = (AbortSignal as any).timeout(timeoutMs) + timeoutSignal.addEventListener( + 'abort', + () => { + if (!res.headersSent) { + sendFailure(ErrorMessage.TIMEOUT_FROM_SIGNER, 500, res, req.url) + } + }, + { once: true } + ) + + await handler(req, res) + } +} + +export function withEnableHandler( + enabled: boolean, + handler: PromiseHandler +): PromiseHandler { + return async (req, res) => { + if (enabled) { + return handler(req, res) + } else { + sendFailure(WarningMessage.API_UNAVAILABLE, 503, res, req.url) + } + } +} + export async function disabledHandler( - _: Request<{}, {}, R>, + req: Request<{}, {}, R>, response: Response, Locals> ): Promise { - sendFailure(WarningMessage.API_UNAVAILABLE, 503, response) + sendFailure(WarningMessage.API_UNAVAILABLE, 503, response, req.url) +} + +export function sendFailure( + error: ErrorType, + status: number, + response: Response, + _endpoint: string, + body?: Record // TODO remove any +) { + send( + response, + { + success: false, + version: getCombinerVersion(), + error, + ...body, + }, + status, + response.locals.logger + ) +} + +export interface Result { + status: number + body: OdisResponse +} + +export type ResultHandler = ( + request: Request<{}, {}, R>, + res: Response, Locals> +) => Promise> + +export function resultHandler( + resHandler: ResultHandler +): PromiseHandler { + return async (req, res) => { + const result = await resHandler(req, res) + send(res, result.body, result.status, res.locals.logger) + } +} + +export function errorResult( + status: number, + error: string, + quotaStatus?: PnpQuotaStatus | { status: SequentialDelayDomainState } +): Result { + // TODO remove any + return { + status, + body: { + success: false, + version: getCombinerVersion(), + error, + ...quotaStatus, + }, + } } diff --git a/packages/phone-number-privacy/combiner/src/domain/endpoints/disable/action.ts b/packages/phone-number-privacy/combiner/src/domain/endpoints/disable/action.ts index 8427bdee30f..2de518eba55 100644 --- a/packages/phone-number-privacy/combiner/src/domain/endpoints/disable/action.ts +++ b/packages/phone-number-privacy/combiner/src/domain/endpoints/disable/action.ts @@ -6,31 +6,28 @@ import { DomainSchema, ErrorMessage, getSignerEndpoint, - send, SequentialDelayDomainStateSchema, verifyDisableDomainRequestAuthenticity, WarningMessage, } from '@celo/phone-number-privacy-common' import { Signer, thresholdCallToSigners } from '../../../common/combine' -import { PromiseHandler } from '../../../common/handlers' -import { getKeyVersionInfo, sendFailure } from '../../../common/io' +import { errorResult, ResultHandler } from '../../../common/handlers' +import { getKeyVersionInfo } from '../../../common/io' import { getCombinerVersion, OdisConfig } from '../../../config' import { logDomainResponseDiscrepancies } from '../../services/log-responses' import { findThresholdDomainState } from '../../services/threshold-state' -export function createDisableDomainHandler( +export function disableDomain( signers: Signer[], config: OdisConfig -): PromiseHandler { +): ResultHandler { return async (request, response) => { if (!disableDomainRequestSchema(DomainSchema).is(request.body)) { - sendFailure(WarningMessage.INVALID_INPUT, 400, response) - return + return errorResult(400, WarningMessage.INVALID_INPUT) } if (!verifyDisableDomainRequestAuthenticity(request.body)) { - sendFailure(WarningMessage.UNAUTHENTICATED_USER, 401, response) - return + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) } // TODO remove? @@ -57,18 +54,14 @@ export function createDisableDomainHandler( signers.length ) if (disableDomainStatus.disabled) { - send( - response, - { + return { + status: 200, + body: { success: true, version: getCombinerVersion(), status: disableDomainStatus, }, - 200, - response.locals.logger - ) - - return + } } } catch (err) { response.locals.logger.error( @@ -77,6 +70,6 @@ export function createDisableDomainHandler( ) } - sendFailure(ErrorMessage.THRESHOLD_DISABLE_DOMAIN_FAILURE, maxErrorCode ?? 500, response) + return errorResult(maxErrorCode ?? 500, ErrorMessage.THRESHOLD_DISABLE_DOMAIN_FAILURE) } } diff --git a/packages/phone-number-privacy/combiner/src/domain/endpoints/quota/action.ts b/packages/phone-number-privacy/combiner/src/domain/endpoints/quota/action.ts index 8d80ee871a5..8112f598a25 100644 --- a/packages/phone-number-privacy/combiner/src/domain/endpoints/quota/action.ts +++ b/packages/phone-number-privacy/combiner/src/domain/endpoints/quota/action.ts @@ -6,31 +6,28 @@ import { DomainSchema, ErrorMessage, getSignerEndpoint, - send, SequentialDelayDomainStateSchema, verifyDomainQuotaStatusRequestAuthenticity, WarningMessage, } from '@celo/phone-number-privacy-common' import { Signer, thresholdCallToSigners } from '../../../common/combine' -import { PromiseHandler } from '../../../common/handlers' -import { getKeyVersionInfo, sendFailure } from '../../../common/io' +import { errorResult, ResultHandler } from '../../../common/handlers' +import { getKeyVersionInfo } from '../../../common/io' import { getCombinerVersion, OdisConfig } from '../../../config' import { logDomainResponseDiscrepancies } from '../../services/log-responses' import { findThresholdDomainState } from '../../services/threshold-state' -export function createDomainQuotaHandler( +export function domainQuota( signers: Signer[], config: OdisConfig -): PromiseHandler { +): ResultHandler { return async (request, response) => { if (!domainQuotaStatusRequestSchema(DomainSchema).is(request.body)) { - sendFailure(WarningMessage.INVALID_INPUT, 400, response) - return + return errorResult(400, WarningMessage.INVALID_INPUT) } if (!verifyDomainQuotaStatusRequestAuthenticity(request.body)) { - sendFailure(WarningMessage.UNAUTHENTICATED_USER, 401, response) - return + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) } // TODO remove? @@ -49,21 +46,18 @@ export function createDomainQuotaHandler( logDomainResponseDiscrepancies(response.locals.logger, signerResponses) if (signerResponses.length >= keyVersionInfo.threshold) { try { - send( - response, - { + return { + status: 200, + body: { success: true, version: getCombinerVersion(), status: findThresholdDomainState(keyVersionInfo, signerResponses, signers.length), }, - 200, - response.locals.logger - ) - return + } } catch (err) { response.locals.logger.error(err, 'Error combining signer quota status responses') } } - sendFailure(ErrorMessage.THRESHOLD_DOMAIN_QUOTA_STATUS_FAILURE, maxErrorCode ?? 500, response) + return errorResult(maxErrorCode ?? 500, ErrorMessage.THRESHOLD_DOMAIN_QUOTA_STATUS_FAILURE) } } diff --git a/packages/phone-number-privacy/combiner/src/domain/endpoints/sign/action.ts b/packages/phone-number-privacy/combiner/src/domain/endpoints/sign/action.ts index fe4ba2c59dc..d696c2f4a06 100644 --- a/packages/phone-number-privacy/combiner/src/domain/endpoints/sign/action.ts +++ b/packages/phone-number-privacy/combiner/src/domain/endpoints/sign/action.ts @@ -8,7 +8,6 @@ import { ErrorType, getSignerEndpoint, OdisResponse, - send, SequentialDelayDomainStateSchema, verifyDomainRestrictedSignatureRequestAuthenticity, WarningMessage, @@ -16,34 +15,31 @@ import { import assert from 'node:assert' import { Signer, thresholdCallToSigners } from '../../../common/combine' import { DomainCryptoClient } from '../../../common/crypto-clients/domain-crypto-client' -import { PromiseHandler } from '../../../common/handlers' -import { getKeyVersionInfo, requestHasSupportedKeyVersion, sendFailure } from '../../../common/io' +import { errorResult, ResultHandler } from '../../../common/handlers' +import { getKeyVersionInfo, requestHasSupportedKeyVersion } from '../../../common/io' import { getCombinerVersion, OdisConfig } from '../../../config' import { logDomainResponseDiscrepancies } from '../../services/log-responses' import { findThresholdDomainState } from '../../services/threshold-state' -export function createDomainSignHandler( +export function domainSign( signers: Signer[], config: OdisConfig -): PromiseHandler { +): ResultHandler { return async (request, response) => { const { logger } = response.locals if (!domainRestrictedSignatureRequestSchema(DomainSchema).is(request.body)) { - sendFailure(WarningMessage.INVALID_INPUT, 400, response) - return + return errorResult(400, WarningMessage.INVALID_INPUT) } if (!requestHasSupportedKeyVersion(request, config, logger)) { - sendFailure(WarningMessage.INVALID_KEY_VERSION_REQUEST, 400, response) - return + return errorResult(400, WarningMessage.INVALID_KEY_VERSION_REQUEST) } // Note that signing requests may include a nonce for replay protection that will be checked by // the signer, but is not checked here. As a result, requests that pass the authentication check // here may still fail when sent to the signer. if (!verifyDomainRestrictedSignatureRequestAuthenticity(request.body)) { - sendFailure(WarningMessage.UNAUTHENTICATED_USER, 401, response) - return + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) } const keyVersionInfo = getKeyVersionInfo(request, config, logger) @@ -96,17 +92,15 @@ export function createDomainSignHandler( logger ) - return send( - response, - { + return { + status: 200, + body: { success: true, version: getCombinerVersion(), signature: combinedSignature, status: findThresholdDomainState(keyVersionInfo, signerResponses, signers.length), }, - 200, - response.locals.logger - ) + } } catch (err) { // May fail upon combining signatures if too many sigs are invalid logger.error('Combining signatures failed in combine') @@ -117,7 +111,7 @@ export function createDomainSignHandler( const errorCode = maxErrorCode ?? 500 const error = errorCodeToError(errorCode) - sendFailure(error, errorCode, response) + return errorResult(errorCode, error) } } diff --git a/packages/phone-number-privacy/combiner/src/index.ts b/packages/phone-number-privacy/combiner/src/index.ts index 42336c696d9..58d5869bf6b 100644 --- a/packages/phone-number-privacy/combiner/src/index.ts +++ b/packages/phone-number-privacy/combiner/src/index.ts @@ -1,4 +1,3 @@ -import { getContractKitWithAgent } from '@celo/phone-number-privacy-common' import * as functions from 'firebase-functions' import config from './config' import { startCombiner } from './server' @@ -12,5 +11,6 @@ export const combiner = functions // Defined check required for running tests vs. deployment minInstances: functions.config().service ? Number(functions.config().service.min_instances) : 0, }) - .https.onRequest(startCombiner(config, getContractKitWithAgent(config.blockchain))) + .https.onRequest(startCombiner(config)) + export * from './config' diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts index 8abf37cc48e..64f31e66ba2 100644 --- a/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts +++ b/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts @@ -9,33 +9,30 @@ import { PnpQuotaRequest, PnpQuotaRequestSchema, PnpQuotaResponseSchema, - send, WarningMessage, } from '@celo/phone-number-privacy-common' import { Request } from 'express' import { Signer, thresholdCallToSigners } from '../../../common/combine' -import { PromiseHandler } from '../../../common/handlers' -import { getKeyVersionInfo, sendFailure } from '../../../common/io' +import { errorResult, ResultHandler } from '../../../common/handlers' +import { getKeyVersionInfo } from '../../../common/io' import { getCombinerVersion, OdisConfig } from '../../../config' import { logPnpSignerResponseDiscrepancies } from '../../services/log-responses' import { findCombinerQuotaState } from '../../services/threshold-state' -export function createPnpQuotaHandler( +export function pnpQuota( signers: Signer[], config: OdisConfig, dekFetcher: DataEncryptionKeyFetcher -): PromiseHandler { +): ResultHandler { return async (request, response) => { const logger = response.locals.logger - if (!validateRequest(request)) { - sendFailure(WarningMessage.INVALID_INPUT, 400, response) - return + if (!isValidRequest(request)) { + return errorResult(400, WarningMessage.INVALID_INPUT) } if (!(await authenticateUser(request, logger, dekFetcher))) { - sendFailure(WarningMessage.UNAUTHENTICATED_USER, 401, response) - return + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) } // TODO remove this, we shouldn't need keyVersionInfo for non-signing endpoints @@ -57,28 +54,24 @@ export function createPnpQuotaHandler( if (signerResponses.length >= threshold) { try { const quotaStatus = findCombinerQuotaState(keyVersionInfo, signerResponses, warnings) - send( - response, - { + return { + status: 200, + body: { success: true, version: getCombinerVersion(), ...quotaStatus, warnings, }, - 200, - logger - ) - - return + } } catch (err) { logger.error(err, 'Error combining signer quota status responses') } } - sendFailure(ErrorMessage.THRESHOLD_PNP_QUOTA_STATUS_FAILURE, maxErrorCode ?? 500, response) + return errorResult(maxErrorCode ?? 500, ErrorMessage.THRESHOLD_PNP_QUOTA_STATUS_FAILURE) } } -function validateRequest( +function isValidRequest( request: Request<{}, {}, unknown> ): request is Request<{}, {}, PnpQuotaRequest> { return ( diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts index eb0f9b6f72d..72c97b3a3a1 100644 --- a/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts +++ b/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts @@ -9,7 +9,6 @@ import { hasValidBlindedPhoneNumberParam, isBodyReasonablySized, OdisResponse, - send, SignMessageRequest, SignMessageRequestSchema, SignMessageResponseSchema, @@ -19,32 +18,29 @@ import { Request } from 'express' import assert from 'node:assert' import { Signer, thresholdCallToSigners } from '../../../common/combine' import { BLSCryptographyClient } from '../../../common/crypto-clients/bls-crypto-client' -import { PromiseHandler } from '../../../common/handlers' -import { getKeyVersionInfo, requestHasSupportedKeyVersion, sendFailure } from '../../../common/io' +import { errorResult, ResultHandler } from '../../../common/handlers' +import { getKeyVersionInfo, requestHasSupportedKeyVersion } from '../../../common/io' import { getCombinerVersion, OdisConfig } from '../../../config' import { logPnpSignerResponseDiscrepancies } from '../../services/log-responses' import { findCombinerQuotaState } from '../../services/threshold-state' -export function createPnpSignHandler( +export function pnpSign( signers: Signer[], config: OdisConfig, dekFetcher: DataEncryptionKeyFetcher -): PromiseHandler { +): ResultHandler { return async (request, response) => { const logger = response.locals.logger - if (!validateRequest(request)) { - sendFailure(WarningMessage.INVALID_INPUT, 400, response) - return + if (!isValidRequest(request)) { + return errorResult(400, WarningMessage.INVALID_INPUT) } if (!requestHasSupportedKeyVersion(request, config, response.locals.logger)) { - sendFailure(WarningMessage.INVALID_KEY_VERSION_REQUEST, 400, response) - return + return errorResult(400, WarningMessage.INVALID_KEY_VERSION_REQUEST) } if (!(await authenticateUser(request, logger, dekFetcher))) { - sendFailure(WarningMessage.UNAUTHENTICATED_USER, 401, response) - return + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) } const keyVersionInfo = getKeyVersionInfo(request, config, logger) const crypto = new BLSCryptographyClient(keyVersionInfo) @@ -93,18 +89,16 @@ export function createPnpSignHandler( logger ) - return send( - response, - { + return { + status: 200, + body: { success: true, version: getCombinerVersion(), signature: combinedSignature, ...findCombinerQuotaState(keyVersionInfo, signerResponses, warnings), warnings, }, - 200, - logger - ) + } } catch (error) { // May fail upon combining signatures if too many sigs are invalid // Fallback to handleMissingSignatures @@ -114,11 +108,11 @@ export function createPnpSignHandler( const errorCode = maxErrorCode ?? 500 const error = errorCodeToError(errorCode) - sendFailure(error, errorCode, response) + return errorResult(errorCode, error) } } -function validateRequest( +function isValidRequest( request: Request<{}, {}, unknown> ): request is Request<{}, {}, SignMessageRequest> { return ( diff --git a/packages/phone-number-privacy/combiner/src/server.ts b/packages/phone-number-privacy/combiner/src/server.ts index cfb5a24eeed..2b191f6122f 100644 --- a/packages/phone-number-privacy/combiner/src/server.ts +++ b/packages/phone-number-privacy/combiner/src/server.ts @@ -1,6 +1,7 @@ import { ContractKit } from '@celo/contractkit' import { CombinerEndpoint, + getContractKitWithAgent, KEY_VERSION_HEADER, loggerMiddleware, newContractKitFetcher, @@ -12,21 +13,27 @@ import { Signer } from './common/combine' import { catchErrorHandler, disabledHandler, + Locals, meteringHandler, - PromiseHandler, + resultHandler, + ResultHandler, + timeoutHandler, + tracingHandler, } from './common/handlers' import { CombinerConfig, getCombinerVersion } from './config' -import { createDisableDomainHandler } from './domain/endpoints/disable/action' -import { createDomainQuotaHandler } from './domain/endpoints/quota/action' -import { createDomainSignHandler } from './domain/endpoints/sign/action' -import { createPnpQuotaHandler } from './pnp/endpoints/quota/action' -import { createPnpSignHandler } from './pnp/endpoints/sign/action' +import { disableDomain } from './domain/endpoints/disable/action' +import { domainQuota } from './domain/endpoints/quota/action' +import { domainSign } from './domain/endpoints/sign/action' +import { pnpQuota } from './pnp/endpoints/quota/action' +import { pnpSign } from './pnp/endpoints/sign/action' require('events').EventEmitter.defaultMaxListeners = 15 -export function startCombiner(config: CombinerConfig, kit: ContractKit) { +export function startCombiner(config: CombinerConfig, kit?: ContractKit) { const logger = rootLogger(config.serviceName) + kit = kit ?? getContractKitWithAgent(config.blockchain) + logger.info('Creating combiner express server') const app = express() @@ -64,26 +71,62 @@ export function startCombiner(config: CombinerConfig, kit: ContractKit) { ) const pnpSigners: Signer[] = JSON.parse(config.phoneNumberPrivacy.odisServices.signers) - const pnpQuota = createPnpQuotaHandler(pnpSigners, config.phoneNumberPrivacy, dekFetcher) - const pnpSign = createPnpSignHandler(pnpSigners, config.phoneNumberPrivacy, dekFetcher) - const domainSigners: Signer[] = JSON.parse(config.domains.odisServices.signers) - const domainQuota = createDomainQuotaHandler(domainSigners, config.domains) - const domainSign = createDomainSignHandler(domainSigners, config.domains) - const domainDisable = createDisableDomainHandler(domainSigners, config.domains) - app.post(CombinerEndpoint.PNP_QUOTA, createHandler(config.phoneNumberPrivacy.enabled, pnpQuota)) - app.post(CombinerEndpoint.PNP_SIGN, createHandler(config.phoneNumberPrivacy.enabled, pnpSign)) - app.post(CombinerEndpoint.DOMAIN_QUOTA_STATUS, createHandler(config.domains.enabled, domainQuota)) - app.post(CombinerEndpoint.DOMAIN_SIGN, createHandler(config.domains.enabled, domainSign)) - app.post(CombinerEndpoint.DISABLE_DOMAIN, createHandler(config.domains.enabled, domainDisable)) + const { domains, phoneNumberPrivacy } = config + + app.post( + CombinerEndpoint.PNP_QUOTA, + createHandler( + phoneNumberPrivacy.odisServices.timeoutMilliSeconds, + phoneNumberPrivacy.enabled, + pnpQuota(pnpSigners, config.phoneNumberPrivacy, dekFetcher) + ) + ) + app.post( + CombinerEndpoint.PNP_SIGN, + createHandler( + phoneNumberPrivacy.odisServices.timeoutMilliSeconds, + phoneNumberPrivacy.enabled, + pnpSign(pnpSigners, config.phoneNumberPrivacy, dekFetcher) + ) + ) + app.post( + CombinerEndpoint.DOMAIN_QUOTA_STATUS, + createHandler( + domains.odisServices.timeoutMilliSeconds, + domains.enabled, + domainQuota(domainSigners, config.domains) + ) + ) + app.post( + CombinerEndpoint.DOMAIN_SIGN, + createHandler( + domains.odisServices.timeoutMilliSeconds, + config.domains.enabled, + domainSign(domainSigners, config.domains) + ) + ) + app.post( + CombinerEndpoint.DISABLE_DOMAIN, + createHandler( + domains.odisServices.timeoutMilliSeconds, + config.domains.enabled, + disableDomain(domainSigners, config.domains) + ) + ) return app } -export function createHandler( +function createHandler( + timeoutMs: number, enabled: boolean, - handler: PromiseHandler -): PromiseHandler { - return meteringHandler(catchErrorHandler(enabled ? handler : disabledHandler)) + action: ResultHandler +): RequestHandler<{}, {}, R, {}, Locals> { + return catchErrorHandler( + tracingHandler( + meteringHandler(timeoutHandler(timeoutMs, enabled ? resultHandler(action) : disabledHandler)) + ) + ) } From 0e6dd147e651dd7ac581e3fb0e9d1d28842bc575 Mon Sep 17 00:00:00 2001 From: Gaston Ponti Date: Fri, 1 Sep 2023 15:37:19 -0300 Subject: [PATCH 02/15] Remove combiner timeouts --- .../combiner/src/common/handlers.ts | 13 ---------- .../combiner/src/common/io.ts | 19 ++------------ .../combiner/src/server.ts | 26 +++---------------- 3 files changed, 6 insertions(+), 52 deletions(-) diff --git a/packages/phone-number-privacy/combiner/src/common/handlers.ts b/packages/phone-number-privacy/combiner/src/common/handlers.ts index f6681450598..36176a18979 100644 --- a/packages/phone-number-privacy/combiner/src/common/handlers.ts +++ b/packages/phone-number-privacy/combiner/src/common/handlers.ts @@ -146,19 +146,6 @@ export function timeoutHandler( } } -export function withEnableHandler( - enabled: boolean, - handler: PromiseHandler -): PromiseHandler { - return async (req, res) => { - if (enabled) { - return handler(req, res) - } else { - sendFailure(WarningMessage.API_UNAVAILABLE, 503, res, req.url) - } - } -} - export async function disabledHandler( req: Request<{}, {}, R>, response: Response, Locals> diff --git a/packages/phone-number-privacy/combiner/src/common/io.ts b/packages/phone-number-privacy/combiner/src/common/io.ts index c43b81f7066..22dfd6a8089 100644 --- a/packages/phone-number-privacy/combiner/src/common/io.ts +++ b/packages/phone-number-privacy/combiner/src/common/io.ts @@ -1,21 +1,19 @@ import { - ErrorType, getRequestKeyVersion, KEY_VERSION_HEADER, KeyVersionInfo, OdisRequest, OdisResponse, requestHasValidKeyVersion, - send, SignerEndpoint, } from '@celo/phone-number-privacy-common' import Logger from 'bunyan' -import { Request, Response } from 'express' +import { Request } from 'express' import * as http from 'http' import * as https from 'https' import fetch, { Response as FetchResponse } from 'node-fetch' import { performance } from 'perf_hooks' -import { getCombinerVersion, OdisConfig } from '../config' +import { OdisConfig } from '../config' import { isAbortError, Signer } from './combine' const httpAgent = new http.Agent({ keepAlive: true }) @@ -116,16 +114,3 @@ async function measureTime(name: string, fn: () => Promise): Promise { performance.measure(name, start, end) } } - -export function sendFailure(error: ErrorType, status: number, response: Response) { - send( - response, - { - success: false, - version: getCombinerVersion(), - error, - }, - status, - response.locals.logger - ) -} diff --git a/packages/phone-number-privacy/combiner/src/server.ts b/packages/phone-number-privacy/combiner/src/server.ts index 2b191f6122f..c77ba8bce79 100644 --- a/packages/phone-number-privacy/combiner/src/server.ts +++ b/packages/phone-number-privacy/combiner/src/server.ts @@ -17,7 +17,6 @@ import { meteringHandler, resultHandler, ResultHandler, - timeoutHandler, tracingHandler, } from './common/handlers' import { CombinerConfig, getCombinerVersion } from './config' @@ -78,7 +77,6 @@ export function startCombiner(config: CombinerConfig, kit?: ContractKit) { app.post( CombinerEndpoint.PNP_QUOTA, createHandler( - phoneNumberPrivacy.odisServices.timeoutMilliSeconds, phoneNumberPrivacy.enabled, pnpQuota(pnpSigners, config.phoneNumberPrivacy, dekFetcher) ) @@ -86,47 +84,31 @@ export function startCombiner(config: CombinerConfig, kit?: ContractKit) { app.post( CombinerEndpoint.PNP_SIGN, createHandler( - phoneNumberPrivacy.odisServices.timeoutMilliSeconds, phoneNumberPrivacy.enabled, pnpSign(pnpSigners, config.phoneNumberPrivacy, dekFetcher) ) ) app.post( CombinerEndpoint.DOMAIN_QUOTA_STATUS, - createHandler( - domains.odisServices.timeoutMilliSeconds, - domains.enabled, - domainQuota(domainSigners, config.domains) - ) + createHandler(domains.enabled, domainQuota(domainSigners, config.domains)) ) app.post( CombinerEndpoint.DOMAIN_SIGN, - createHandler( - domains.odisServices.timeoutMilliSeconds, - config.domains.enabled, - domainSign(domainSigners, config.domains) - ) + createHandler(domains.enabled, domainSign(domainSigners, domains)) ) app.post( CombinerEndpoint.DISABLE_DOMAIN, - createHandler( - domains.odisServices.timeoutMilliSeconds, - config.domains.enabled, - disableDomain(domainSigners, config.domains) - ) + createHandler(domains.enabled, disableDomain(domainSigners, domains)) ) return app } function createHandler( - timeoutMs: number, enabled: boolean, action: ResultHandler ): RequestHandler<{}, {}, R, {}, Locals> { return catchErrorHandler( - tracingHandler( - meteringHandler(timeoutHandler(timeoutMs, enabled ? resultHandler(action) : disabledHandler)) - ) + tracingHandler(meteringHandler(enabled ? resultHandler(action) : disabledHandler)) ) } From 248d0928bc115ea8d0a6d77e763a1272ad8361aa Mon Sep 17 00:00:00 2001 From: alecps Date: Wed, 6 Sep 2023 21:39:31 -0400 Subject: [PATCH 03/15] fix lint --- .../combiner/src/common/handlers.ts | 5 +++-- packages/phone-number-privacy/combiner/src/server.ts | 12 +++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/phone-number-privacy/combiner/src/common/handlers.ts b/packages/phone-number-privacy/combiner/src/common/handlers.ts index 36176a18979..a8323540fcd 100644 --- a/packages/phone-number-privacy/combiner/src/common/handlers.ts +++ b/packages/phone-number-privacy/combiner/src/common/handlers.ts @@ -4,15 +4,16 @@ import { OdisRequest, OdisResponse, PnpQuotaStatus, + send, + // tslint:disable-next-line: ordered-imports SequentialDelayDomainState, WarningMessage, - send, } from '@celo/phone-number-privacy-common' import opentelemetry, { SpanStatusCode } from '@opentelemetry/api' import { SemanticAttributes } from '@opentelemetry/semantic-conventions' import Logger from 'bunyan' import { Request, Response } from 'express' -import { PerformanceObserver, performance } from 'perf_hooks' +import { performance, PerformanceObserver } from 'perf_hooks' import { getCombinerVersion } from '../config' import { OdisError } from './error' diff --git a/packages/phone-number-privacy/combiner/src/server.ts b/packages/phone-number-privacy/combiner/src/server.ts index c77ba8bce79..94ca03ffcf1 100644 --- a/packages/phone-number-privacy/combiner/src/server.ts +++ b/packages/phone-number-privacy/combiner/src/server.ts @@ -76,21 +76,15 @@ export function startCombiner(config: CombinerConfig, kit?: ContractKit) { app.post( CombinerEndpoint.PNP_QUOTA, - createHandler( - phoneNumberPrivacy.enabled, - pnpQuota(pnpSigners, config.phoneNumberPrivacy, dekFetcher) - ) + createHandler(phoneNumberPrivacy.enabled, pnpQuota(pnpSigners, phoneNumberPrivacy, dekFetcher)) ) app.post( CombinerEndpoint.PNP_SIGN, - createHandler( - phoneNumberPrivacy.enabled, - pnpSign(pnpSigners, config.phoneNumberPrivacy, dekFetcher) - ) + createHandler(phoneNumberPrivacy.enabled, pnpSign(pnpSigners, phoneNumberPrivacy, dekFetcher)) ) app.post( CombinerEndpoint.DOMAIN_QUOTA_STATUS, - createHandler(domains.enabled, domainQuota(domainSigners, config.domains)) + createHandler(domains.enabled, domainQuota(domainSigners, domains)) ) app.post( CombinerEndpoint.DOMAIN_SIGN, From 64e2f17b49f4bddfc77f5f2ab0cf8b155aa1c972 Mon Sep 17 00:00:00 2001 From: alecps Date: Wed, 6 Sep 2023 23:56:15 -0400 Subject: [PATCH 04/15] fix combiner e2e tests --- .../combiner/test/end-to-end/pnp.test.ts | 54 +++++++++++-------- .../combiner/test/end-to-end/resources.ts | 4 +- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts b/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts index 4a315aacf20..3ba6dfdcfb1 100644 --- a/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts +++ b/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts @@ -9,6 +9,7 @@ import { SignMessageRequest, SignMessageResponseSchema, } from '@celo/phone-number-privacy-common' +import { normalizeAddressWith0x } from '@celo/utils/lib/address' import threshold_bls from 'blind-threshold-bls' import { randomBytes } from 'crypto' import 'isomorphic-fetch' @@ -19,6 +20,7 @@ import { ACCOUNT_ADDRESS_NO_QUOTA, BLINDED_PHONE_NUMBER, dekAuthSigner, + deks, getTestContextName, PHONE_NUMBER, walletAuthSigner, @@ -36,9 +38,17 @@ const fullNodeUrl = process.env.ODIS_BLOCKCHAIN_PROVIDER const expectedVersion = getCombinerVersion() -// TODO fix combiner e2e tests - describe(`Running against service deployed at ${combinerUrl} w/ blockchain provider ${fullNodeUrl}`, () => { + beforeAll(async () => { + const accounts = await walletAuthSigner.contractKit.contracts.getAccounts() + const dekPublicKey = normalizeAddressWith0x(deks[0].publicKey) + if ((await accounts.getDataEncryptionKey(ACCOUNT_ADDRESS)) !== dekPublicKey) { + await accounts + .setAccountDataEncryptionKey(dekPublicKey) + .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) + } + }) + it('Service is deployed at correct version', async () => { const response = await fetch(combinerUrl + CombinerEndpoint.STATUS, { method: 'GET', @@ -140,27 +150,27 @@ describe(`Running against service deployed at ${combinerUrl} w/ blockchain provi }) describe(`${CombinerEndpoint.PNP_SIGN}`, () => { - describe('new requests', () => { - beforeAll(async () => { - // Replenish quota for ACCOUNT_ADDRESS - // If this fails, may be necessary to faucet ACCOUNT_ADDRESS more funds - const numQueriesToReplenish = 2 - const amountInWei = signerConfig.quota.queryPriceInCUSD - .times(1e18) - .times(numQueriesToReplenish) - .toString() - const stableToken = await walletAuthSigner.contractKit.contracts.getStableToken( - StableToken.cUSD - ) - const odisPayments = await walletAuthSigner.contractKit.contracts.getOdisPayments() - await stableToken - .approve(odisPayments.address, amountInWei) - .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) - await odisPayments - .payInCUSD(ACCOUNT_ADDRESS, amountInWei) - .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) - }) + beforeAll(async () => { + // Replenish quota for ACCOUNT_ADDRESS + // If this fails, may be necessary to faucet ACCOUNT_ADDRESS more funds + const numQueriesToReplenish = 100 + const amountInWei = signerConfig.quota.queryPriceInCUSD + .times(1e18) + .times(numQueriesToReplenish) + .toString() + const stableToken = await walletAuthSigner.contractKit.contracts.getStableToken( + StableToken.cUSD + ) + const odisPayments = await walletAuthSigner.contractKit.contracts.getOdisPayments() + await stableToken + .approve(odisPayments.address, amountInWei) + .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) + await odisPayments + .payInCUSD(ACCOUNT_ADDRESS, amountInWei) + .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) + }) + describe('new requests', () => { // Requests made for PHONE_NUMBER from ACCOUNT_ADDRESS & same blinding factor // are replayed from previous test runs (for every run after the very first) let startingPerformedQueryCount: number diff --git a/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts b/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts index c90c4a7b9be..bd93afa69db 100644 --- a/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts +++ b/packages/phone-number-privacy/combiner/test/end-to-end/resources.ts @@ -34,8 +34,8 @@ export const getTestContextName = (): OdisContextName => { export const DEFAULT_FORNO_URL = process.env.ODIS_BLOCKCHAIN_PROVIDER ?? 'https://alfajores-forno.celo-testnet.org' -export const PRIVATE_KEY = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef' -export const ACCOUNT_ADDRESS = normalizeAddressWith0x(privateKeyToAddress(PRIVATE_KEY)) // 0x1be31a94361a391bbafb2a4ccd704f57dc04d4bb +export const PRIVATE_KEY = '2c63bf6d60b16c8afa13e1069dbe92fef337c23855fff8b27732b3e9c6e7efd4' +export const ACCOUNT_ADDRESS = normalizeAddressWith0x(privateKeyToAddress(PRIVATE_KEY)) // 0x6037800e91eaa703e38bad40c01410bbdf0fea7e export const PRIVATE_KEY_NO_QUOTA = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890000000' From 4c9c3676d5a32df2aa1978cabf98bb89d87780b2 Mon Sep 17 00:00:00 2001 From: alecps Date: Wed, 6 Sep 2023 23:57:24 -0400 Subject: [PATCH 05/15] point to published dependencies --- .../combiner/package.json | 14 +- .../phone-number-privacy/signer/package.json | 10 +- yarn.lock | 201 ++++++++++++++++++ 3 files changed, 213 insertions(+), 12 deletions(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index fd432f11a9f..0fcb56e17cb 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-combiner", - "version": "3.0.0-beta.8-dev", + "version": "3.0.0-beta.8", "description": "Orchestrates and combines threshold signatures for use in ODIS", "author": "Celo", "license": "Apache-2.0", @@ -28,10 +28,10 @@ "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e" }, "dependencies": { - "@celo/contractkit": "^4.1.2-dev", - "@celo/phone-number-privacy-common": "^3.0.1-dev", - "@celo/identity": "^4.1.2-dev", - "@celo/encrypted-backup": "^4.1.2-dev", + "@celo/contractkit": "^4.1.1", + "@celo/phone-number-privacy-common": "^3.0.0", + "@celo/identity": "^4.1.1", + "@celo/encrypted-backup": "^4.1.1", "@celo/poprf": "^0.1.9", "@types/bunyan": "^1.8.8", "@opentelemetry/api": "^1.4.1", @@ -54,8 +54,8 @@ }, "devDependencies": { "@types/node": "18.15.13", - "@celo/utils": "^4.1.2-dev", - "@celo/phone-utils": "^4.1.2-dev", + "@celo/utils": "^4.1.1", + "@celo/phone-utils": "^4.1.1", "@types/express": "^4.17.6", "@types/supertest": "^2.0.12", "@types/uuid": "^7.0.3", diff --git a/packages/phone-number-privacy/signer/package.json b/packages/phone-number-privacy/signer/package.json index c7f9e5b59fe..808ecd48a92 100644 --- a/packages/phone-number-privacy/signer/package.json +++ b/packages/phone-number-privacy/signer/package.json @@ -37,12 +37,12 @@ "ssl:keygen": "./scripts/create-ssl-cert.sh" }, "dependencies": { - "@celo/base": "^4.1.2-dev", - "@celo/contractkit": "^4.1.2-dev", - "@celo/phone-number-privacy-common": "^3.0.1-dev", + "@celo/base": "^4.1.1", + "@celo/contractkit": "^4.1.1", + "@celo/phone-number-privacy-common": "^3.0.0", "@celo/poprf": "^0.1.9", - "@celo/utils": "^4.1.2-dev", - "@celo/wallet-hsm-azure": "^4.1.2-dev", + "@celo/utils": "^4.1.1", + "@celo/wallet-hsm-azure": "^4.1.1", "@google-cloud/secret-manager": "3.0.0", "@opentelemetry/api": "^1.4.1", "@opentelemetry/auto-instrumentations-node": "^0.38.0", diff --git a/yarn.lock b/yarn.lock index bc941833aaa..9ecd1e1572e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -933,6 +933,11 @@ resolved "https://registry.yarnpkg.com/@celo/base/-/base-1.5.2.tgz#168ab5e4e30b374079d8d139fafc52ca6bfd4100" integrity sha512-KGf6Dl9E6D01vAfkgkjL2sG+zqAjspAogILIpWstljWdG5ifyA75jihrnDEHaMCoQS0KxHvTdP1XYS/GS6BEyQ== +"@celo/base@4.1.1", "@celo/base@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/base/-/base-4.1.1.tgz#2a6c38deaf2a82dbe10bf3dea4881e4452c11803" + integrity sha512-M7qpbDX/Bhmzcr/spZ7w9jkKfnMnv5zAnUFKoLrFsM8DB0NFcH95YAyeG1eVTfWmD7nu8RG1sASFPGGOAji2Iw== + "@celo/bls12377js@0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@celo/bls12377js/-/bls12377js-0.1.1.tgz#ba3574f41697cdba96c10ae96bb1aac057285798" @@ -953,6 +958,19 @@ debug "^4.1.1" utf8 "3.0.0" +"@celo/connect@4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/connect/-/connect-4.1.1.tgz#7a631a5fc832b85eabb7f5e9a5fb95dbd620ee67" + integrity sha512-yIdrh4OiO2MEuDrgkALflor8oAx6YMAqy9/EUcDHql+BxKkDjSA8yxggqiOQzmZJ7GHKRzpwOpx5bey9siP3pQ== + dependencies: + "@celo/base" "4.1.1" + "@celo/utils" "4.1.1" + "@types/debug" "^4.1.5" + "@types/utf8" "^2.1.6" + bignumber.js "^9.0.0" + debug "^4.1.1" + utf8 "3.0.0" + "@celo/contractkit@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/contractkit/-/contractkit-1.5.2.tgz#be15d570f3044a190dabb6bbe53d5081c78ea605" @@ -971,6 +989,59 @@ semver "^7.3.5" web3 "1.3.6" +"@celo/contractkit@4.1.1", "@celo/contractkit@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/contractkit/-/contractkit-4.1.1.tgz#2ace459e9b6b599ec535f167a3b1e0c461fce0b1" + integrity sha512-1QBCyFOlIuCX9m2LzFXwfs4LNrZaYhuCUleuKPlLVZiw17uhv7Bq1mEPWAmV7SckLzTjYFvggzIg42U01g8brg== + dependencies: + "@celo/base" "4.1.1" + "@celo/connect" "4.1.1" + "@celo/utils" "4.1.1" + "@celo/wallet-local" "4.1.1" + "@types/bn.js" "^5.1.0" + "@types/debug" "^4.1.5" + bignumber.js "^9.0.0" + cross-fetch "3.0.6" + debug "^4.1.1" + fp-ts "2.1.1" + io-ts "2.0.1" + semver "^7.3.5" + web3 "1.10.0" + +"@celo/encrypted-backup@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/encrypted-backup/-/encrypted-backup-4.1.1.tgz#24603821eddcd15b73ef693b9119d7e7d3ac7a35" + integrity sha512-/WXhPwWpTlx/tQxpoeGTX4Nwy5XoZu6nOkWQ/30NWKFc79GG/jEFpz8OQAg/izVaAWp82RrzpIuQq5U5KwaI7A== + dependencies: + "@celo/base" "4.1.1" + "@celo/identity" "4.1.1" + "@celo/phone-number-privacy-common" "^3.0.0" + "@celo/poprf" "^0.1.9" + "@celo/utils" "4.1.1" + "@types/debug" "^4.1.5" + debug "^4.1.1" + fp-ts "2.1.1" + io-ts "2.0.1" + +"@celo/identity@4.1.1", "@celo/identity@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/identity/-/identity-4.1.1.tgz#d0c541c344db466067d5baeba0b08d8fd3191053" + integrity sha512-tFj9vd6j4PIzqQDWjcolbH4cKOaKZkMLT+XxNesxHpXbyNhGm+4zMADbtyN4BWLTZB7dWVwgld+5EtMQQ9uSxw== + dependencies: + "@celo/base" "4.1.1" + "@celo/contractkit" "4.1.1" + "@celo/phone-number-privacy-common" "^3.0.0" + "@celo/utils" "4.1.1" + "@types/debug" "^4.1.5" + bignumber.js "^9.0.0" + blind-threshold-bls "https://github.com/celo-org/blind-threshold-bls-wasm#e1e2f8a" + cross-fetch "3.0.6" + debug "^4.1.1" + elliptic "^6.5.4" + ethereum-cryptography "1.2.0" + fp-ts "2.1.1" + io-ts "2.0.1" + "@celo/phone-number-privacy-common@1.0.39": version "1.0.39" resolved "https://registry.yarnpkg.com/@celo/phone-number-privacy-common/-/phone-number-privacy-common-1.0.39.tgz#3c9568f70378d24d11afcc4306024c5cf4f8efe9" @@ -989,6 +1060,47 @@ elliptic "^6.5.4" is-base64 "^1.1.0" +"@celo/phone-number-privacy-common@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@celo/phone-number-privacy-common/-/phone-number-privacy-common-3.0.0.tgz#b6041820ff4e954a19434163d7bdb9f005139b26" + integrity sha512-lxzYb3iNU85RA7vf0YAGCT904uPeinvR4MWSWyefTrH/OR6rdXgWV7MjptxM72mSTFGRPpnQLmhRD6b7O1UYqA== + dependencies: + "@celo/base" "^4.1.1" + "@celo/contractkit" "^4.1.1" + "@celo/phone-utils" "^4.1.1" + "@celo/utils" "^4.1.1" + "@opentelemetry/api" "^1.4.1" + "@opentelemetry/auto-instrumentations-node" "^0.38.0" + "@opentelemetry/propagator-ot-trace" "^0.27.0" + "@opentelemetry/sdk-metrics" "^1.15.1" + "@opentelemetry/sdk-node" "^0.41.1" + "@opentelemetry/sdk-trace-web" "^1.15.1" + "@opentelemetry/semantic-conventions" "^1.15.1" + "@types/bunyan" "1.8.8" + bignumber.js "^9.0.0" + bunyan "1.8.12" + bunyan-debug-stream "2.0.0" + bunyan-gke-stackdriver "0.1.2" + dotenv "^8.2.0" + elliptic "^6.5.4" + io-ts "2.0.1" + is-base64 "^1.1.0" + +"@celo/phone-utils@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/phone-utils/-/phone-utils-4.1.1.tgz#2c99f409d59c84964410b6f2f509bed415df4ba4" + integrity sha512-KTLgDWjORlLBov77uRmNbhgFQ1wycYD6+l1yigrXr651fPgejQeFXraF8mYy2eCOkehOR8e7GCmfOuoR3+VCzA== + dependencies: + "@celo/base" "4.1.1" + "@celo/utils" "4.1.1" + "@types/country-data" "^0.0.0" + "@types/google-libphonenumber" "^7.4.23" + "@types/node" "^10.12.18" + country-data "^0.0.31" + fp-ts "2.1.1" + google-libphonenumber "^3.2.27" + io-ts "2.0.1" + "@celo/poprf@^0.1.9": version "0.1.9" resolved "https://registry.yarnpkg.com/@celo/poprf/-/poprf-0.1.9.tgz#38c514ce0f572b80edeb9dc280b6cf5e9d7c2a75" @@ -1035,6 +1147,23 @@ web3-eth-abi "1.3.6" web3-utils "1.3.6" +"@celo/utils@4.1.1", "@celo/utils@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/utils/-/utils-4.1.1.tgz#4423b64ad5f9028ba96af3023c32e950b152d68a" + integrity sha512-NZJqkvrV4ADNU0JGi1NkjWWDFL3NFZpHdQC57O+GNahE0uvC2ylCjCMQyyrDuFjyJwsIHPV2zNdVt4LRM2NP7g== + dependencies: + "@celo/base" "4.1.1" + "@ethereumjs/util" "8.0.5" + "@types/bn.js" "^5.1.0" + "@types/elliptic" "^6.4.9" + "@types/node" "^10.12.18" + bignumber.js "^9.0.0" + elliptic "^6.5.4" + ethereum-cryptography "1.2.0" + io-ts "2.0.1" + web3-eth-abi "1.10.0" + web3-utils "1.10.0" + "@celo/wallet-base@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/wallet-base/-/wallet-base-1.5.2.tgz#ae8df425bf3c702277bb1b63a761a2ec8429e7aa" @@ -1050,6 +1179,55 @@ eth-lib "^0.2.8" ethereumjs-util "^5.2.0" +"@celo/wallet-base@4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/wallet-base/-/wallet-base-4.1.1.tgz#70a955193a75e1c54f5111f03a5f2bfd0fb1f47a" + integrity sha512-WbZyXnZnFQ13AmSYXTT5JEtFFZiGWquBKMfPXhRreRiNHqtwGSxq+bUaDmYIFKq6f4DU4gFV3UwdpW7iNbSv2w== + dependencies: + "@celo/base" "4.1.1" + "@celo/connect" "4.1.1" + "@celo/utils" "4.1.1" + "@ethereumjs/util" "8.0.5" + "@types/debug" "^4.1.5" + bignumber.js "^9.0.0" + debug "^4.1.1" + eth-lib "^0.2.8" + +"@celo/wallet-hsm-azure@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/wallet-hsm-azure/-/wallet-hsm-azure-4.1.1.tgz#9dcea4630d9670999410f4be5cf409368b769e71" + integrity sha512-3CnaOgClOEy0HIW8SIinPz21mcpp4Ps9zOcamfzs5ICWld4rPvk0DfKIW+8TQrHSxJWRRdZQNfbdVSKrQ6gZCg== + dependencies: + "@azure/identity" "^1.1.0" + "@azure/keyvault-keys" "^4.1.0" + "@azure/keyvault-secrets" "^4.1.0" + "@celo/connect" "4.1.1" + "@celo/utils" "4.1.1" + "@celo/wallet-base" "4.1.1" + "@celo/wallet-hsm" "4.1.1" + "@celo/wallet-remote" "4.1.1" + "@ethereumjs/util" "8.0.5" + "@types/secp256k1" "^4.0.0" + bignumber.js "^9.0.0" + debug "^4.1.1" + eth-lib "^0.2.8" + secp256k1 "^4.0.0" + +"@celo/wallet-hsm@4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/wallet-hsm/-/wallet-hsm-4.1.1.tgz#9d7f292027f14c172f406b5159f654a32c3405d0" + integrity sha512-pbTD/+G9i28HRb1KS1IHd+AkwspG8nqRmK59hH907BFcsUyVQ7a6HIs8s+i63/tBFHpnNIcP/5cGXW8V0VsnUg== + dependencies: + "@celo/base" "4.1.1" + "@ethereumjs/util" "8.0.5" + "@types/asn1js" "^0.0.2" + "@types/debug" "^4.1.5" + "@types/secp256k1" "^4.0.0" + asn1js "^2.0.26" + elliptic "^6.5.4" + eth-lib "^0.2.8" + secp256k1 "^4.0.0" + "@celo/wallet-local@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/wallet-local/-/wallet-local-1.5.2.tgz#66ea5fb763e19724309e3d56f312f1a342e12b91" @@ -1062,6 +1240,29 @@ eth-lib "^0.2.8" ethereumjs-util "^5.2.0" +"@celo/wallet-local@4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/wallet-local/-/wallet-local-4.1.1.tgz#8b7748ccb030dcc64e62b533af79c9ff3f602f36" + integrity sha512-+eyBOkir3AaGbBGg0HIEpjfZ+G3yIeYy9ekJAZbDkCH81K0dlAHKVGnylITVORR+lQsJeHDiciREVcp5Zq3aHw== + dependencies: + "@celo/connect" "4.1.1" + "@celo/utils" "4.1.1" + "@celo/wallet-base" "4.1.1" + "@ethereumjs/util" "8.0.5" + eth-lib "^0.2.8" + +"@celo/wallet-remote@4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@celo/wallet-remote/-/wallet-remote-4.1.1.tgz#c717ef53a160f07d6402a7576122496d644c44de" + integrity sha512-cG85ryLRz+xANioZBDsGgCjY/mgh4X5bbCWmEWh5Jd7QHsbbkFgNCs7rcNEPjgxB7q2DfbRT0oUzcETqIWmXIA== + dependencies: + "@celo/connect" "4.1.1" + "@celo/utils" "4.1.1" + "@celo/wallet-base" "4.1.1" + "@ethereumjs/util" "8.0.5" + "@types/debug" "^4.1.5" + eth-lib "^0.2.8" + "@chainsafe/as-sha256@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.1.tgz#3639df0e1435cab03f4d9870cc3ac079e57a6fc9" From 2053f847b195e7b7609036cb10722f3b91eb090f Mon Sep 17 00:00:00 2001 From: alecps Date: Thu, 7 Sep 2023 14:59:39 -0400 Subject: [PATCH 06/15] put auth behind flag --- packages/phone-number-privacy/combiner/src/config.ts | 7 ++++++- .../combiner/src/pnp/endpoints/quota/action.ts | 6 ++++-- .../combiner/src/pnp/endpoints/sign/action.ts | 7 +++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/phone-number-privacy/combiner/src/config.ts b/packages/phone-number-privacy/combiner/src/config.ts index a1b9829f230..3be92b106da 100644 --- a/packages/phone-number-privacy/combiner/src/config.ts +++ b/packages/phone-number-privacy/combiner/src/config.ts @@ -39,6 +39,7 @@ export interface OdisConfig { fullNodeTimeoutMs: number fullNodeRetryCount: number fullNodeRetryDelayMs: number + shouldAuthenticate: boolean } export interface CombinerConfig { @@ -106,6 +107,7 @@ if (DEV_MODE) { fullNodeTimeoutMs: FULL_NODE_TIMEOUT_IN_MS, fullNodeRetryCount: RETRY_COUNT, fullNodeRetryDelayMs: RETRY_DELAY_IN_MS, + shouldAuthenticate: true, }, domains: { serviceName: defaultServiceName, @@ -140,6 +142,7 @@ if (DEV_MODE) { fullNodeTimeoutMs: FULL_NODE_TIMEOUT_IN_MS, fullNodeRetryCount: RETRY_COUNT, fullNodeRetryDelayMs: RETRY_DELAY_IN_MS, + shouldAuthenticate: true, }, } } else { @@ -168,6 +171,7 @@ if (DEV_MODE) { fullNodeRetryDelayMs: Number( functionConfig.pnp.full_node_retry_delay_ms ?? RETRY_DELAY_IN_MS ), + shouldAuthenticate: toBool(functionConfig.pnp.shouldAuthenticate, true), }, domains: { serviceName: functionConfig.domains.service_name ?? defaultServiceName, @@ -182,11 +186,12 @@ if (DEV_MODE) { currentVersion: Number(functionConfig.domains_keys.current_version), versions: functionConfig.domains_keys.versions, }, - fullNodeTimeoutMs: Number(functionConfig.pnp.full_node_timeout_ms ?? FULL_NODE_TIMEOUT_IN_MS), + fullNodeTimeoutMs: Number(functionConfig.pnp.full_node_timeout_ms ?? FULL_NODE_TIMEOUT_IN_MS), // TODO refactor config - domains endpoints don't use full node fullNodeRetryCount: Number(functionConfig.pnp.full_node_retry_count ?? RETRY_COUNT), fullNodeRetryDelayMs: Number( functionConfig.pnp.full_node_retry_delay_ms ?? RETRY_DELAY_IN_MS ), + shouldAuthenticate: true, }, } } diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts index 64f31e66ba2..3f06d1a87b8 100644 --- a/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts +++ b/packages/phone-number-privacy/combiner/src/pnp/endpoints/quota/action.ts @@ -31,8 +31,10 @@ export function pnpQuota( return errorResult(400, WarningMessage.INVALID_INPUT) } - if (!(await authenticateUser(request, logger, dekFetcher))) { - return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) + if (config.shouldAuthenticate) { + if (!(await authenticateUser(request, logger, dekFetcher))) { + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) + } } // TODO remove this, we shouldn't need keyVersionInfo for non-signing endpoints diff --git a/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts b/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts index 72c97b3a3a1..9109bfc2863 100644 --- a/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts +++ b/packages/phone-number-privacy/combiner/src/pnp/endpoints/sign/action.ts @@ -39,9 +39,12 @@ export function pnpSign( return errorResult(400, WarningMessage.INVALID_KEY_VERSION_REQUEST) } - if (!(await authenticateUser(request, logger, dekFetcher))) { - return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) + if (config.shouldAuthenticate) { + if (!(await authenticateUser(request, logger, dekFetcher))) { + return errorResult(401, WarningMessage.UNAUTHENTICATED_USER) + } } + const keyVersionInfo = getKeyVersionInfo(request, config, logger) const crypto = new BLSCryptographyClient(keyVersionInfo) From 23b8c88e62fb03cc8564fcc7d2f360c80e9f7a30 Mon Sep 17 00:00:00 2001 From: alecps Date: Thu, 7 Sep 2023 15:01:17 -0400 Subject: [PATCH 07/15] fix config casing --- packages/phone-number-privacy/combiner/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phone-number-privacy/combiner/src/config.ts b/packages/phone-number-privacy/combiner/src/config.ts index 3be92b106da..2097dbfb5e2 100644 --- a/packages/phone-number-privacy/combiner/src/config.ts +++ b/packages/phone-number-privacy/combiner/src/config.ts @@ -171,7 +171,7 @@ if (DEV_MODE) { fullNodeRetryDelayMs: Number( functionConfig.pnp.full_node_retry_delay_ms ?? RETRY_DELAY_IN_MS ), - shouldAuthenticate: toBool(functionConfig.pnp.shouldAuthenticate, true), + shouldAuthenticate: toBool(functionConfig.pnp.should_authenticate, true), }, domains: { serviceName: functionConfig.domains.service_name ?? defaultServiceName, From 5e47125be6667c05a5e1aab2de132fbcec99db2a Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:03:46 -0400 Subject: [PATCH 08/15] added memory allocation to function --- packages/phone-number-privacy/combiner/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/phone-number-privacy/combiner/src/index.ts b/packages/phone-number-privacy/combiner/src/index.ts index 58d5869bf6b..02297508a86 100644 --- a/packages/phone-number-privacy/combiner/src/index.ts +++ b/packages/phone-number-privacy/combiner/src/index.ts @@ -10,6 +10,7 @@ export const combiner = functions // Keep instances warm for mainnet functions // Defined check required for running tests vs. deployment minInstances: functions.config().service ? Number(functions.config().service.min_instances) : 0, + memory: functions.config().service ? functions.config().service.memory : '512MB', }) .https.onRequest(startCombiner(config)) From 1cc61143bb4e65441d5e43fe34713c483bb1b32a Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:10:53 -0400 Subject: [PATCH 09/15] bump version --- packages/phone-number-privacy/combiner/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index 0fcb56e17cb..a56473255ad 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-combiner", - "version": "3.0.0-beta.8", + "version": "3.0.0-beta.9", "description": "Orchestrates and combines threshold signatures for use in ODIS", "author": "Celo", "license": "Apache-2.0", From 3e4cc9fada106ac502867b257a4396df2c8f2345 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 8 Sep 2023 10:27:07 -0400 Subject: [PATCH 10/15] bump combiner version --- packages/phone-number-privacy/combiner/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index a56473255ad..4970c8e47a1 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-combiner", - "version": "3.0.0-beta.9", + "version": "3.0.0", "description": "Orchestrates and combines threshold signatures for use in ODIS", "author": "Celo", "license": "Apache-2.0", From 69cb7009fcd7fbd170927422661e7a02a99c47e9 Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:50:17 -0400 Subject: [PATCH 11/15] switched to dev dependencies --- .../combiner/package.json | 14 +- .../phone-number-privacy/signer/package.json | 10 +- yarn.lock | 201 ------------------ 3 files changed, 12 insertions(+), 213 deletions(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index 4970c8e47a1..11a8cf0a9b8 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -1,6 +1,6 @@ { "name": "@celo/phone-number-privacy-combiner", - "version": "3.0.0", + "version": "3.0.1-dev", "description": "Orchestrates and combines threshold signatures for use in ODIS", "author": "Celo", "license": "Apache-2.0", @@ -28,10 +28,10 @@ "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e" }, "dependencies": { - "@celo/contractkit": "^4.1.1", - "@celo/phone-number-privacy-common": "^3.0.0", - "@celo/identity": "^4.1.1", - "@celo/encrypted-backup": "^4.1.1", + "@celo/contractkit": "^4.1.2-dev", + "@celo/phone-number-privacy-common": "^3.0.1-dev", + "@celo/identity": "^4.1.2-dev", + "@celo/encrypted-backup": "^4.1.2-dev", "@celo/poprf": "^0.1.9", "@types/bunyan": "^1.8.8", "@opentelemetry/api": "^1.4.1", @@ -54,8 +54,8 @@ }, "devDependencies": { "@types/node": "18.15.13", - "@celo/utils": "^4.1.1", - "@celo/phone-utils": "^4.1.1", + "@celo/utils": "^4.1.2-dev", + "@celo/phone-utils": "^4.1.2-dev", "@types/express": "^4.17.6", "@types/supertest": "^2.0.12", "@types/uuid": "^7.0.3", diff --git a/packages/phone-number-privacy/signer/package.json b/packages/phone-number-privacy/signer/package.json index 808ecd48a92..c7f9e5b59fe 100644 --- a/packages/phone-number-privacy/signer/package.json +++ b/packages/phone-number-privacy/signer/package.json @@ -37,12 +37,12 @@ "ssl:keygen": "./scripts/create-ssl-cert.sh" }, "dependencies": { - "@celo/base": "^4.1.1", - "@celo/contractkit": "^4.1.1", - "@celo/phone-number-privacy-common": "^3.0.0", + "@celo/base": "^4.1.2-dev", + "@celo/contractkit": "^4.1.2-dev", + "@celo/phone-number-privacy-common": "^3.0.1-dev", "@celo/poprf": "^0.1.9", - "@celo/utils": "^4.1.1", - "@celo/wallet-hsm-azure": "^4.1.1", + "@celo/utils": "^4.1.2-dev", + "@celo/wallet-hsm-azure": "^4.1.2-dev", "@google-cloud/secret-manager": "3.0.0", "@opentelemetry/api": "^1.4.1", "@opentelemetry/auto-instrumentations-node": "^0.38.0", diff --git a/yarn.lock b/yarn.lock index 9ecd1e1572e..bc941833aaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -933,11 +933,6 @@ resolved "https://registry.yarnpkg.com/@celo/base/-/base-1.5.2.tgz#168ab5e4e30b374079d8d139fafc52ca6bfd4100" integrity sha512-KGf6Dl9E6D01vAfkgkjL2sG+zqAjspAogILIpWstljWdG5ifyA75jihrnDEHaMCoQS0KxHvTdP1XYS/GS6BEyQ== -"@celo/base@4.1.1", "@celo/base@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/base/-/base-4.1.1.tgz#2a6c38deaf2a82dbe10bf3dea4881e4452c11803" - integrity sha512-M7qpbDX/Bhmzcr/spZ7w9jkKfnMnv5zAnUFKoLrFsM8DB0NFcH95YAyeG1eVTfWmD7nu8RG1sASFPGGOAji2Iw== - "@celo/bls12377js@0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@celo/bls12377js/-/bls12377js-0.1.1.tgz#ba3574f41697cdba96c10ae96bb1aac057285798" @@ -958,19 +953,6 @@ debug "^4.1.1" utf8 "3.0.0" -"@celo/connect@4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/connect/-/connect-4.1.1.tgz#7a631a5fc832b85eabb7f5e9a5fb95dbd620ee67" - integrity sha512-yIdrh4OiO2MEuDrgkALflor8oAx6YMAqy9/EUcDHql+BxKkDjSA8yxggqiOQzmZJ7GHKRzpwOpx5bey9siP3pQ== - dependencies: - "@celo/base" "4.1.1" - "@celo/utils" "4.1.1" - "@types/debug" "^4.1.5" - "@types/utf8" "^2.1.6" - bignumber.js "^9.0.0" - debug "^4.1.1" - utf8 "3.0.0" - "@celo/contractkit@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/contractkit/-/contractkit-1.5.2.tgz#be15d570f3044a190dabb6bbe53d5081c78ea605" @@ -989,59 +971,6 @@ semver "^7.3.5" web3 "1.3.6" -"@celo/contractkit@4.1.1", "@celo/contractkit@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/contractkit/-/contractkit-4.1.1.tgz#2ace459e9b6b599ec535f167a3b1e0c461fce0b1" - integrity sha512-1QBCyFOlIuCX9m2LzFXwfs4LNrZaYhuCUleuKPlLVZiw17uhv7Bq1mEPWAmV7SckLzTjYFvggzIg42U01g8brg== - dependencies: - "@celo/base" "4.1.1" - "@celo/connect" "4.1.1" - "@celo/utils" "4.1.1" - "@celo/wallet-local" "4.1.1" - "@types/bn.js" "^5.1.0" - "@types/debug" "^4.1.5" - bignumber.js "^9.0.0" - cross-fetch "3.0.6" - debug "^4.1.1" - fp-ts "2.1.1" - io-ts "2.0.1" - semver "^7.3.5" - web3 "1.10.0" - -"@celo/encrypted-backup@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/encrypted-backup/-/encrypted-backup-4.1.1.tgz#24603821eddcd15b73ef693b9119d7e7d3ac7a35" - integrity sha512-/WXhPwWpTlx/tQxpoeGTX4Nwy5XoZu6nOkWQ/30NWKFc79GG/jEFpz8OQAg/izVaAWp82RrzpIuQq5U5KwaI7A== - dependencies: - "@celo/base" "4.1.1" - "@celo/identity" "4.1.1" - "@celo/phone-number-privacy-common" "^3.0.0" - "@celo/poprf" "^0.1.9" - "@celo/utils" "4.1.1" - "@types/debug" "^4.1.5" - debug "^4.1.1" - fp-ts "2.1.1" - io-ts "2.0.1" - -"@celo/identity@4.1.1", "@celo/identity@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/identity/-/identity-4.1.1.tgz#d0c541c344db466067d5baeba0b08d8fd3191053" - integrity sha512-tFj9vd6j4PIzqQDWjcolbH4cKOaKZkMLT+XxNesxHpXbyNhGm+4zMADbtyN4BWLTZB7dWVwgld+5EtMQQ9uSxw== - dependencies: - "@celo/base" "4.1.1" - "@celo/contractkit" "4.1.1" - "@celo/phone-number-privacy-common" "^3.0.0" - "@celo/utils" "4.1.1" - "@types/debug" "^4.1.5" - bignumber.js "^9.0.0" - blind-threshold-bls "https://github.com/celo-org/blind-threshold-bls-wasm#e1e2f8a" - cross-fetch "3.0.6" - debug "^4.1.1" - elliptic "^6.5.4" - ethereum-cryptography "1.2.0" - fp-ts "2.1.1" - io-ts "2.0.1" - "@celo/phone-number-privacy-common@1.0.39": version "1.0.39" resolved "https://registry.yarnpkg.com/@celo/phone-number-privacy-common/-/phone-number-privacy-common-1.0.39.tgz#3c9568f70378d24d11afcc4306024c5cf4f8efe9" @@ -1060,47 +989,6 @@ elliptic "^6.5.4" is-base64 "^1.1.0" -"@celo/phone-number-privacy-common@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@celo/phone-number-privacy-common/-/phone-number-privacy-common-3.0.0.tgz#b6041820ff4e954a19434163d7bdb9f005139b26" - integrity sha512-lxzYb3iNU85RA7vf0YAGCT904uPeinvR4MWSWyefTrH/OR6rdXgWV7MjptxM72mSTFGRPpnQLmhRD6b7O1UYqA== - dependencies: - "@celo/base" "^4.1.1" - "@celo/contractkit" "^4.1.1" - "@celo/phone-utils" "^4.1.1" - "@celo/utils" "^4.1.1" - "@opentelemetry/api" "^1.4.1" - "@opentelemetry/auto-instrumentations-node" "^0.38.0" - "@opentelemetry/propagator-ot-trace" "^0.27.0" - "@opentelemetry/sdk-metrics" "^1.15.1" - "@opentelemetry/sdk-node" "^0.41.1" - "@opentelemetry/sdk-trace-web" "^1.15.1" - "@opentelemetry/semantic-conventions" "^1.15.1" - "@types/bunyan" "1.8.8" - bignumber.js "^9.0.0" - bunyan "1.8.12" - bunyan-debug-stream "2.0.0" - bunyan-gke-stackdriver "0.1.2" - dotenv "^8.2.0" - elliptic "^6.5.4" - io-ts "2.0.1" - is-base64 "^1.1.0" - -"@celo/phone-utils@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/phone-utils/-/phone-utils-4.1.1.tgz#2c99f409d59c84964410b6f2f509bed415df4ba4" - integrity sha512-KTLgDWjORlLBov77uRmNbhgFQ1wycYD6+l1yigrXr651fPgejQeFXraF8mYy2eCOkehOR8e7GCmfOuoR3+VCzA== - dependencies: - "@celo/base" "4.1.1" - "@celo/utils" "4.1.1" - "@types/country-data" "^0.0.0" - "@types/google-libphonenumber" "^7.4.23" - "@types/node" "^10.12.18" - country-data "^0.0.31" - fp-ts "2.1.1" - google-libphonenumber "^3.2.27" - io-ts "2.0.1" - "@celo/poprf@^0.1.9": version "0.1.9" resolved "https://registry.yarnpkg.com/@celo/poprf/-/poprf-0.1.9.tgz#38c514ce0f572b80edeb9dc280b6cf5e9d7c2a75" @@ -1147,23 +1035,6 @@ web3-eth-abi "1.3.6" web3-utils "1.3.6" -"@celo/utils@4.1.1", "@celo/utils@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/utils/-/utils-4.1.1.tgz#4423b64ad5f9028ba96af3023c32e950b152d68a" - integrity sha512-NZJqkvrV4ADNU0JGi1NkjWWDFL3NFZpHdQC57O+GNahE0uvC2ylCjCMQyyrDuFjyJwsIHPV2zNdVt4LRM2NP7g== - dependencies: - "@celo/base" "4.1.1" - "@ethereumjs/util" "8.0.5" - "@types/bn.js" "^5.1.0" - "@types/elliptic" "^6.4.9" - "@types/node" "^10.12.18" - bignumber.js "^9.0.0" - elliptic "^6.5.4" - ethereum-cryptography "1.2.0" - io-ts "2.0.1" - web3-eth-abi "1.10.0" - web3-utils "1.10.0" - "@celo/wallet-base@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/wallet-base/-/wallet-base-1.5.2.tgz#ae8df425bf3c702277bb1b63a761a2ec8429e7aa" @@ -1179,55 +1050,6 @@ eth-lib "^0.2.8" ethereumjs-util "^5.2.0" -"@celo/wallet-base@4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/wallet-base/-/wallet-base-4.1.1.tgz#70a955193a75e1c54f5111f03a5f2bfd0fb1f47a" - integrity sha512-WbZyXnZnFQ13AmSYXTT5JEtFFZiGWquBKMfPXhRreRiNHqtwGSxq+bUaDmYIFKq6f4DU4gFV3UwdpW7iNbSv2w== - dependencies: - "@celo/base" "4.1.1" - "@celo/connect" "4.1.1" - "@celo/utils" "4.1.1" - "@ethereumjs/util" "8.0.5" - "@types/debug" "^4.1.5" - bignumber.js "^9.0.0" - debug "^4.1.1" - eth-lib "^0.2.8" - -"@celo/wallet-hsm-azure@^4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/wallet-hsm-azure/-/wallet-hsm-azure-4.1.1.tgz#9dcea4630d9670999410f4be5cf409368b769e71" - integrity sha512-3CnaOgClOEy0HIW8SIinPz21mcpp4Ps9zOcamfzs5ICWld4rPvk0DfKIW+8TQrHSxJWRRdZQNfbdVSKrQ6gZCg== - dependencies: - "@azure/identity" "^1.1.0" - "@azure/keyvault-keys" "^4.1.0" - "@azure/keyvault-secrets" "^4.1.0" - "@celo/connect" "4.1.1" - "@celo/utils" "4.1.1" - "@celo/wallet-base" "4.1.1" - "@celo/wallet-hsm" "4.1.1" - "@celo/wallet-remote" "4.1.1" - "@ethereumjs/util" "8.0.5" - "@types/secp256k1" "^4.0.0" - bignumber.js "^9.0.0" - debug "^4.1.1" - eth-lib "^0.2.8" - secp256k1 "^4.0.0" - -"@celo/wallet-hsm@4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/wallet-hsm/-/wallet-hsm-4.1.1.tgz#9d7f292027f14c172f406b5159f654a32c3405d0" - integrity sha512-pbTD/+G9i28HRb1KS1IHd+AkwspG8nqRmK59hH907BFcsUyVQ7a6HIs8s+i63/tBFHpnNIcP/5cGXW8V0VsnUg== - dependencies: - "@celo/base" "4.1.1" - "@ethereumjs/util" "8.0.5" - "@types/asn1js" "^0.0.2" - "@types/debug" "^4.1.5" - "@types/secp256k1" "^4.0.0" - asn1js "^2.0.26" - elliptic "^6.5.4" - eth-lib "^0.2.8" - secp256k1 "^4.0.0" - "@celo/wallet-local@1.5.2": version "1.5.2" resolved "https://registry.yarnpkg.com/@celo/wallet-local/-/wallet-local-1.5.2.tgz#66ea5fb763e19724309e3d56f312f1a342e12b91" @@ -1240,29 +1062,6 @@ eth-lib "^0.2.8" ethereumjs-util "^5.2.0" -"@celo/wallet-local@4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/wallet-local/-/wallet-local-4.1.1.tgz#8b7748ccb030dcc64e62b533af79c9ff3f602f36" - integrity sha512-+eyBOkir3AaGbBGg0HIEpjfZ+G3yIeYy9ekJAZbDkCH81K0dlAHKVGnylITVORR+lQsJeHDiciREVcp5Zq3aHw== - dependencies: - "@celo/connect" "4.1.1" - "@celo/utils" "4.1.1" - "@celo/wallet-base" "4.1.1" - "@ethereumjs/util" "8.0.5" - eth-lib "^0.2.8" - -"@celo/wallet-remote@4.1.1": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@celo/wallet-remote/-/wallet-remote-4.1.1.tgz#c717ef53a160f07d6402a7576122496d644c44de" - integrity sha512-cG85ryLRz+xANioZBDsGgCjY/mgh4X5bbCWmEWh5Jd7QHsbbkFgNCs7rcNEPjgxB7q2DfbRT0oUzcETqIWmXIA== - dependencies: - "@celo/connect" "4.1.1" - "@celo/utils" "4.1.1" - "@celo/wallet-base" "4.1.1" - "@ethereumjs/util" "8.0.5" - "@types/debug" "^4.1.5" - eth-lib "^0.2.8" - "@chainsafe/as-sha256@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.1.tgz#3639df0e1435cab03f4d9870cc3ac079e57a6fc9" From 21a1322f883b3280772af82dee9086cc63166fb0 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 8 Sep 2023 16:00:36 -0400 Subject: [PATCH 12/15] improve signer e2e tests --- .../phone-number-privacy/combiner/test/end-to-end/pnp.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts b/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts index 3ba6dfdcfb1..4dff6569640 100644 --- a/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts +++ b/packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts @@ -25,6 +25,7 @@ import { PHONE_NUMBER, walletAuthSigner, } from './resources' +import { sleep } from '@celo/base' const { IdentifierPrefix } = OdisUtils.Identifier @@ -168,6 +169,9 @@ describe(`Running against service deployed at ${combinerUrl} w/ blockchain provi await odisPayments .payInCUSD(ACCOUNT_ADDRESS, amountInWei) .sendAndWaitForReceipt({ from: ACCOUNT_ADDRESS }) + // wait for cache to expire and then query to refresh + await sleep(5 * 1000) + await OdisUtils.Quota.getPnpQuotaStatus(ACCOUNT_ADDRESS, dekAuthSigner(0), SERVICE_CONTEXT) }) describe('new requests', () => { From 464ac48ef5dc25e7c3d80c8672fd55ccfa3044aa Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 8 Sep 2023 16:02:06 -0400 Subject: [PATCH 13/15] add mainnet e2e command --- packages/phone-number-privacy/combiner/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index 4970c8e47a1..70ea98931aa 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -23,9 +23,10 @@ "test": "jest --runInBand --testPathIgnorePatterns test/end-to-end", "test:coverage": "yarn test --coverage", "test:integration": "jest --runInBand test/integration", - "test:e2e": "jest test/end-to-end --verbose", + "test:e2e": "jest --runInBand test/end-to-end --verbose", "test:e2e:staging": "CONTEXT_NAME=staging yarn test:e2e", - "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e" + "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e", + "test:e2e:mainnet": "CONTEXT_NAME=alfajores yarn test:e2e" }, "dependencies": { "@celo/contractkit": "^4.1.1", From 0427325fd30055602b2d1cc2455468e0ff66ffe0 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 8 Sep 2023 17:45:51 -0400 Subject: [PATCH 14/15] add privateKey param to loadTest script --- .../phone-number-privacy/monitor/src/query.ts | 12 +++++---- .../monitor/src/scripts/run-load-test.ts | 7 ++++- .../phone-number-privacy/monitor/src/test.ts | 26 ++++++++++++++----- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/phone-number-privacy/monitor/src/query.ts b/packages/phone-number-privacy/monitor/src/query.ts index 99b5f221b53..6f3d7851ef5 100644 --- a/packages/phone-number-privacy/monitor/src/query.ts +++ b/packages/phone-number-privacy/monitor/src/query.ts @@ -32,7 +32,8 @@ export const queryOdisForSalt = async ( contextName: OdisContextName, timeoutMs: number = 10000, bypassQuota: boolean = false, - useDEK: boolean = false + useDEK: boolean = false, + privateKey?: string ) => { let authSigner: AuthSigner let accountAddress: string @@ -43,12 +44,12 @@ export const queryOdisForSalt = async ( if (useDEK) { accountAddress = ACCOUNT_ADDRESS - contractKit.connection.addAccount(PRIVATE_KEY) + contractKit.connection.addAccount(privateKey ?? PRIVATE_KEY) contractKit.defaultAccount = accountAddress authSigner = dekAuthSigner(0) phoneNumber = generateRandomPhoneNumber() } else { - const privateKey = await newPrivateKey() + privateKey ??= await newPrivateKey() accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey)) contractKit.connection.addAccount(privateKey) contractKit.defaultAccount = accountAddress @@ -90,7 +91,8 @@ export const queryOdisForSalt = async ( export const queryOdisForQuota = async ( blockchainProvider: string, contextName: OdisContextName, - timeoutMs: number = 10000 + timeoutMs: number = 10000, + privateKey?: string ) => { console.log(`contextName: ${contextName}`) // tslint:disable-line:no-console console.log(`blockchain provider: ${blockchainProvider}`) // tslint:disable-line:no-console @@ -98,7 +100,7 @@ export const queryOdisForQuota = async ( const serviceContext = getServiceContext(contextName, OdisAPI.PNP) const contractKit = newKit(blockchainProvider, new LocalWallet()) - const privateKey = await newPrivateKey() + privateKey ??= await newPrivateKey() const accountAddress = normalizeAddressWith0x(privateKeyToAddress(privateKey)) contractKit.connection.addAccount(privateKey) contractKit.defaultAccount = accountAddress diff --git a/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts b/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts index fc5dea21dfe..b2023ae7e36 100644 --- a/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts +++ b/packages/phone-number-privacy/monitor/src/scripts/run-load-test.ts @@ -44,6 +44,10 @@ yargs type: 'number', description: 'number of requests to use when calculating latency moving average', default: 50, + }) + .option('privateKey', { + type: 'string', + description: 'optional private key to send requests from', }), (args) => { if (args.rps == null || args.contextName == null) { @@ -82,7 +86,8 @@ yargs args.duration, args.bypassQuota, args.useDEK, - args.movingAvgRequests + args.movingAvgRequests, + args.privateKey ) // tslint:disable-line:no-floating-promises } ).argv diff --git a/packages/phone-number-privacy/monitor/src/test.ts b/packages/phone-number-privacy/monitor/src/test.ts index 0431f2aedda..a1590406a6b 100644 --- a/packages/phone-number-privacy/monitor/src/test.ts +++ b/packages/phone-number-privacy/monitor/src/test.ts @@ -15,7 +15,8 @@ export async function testPNPSignQuery( contextName: OdisContextName, timeoutMs?: number, bypassQuota?: boolean, - useDEK?: boolean + useDEK?: boolean, + privateKey?: string ) { try { const odisResponse: IdentifierHashDetails = await queryOdisForSalt( @@ -23,7 +24,8 @@ export async function testPNPSignQuery( contextName, timeoutMs, bypassQuota, - useDEK + useDEK, + privateKey ) logger.debug({ odisResponse }, 'ODIS salt request successful. System is healthy.') } catch (err) { @@ -43,14 +45,16 @@ export async function testPNPSignQuery( export async function testPNPQuotaQuery( blockchainProvider: string, contextName: OdisContextName, - timeoutMs?: number + timeoutMs?: number, + privateKey?: string ) { logger.info(`Performing test PNP query for ${CombinerEndpointPNP.PNP_QUOTA}`) try { const odisResponse: PnpClientQuotaStatus = await queryOdisForQuota( blockchainProvider, contextName, - timeoutMs + timeoutMs, + privateKey ) logger.info({ odisResponse }, 'ODIS quota request successful. System is healthy.') } catch (err) { @@ -88,7 +92,8 @@ export async function concurrentRPSLoadTest( duration: number = 0, bypassQuota: boolean = false, useDEK: boolean = false, - movingAverageRequests: number = 50 + movingAverageRequests: number = 50, + privateKey?: string ) { const latencyQueue: number[] = [] let movingAvgLatencySum = 0 @@ -128,8 +133,15 @@ export async function concurrentRPSLoadTest( const testFn = async () => { try { await (endpoint === CombinerEndpointPNP.PNP_SIGN - ? testPNPSignQuery(blockchainProvider, contextName, undefined, bypassQuota, useDEK) - : testPNPQuotaQuery(blockchainProvider, contextName)) + ? testPNPSignQuery( + blockchainProvider, + contextName, + undefined, + bypassQuota, + useDEK, + privateKey + ) + : testPNPQuotaQuery(blockchainProvider, contextName, undefined, privateKey)) } catch (_) { logger.error('load test request failed') } From 5972548061f5850532a402b8b1ce1996694d8f94 Mon Sep 17 00:00:00 2001 From: soloseng <102702451+soloseng@users.noreply.github.com> Date: Fri, 8 Sep 2023 17:53:51 -0400 Subject: [PATCH 15/15] update context name --- packages/phone-number-privacy/combiner/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phone-number-privacy/combiner/package.json b/packages/phone-number-privacy/combiner/package.json index c90332cd99f..4ad16b5a842 100644 --- a/packages/phone-number-privacy/combiner/package.json +++ b/packages/phone-number-privacy/combiner/package.json @@ -26,7 +26,7 @@ "test:e2e": "jest --runInBand test/end-to-end --verbose", "test:e2e:staging": "CONTEXT_NAME=staging yarn test:e2e", "test:e2e:alfajores": "CONTEXT_NAME=alfajores yarn test:e2e", - "test:e2e:mainnet": "CONTEXT_NAME=alfajores yarn test:e2e" + "test:e2e:mainnet": "CONTEXT_NAME=mainnet yarn test:e2e" }, "dependencies": { "@celo/contractkit": "^4.1.2-dev",