diff --git a/stacks/ucan-invocation-stack.js b/stacks/ucan-invocation-stack.js index c512f57e..cd414b9e 100644 --- a/stacks/ucan-invocation-stack.js +++ b/stacks/ucan-invocation-stack.js @@ -189,8 +189,17 @@ export function UcanInvocationStack({ stack, app }) { deadLetterQueue: spaceMetricsDLQ.cdk.queue, }) + // TODO: keep for historical content that we might want to process + new KinesisStream(stack, 'ucan-stream', { + cdk: { + stream: { + retentionPeriod: Duration.days(365) + } + }, + }) + // create a kinesis stream - const ucanStream = new KinesisStream(stack, 'ucan-stream', { + const ucanStream = new KinesisStream(stack, 'ucan-stream-v2', { cdk: { stream: { retentionPeriod: Duration.days(365) diff --git a/ucan-invocation/constants.js b/ucan-invocation/constants.js index 07f4719a..69f9f0f0 100644 --- a/ucan-invocation/constants.js +++ b/ucan-invocation/constants.js @@ -7,6 +7,11 @@ import { remove as uploadRemove } from '@web3-storage/capabilities/upload' +export const CONTENT_TYPE = { + WORKFLOW: 'application/invocations+car', + RECEIPT: 'application/receipt+dag-cbor' +} + // UCAN protocol export const STORE_ADD = storeAdd.can export const STORE_REMOVE = storeRemove.can diff --git a/ucan-invocation/functions/metrics-store-add-size-total.js b/ucan-invocation/functions/metrics-store-add-size-total.js index c613edb5..92e8eae9 100644 --- a/ucan-invocation/functions/metrics-store-add-size-total.js +++ b/ucan-invocation/functions/metrics-store-add-size-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_ADD } from '../constants.js' +import { STORE_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateSizeTotal (ucanInvocations, ctx) { const invocationsWithStoreAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_ADD) + inv => inv.value.att.find(a => a.can === STORE_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.metricsTable.incrementStoreAddSizeTotal(invocationsWithStoreAdd) diff --git a/ucan-invocation/functions/metrics-store-add-total.js b/ucan-invocation/functions/metrics-store-add-total.js index 205e8e20..ebf8123d 100644 --- a/ucan-invocation/functions/metrics-store-add-total.js +++ b/ucan-invocation/functions/metrics-store-add-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_ADD } from '../constants.js' +import { STORE_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateStoreAddTotal (ucanInvocations, ctx) { const invocationsWithStoreAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_ADD) + inv => inv.value.att.find(a => a.can === STORE_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.metricsTable.incrementStoreAddTotal(invocationsWithStoreAdd) diff --git a/ucan-invocation/functions/metrics-store-remove-size-total.js b/ucan-invocation/functions/metrics-store-remove-size-total.js index 8952b67a..026741dc 100644 --- a/ucan-invocation/functions/metrics-store-remove-size-total.js +++ b/ucan-invocation/functions/metrics-store-remove-size-total.js @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/serverless' import { createCarStore } from '../buckets/car-store.js' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_REMOVE } from '../constants.js' +import { STORE_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -36,7 +36,7 @@ async function handler(event) { */ export async function updateRemoveSizeTotal (ucanInvocations, ctx) { const invocationsWithStoreRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_REMOVE) + inv => inv.value.att.find(a => a.can === STORE_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) // TODO: once we have receipts for store/remove, replace this diff --git a/ucan-invocation/functions/metrics-store-remove-total.js b/ucan-invocation/functions/metrics-store-remove-total.js index 3728d906..c9e2b89a 100644 --- a/ucan-invocation/functions/metrics-store-remove-total.js +++ b/ucan-invocation/functions/metrics-store-remove-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_REMOVE } from '../constants.js' +import { STORE_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateStoreRemoveTotal (ucanInvocations, ctx) { const invocationsWithStoreRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_REMOVE) + inv => inv.value.att.find(a => a.can === STORE_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.metricsTable.incrementStoreRemoveTotal(invocationsWithStoreRemove) diff --git a/ucan-invocation/functions/metrics-upload-add-total.js b/ucan-invocation/functions/metrics-upload-add-total.js index 309d5422..25328501 100644 --- a/ucan-invocation/functions/metrics-upload-add-total.js +++ b/ucan-invocation/functions/metrics-upload-add-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { UPLOAD_ADD } from '../constants.js' +import { UPLOAD_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateUploadAddTotal (ucanInvocations, ctx) { const invocationsWithUploadAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === UPLOAD_ADD) + inv => inv.value.att.find(a => a.can === UPLOAD_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.metricsTable.incrementUploadAddTotal(invocationsWithUploadAdd) diff --git a/ucan-invocation/functions/metrics-upload-remove-total.js b/ucan-invocation/functions/metrics-upload-remove-total.js index 2be27caf..cc2ff16e 100644 --- a/ucan-invocation/functions/metrics-upload-remove-total.js +++ b/ucan-invocation/functions/metrics-upload-remove-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createMetricsTable } from '../tables/metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { UPLOAD_REMOVE } from '../constants.js' +import { UPLOAD_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateUploadRemoveTotal (ucanInvocations, ctx) { const invocationsWithUploadRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === UPLOAD_REMOVE) + inv => inv.value.att.find(a => a.can === UPLOAD_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.metricsTable.incrementUploadRemoveTotal(invocationsWithUploadRemove) diff --git a/ucan-invocation/functions/space-metrics-store-add-size-total.js b/ucan-invocation/functions/space-metrics-store-add-size-total.js index 7ea5fdf0..3daac1de 100644 --- a/ucan-invocation/functions/space-metrics-store-add-size-total.js +++ b/ucan-invocation/functions/space-metrics-store-add-size-total.js @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/serverless' import { createCarStore } from '../buckets/car-store.js' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_ADD } from '../constants.js' +import { STORE_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -36,7 +36,7 @@ async function handler(event) { */ export async function updateAddSizeTotal (ucanInvocations, ctx) { const invocationsWithStoreAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_ADD) + inv => inv.value.att.find(a => a.can === STORE_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.spaceMetricsTable.incrementStoreAddSizeTotal(invocationsWithStoreAdd) diff --git a/ucan-invocation/functions/space-metrics-store-add-total.js b/ucan-invocation/functions/space-metrics-store-add-total.js index 85f22197..6e6ce2e5 100644 --- a/ucan-invocation/functions/space-metrics-store-add-total.js +++ b/ucan-invocation/functions/space-metrics-store-add-total.js @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' +import { STORE_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -9,7 +10,6 @@ Sentry.AWSLambda.init({ tracesSampleRate: 1.0, }) -const STORE_ADD = 'store/add' const AWS_REGION = process.env.AWS_REGION || 'us-west-2' /** @@ -43,7 +43,7 @@ async function handler(event) { */ export async function updateStoreCount (ucanInvocations, ctx) { const invocationsWithStoreAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_ADD) + inv => inv.value.att.find(a => a.can === STORE_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.spaceMetricsTable.incrementStoreAddCount(invocationsWithStoreAdd) diff --git a/ucan-invocation/functions/space-metrics-store-remove-size-total.js b/ucan-invocation/functions/space-metrics-store-remove-size-total.js index 89a1d583..743fb751 100644 --- a/ucan-invocation/functions/space-metrics-store-remove-size-total.js +++ b/ucan-invocation/functions/space-metrics-store-remove-size-total.js @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/serverless' import { createCarStore } from '../buckets/car-store.js' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_REMOVE } from '../constants.js' +import { STORE_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -36,7 +36,7 @@ async function handler(event) { */ export async function updateRemoveSizeTotal (ucanInvocations, ctx) { const invocationsWithStoreRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_REMOVE) + inv => inv.value.att.find(a => a.can === STORE_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) // TODO: once we have receipts for store/remove, replace this diff --git a/ucan-invocation/functions/space-metrics-store-remove-total.js b/ucan-invocation/functions/space-metrics-store-remove-total.js index f6ca3c6f..a4053956 100644 --- a/ucan-invocation/functions/space-metrics-store-remove-total.js +++ b/ucan-invocation/functions/space-metrics-store-remove-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { STORE_REMOVE } from '../constants.js' +import { STORE_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -43,7 +43,7 @@ async function handler(event) { */ export async function updateStoreCount (ucanInvocations, ctx) { const invocationsWithStoreRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === STORE_REMOVE) + inv => inv.value.att.find(a => a.can === STORE_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.spaceMetricsTable.incrementStoreRemoveCount(invocationsWithStoreRemove) diff --git a/ucan-invocation/functions/space-metrics-upload-add-total.js b/ucan-invocation/functions/space-metrics-upload-add-total.js index da99409e..99f7183f 100644 --- a/ucan-invocation/functions/space-metrics-upload-add-total.js +++ b/ucan-invocation/functions/space-metrics-upload-add-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { UPLOAD_ADD } from '../constants.js' +import { UPLOAD_ADD, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -43,7 +43,7 @@ async function handler(event) { */ export async function updateUploadCount (ucanInvocations, ctx) { const invocationsWithUploadAdd = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === UPLOAD_ADD) + inv => inv.value.att.find(a => a.can === UPLOAD_ADD) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.spaceMetricsTable.incrementUploadAddCount(invocationsWithUploadAdd) diff --git a/ucan-invocation/functions/space-metrics-upload-remove-total.js b/ucan-invocation/functions/space-metrics-upload-remove-total.js index 299a3981..387b907d 100644 --- a/ucan-invocation/functions/space-metrics-upload-remove-total.js +++ b/ucan-invocation/functions/space-metrics-upload-remove-total.js @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import { createSpaceMetricsTable } from '../tables/space-metrics.js' import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' -import { UPLOAD_REMOVE } from '../constants.js' +import { UPLOAD_REMOVE, CONTENT_TYPE } from '../constants.js' Sentry.AWSLambda.init({ environment: process.env.SST_STAGE, @@ -37,7 +37,7 @@ async function handler(event) { */ export async function updateUploadCount (ucanInvocations, ctx) { const invocationsWithUploadRemove = ucanInvocations.filter( - inv => inv.value.att.find(a => a.can === UPLOAD_REMOVE) + inv => inv.value.att.find(a => a.can === UPLOAD_REMOVE) && inv.type === CONTENT_TYPE.RECEIPT ).flatMap(inv => inv.value.att) await ctx.spaceMetricsTable.incrementUploadRemoveCount(invocationsWithUploadRemove) diff --git a/ucan-invocation/test/functions/metrics-store-add-size-total.test.js b/ucan-invocation/test/functions/metrics-store-add-size-total.test.js index ceef9c6f..900be131 100644 --- a/ucan-invocation/test/functions/metrics-store-add-size-total.test.js +++ b/ucan-invocation/test/functions/metrics-store-add-size-total.test.js @@ -11,7 +11,7 @@ import { adminMetricsTableProps } from '../../tables/index.js' import { updateSizeTotal } from '../../functions/metrics-store-add-size-total.js' import { createMetricsTable } from '../../tables/metrics.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -52,6 +52,7 @@ test('handles a batch of single invocation with store/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -95,6 +96,7 @@ test('handles batch of single invocations with multiple store/add attributes', a aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -112,6 +114,91 @@ test('handles batch of single invocations with multiple store/add attributes', a t.is(item?.value, cars.reduce((acc, c) => acc + c.size, 0)) }) +test('handles a batch of single invocation without store/add', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid, + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateSizeTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_SIZE_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipts', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.add.create({ + with: spaceDid, + nb: { + link: car.cid, + size: car.size + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, + ts: Date.now() + }] + + // @ts-expect-error + await updateSizeTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_SIZE_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) + t.is(item?.value, 0) +}) + /** * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamoClient */ diff --git a/ucan-invocation/test/functions/metrics-store-add-total.test.js b/ucan-invocation/test/functions/metrics-store-add-total.test.js index fec105d9..a5f40a34 100644 --- a/ucan-invocation/test/functions/metrics-store-add-total.test.js +++ b/ucan-invocation/test/functions/metrics-store-add-total.test.js @@ -11,7 +11,7 @@ import { adminMetricsTableProps } from '../../tables/index.js' import { updateStoreAddTotal } from '../../functions/metrics-store-add-total.js' import { createMetricsTable } from '../../tables/metrics.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -52,6 +52,7 @@ test('handles a batch of single invocation with store/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -95,6 +96,7 @@ test('handles batch of single invocations with multiple store/add attributes', a aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -137,6 +139,50 @@ test('handles a batch of single invocation without store/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreAddTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipts', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.add.create({ + with: spaceDid, + nb: { + link: car.cid, + size: car.size + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, ts: Date.now() }] diff --git a/ucan-invocation/test/functions/metrics-store-remove-size-total.test.js b/ucan-invocation/test/functions/metrics-store-remove-size-total.test.js index 67292e12..730c087e 100644 --- a/ucan-invocation/test/functions/metrics-store-remove-size-total.test.js +++ b/ucan-invocation/test/functions/metrics-store-remove-size-total.test.js @@ -17,7 +17,7 @@ import { import { updateRemoveSizeTotal } from '../../functions/metrics-store-remove-size-total.js' import { createMetricsTable } from '../../tables/metrics.js' import { createCarStore } from '../../buckets/car-store.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -71,6 +71,7 @@ test('handles a batch of single invocation with store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -126,6 +127,7 @@ test('handles batch of single invocations with multiple store/remove attributes' aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -181,6 +183,62 @@ test('handles a batch of single invocation without store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateRemoveSizeTotal(invocations, { + metricsTable, + carStoreBucket + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_REMOVE_SIZE_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_REMOVE_SIZE_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipts', async t => { + const { tableName, bucketName } = await prepareResources(t.context.dynamoClient, t.context.s3) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + // Put CAR to bucket + const putObjectCmd = new PutObjectCommand({ + Key: `${car.cid.toString()}/${car.cid.toString()}.car`, + Bucket: bucketName, + Body: Buffer.from( + await car.arrayBuffer() + ) + }) + await t.context.s3.send(putObjectCmd) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + const carStoreBucket = createCarStore(REGION, bucketName, t.context.s3Opts) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, ts: Date.now() }] diff --git a/ucan-invocation/test/functions/metrics-store-remove-total.test.js b/ucan-invocation/test/functions/metrics-store-remove-total.test.js index e7f41458..ed7c72d1 100644 --- a/ucan-invocation/test/functions/metrics-store-remove-total.test.js +++ b/ucan-invocation/test/functions/metrics-store-remove-total.test.js @@ -11,7 +11,7 @@ import { adminMetricsTableProps } from '../../tables/index.js' import { updateStoreRemoveTotal } from '../../functions/metrics-store-remove-total.js' import { createMetricsTable } from '../../tables/metrics.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -51,6 +51,7 @@ test('handles a batch of single invocation with store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -93,6 +94,7 @@ test('handles batch of single invocations with multiple store/remove attributes' aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -136,6 +138,50 @@ test('handles a batch of single invocation without store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreRemoveTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_REMOVE_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_REMOVE_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipts', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, ts: Date.now() }] diff --git a/ucan-invocation/test/functions/metrics-upload-add-total.test.js b/ucan-invocation/test/functions/metrics-upload-add-total.test.js index cd38fa56..667e92ab 100644 --- a/ucan-invocation/test/functions/metrics-upload-add-total.test.js +++ b/ucan-invocation/test/functions/metrics-upload-add-total.test.js @@ -11,7 +11,7 @@ import { adminMetricsTableProps } from '../../tables/index.js' import { updateUploadAddTotal } from '../../functions/metrics-upload-add-total.js' import { createMetricsTable } from '../../tables/metrics.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -52,6 +52,7 @@ test('handles a batch of single invocation with upload/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -95,6 +96,7 @@ test('handles batch of single invocations with multiple upload/add attributes', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -137,6 +139,51 @@ test('handles a batch of single invocation without upload/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateUploadAddTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.UPLOAD_ADD_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.UPLOAD_ADD_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipts', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + UploadCapabilities.add.create({ + with: spaceDid, + nb: { + root: car.cid, + shards: [car.cid] + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, ts: Date.now() }] diff --git a/ucan-invocation/test/functions/metrics-upload-remove-total.test.js b/ucan-invocation/test/functions/metrics-upload-remove-total.test.js index e2b4aa26..e31b66ce 100644 --- a/ucan-invocation/test/functions/metrics-upload-remove-total.test.js +++ b/ucan-invocation/test/functions/metrics-upload-remove-total.test.js @@ -11,7 +11,7 @@ import { adminMetricsTableProps } from '../../tables/index.js' import { updateUploadRemoveTotal } from '../../functions/metrics-upload-remove-total.js' import { createMetricsTable } from '../../tables/metrics.js' -import { METRICS_NAMES } from '../../constants.js' +import { METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' const REGION = 'us-west-2' @@ -51,6 +51,7 @@ test('handles a batch of single invocation with upload/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -93,6 +94,7 @@ test('handles batch of single invocations with multiple upload/remove attributes aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -136,6 +138,50 @@ test('handles a batch of single invocation without upload/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, + ts: Date.now() + }] + + // @ts-expect-error + await updateUploadRemoveTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.UPLOAD_REMOVE_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.UPLOAD_REMOVE_TOTAL) + t.is(item?.value, 0) +}) + +test('handles a batch of single invocation without receipt', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + UploadCapabilities.remove.create({ + with: spaceDid, + nb: { + root: car.cid + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + type: CONTENT_TYPE.WORKFLOW, ts: Date.now() }] diff --git a/ucan-invocation/test/functions/space-metrics-store-add-size-total.test.js b/ucan-invocation/test/functions/space-metrics-store-add-size-total.test.js index 02c8cfa8..284d3f73 100644 --- a/ucan-invocation/test/functions/space-metrics-store-add-size-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-store-add-size-total.test.js @@ -15,7 +15,7 @@ import { createSpace } from '../helpers/ucanto.js' import { randomCAR } from '../helpers/random.js' import { updateAddSizeTotal } from '../../functions/space-metrics-store-add-size-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' import { createCarStore } from '../../buckets/car-store.js' @@ -74,6 +74,7 @@ test('handles a batch of single invocation with store/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -131,6 +132,7 @@ test('handles batch of single invocation with multiple store/add attributes', as aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -190,6 +192,7 @@ test('handles batch of multiple invocations with store/add in same space', async aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -247,6 +250,7 @@ test('handles batch of multiple invocations with store/add in multiple spaces', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -310,6 +314,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/test/functions/space-metrics-store-add-total.test.js b/ucan-invocation/test/functions/space-metrics-store-add-total.test.js index b167e1e1..4b9b9f9c 100644 --- a/ucan-invocation/test/functions/space-metrics-store-add-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-store-add-total.test.js @@ -10,7 +10,7 @@ import { createSpace } from '../helpers/ucanto.js' import { randomCAR } from '../helpers/random.js' import { updateStoreCount } from '../../functions/space-metrics-store-add-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' const REGION = 'us-west-2' @@ -52,6 +52,7 @@ test('handles a batch of single invocation with store/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -96,6 +97,7 @@ test('handles batch of single invocation with multiple store/add attributes', as aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -142,6 +144,7 @@ test('handles batch of multiple invocations with store/add in same space', async aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -187,6 +190,7 @@ test('handles batch of multiple invocations with store/add in multiple spaces', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -238,6 +242,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/test/functions/space-metrics-store-remove-size-total.test.js b/ucan-invocation/test/functions/space-metrics-store-remove-size-total.test.js index d18083b2..c8c18188 100644 --- a/ucan-invocation/test/functions/space-metrics-store-remove-size-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-store-remove-size-total.test.js @@ -15,7 +15,7 @@ import { createSpace } from '../helpers/ucanto.js' import { randomCAR } from '../helpers/random.js' import { updateRemoveSizeTotal } from '../../functions/space-metrics-store-remove-size-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' import { createCarStore } from '../../buckets/car-store.js' @@ -72,6 +72,7 @@ test('handles a batch of single invocation with store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -128,6 +129,7 @@ test('handles batch of single invocation with multiple store/remove attributes', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -186,6 +188,7 @@ test('handles batch of multiple invocations with store/remove in same space', as aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -242,6 +245,7 @@ test('handles batch of multiple invocations with store/remove in multiple spaces aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -304,6 +308,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/test/functions/space-metrics-store-remove-total.test.js b/ucan-invocation/test/functions/space-metrics-store-remove-total.test.js index 89330477..5186c2a4 100644 --- a/ucan-invocation/test/functions/space-metrics-store-remove-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-store-remove-total.test.js @@ -10,7 +10,7 @@ import { createSpace } from '../helpers/ucanto.js' import { randomCAR } from '../helpers/random.js' import { updateStoreCount } from '../../functions/space-metrics-store-remove-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' const REGION = 'us-west-2' @@ -51,6 +51,7 @@ test('handles a batch of single invocation with store/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -94,6 +95,7 @@ test('handles batch of single invocation with multiple store/remove attributes', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -139,6 +141,7 @@ test('handles batch of multiple invocations with store/remove in same space', as aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -183,6 +186,7 @@ test('handles batch of multiple invocations with store/remove in multiple spaces aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -233,6 +237,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/test/functions/space-metrics-upload-add-total.test.js b/ucan-invocation/test/functions/space-metrics-upload-add-total.test.js index 54b7b94e..e0aa5467 100644 --- a/ucan-invocation/test/functions/space-metrics-upload-add-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-upload-add-total.test.js @@ -10,7 +10,7 @@ import { createDynamoTable, getItemFromTable} from '../helpers/tables.js' import { spaceMetricsTableProps } from '../../tables/index.js' import { updateUploadCount } from '../../functions/space-metrics-upload-add-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' const REGION = 'us-west-2' @@ -52,6 +52,7 @@ test('handles a batch of single invocation with upload/add', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -96,6 +97,7 @@ test('handles batch of single invocation with multiple upload/add attributes', a aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -142,6 +144,7 @@ test('handles batch of multiple invocations with upload/add in same space', asyn aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -187,6 +190,7 @@ test('handles batch of multiple invocations with upload/add in multiple spaces', aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -237,6 +241,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/test/functions/space-metrics-upload-remove-total.test.js b/ucan-invocation/test/functions/space-metrics-upload-remove-total.test.js index 3c8c41eb..ac8e280c 100644 --- a/ucan-invocation/test/functions/space-metrics-upload-remove-total.test.js +++ b/ucan-invocation/test/functions/space-metrics-upload-remove-total.test.js @@ -10,7 +10,7 @@ import { createDynamoTable, getItemFromTable} from '../helpers/tables.js' import { spaceMetricsTableProps } from '../../tables/index.js' import { updateUploadCount } from '../../functions/space-metrics-upload-remove-total.js' -import { SPACE_METRICS_NAMES } from '../../constants.js' +import { SPACE_METRICS_NAMES, CONTENT_TYPE } from '../../constants.js' import { createSpaceMetricsTable } from '../../tables/space-metrics.js' const REGION = 'us-west-2' @@ -51,6 +51,7 @@ test('handles a batch of single invocation with upload/remove', async t => { aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -94,6 +95,7 @@ test('handles batch of single invocation with multiple upload/remove attributes' aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() }] @@ -139,6 +141,7 @@ test('handles batch of multiple invocations with upload/remove in same space', a aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -183,6 +186,7 @@ test('handles batch of multiple invocations with upload/remove in multiple space aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) @@ -232,6 +236,7 @@ test('errors handling batch of multiple invocations with more transactions than aud: uploadService.did(), iss: alice.did() }, + type: CONTENT_TYPE.RECEIPT, ts: Date.now() })) diff --git a/ucan-invocation/types.ts b/ucan-invocation/types.ts index 0c6bbbe4..37c2adf7 100644 --- a/ucan-invocation/types.ts +++ b/ucan-invocation/types.ts @@ -44,10 +44,13 @@ export interface MetricsBySpaceWithBucketCtx { carStoreBucket: CarStoreBucket } +export type UcanInvocationType = 'application/invocations+car' | 'application/receipt+dag-cbor' + export interface UcanInvocation { - carCid: string, - value: UcanInvocationValue, + carCid: string + value: UcanInvocationValue ts: number + type: UcanInvocationType } export interface UcanInvocationValue {