From 150b46bc2f20549bff45424f426f079c37cc43a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 13 Dec 2024 14:23:07 +0100 Subject: [PATCH 01/19] fix(db-dynamodb): tools for batch write --- .../src/utils/batch/EntityWriteBatch.ts | 68 ++++++++++++++++++ .../utils/batch/EntityWriteBatchBuilder.ts | 30 ++++++++ .../src/utils/batch/TableWriteBatch.ts | 71 +++++++++++++++++++ .../src/utils/{ => batch}/batchRead.ts | 0 .../src/utils/{ => batch}/batchWrite.ts | 21 +----- packages/db-dynamodb/src/utils/batch/index.ts | 6 ++ packages/db-dynamodb/src/utils/batch/types.ts | 58 +++++++++++++++ packages/db-dynamodb/src/utils/index.ts | 3 +- packages/db-dynamodb/src/utils/put.ts | 19 ++--- packages/db-dynamodb/src/utils/update.ts | 4 +- 10 files changed, 249 insertions(+), 31 deletions(-) create mode 100644 packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts create mode 100644 packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts create mode 100644 packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts rename packages/db-dynamodb/src/utils/{ => batch}/batchRead.ts (100%) rename packages/db-dynamodb/src/utils/{ => batch}/batchWrite.ts (79%) create mode 100644 packages/db-dynamodb/src/utils/batch/index.ts create mode 100644 packages/db-dynamodb/src/utils/batch/types.ts diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts new file mode 100644 index 00000000000..88527554195 --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -0,0 +1,68 @@ +import type { Entity } from "dynamodb-toolbox"; +import { batchWriteAll } from "./batchWrite"; +import type { + BatchWriteItem, + BatchWriteResult, + IDeleteBatchItem, + IEntityWriteBatch, + IEntityWriteBatchBuilder, + IPutBatchItem, + ITableWriteBatch +} from "./types"; +import { createTableWriteBatch } from "./TableWriteBatch"; +import type { TableDef } from "~/toolbox"; +import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; + +export interface IEntityWriteBatchParams { + entity: Entity; + put?: IPutBatchItem[]; + delete?: IDeleteBatchItem[]; +} + +export class EntityWriteBatch implements IEntityWriteBatch { + public readonly entity: Entity; + public readonly items: BatchWriteItem[] = []; + public readonly builder: IEntityWriteBatchBuilder; + + public constructor(params: IEntityWriteBatchParams) { + if (!params.entity.name) { + throw new Error(`No name provided for entity.`); + } else if (!params.entity.table) { + throw new Error(`No table provided for entity ${params.entity.name}.`); + } + this.entity = params.entity; + this.builder = createEntityWriteBatchBuilder(this.entity); + for (const item of params.put || []) { + this.put(item); + } + for (const item of params.delete || []) { + this.delete(item); + } + } + + public put>(item: IPutBatchItem): void { + this.items.push(this.builder.put(item)); + } + + public delete(item: IDeleteBatchItem): void { + this.items.push(this.builder.delete(item)); + } + + public combine(items: BatchWriteItem[]): ITableWriteBatch { + return createTableWriteBatch({ + table: this.entity!.table as TableDef, + items: this.items.concat(items) + }); + } + + public async execute(): Promise { + return await batchWriteAll({ + items: this.items, + table: this.entity.table + }); + } +} + +export const createEntityWriteBatch = (params: IEntityWriteBatchParams): IEntityWriteBatch => { + return new EntityWriteBatch(params); +}; diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts new file mode 100644 index 00000000000..c344f5dfb81 --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts @@ -0,0 +1,30 @@ +import type { Entity } from "~/toolbox"; +import type { + BatchWriteItem, + IDeleteBatchItem, + IEntityWriteBatchBuilder, + IPutBatchItem +} from "./types"; + +export class EntityWriteBatchBuilder implements IEntityWriteBatchBuilder { + public readonly entity: Entity; + + public constructor(entity: Entity) { + this.entity = entity; + } + + public put>(item: IPutBatchItem): BatchWriteItem { + return this.entity.putBatch(item, { + execute: true, + strictSchemaCheck: false + }); + } + + public delete(item: IDeleteBatchItem): BatchWriteItem { + return this.entity.deleteBatch(item); + } +} + +export const createEntityWriteBatchBuilder = (entity: Entity): IEntityWriteBatchBuilder => { + return new EntityWriteBatchBuilder(entity); +}; diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts new file mode 100644 index 00000000000..f9e0b24e9ff --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -0,0 +1,71 @@ +import { Entity, TableDef } from "~/toolbox"; +import { + BatchWriteItem, + BatchWriteResult, + IDeleteBatchItem, + IEntityWriteBatchBuilder, + IPutBatchItem, + ITableWriteBatch +} from "./types"; +import { batchWriteAll } from "./batchWrite"; +import { createEntityWriteBatchBuilder } from "~/utils/batch/EntityWriteBatchBuilder"; + +export interface ITableWriteBatchParams { + table: TableDef; + items?: BatchWriteItem[]; +} + +export class TableWriteBatch implements ITableWriteBatch { + public readonly table: TableDef; + public readonly items: BatchWriteItem[] = []; + private readonly builders: Map = new Map(); + + public constructor(params: ITableWriteBatchParams) { + this.table = params.table; + if (!params.items?.length) { + return; + } + this.items.push(...params.items); + } + + public put(entity: Entity, item: IPutBatchItem): void { + const builder = this.getBuilder(entity); + this.items.push(builder.put(item)); + } + + public delete(entity: Entity, item: IDeleteBatchItem): void { + const builder = this.getBuilder(entity); + this.items.push(builder.delete(item)); + } + + public combine(items: BatchWriteItem[]): ITableWriteBatch { + return createTableWriteBatch({ + table: this.table, + items: this.items.concat(items) + }); + } + + public async execute(): Promise { + return await batchWriteAll({ + items: this.items, + table: this.table + }); + } + + private getBuilder(entity: Entity): IEntityWriteBatchBuilder { + if (!entity.name) { + throw new Error("Entity must have a name."); + } + const builder = this.builders.get(entity.name); + if (builder) { + return builder; + } + const newBuilder = createEntityWriteBatchBuilder(entity); + this.builders.set(entity.name, newBuilder); + return newBuilder; + } +} + +export const createTableWriteBatch = (params: ITableWriteBatchParams): ITableWriteBatch => { + return new TableWriteBatch(params); +}; diff --git a/packages/db-dynamodb/src/utils/batchRead.ts b/packages/db-dynamodb/src/utils/batch/batchRead.ts similarity index 100% rename from packages/db-dynamodb/src/utils/batchRead.ts rename to packages/db-dynamodb/src/utils/batch/batchRead.ts diff --git a/packages/db-dynamodb/src/utils/batchWrite.ts b/packages/db-dynamodb/src/utils/batch/batchWrite.ts similarity index 79% rename from packages/db-dynamodb/src/utils/batchWrite.ts rename to packages/db-dynamodb/src/utils/batch/batchWrite.ts index b65e5afe94a..6c753bd46dc 100644 --- a/packages/db-dynamodb/src/utils/batchWrite.ts +++ b/packages/db-dynamodb/src/utils/batch/batchWrite.ts @@ -1,31 +1,12 @@ import lodashChunk from "lodash/chunk"; import { TableDef } from "~/toolbox"; -import { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; - -export interface BatchWriteItem { - [key: string]: WriteRequest; -} +import { BatchWriteItem, BatchWriteResponse, BatchWriteResult } from "./types"; export interface BatchWriteParams { table?: TableDef; items: BatchWriteItem[]; } -export interface BatchWriteResponse { - next?: () => Promise; - $metadata: { - httpStatusCode: number; - requestId: string; - attempts: number; - totalRetryDelay: number; - }; - UnprocessedItems?: { - [table: string]: WriteRequest[]; - }; -} - -export type BatchWriteResult = BatchWriteResponse[]; - const hasUnprocessedItems = (result: BatchWriteResponse): boolean => { if (typeof result.next !== "function") { return false; diff --git a/packages/db-dynamodb/src/utils/batch/index.ts b/packages/db-dynamodb/src/utils/batch/index.ts new file mode 100644 index 00000000000..58cdc76c1ff --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/index.ts @@ -0,0 +1,6 @@ +export * from "./batchRead"; +export * from "./batchWrite"; +export * from "./EntityWriteBatch"; +export * from "./EntityWriteBatchBuilder"; +export * from "./TableWriteBatch"; +export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts new file mode 100644 index 00000000000..eeb7ad46b9a --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -0,0 +1,58 @@ +import type { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; +import { Entity, TableDef } from "~/toolbox"; + +export interface BatchWriteResponse { + next?: () => Promise; + $metadata: { + httpStatusCode: number; + requestId: string; + attempts: number; + totalRetryDelay: number; + }; + UnprocessedItems?: { + [table: string]: WriteRequest[]; + }; +} + +export type BatchWriteResult = BatchWriteResponse[]; + +export interface IDeleteBatchItem { + PK: string; + SK: string; +} + +export type IPutBatchItem = Record> = { + PK: string; + SK: string; + TYPE: string; +} & T; + +export interface BatchWriteItem { + [key: string]: WriteRequest; +} + +export interface IEntityWriteBatchBuilder { + readonly entity: Entity; + put>(item: IPutBatchItem): BatchWriteItem; + delete(item: IDeleteBatchItem): BatchWriteItem; +} + +export interface IEntityWriteBatch { + readonly entity: Entity; + readonly items: BatchWriteItem[]; + readonly builder: IEntityWriteBatchBuilder; + + put(item: IPutBatchItem): void; + delete(item: IDeleteBatchItem): void; + execute(): Promise; + combine(items: BatchWriteItem[]): ITableWriteBatch; +} + +export interface ITableWriteBatch { + readonly table: TableDef; + readonly items: BatchWriteItem[]; + put(entity: Entity, item: IPutBatchItem): void; + delete(entity: Entity, item: IDeleteBatchItem): void; + execute(): Promise; + combine(items: BatchWriteItem[]): ITableWriteBatch; +} diff --git a/packages/db-dynamodb/src/utils/index.ts b/packages/db-dynamodb/src/utils/index.ts index b3823773402..087e51b7f56 100644 --- a/packages/db-dynamodb/src/utils/index.ts +++ b/packages/db-dynamodb/src/utils/index.ts @@ -1,5 +1,3 @@ -export * from "./batchRead"; -export * from "./batchWrite"; export * from "./cleanup"; export * from "./createEntity"; export * from "./createTable"; @@ -15,3 +13,4 @@ export * from "./scan"; export * from "./sort"; export * from "./table"; export * from "./update"; +export * from "./batch"; diff --git a/packages/db-dynamodb/src/utils/put.ts b/packages/db-dynamodb/src/utils/put.ts index 65f1a193552..f426445103c 100644 --- a/packages/db-dynamodb/src/utils/put.ts +++ b/packages/db-dynamodb/src/utils/put.ts @@ -1,18 +1,21 @@ import { Entity } from "~/toolbox"; -interface Params { +export interface IPutParamsItem { + PK: string; + SK: string; + [key: string]: any; +} + +export interface IPutParams { entity: Entity; - item: { - PK: string; - SK: string; - [key: string]: any; - }; + item: IPutParamsItem; } -export const put = async (params: Params) => { +export const put = async (params: IPutParams) => { const { entity, item } = params; return await entity.put(item, { - execute: true + execute: true, + strictSchemaCheck: false }); }; diff --git a/packages/db-dynamodb/src/utils/update.ts b/packages/db-dynamodb/src/utils/update.ts index b7fad30da83..5da727ccc48 100644 --- a/packages/db-dynamodb/src/utils/update.ts +++ b/packages/db-dynamodb/src/utils/update.ts @@ -5,6 +5,7 @@ interface Params { item: { PK: string; SK: string; + TYPE: string; [key: string]: any; }; } @@ -13,6 +14,7 @@ export const update = async (params: Params) => { const { entity, item } = params; return await entity.update(item, { - execute: true + execute: true, + strictSchemaCheck: false }); }; From b0981d18a2c217ade7d33242d08f8326f7aeac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 13 Dec 2024 14:56:18 +0100 Subject: [PATCH 02/19] wip: go through packages and update with new batch write tool --- .../src/definitions/entry.ts | 6 + .../tasks/reindexing/ReindexingTaskRunner.ts | 20 +- packages/api-elasticsearch-tasks/src/types.ts | 6 +- .../operations/AliasesStorageOperations.ts | 52 ++-- .../src/definitions/elasticsearch.ts | 1 - .../src/operations/form/index.ts | 266 ++++++++-------- .../src/operations/submission/index.ts | 2 +- .../src/operations/form/index.ts | 285 +++++++++--------- .../src/definitions/entryElasticsearch.ts | 3 + .../dataLoader/getLatestRevisionByEntryId.ts | 2 +- .../getPublishedRevisionByEntryId.ts | 2 +- .../entry/dataLoader/getRevisionById.ts | 2 +- .../src/operations/entry/index.ts | 2 +- .../dataLoader/getLatestRevisionByEntryId.ts | 2 +- .../getPublishedRevisionByEntryId.ts | 2 +- .../entry/dataLoader/getRevisionById.ts | 2 +- .../src/operations/entry/index.ts | 2 +- .../definitions/pageElasticsearchEntity.ts | 3 + .../operations/blockCategory/dataLoader.ts | 2 +- .../src/operations/category/dataLoader.ts | 2 +- .../src/operations/pageBlock/dataLoader.ts | 2 +- .../src/operations/pageTemplate/dataLoader.ts | 2 +- .../src/operations/pages/index.ts | 2 +- .../operations/blockCategory/dataLoader.ts | 2 +- .../src/operations/category/dataLoader.ts | 2 +- .../src/operations/pageBlock/dataLoader.ts | 2 +- .../src/operations/pageTemplate/dataLoader.ts | 2 +- .../src/operations/pages/index.ts | 2 +- .../src/operations/queueJob.ts | 2 +- .../src/operations/render.ts | 101 ++++--- packages/api-security-so-ddb/src/index.ts | 2 +- packages/api-tenancy-so-ddb/src/index.ts | 109 +++---- packages/db-dynamodb/src/utils/batch/types.ts | 1 - 33 files changed, 451 insertions(+), 444 deletions(-) diff --git a/packages/api-elasticsearch-tasks/src/definitions/entry.ts b/packages/api-elasticsearch-tasks/src/definitions/entry.ts index fa3c9bb7afd..71c11511ff9 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/entry.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/entry.ts @@ -1,3 +1,6 @@ +/** + * TODO If adding GSIs to the Elasticsearch table, add them here. + */ import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; interface Params { @@ -24,6 +27,9 @@ export const createEntry = (params: Params): Entity => { }, data: { type: "map" + }, + TYPE: { + type: "string" } } }); diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts index 908d8d911b7..f38ae06b763 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts @@ -6,7 +6,7 @@ import { } from "~/types"; import { ITaskResponse, ITaskResponseResult } from "@webiny/tasks/response/abstractions"; import { scan } from "~/helpers/scan"; -import { BatchWriteItem, ScanResponse } from "@webiny/db-dynamodb"; +import { createTableWriteBatch, ScanResponse } from "@webiny/db-dynamodb"; import { IndexManager } from "~/settings"; import { IIndexManager } from "~/settings/types"; @@ -73,7 +73,10 @@ export class ReindexingTaskRunner { return this.response.done("No more items to process."); } - const batch: BatchWriteItem[] = []; + const tableWriteBatch = createTableWriteBatch({ + table: this.manager.table + }); + for (const item of results.items) { /** * No index defined? Impossible but let's skip if really happens. @@ -110,14 +113,13 @@ export class ReindexingTaskRunner { /** * Reindexing will be triggered by the `putBatch` method. */ - batch.push( - entity.putBatch({ - ...item, - modified: new Date().toISOString() - }) - ); + tableWriteBatch.put(entity, { + ...item, + TYPE: item.TYPE || "unknown", + modified: new Date().toISOString() + }); } - await this.manager.write(batch); + await tableWriteBatch.execute(); /** * We always store the index settings, so we can restore them later. * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point. diff --git a/packages/api-elasticsearch-tasks/src/types.ts b/packages/api-elasticsearch-tasks/src/types.ts index b2d5b276551..9265a282b2c 100644 --- a/packages/api-elasticsearch-tasks/src/types.ts +++ b/packages/api-elasticsearch-tasks/src/types.ts @@ -12,6 +12,7 @@ import { ITaskResponse } from "@webiny/tasks/response/abstractions"; import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; import { ITimer } from "@webiny/handler-aws"; +import { GenericRecord } from "@webiny/api/types"; export interface Context extends ElasticsearchContext, TasksContext {} @@ -42,17 +43,18 @@ export interface IElasticsearchIndexingTaskValues { } export interface AugmentedError extends Error { - data?: Record; + data?: GenericRecord; [key: string]: any; } export interface IDynamoDbElasticsearchRecord { PK: string; SK: string; + TYPE?: string; index: string; _et?: string; entity: string; - data: Record; + data: GenericRecord; modified: string; } diff --git a/packages/api-file-manager-ddb/src/operations/AliasesStorageOperations.ts b/packages/api-file-manager-ddb/src/operations/AliasesStorageOperations.ts index 47d2ed6280b..af1f8aba00d 100644 --- a/packages/api-file-manager-ddb/src/operations/AliasesStorageOperations.ts +++ b/packages/api-file-manager-ddb/src/operations/AliasesStorageOperations.ts @@ -1,13 +1,12 @@ -import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; -import { - FileManagerAliasesStorageOperations, +import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; +import type { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import type { File, - FileAlias + FileAlias, + FileManagerAliasesStorageOperations } from "@webiny/api-file-manager/types"; import { - BatchWriteItem, - batchWriteAll, + createEntityWriteBatch, createStandardEntity, createTable, DbItem, @@ -39,52 +38,49 @@ export class AliasesStorageOperations implements FileManagerAliasesStorageOperat async deleteAliases(file: File): Promise { const aliasItems = await this.getExistingAliases(file); - const items: BatchWriteItem[] = []; - aliasItems.forEach(item => { - items.push( - this.aliasEntity.deleteBatch({ + const batchWrite = createEntityWriteBatch({ + entity: this.aliasEntity, + delete: aliasItems.map(item => { + return { PK: this.createPartitionKey({ id: item.fileId, tenant: item.tenant, locale: item.locale }), SK: `ALIAS#${item.alias}` - }) - ); + }; + }) }); - await batchWriteAll({ table: this.table, items }); + await batchWrite.execute(); } async storeAliases(file: File): Promise { - const items: BatchWriteItem[] = []; const existingAliases = await this.getExistingAliases(file); const newAliases = this.createNewAliasesRecords(file, existingAliases); - newAliases.forEach(alias => { - items.push(this.aliasEntity.putBatch(alias)); + const batchWrite = createEntityWriteBatch({ + entity: this.aliasEntity }); + for (const alias of newAliases) { + batchWrite.put(alias); + } // Delete aliases that are in the DB but are NOT in the file. for (const data of existingAliases) { if (!file.aliases.some(alias => data.alias === alias)) { - items.push( - this.aliasEntity.deleteBatch({ - PK: this.createPartitionKey(file), - SK: `ALIAS#${data.alias}` - }) - ); + batchWrite.delete({ + PK: this.createPartitionKey(file), + SK: `ALIAS#${data.alias}` + }); } } - await batchWriteAll({ - table: this.table, - items - }); + await batchWrite.execute(); } - private async getExistingAliases(file: File) { + private async getExistingAliases(file: File): Promise { const aliases = await queryAll<{ data: FileAlias }>({ entity: this.aliasEntity, partitionKey: this.createPartitionKey(file), diff --git a/packages/api-form-builder-so-ddb-es/src/definitions/elasticsearch.ts b/packages/api-form-builder-so-ddb-es/src/definitions/elasticsearch.ts index 84f28c9538c..9236ce07eee 100644 --- a/packages/api-form-builder-so-ddb-es/src/definitions/elasticsearch.ts +++ b/packages/api-form-builder-so-ddb-es/src/definitions/elasticsearch.ts @@ -28,7 +28,6 @@ export const createElasticsearchEntity = (params: Params) => { TYPE: { type: "string" }, - ...(attributes || {}) } }); diff --git a/packages/api-form-builder-so-ddb-es/src/operations/form/index.ts b/packages/api-form-builder-so-ddb-es/src/operations/form/index.ts index db1e3cf4c05..5ca078d4910 100644 --- a/packages/api-form-builder-so-ddb-es/src/operations/form/index.ts +++ b/packages/api-form-builder-so-ddb-es/src/operations/form/index.ts @@ -17,7 +17,7 @@ import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; import { queryAll, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; import WebinyError from "@webiny/error"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { createEntityWriteBatch, getClean, IPutParamsItem, put } from "@webiny/db-dynamodb"; import { configurations } from "~/configurations"; import { filterItems } from "@webiny/db-dynamodb/utils/filter"; import fields from "./fields"; @@ -28,7 +28,6 @@ import { decodeCursor, encodeCursor } from "@webiny/api-elasticsearch"; import { PluginsContainer } from "@webiny/plugins"; import { FormBuilderFormCreateKeyParams, FormBuilderFormStorageOperations } from "~/types"; import { ElasticsearchSearchResponse } from "@webiny/api-elasticsearch/types"; -import { deleteItem, getClean, put } from "@webiny/db-dynamodb"; export type DbRecord = T & { PK: string; @@ -71,7 +70,7 @@ const getESDataForLatestRevision = (form: FbForm): FbFormElastic => ({ export const createFormStorageOperations = ( params: CreateFormStorageOperationsParams ): FormBuilderFormStorageOperations => { - const { entity, esEntity, table, plugins, elasticsearch } = params; + const { entity, esEntity, plugins, elasticsearch } = params; const formDynamoDbFields = fields(); @@ -123,24 +122,24 @@ export const createFormStorageOperations = ( SK: createLatestSortKey() }; - const items = [ - entity.putBatch({ - ...form, - TYPE: createFormType(), - ...revisionKeys - }), - entity.putBatch({ - ...form, - TYPE: createFormLatestType(), - ...latestKeys - }) - ]; + const itemsBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + TYPE: createFormType(), + ...revisionKeys + }, + { + ...form, + TYPE: createFormLatestType(), + ...latestKeys + } + ] + }); try { - await batchWriteAll({ - table, - items - }); + await itemsBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not insert form data into regular table.", @@ -194,24 +193,24 @@ export const createFormStorageOperations = ( SK: createLatestSortKey() }; - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - TYPE: createFormType() - }), - entity.putBatch({ - ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + TYPE: createFormType() + }, + { + ...form, + ...latestKeys, + TYPE: createFormLatestType() + } + ] + }); try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || @@ -283,27 +282,26 @@ export const createFormStorageOperations = ( }); const isLatestForm = latestForm ? latestForm.id === form.id : false; - const items = [ - entity.putBatch({ - ...form, - TYPE: createFormType(), - ...revisionKeys - }) - ]; - if (isLatestForm) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...form, - TYPE: createFormLatestType(), - ...latestKeys - }) - ); + TYPE: createFormType(), + ...revisionKeys + } + ] + }); + + if (isLatestForm) { + entityBatch.put({ + ...form, + TYPE: createFormLatestType(), + ...latestKeys + }); } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update form data in the regular table.", @@ -547,17 +545,18 @@ export const createFormStorageOperations = ( ); } - const deleteItems = items.map(item => { - return entity.deleteBatch({ - PK: item.PK, - SK: item.SK - }); + const deleteBatch = createEntityWriteBatch({ + entity, + delete: items.map(item => { + return { + PK: item.PK, + SK: item.SK + }; + }) }); + try { - await batchWriteAll({ - table, - items: deleteItems - }); + await deleteBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete form and it's submissions.", @@ -569,11 +568,13 @@ export const createFormStorageOperations = ( PK: createFormPartitionKey(form), SK: createLatestSortKey() }; + const deleteEsBatch = createEntityWriteBatch({ + entity: esEntity, + delete: [latestKeys] + }); + try { - await deleteItem({ - entity: esEntity, - keys: latestKeys - }); + await deleteEsBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete latest form record from Elasticsearch.", @@ -612,8 +613,12 @@ export const createFormStorageOperations = ( const isLatest = latestForm ? latestForm.id === form.id : false; const isLatestPublished = latestPublishedForm ? latestPublishedForm.id === form.id : false; - const items = [entity.deleteBatch(revisionKeys)]; - let esDataItem = undefined; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [revisionKeys] + }); + + let esDataItem: IPutParamsItem | undefined = undefined; if (isLatest || isLatestPublished) { /** @@ -630,34 +635,28 @@ export const createFormStorageOperations = ( }) .shift(); if (previouslyPublishedForm) { - items.push( - entity.putBatch({ - ...previouslyPublishedForm, - PK: createFormPartitionKey(previouslyPublishedForm), - SK: createLatestPublishedSortKey(), - TYPE: createFormLatestPublishedType() - }) - ); + entityBatch.put({ + ...previouslyPublishedForm, + PK: createFormPartitionKey(previouslyPublishedForm), + SK: createLatestPublishedSortKey(), + TYPE: createFormLatestPublishedType() + }); } else { - items.push( - entity.deleteBatch({ - PK: createFormPartitionKey(form), - SK: createLatestPublishedSortKey() - }) - ); + entityBatch.delete({ + PK: createFormPartitionKey(form), + SK: createLatestPublishedSortKey() + }); } } /** * Sort out the latest record. */ if (isLatest && previous) { - items.push( - entity.putBatch({ - ...previous, - ...latestKeys, - TYPE: createFormLatestType() - }) - ); + entityBatch.put({ + ...previous, + ...latestKeys, + TYPE: createFormLatestType() + }); const { index } = configurations.es({ tenant: previous.tenant, @@ -675,10 +674,7 @@ export const createFormStorageOperations = ( * Now save the batch data. */ try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete form revision from regular table.", @@ -759,36 +755,35 @@ export const createFormStorageOperations = ( /** * Update revision and latest published records */ - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - TYPE: createFormType() - }), - entity.putBatch({ - ...form, - ...latestPublishedKeys, - TYPE: createFormLatestPublishedType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + TYPE: createFormType() + }, + { + ...form, + ...latestPublishedKeys, + TYPE: createFormLatestPublishedType() + } + ] + }); + /** * Update the latest form as well */ if (isLatestForm) { - items.push( - entity.putBatch({ - ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ); + entityBatch.put({ + ...form, + ...latestKeys, + TYPE: createFormLatestType() + }); } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not publish form.", @@ -887,14 +882,18 @@ export const createFormStorageOperations = ( const isLatest = latestForm ? latestForm.id === form.id : false; const isLatestPublished = latestPublishedForm ? latestPublishedForm.id === form.id : false; - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - TYPE: createFormType() - }) - ]; - let esData: any = undefined; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + TYPE: createFormType() + } + ] + }); + + let esData: FbFormElastic | undefined = undefined; if (isLatest) { esData = getESDataForLatestRevision(form); } @@ -916,23 +915,18 @@ export const createFormStorageOperations = ( const previouslyPublishedRevision = revisions.shift(); if (previouslyPublishedRevision) { - items.push( - entity.putBatch({ - ...previouslyPublishedRevision, - ...latestPublishedKeys, - TYPE: createFormLatestPublishedType() - }) - ); + entityBatch.put({ + ...previouslyPublishedRevision, + ...latestPublishedKeys, + TYPE: createFormLatestPublishedType() + }); } else { - items.push(entity.deleteBatch(latestPublishedKeys)); + entityBatch.delete(latestPublishedKeys); } } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not unpublish form.", diff --git a/packages/api-form-builder-so-ddb-es/src/operations/submission/index.ts b/packages/api-form-builder-so-ddb-es/src/operations/submission/index.ts index a1e1a00aafa..e93ee28b580 100644 --- a/packages/api-form-builder-so-ddb-es/src/operations/submission/index.ts +++ b/packages/api-form-builder-so-ddb-es/src/operations/submission/index.ts @@ -10,7 +10,7 @@ import { import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; import WebinyError from "@webiny/error"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { sortItems } from "@webiny/db-dynamodb/utils/sort"; import { createLimit, decodeCursor, encodeCursor } from "@webiny/api-elasticsearch"; import { diff --git a/packages/api-form-builder-so-ddb/src/operations/form/index.ts b/packages/api-form-builder-so-ddb/src/operations/form/index.ts index 3ade575baa2..c25f60780dc 100644 --- a/packages/api-form-builder-so-ddb/src/operations/form/index.ts +++ b/packages/api-form-builder-so-ddb/src/operations/form/index.ts @@ -1,5 +1,5 @@ import WebinyError from "@webiny/error"; -import { +import type { FbForm, FormBuilderStorageOperationsCreateFormFromParams, FormBuilderStorageOperationsCreateFormParams, @@ -14,14 +14,14 @@ import { FormBuilderStorageOperationsUnpublishFormParams, FormBuilderStorageOperationsUpdateFormParams } from "@webiny/api-form-builder/types"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import type { Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { queryAll, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; import { filterItems } from "@webiny/db-dynamodb/utils/filter"; import { sortItems } from "@webiny/db-dynamodb/utils/sort"; import { createIdentifier, parseIdentifier } from "@webiny/utils"; -import { PluginsContainer } from "@webiny/plugins"; -import { +import type { PluginsContainer } from "@webiny/plugins"; +import type { FormBuilderFormCreateGSIPartitionKeyParams, FormBuilderFormCreatePartitionKeyParams, FormBuilderFormStorageOperations @@ -60,7 +60,7 @@ export interface CreateFormStorageOperationsParams { export const createFormStorageOperations = ( params: CreateFormStorageOperationsParams ): FormBuilderFormStorageOperations => { - const { entity, table, plugins } = params; + const { entity, plugins } = params; const formDynamoDbFields = plugins.byType( FormDynamoDbFieldPlugin.type @@ -123,28 +123,30 @@ export const createFormStorageOperations = ( return "fb.form.latestPublished"; }; - const createRevisionKeys = (form: FbForm): Keys => { + const createRevisionKeys = (form: Pick): Keys => { return { PK: createFormPartitionKey(form), SK: createRevisionSortKey(form) }; }; - const createLatestKeys = (form: FbForm): Keys => { + const createLatestKeys = (form: Pick): Keys => { return { PK: createFormLatestPartitionKey(form), SK: createFormLatestSortKey(form) }; }; - const createLatestPublishedKeys = (form: FbForm): Keys => { + const createLatestPublishedKeys = ( + form: Pick + ): Keys => { return { PK: createFormLatestPublishedPartitionKey(form), SK: createLatestPublishedSortKey(form) }; }; - const createGSIKeys = (form: FbForm): GsiKeys => { + const createGSIKeys = (form: Pick): GsiKeys => { return { GSI1_PK: createFormGSIPartitionKey(form), GSI1_SK: createGSISortKey(form.version) @@ -160,25 +162,25 @@ export const createFormStorageOperations = ( const latestKeys = createLatestKeys(form); const gsiKeys = createGSIKeys(form); - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - ...gsiKeys, - TYPE: createFormType() - }), - entity.putBatch({ - ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + ...gsiKeys, + TYPE: createFormType() + }, + { + ...form, + ...latestKeys, + TYPE: createFormLatestType() + } + ] + }); try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not insert form data into table.", @@ -202,25 +204,25 @@ export const createFormStorageOperations = ( const latestKeys = createLatestKeys(form); const gsiKeys = createGSIKeys(form); - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - ...gsiKeys, - TYPE: createFormType() - }), - entity.putBatch({ - ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + ...gsiKeys, + TYPE: createFormType() + }, + { + ...form, + ...latestKeys, + TYPE: createFormLatestType() + } + ] + }); try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not create form data in the table, from existing form.", @@ -259,28 +261,27 @@ export const createFormStorageOperations = ( }); const isLatestForm = latestForm ? latestForm.id === form.id : false; - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - ...gsiKeys, - TYPE: createFormType() - }) - ]; - if (isLatestForm) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ); + ...revisionKeys, + ...gsiKeys, + TYPE: createFormType() + } + ] + }); + + if (isLatestForm) { + entityBatch.put({ + ...form, + ...latestKeys, + TYPE: createFormLatestType() + }); } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update form data in the table.", @@ -510,29 +511,27 @@ export const createFormStorageOperations = ( } ); } + let latestPublishedKeys: Keys | undefined; + const entityBatch = createEntityWriteBatch({ + entity + }); - let hasLatestPublishedRecord = false; - - const deleteItems = items.map(item => { - if (!hasLatestPublishedRecord && item.published) { - hasLatestPublishedRecord = true; + for (const item of items) { + if (!latestPublishedKeys && item.published) { + latestPublishedKeys = createLatestPublishedKeys(item); } - return entity.deleteBatch({ + entityBatch.delete({ PK: item.PK, SK: item.SK }); - }); - if (hasLatestPublishedRecord) { - deleteItems.push(entity.deleteBatch(createLatestPublishedKeys(items[0]))); } - deleteItems.push(entity.deleteBatch(createLatestKeys(items[0]))); + if (latestPublishedKeys) { + entityBatch.delete(latestPublishedKeys); + } try { - await batchWriteAll({ - table, - items: deleteItems - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete form and it's submissions.", @@ -561,7 +560,10 @@ export const createFormStorageOperations = ( const isLatest = latestForm ? latestForm.id === form.id : false; const isLatestPublished = latestPublishedForm ? latestPublishedForm.id === form.id : false; - const items = [entity.deleteBatch(revisionKeys)]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [revisionKeys] + }); if (isLatest || isLatestPublished) { /** @@ -578,42 +580,36 @@ export const createFormStorageOperations = ( }) .shift(); if (previouslyPublishedForm) { - items.push( - entity.putBatch({ - ...previouslyPublishedForm, - ...createLatestPublishedKeys(previouslyPublishedForm), - GSI1_PK: null, - GSI1_SK: null, - TYPE: createFormLatestPublishedType() - }) - ); + entityBatch.put({ + ...previouslyPublishedForm, + ...createLatestPublishedKeys(previouslyPublishedForm), + GSI1_PK: null, + GSI1_SK: null, + TYPE: createFormLatestPublishedType() + }); } else { - items.push(entity.deleteBatch(createLatestPublishedKeys(form))); + entityBatch.delete(createLatestPublishedKeys(form)); } } /** * Sort out the latest record. */ if (isLatest) { - items.push( - entity.putBatch({ - ...previous, - ...latestKeys, - GSI1_PK: null, - GSI1_SK: null, - TYPE: createFormLatestType() - }) - ); + entityBatch.put({ + ...previous, + ...latestKeys, + GSI1_PK: null, + GSI1_SK: null, + TYPE: createFormLatestType() + }); } } /** * Now save the batch data. */ try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); + return form; } catch (ex) { throw new WebinyError( @@ -667,37 +663,36 @@ export const createFormStorageOperations = ( /** * Update revision and latest published records */ - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - ...gsiKeys, - TYPE: createFormType() - }), - entity.putBatch({ - ...form, - ...latestPublishedKeys, - TYPE: createFormLatestPublishedType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + ...gsiKeys, + TYPE: createFormType() + }, + { + ...form, + ...latestPublishedKeys, + TYPE: createFormLatestPublishedType() + } + ] + }); + /** * Update the latest form as well */ if (isLatestForm) { - items.push( - entity.putBatch({ - ...form, - ...latestKeys, - TYPE: createFormLatestType() - }) - ); + entityBatch.put({ + ...form, + ...latestKeys, + TYPE: createFormLatestType() + }); } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not publish form.", @@ -754,17 +749,20 @@ export const createFormStorageOperations = ( const isLatest = latestForm ? latestForm.id === form.id : false; const isLatestPublished = latestPublishedForm ? latestPublishedForm.id === form.id : false; - const items = [ - entity.putBatch({ - ...form, - ...revisionKeys, - ...gsiKeys, - TYPE: createFormType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...form, + ...revisionKeys, + ...gsiKeys, + TYPE: createFormType() + } + ] + }); if (isLatest) { - entity.putBatch({ + entityBatch.put({ ...form, ...latestKeys, TYPE: createFormLatestType() @@ -788,23 +786,18 @@ export const createFormStorageOperations = ( const previouslyPublishedRevision = revisions.shift(); if (previouslyPublishedRevision) { - items.push( - entity.putBatch({ - ...previouslyPublishedRevision, - ...latestPublishedKeys, - TYPE: createFormLatestPublishedType() - }) - ); + entityBatch.put({ + ...previouslyPublishedRevision, + ...latestPublishedKeys, + TYPE: createFormLatestPublishedType() + }); } else { - items.push(entity.deleteBatch(latestPublishedKeys)); + entityBatch.delete(latestPublishedKeys); } } try { - await batchWriteAll({ - table, - items - }); + await entityBatch.execute(); return form; } catch (ex) { throw new WebinyError( diff --git a/packages/api-headless-cms-ddb-es/src/definitions/entryElasticsearch.ts b/packages/api-headless-cms-ddb-es/src/definitions/entryElasticsearch.ts index 97bffededb1..338b1b42af0 100644 --- a/packages/api-headless-cms-ddb-es/src/definitions/entryElasticsearch.ts +++ b/packages/api-headless-cms-ddb-es/src/definitions/entryElasticsearch.ts @@ -28,6 +28,9 @@ export const createEntryElasticsearchEntity = ( data: { type: "map" }, + TYPE: { + type: "string" + }, ...(attributes || {}) } }); diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts index 1e35ed56305..6338066f712 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { createBatchScheduleFn } from "./createBatchScheduleFn"; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts index 5e510a2a714..d6a409bd913 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { createPartitionKey, createPublishedSortKey } from "~/operations/entry/keys"; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getRevisionById.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getRevisionById.ts index 8374efbcb72..4458e9a34fc 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getRevisionById.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/dataLoader/getRevisionById.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { createPartitionKey, createRevisionSortKey } from "~/operations/entry/keys"; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index 574423f8119..41b7fbdfc48 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -11,7 +11,7 @@ import { configurations } from "~/configurations"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; import { PluginsContainer } from "@webiny/plugins"; -import { batchWriteAll, BatchWriteItem } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll, BatchWriteItem } from "@webiny/db-dynamodb"; import { DataLoadersHandler } from "./dataLoaders"; import { createLatestSortKey, diff --git a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts index b35994d54f1..fc13f4270d4 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getLatestRevisionByEntryId.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { createBatchScheduleFn } from "./createBatchScheduleFn"; diff --git a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts index 062ebae0557..d3b97b70eae 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getPublishedRevisionByEntryId.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { createPartitionKey, createPublishedSortKey } from "~/operations/entry/keys"; diff --git a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getRevisionById.ts b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getRevisionById.ts index 7c020d82574..04b5783ace6 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getRevisionById.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/dataLoader/getRevisionById.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { CmsStorageEntry } from "@webiny/api-headless-cms/types"; import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; import { createPartitionKey, createRevisionSortKey } from "~/operations/entry/keys"; diff --git a/packages/api-headless-cms-ddb/src/operations/entry/index.ts b/packages/api-headless-cms-ddb/src/operations/entry/index.ts index 9c5f84e5b90..7511895b32f 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/index.ts @@ -18,7 +18,7 @@ import { createPublishedSortKey, createRevisionSortKey } from "~/operations/entry/keys"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll } from "@webiny/db-dynamodb"; import { DbItem, queryAll, diff --git a/packages/api-page-builder-so-ddb-es/src/definitions/pageElasticsearchEntity.ts b/packages/api-page-builder-so-ddb-es/src/definitions/pageElasticsearchEntity.ts index 0d0bf837a21..5d21fc487f1 100644 --- a/packages/api-page-builder-so-ddb-es/src/definitions/pageElasticsearchEntity.ts +++ b/packages/api-page-builder-so-ddb-es/src/definitions/pageElasticsearchEntity.ts @@ -25,6 +25,9 @@ export const createPageElasticsearchEntity = (params: Params): Entity => { data: { type: "map" }, + TYPE: { + type: "string" + }, ...(attributes || {}) } }); diff --git a/packages/api-page-builder-so-ddb-es/src/operations/blockCategory/dataLoader.ts b/packages/api-page-builder-so-ddb-es/src/operations/blockCategory/dataLoader.ts index 5a158ff5aad..1fd149702d9 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/blockCategory/dataLoader.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/blockCategory/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { BlockCategory } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb-es/src/operations/category/dataLoader.ts b/packages/api-page-builder-so-ddb-es/src/operations/category/dataLoader.ts index 50a0d6a8358..913ee4fc3b0 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/category/dataLoader.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/category/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { Category } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts index c9505ba7858..b3614d766f6 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/pageBlock/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { PageBlock } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pageTemplate/dataLoader.ts b/packages/api-page-builder-so-ddb-es/src/operations/pageTemplate/dataLoader.ts index 32437cbacc9..248b6ffe129 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/pageTemplate/dataLoader.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/pageTemplate/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { PageTemplate } from "@webiny/api-page-builder/types"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { createPrimaryPK } from "./keys"; diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts b/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts index 2bde41b247c..b11f059e46c 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts @@ -30,7 +30,7 @@ import { SearchLatestPagesPlugin } from "~/plugins/definitions/SearchLatestPages import { SearchPublishedPagesPlugin } from "~/plugins/definitions/SearchPublishedPagesPlugin"; import { DbItem, queryAll, QueryAllParams, queryOne } from "@webiny/db-dynamodb/utils/query"; import { SearchPagesPlugin } from "~/plugins/definitions/SearchPagesPlugin"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll } from "@webiny/db-dynamodb"; import { getESLatestPageData, getESPublishedPageData } from "./helpers"; import { PluginsContainer } from "@webiny/plugins"; import { diff --git a/packages/api-page-builder-so-ddb/src/operations/blockCategory/dataLoader.ts b/packages/api-page-builder-so-ddb/src/operations/blockCategory/dataLoader.ts index a8112be7e37..cd030e6a912 100644 --- a/packages/api-page-builder-so-ddb/src/operations/blockCategory/dataLoader.ts +++ b/packages/api-page-builder-so-ddb/src/operations/blockCategory/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { BlockCategory } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb/src/operations/category/dataLoader.ts b/packages/api-page-builder-so-ddb/src/operations/category/dataLoader.ts index 2c9ebc9c70a..661b32359d6 100644 --- a/packages/api-page-builder-so-ddb/src/operations/category/dataLoader.ts +++ b/packages/api-page-builder-so-ddb/src/operations/category/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { Category } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts b/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts index c9505ba7858..b3614d766f6 100644 --- a/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts +++ b/packages/api-page-builder-so-ddb/src/operations/pageBlock/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { PageBlock } from "@webiny/api-page-builder/types"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; import { Entity } from "@webiny/db-dynamodb/toolbox"; diff --git a/packages/api-page-builder-so-ddb/src/operations/pageTemplate/dataLoader.ts b/packages/api-page-builder-so-ddb/src/operations/pageTemplate/dataLoader.ts index 32437cbacc9..248b6ffe129 100644 --- a/packages/api-page-builder-so-ddb/src/operations/pageTemplate/dataLoader.ts +++ b/packages/api-page-builder-so-ddb/src/operations/pageTemplate/dataLoader.ts @@ -1,5 +1,5 @@ import DataLoader from "dataloader"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; +import { batchReadAll } from "@webiny/db-dynamodb"; import { PageTemplate } from "@webiny/api-page-builder/types"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { createPrimaryPK } from "./keys"; diff --git a/packages/api-page-builder-so-ddb/src/operations/pages/index.ts b/packages/api-page-builder-so-ddb/src/operations/pages/index.ts index f2619b1fdb7..7130312a8f3 100644 --- a/packages/api-page-builder-so-ddb/src/operations/pages/index.ts +++ b/packages/api-page-builder-so-ddb/src/operations/pages/index.ts @@ -27,7 +27,7 @@ import { queryOne, queryOneClean } from "@webiny/db-dynamodb/utils/query"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll } from "@webiny/db-dynamodb"; import { filterItems } from "@webiny/db-dynamodb/utils/filter"; import { sortItems } from "@webiny/db-dynamodb/utils/sort"; import { decodeCursor, encodeCursor } from "@webiny/db-dynamodb/utils/cursor"; diff --git a/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts b/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts index 3499661f229..2c170040860 100644 --- a/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts +++ b/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts @@ -6,7 +6,7 @@ import { QueueJob } from "@webiny/api-prerendering-service/types"; import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll } from "@webiny/db-dynamodb"; import { queryAllClean, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; import { put } from "@webiny/db-dynamodb"; diff --git a/packages/api-prerendering-service-so-ddb/src/operations/render.ts b/packages/api-prerendering-service-so-ddb/src/operations/render.ts index 07565470dc1..b9fb75a89b9 100644 --- a/packages/api-prerendering-service-so-ddb/src/operations/render.ts +++ b/packages/api-prerendering-service-so-ddb/src/operations/render.ts @@ -1,5 +1,5 @@ import WebinyError from "@webiny/error"; -import { +import type { PrerenderingServiceRenderStorageOperations, PrerenderingServiceStorageOperationsCreateRenderParams, PrerenderingServiceStorageOperationsCreateTagPathLinksParams, @@ -12,18 +12,24 @@ import { Tag, TagPathLink } from "@webiny/api-prerendering-service/types"; -import { Entity, EntityQueryOptions } from "@webiny/db-dynamodb/toolbox"; -import { get } from "@webiny/db-dynamodb/utils/get"; -import { queryAll, queryAllClean, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; -import { cleanupItem, cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; -import { DataContainer } from "~/types"; -import { deleteItem, put } from "@webiny/db-dynamodb"; +import type { Entity, EntityQueryOptions } from "@webiny/db-dynamodb/toolbox"; +import { + batchReadAll, + cleanupItem, + cleanupItems, + createEntityWriteBatch, + deleteItem, + get, + put, + queryAll, + queryAllClean +} from "@webiny/db-dynamodb"; +import type { QueryAllParams } from "@webiny/db-dynamodb"; +import type { DataContainer } from "~/types"; export interface CreateRenderStorageOperationsParams { - entity: Entity; - tagPathLinkEntity: Entity; + entity: Entity; + tagPathLinkEntity: Entity; } export interface CreateTagPathLinkPartitionKeyParams { @@ -276,29 +282,29 @@ export const createRenderStorageOperations = ( ) => { const { tagPathLinks } = params; - const items = tagPathLinks.map(item => { - return tagPathLinkEntity.putBatch({ - data: item, - TYPE: createTagPathLinkType(), - PK: createTagPathLinkPartitionKey({ - tenant: item.tenant, - tag: item, - path: item.path - }), - SK: createTagPathLinkSortKey({ - tag: item, - path: item.path - }), - GSI1_PK: createTagPathLinkGSI1PartitionKey({ tag: item, tenant: item.tenant }), - GSI1_SK: createTagPathLinkGSI1SortKey({ tag: item, path: item.path }) - }); + const tagPathLinksBatch = createEntityWriteBatch({ + entity: tagPathLinkEntity, + put: tagPathLinks.map(item => { + return { + data: item, + TYPE: createTagPathLinkType(), + PK: createTagPathLinkPartitionKey({ + tenant: item.tenant, + tag: item, + path: item.path + }), + SK: createTagPathLinkSortKey({ + tag: item, + path: item.path + }), + GSI1_PK: createTagPathLinkGSI1PartitionKey({ tag: item, tenant: item.tenant }), + GSI1_SK: createTagPathLinkGSI1SortKey({ tag: item, path: item.path }) + }; + }) }); try { - await batchWriteAll({ - table: tagPathLinkEntity.table, - items - }); + await tagPathLinksBatch.execute(); return tagPathLinks; } catch (ex) { throw new WebinyError( @@ -315,25 +321,26 @@ export const createRenderStorageOperations = ( params: PrerenderingServiceStorageOperationsDeleteTagPathLinksParams ): Promise => { const { tenant, tags, path } = params; - const items = tags.map(tag => { - return tagPathLinkEntity.deleteBatch({ - PK: createTagPathLinkPartitionKey({ - tag, - tenant, - path - }), - SK: createTagPathLinkSortKey({ - tag, - path - }) - }); + + const tagPathLinksBatch = createEntityWriteBatch({ + entity: tagPathLinkEntity, + delete: tags.map(tag => { + return { + PK: createTagPathLinkPartitionKey({ + tag, + tenant, + path + }), + SK: createTagPathLinkSortKey({ + tag, + path + }) + }; + }) }); try { - await batchWriteAll({ - table: tagPathLinkEntity.table, - items - }); + await tagPathLinksBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete tagPathLink records.", diff --git a/packages/api-security-so-ddb/src/index.ts b/packages/api-security-so-ddb/src/index.ts index c900f2e9261..89007d6c01d 100644 --- a/packages/api-security-so-ddb/src/index.ts +++ b/packages/api-security-so-ddb/src/index.ts @@ -26,7 +26,7 @@ import { QueryOneParams } from "@webiny/db-dynamodb/utils/query"; import { sortItems } from "@webiny/db-dynamodb/utils/sort"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { batchWriteAll } from "@webiny/db-dynamodb"; import { deleteItem, getClean, put } from "@webiny/db-dynamodb"; const reservedFields: string[] = ["PK", "SK", "index", "data"]; diff --git a/packages/api-tenancy-so-ddb/src/index.ts b/packages/api-tenancy-so-ddb/src/index.ts index 9e8a3cca180..a81ea07bf68 100644 --- a/packages/api-tenancy-so-ddb/src/index.ts +++ b/packages/api-tenancy-so-ddb/src/index.ts @@ -1,5 +1,10 @@ -import { batchReadAll } from "@webiny/db-dynamodb/utils/batchRead"; -import { batchWriteAll } from "@webiny/db-dynamodb/utils/batchWrite"; +import { + batchReadAll, + createEntityWriteBatch, + createTableWriteBatch, + getClean, + put +} from "@webiny/db-dynamodb"; import { queryAll, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; import WebinyError from "@webiny/error"; import { createTable } from "~/definitions/table"; @@ -7,8 +12,7 @@ import { createTenantEntity } from "~/definitions/tenantEntity"; import { createSystemEntity } from "~/definitions/systemEntity"; import { createDomainEntity } from "~/definitions/domainEntity"; import { CreateTenancyStorageOperations, ENTITIES } from "~/types"; -import { ListTenantsParams, System, Tenant, TenantDomain } from "@webiny/api-tenancy/types"; -import { getClean, put } from "@webiny/db-dynamodb"; +import type { ListTenantsParams, System, Tenant, TenantDomain } from "@webiny/api-tenancy/types"; interface TenantDomainRecord { PK: string; @@ -181,20 +185,24 @@ export const createStorageOperations: CreateTenancyStorageOperations = params => }; try { - const items: any[] = [ - entities.tenants.putBatch({ TYPE: "tenancy.tenant", ...keys, data }) - ]; - const newDomains = createNewDomainsRecords(data); - - newDomains.forEach(record => { - items.push(entities.domains.putBatch(record)); + const tableWrite = createTableWriteBatch({ + table: tableInstance }); - await batchWriteAll({ - table: tableInstance, - items: items + tableWrite.put(entities.tenants, { + TYPE: "tenancy.tenant", + ...keys, + data }); + const newDomains = createNewDomainsRecords(data); + + for (const domain of newDomains) { + tableWrite.put(entities.domains, domain); + } + + await tableWrite.execute(); + return data as TTenant; } catch (err) { throw WebinyError.from(err, { @@ -215,8 +223,6 @@ export const createStorageOperations: CreateTenancyStorageOperations = params => GSI1_SK: `T#${data.parent}#${data.createdOn}` }; - const items: any[] = [entities.tenants.putBatch({ ...keys, data })]; - const existingDomains = await queryAll({ entity: entities.domains, partitionKey: "DOMAINS", @@ -228,31 +234,30 @@ export const createStorageOperations: CreateTenancyStorageOperations = params => const newDomains = createNewDomainsRecords(data, existingDomains); + const tableWrite = createTableWriteBatch({ + table: tableInstance + }); + + tableWrite.put(entities.tenants, { ...keys, data }); + // Delete domains that are in the DB but are NOT in the settings. - const deleteDomains = []; + for (const { fqdn } of existingDomains) { - if (!data.settings.domains.find(d => d.fqdn === fqdn)) { - deleteDomains.push({ - PK: `DOMAIN#${fqdn}`, - SK: `A` - }); + if (data.settings.domains.some(d => d.fqdn === fqdn)) { + continue; } - } - - try { - newDomains.forEach(record => { - items.push(entities.domains.putBatch(record)); - }); - - deleteDomains.forEach(item => { - items.push(entities.domains.deleteBatch(item)); + tableWrite.delete(entities.domains, { + PK: `DOMAIN#${fqdn}`, + SK: `A` }); + } - await batchWriteAll({ - table: tableInstance, - items: items - }); + for (const domain of newDomains) { + tableWrite.put(entities.domains, domain); + } + try { + await tableWrite.execute(); return data as TTenant; } catch (err) { throw WebinyError.from(err, { @@ -273,27 +278,25 @@ export const createStorageOperations: CreateTenancyStorageOperations = params => } }); - const items = [ - entities.tenants.deleteBatch({ - PK: `T#${id}`, - SK: "A" - }) - ]; - - existingDomains.forEach(domain => { - items.push( - entities.domains.deleteBatch({ - PK: domain.PK, - SK: domain.SK - }) - ); + const batchWrite = createEntityWriteBatch({ + entity: entities.tenants, + delete: [ + { + PK: `T#${id}`, + SK: "A" + } + ] }); + for (const domain of existingDomains) { + batchWrite.delete({ + PK: domain.PK, + SK: domain.SK + }); + } + // Delete tenant and domain items - await batchWriteAll({ - table: tableInstance, - items - }); + await batchWrite.execute(); } }; }; diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index eeb7ad46b9a..6cdf4c1d299 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -24,7 +24,6 @@ export interface IDeleteBatchItem { export type IPutBatchItem = Record> = { PK: string; SK: string; - TYPE: string; } & T; export interface BatchWriteItem { From b124625f0cba8d1b4cba9c06ab742ac91f889231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 13 Dec 2024 15:03:41 +0100 Subject: [PATCH 03/19] fix(api-security-so-ddb): implement batch write tools --- packages/api-security-so-ddb/src/index.ts | 75 +++++++++++++---------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/packages/api-security-so-ddb/src/index.ts b/packages/api-security-so-ddb/src/index.ts index 89007d6c01d..628e339a5ce 100644 --- a/packages/api-security-so-ddb/src/index.ts +++ b/packages/api-security-so-ddb/src/index.ts @@ -1,5 +1,5 @@ import { ENTITIES, SecurityStorageParams } from "~/types"; -import { +import type { ApiKey, Group, ListTenantLinksByTypeParams, @@ -18,16 +18,19 @@ import { createTeamEntity, createTenantLinkEntity } from "~/definitions/entities"; -import { cleanupItem, cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; +import type { QueryOneParams } from "@webiny/db-dynamodb"; import { + cleanupItem, + cleanupItems, + createEntityWriteBatch, + deleteItem, + getClean, + put, queryAll, queryAllClean, queryOneClean, - QueryOneParams -} from "@webiny/db-dynamodb/utils/query"; -import { sortItems } from "@webiny/db-dynamodb/utils/sort"; -import { batchWriteAll } from "@webiny/db-dynamodb"; -import { deleteItem, getClean, put } from "@webiny/db-dynamodb"; + sortItems +} from "@webiny/db-dynamodb"; const reservedFields: string[] = ["PK", "SK", "index", "data"]; @@ -187,17 +190,20 @@ export const createStorageOperations = ( } }, async createTenantLinks(links): Promise { - const items = links.map(link => { - return entities.tenantLinks.putBatch({ - PK: `IDENTITY#${link.identity}`, - SK: `LINK#T#${link.tenant}`, - GSI1_PK: `T#${link.tenant}`, - GSI1_SK: `TYPE#${link.type}#IDENTITY#${link.identity}`, - ...cleanupItem(entities.tenantLinks, link) - }); + const batchWrite = createEntityWriteBatch({ + entity: entities.tenantLinks, + put: links.map(link => { + return { + PK: `IDENTITY#${link.identity}`, + SK: `LINK#T#${link.tenant}`, + GSI1_PK: `T#${link.tenant}`, + GSI1_SK: `TYPE#${link.type}#IDENTITY#${link.identity}`, + ...cleanupItem(entities.tenantLinks, link) + }; + }) }); - await batchWriteAll({ table, items }); + await batchWrite.execute(); }, async deleteApiKey({ apiKey }) { const keys = createApiKeyKeys(apiKey); @@ -248,14 +254,16 @@ export const createStorageOperations = ( } }, async deleteTenantLinks(links): Promise { - const items = links.map(link => { - return entities.tenantLinks.deleteBatch({ - PK: `IDENTITY#${link.identity}`, - SK: `LINK#T#${link.tenant}` - }); + const batchWrite = createEntityWriteBatch({ + entity: entities.tenantLinks, + delete: links.map(link => { + return { + PK: `IDENTITY#${link.identity}`, + SK: `LINK#T#${link.tenant}` + }; + }) }); - - await batchWriteAll({ table, items }); + await batchWrite.execute(); }, async getApiKey({ id, tenant }) { const keys = createApiKeyKeys({ id, tenant }); @@ -586,17 +594,20 @@ export const createStorageOperations = ( } }, async updateTenantLinks(links): Promise { - const items = links.map(link => { - return entities.tenantLinks.putBatch({ - PK: `IDENTITY#${link.identity}`, - SK: `LINK#T#${link.tenant}`, - GSI1_PK: `T#${link.tenant}`, - GSI1_SK: `TYPE#${link.type}#IDENTITY#${link.identity}`, - ...cleanupItem(entities.tenantLinks, link) - }); + const batchWrite = createEntityWriteBatch({ + entity: entities.tenantLinks, + put: links.map(link => { + return { + PK: `IDENTITY#${link.identity}`, + SK: `LINK#T#${link.tenant}`, + GSI1_PK: `T#${link.tenant}`, + GSI1_SK: `TYPE#${link.type}#IDENTITY#${link.identity}`, + ...cleanupItem(entities.tenantLinks, link) + }; + }) }); - await batchWriteAll({ table, items }); + await batchWrite.execute(); } }; }; From 362b13021bdbb706d18f69ae2a41873ca0f84e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 13 Dec 2024 15:08:57 +0100 Subject: [PATCH 04/19] fix(api-elasticsearch-tasks): implement batch write tools --- .../src/tasks/Manager.ts | 17 ++------------ packages/api-elasticsearch-tasks/src/types.ts | 22 +++++++++---------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts index a62664742a0..7155aa7f042 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts @@ -6,13 +6,7 @@ import { createEntry } from "~/definitions/entry"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { ITaskResponse } from "@webiny/tasks/response/abstractions"; import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; -import { - batchReadAll, - BatchReadItem, - batchWriteAll, - BatchWriteItem, - BatchWriteResult -} from "@webiny/db-dynamodb"; +import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; import { ITimer } from "@webiny/handler-aws/utils"; export interface ManagerParams { @@ -75,17 +69,10 @@ export class Manager implements IManager { })); } - public async read(items: BatchReadItem[]) { + public async read(items: BatchReadItem[]): Promise { return await batchReadAll({ table: this.table, items }); } - - public async write(items: BatchWriteItem[]): Promise { - return await batchWriteAll({ - table: this.table, - items - }); - } } diff --git a/packages/api-elasticsearch-tasks/src/types.ts b/packages/api-elasticsearch-tasks/src/types.ts index 9265a282b2c..87a0f5cbd49 100644 --- a/packages/api-elasticsearch-tasks/src/types.ts +++ b/packages/api-elasticsearch-tasks/src/types.ts @@ -1,18 +1,18 @@ -import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { +import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; +import type { Context as TasksContext, IIsCloseToTimeoutCallable, + ITaskManagerStore, + ITaskResponse, ITaskResponseDoneResultOutput } from "@webiny/tasks/types"; -import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Client } from "@webiny/api-elasticsearch"; +import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; +import type { Client } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; -import { ITaskResponse } from "@webiny/tasks/response/abstractions"; -import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; -import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; -import { ITimer } from "@webiny/handler-aws"; -import { GenericRecord } from "@webiny/api/types"; +import type { BatchReadItem } from "@webiny/db-dynamodb"; +import type { ITimer } from "@webiny/handler-aws"; +import type { GenericRecord } from "@webiny/api/types"; export interface Context extends ElasticsearchContext, TasksContext {} @@ -74,5 +74,5 @@ export interface IManager< getEntity: (name: string) => Entity; - write: (items: BatchWriteItem[]) => Promise; + read(items: BatchReadItem[]): Promise; } From f28bb513832b11f3e8ee4d2da1cc60d8422a1901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 13 Dec 2024 15:51:18 +0100 Subject: [PATCH 05/19] fix(api-headless-cms-ddb-es): implement batch write tools --- .../src/operations/entry/index.ts | 1034 ++++++++--------- packages/db-dynamodb/src/index.ts | 2 +- .../src/utils/batch/EntityWriteBatch.ts | 6 +- .../utils/batch/EntityWriteBatchBuilder.ts | 2 +- .../src/utils/batch/TableWriteBatch.ts | 4 +- packages/db-dynamodb/src/utils/batch/types.ts | 14 +- 6 files changed, 497 insertions(+), 565 deletions(-) diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index 41b7fbdfc48..f5d098b658e 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -1,17 +1,25 @@ import WebinyError from "@webiny/error"; -import { +import type { CmsEntry, CmsModel, CmsStorageEntry, - CONTENT_ENTRY_STATUS, StorageOperationsCmsModel } from "@webiny/api-headless-cms/types"; +import { CONTENT_ENTRY_STATUS } from "@webiny/api-headless-cms/types"; import { extractEntriesFromIndex } from "~/helpers"; import { configurations } from "~/configurations"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { Client } from "@elastic/elasticsearch"; -import { PluginsContainer } from "@webiny/plugins"; -import { batchWriteAll, BatchWriteItem } from "@webiny/db-dynamodb"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; +import type { Client } from "@elastic/elasticsearch"; +import type { PluginsContainer } from "@webiny/plugins"; +import type { BatchReadItem, QueryAllParams, QueryOneParams } from "@webiny/db-dynamodb"; +import { + batchReadAll, + cleanupItem, + createEntityWriteBatch, + getClean, + queryAll, + queryOne +} from "@webiny/db-dynamodb"; import { DataLoadersHandler } from "./dataLoaders"; import { createLatestSortKey, @@ -19,12 +27,6 @@ import { createPublishedSortKey, createRevisionSortKey } from "./keys"; -import { - queryAll, - QueryAllParams, - queryOne, - QueryOneParams -} from "@webiny/db-dynamodb/utils/query"; import { compress, createLimit, @@ -32,21 +34,17 @@ import { decompress, encodeCursor } from "@webiny/api-elasticsearch"; -import { getClean } from "@webiny/db-dynamodb/utils/get"; import { zeroPad } from "@webiny/utils"; -import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; -import { +import type { ElasticsearchSearchResponse, SearchBody as ElasticsearchSearchBody } from "@webiny/api-elasticsearch/types"; -import { CmsEntryStorageOperations, CmsIndexEntry } from "~/types"; +import type { CmsEntryStorageOperations, CmsIndexEntry } from "~/types"; import { createElasticsearchBody } from "./elasticsearch/body"; import { logIgnoredEsResponseError } from "./elasticsearch/logIgnoredEsResponseError"; import { shouldIgnoreEsResponseError } from "./elasticsearch/shouldIgnoreEsResponseError"; import { createLatestRecordType, createPublishedRecordType, createRecordType } from "./recordType"; import { StorageOperationsCmsModelPlugin } from "@webiny/api-headless-cms"; -import { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; import { createTransformer } from "./transformations"; import { convertEntryKeysFromStorage } from "./transformations/convertEntryKeys"; import { @@ -57,6 +55,9 @@ import { } from "@webiny/api-headless-cms/constants"; interface ElasticsearchDbRecord { + PK: string; + SK: string; + TYPE: string; index: string; data: Record; } @@ -164,37 +165,35 @@ export const createEntriesStorageOperations = ( SK: createPublishedSortKey() }; - const items = [ - entity.putBatch({ - ...storageEntry, - locked, - ...revisionKeys, - TYPE: createRecordType() - }), - entity.putBatch({ - ...storageEntry, - locked, - ...latestKeys, - TYPE: createLatestRecordType() - }) - ]; - - if (isPublished) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...storageEntry, locked, - ...publishedKeys, - TYPE: createPublishedRecordType() - }) - ); + ...revisionKeys, + TYPE: createRecordType() + }, + { + ...storageEntry, + locked, + ...latestKeys, + TYPE: createLatestRecordType() + } + ] + }); + + if (isPublished) { + entityBatch.put({ + ...storageEntry, + locked, + ...publishedKeys, + TYPE: createPublishedRecordType() + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -211,29 +210,29 @@ export const createEntriesStorageOperations = ( } const esLatestData = await transformer.getElasticsearchLatestEntryData(); - const esItems: BatchWriteItem[] = [ - esEntity.putBatch({ - ...latestKeys, - index: esIndex, - data: esLatestData - }) - ]; + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + put: [ + { + ...latestKeys, + index: esIndex, + data: esLatestData + } + ] + }); + if (isPublished) { const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); - esItems.push( - esEntity.putBatch({ - ...publishedKeys, - index: esIndex, - data: esPublishedData - }) - ); + elasticsearchEntityBatch.put({ + ...publishedKeys, + index: esIndex, + data: esPublishedData + }); } try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not insert entry data into the Elasticsearch DynamoDB table.", @@ -295,27 +294,28 @@ export const createEntriesStorageOperations = ( const esLatestData = await transformer.getElasticsearchLatestEntryData(); - const items = [ - entity.putBatch({ - ...storageEntry, - TYPE: createRecordType(), - ...revisionKeys - }), - entity.putBatch({ - ...storageEntry, - TYPE: createLatestRecordType(), - ...latestKeys - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...storageEntry, + TYPE: createRecordType(), + ...revisionKeys + }, + { + ...storageEntry, + TYPE: createLatestRecordType(), + ...latestKeys + } + ] + }); if (isPublished) { - items.push( - entity.putBatch({ - ...storageEntry, - TYPE: createPublishedRecordType(), - ...publishedKeys - }) - ); + entityBatch.put({ + ...storageEntry, + TYPE: createPublishedRecordType(), + ...publishedKeys + }); // Unpublish previously published revision (if any). const [publishedRevisionStorageEntry] = await dataLoaders.getPublishedRevisionByEntryId( @@ -326,27 +326,23 @@ export const createEntriesStorageOperations = ( ); if (publishedRevisionStorageEntry) { - items.push( - entity.putBatch({ - ...publishedRevisionStorageEntry, - PK: createPartitionKey({ - id: publishedRevisionStorageEntry.id, - locale: model.locale, - tenant: model.tenant - }), - SK: createRevisionSortKey(publishedRevisionStorageEntry), - TYPE: createRecordType(), - status: CONTENT_ENTRY_STATUS.UNPUBLISHED - }) - ); + entityBatch.put({ + ...publishedRevisionStorageEntry, + PK: createPartitionKey({ + id: publishedRevisionStorageEntry.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createRevisionSortKey(publishedRevisionStorageEntry), + TYPE: createRecordType(), + status: CONTENT_ENTRY_STATUS.UNPUBLISHED + }); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -366,30 +362,28 @@ export const createEntriesStorageOperations = ( model }); - const esItems: BatchWriteItem[] = [ - esEntity.putBatch({ - ...latestKeys, - index: esIndex, - data: esLatestData - }) - ]; + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + put: [ + { + ...latestKeys, + index: esIndex, + data: esLatestData + } + ] + }); if (isPublished) { const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); - esItems.push( - esEntity.putBatch({ - ...publishedKeys, - index: esIndex, - data: esPublishedData - }) - ); + elasticsearchEntityBatch.put({ + ...publishedKeys, + index: esIndex, + data: esPublishedData + }); } try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update latest entry in the DynamoDB Elasticsearch table.", @@ -461,26 +455,30 @@ export const createEntriesStorageOperations = ( ids: [entry.id] }); - const items = [ - entity.putBatch({ - ...storageEntry, - locked, - ...revisionKeys, - TYPE: createRecordType() - }) - ]; - if (isPublished) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...storageEntry, locked, - ...publishedKeys, - TYPE: createPublishedRecordType() - }) - ); + ...revisionKeys, + TYPE: createRecordType() + } + ] + }); + + if (isPublished) { + entityBatch.put({ + ...storageEntry, + locked, + ...publishedKeys, + TYPE: createPublishedRecordType() + }); } - const esItems: BatchWriteItem[] = []; + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); const { index: esIndex } = configurations.es({ model @@ -495,26 +493,22 @@ export const createEntriesStorageOperations = ( /** * First we update the regular DynamoDB table. */ - items.push( - entity.putBatch({ - ...storageEntry, - ...latestKeys, - TYPE: createLatestRecordType() - }) - ); + entityBatch.put({ + ...storageEntry, + ...latestKeys, + TYPE: createLatestRecordType() + }); /** * And then update the Elasticsearch table to propagate changes to the Elasticsearch */ const elasticsearchLatestData = await transformer.getElasticsearchLatestEntryData(); - esItems.push( - esEntity.putBatch({ - ...latestKeys, - index: esIndex, - data: elasticsearchLatestData - }) - ); + elasticsearchEntityBatch.put({ + ...latestKeys, + index: esIndex, + data: elasticsearchLatestData + }); } else { /** * If not updating latest revision, we still want to update the latest revision's @@ -536,25 +530,21 @@ export const createEntriesStorageOperations = ( * - one for the actual revision record * - one for the latest record */ - items.push( - entity.putBatch({ - ...updatedLatestStorageEntry, - PK: createPartitionKey({ - id: latestStorageEntry.id, - locale: model.locale, - tenant: model.tenant - }), - SK: createRevisionSortKey(latestStorageEntry), - TYPE: createRecordType() - }) - ); + entityBatch.put({ + ...updatedLatestStorageEntry, + PK: createPartitionKey({ + id: latestStorageEntry.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createRevisionSortKey(latestStorageEntry), + TYPE: createRecordType() + }); - items.push( - entity.putBatch({ - ...updatedLatestStorageEntry, - TYPE: createLatestRecordType() - }) - ); + entityBatch.put({ + ...updatedLatestStorageEntry, + TYPE: createLatestRecordType() + }); /** * Update the Elasticsearch table to propagate changes to the Elasticsearch. @@ -575,13 +565,11 @@ export const createEntriesStorageOperations = ( ...updatedEntryLevelMetaFields }); - esItems.push( - esEntity.putBatch({ - ...latestKeys, - index: esIndex, - data: updatedLatestEntry - }) - ); + elasticsearchEntityBatch.put({ + ...latestKeys, + index: esIndex, + data: updatedLatestEntry + }); } } } @@ -589,19 +577,15 @@ export const createEntriesStorageOperations = ( if (isPublished && publishedStorageEntry?.id === entry.id) { const elasticsearchPublishedData = await transformer.getElasticsearchPublishedEntryData(); - esItems.push( - esEntity.putBatch({ - ...publishedKeys, - index: esIndex, - data: elasticsearchPublishedData - }) - ); + elasticsearchEntityBatch.put({ + ...publishedKeys, + index: esIndex, + data: elasticsearchPublishedData + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -616,15 +600,8 @@ export const createEntriesStorageOperations = ( } ); } - if (esItems.length === 0) { - return initialStorageEntry; - } - try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update entry DynamoDB Elasticsearch records.", @@ -664,17 +641,19 @@ export const createEntriesStorageOperations = ( */ let latestRecord: CmsEntry | undefined = undefined; let publishedRecord: CmsEntry | undefined = undefined; - const items: BatchWriteItem[] = []; + const entityBatch = createEntityWriteBatch({ + entity + }); + for (const record of records) { - items.push( - entity.putBatch({ - ...record, - location: { - ...record?.location, - folderId - } - }) - ); + entityBatch.put({ + ...record, + location: { + ...record?.location, + folderId + } + }); + /** * We need to get the published and latest records, so we can update the Elasticsearch. */ @@ -685,10 +664,7 @@ export const createEntriesStorageOperations = ( } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -743,27 +719,27 @@ export const createEntriesStorageOperations = ( if (esItems.length === 0) { return; } - const esUpdateItems: BatchWriteItem[] = []; - for (const item of esItems) { - esUpdateItems.push( - esEntity.putBatch({ - ...item, - data: await compress(plugins, { - ...item.data, - location: { - ...item.data?.location, - folderId - } - }) + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + put: await Promise.all( + esItems.map(async item => { + return { + ...item, + data: await compress(plugins, { + ...item.data, + location: { + ...item.data?.location, + folderId + } + }) + }; }) - ); - } + ) + }); try { - await batchWriteAll({ - table: esEntity.table, - items: esUpdateItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not move entry DynamoDB Elasticsearch records.", @@ -820,18 +796,20 @@ export const createEntriesStorageOperations = ( */ let latestRecord: CmsEntry | undefined = undefined; let publishedRecord: CmsEntry | undefined = undefined; - const items: BatchWriteItem[] = []; + + const entityBatch = createEntityWriteBatch({ + entity + }); for (const record of records) { - items.push( - entity.putBatch({ - ...record, - ...updatedEntryMetaFields, - wbyDeleted: storageEntry.wbyDeleted, - location: storageEntry.location, - binOriginalFolderId: storageEntry.binOriginalFolderId - }) - ); + entityBatch.put({ + ...record, + ...updatedEntryMetaFields, + wbyDeleted: storageEntry.wbyDeleted, + location: storageEntry.location, + binOriginalFolderId: storageEntry.binOriginalFolderId + }); + /** * We need to get the published and latest records, so we can update the Elasticsearch. */ @@ -846,10 +824,8 @@ export const createEntriesStorageOperations = ( * We write the records back to the primary DynamoDB table. */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -915,30 +891,28 @@ export const createEntriesStorageOperations = ( /** * We update all ES records with data received. */ - const esUpdateItems: BatchWriteItem[] = []; + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); + for (const item of esItems) { - esUpdateItems.push( - esEntity.putBatch({ - ...item, - data: await compress(plugins, { - ...item.data, - ...updatedEntryMetaFields, - wbyDeleted: entry.wbyDeleted, - location: entry.location, - binOriginalFolderId: entry.binOriginalFolderId - }) + elasticsearchEntityBatch.put({ + ...item, + data: await compress(plugins, { + ...item.data, + ...updatedEntryMetaFields, + wbyDeleted: entry.wbyDeleted, + location: entry.location, + binOriginalFolderId: entry.binOriginalFolderId }) - ); + }); } /** * We write the records back to the primary DynamoDB Elasticsearch table. */ try { - await batchWriteAll({ - table: esEntity.table, - items: esUpdateItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || @@ -1000,18 +974,20 @@ export const createEntriesStorageOperations = ( */ let latestRecord: CmsEntry | undefined = undefined; let publishedRecord: CmsEntry | undefined = undefined; - const items: BatchWriteItem[] = []; + + const entityBatch = createEntityWriteBatch({ + entity + }); for (const record of records) { - items.push( - entity.putBatch({ - ...record, - ...updatedEntryMetaFields, - wbyDeleted: storageEntry.wbyDeleted, - location: storageEntry.location, - binOriginalFolderId: storageEntry.binOriginalFolderId - }) - ); + entityBatch.put({ + ...record, + ...updatedEntryMetaFields, + wbyDeleted: storageEntry.wbyDeleted, + location: storageEntry.location, + binOriginalFolderId: storageEntry.binOriginalFolderId + }); + /** * We need to get the published and latest records, so we can update the Elasticsearch. */ @@ -1026,10 +1002,8 @@ export const createEntriesStorageOperations = ( * We write the records back to the primary DynamoDB table. */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -1092,30 +1066,27 @@ export const createEntriesStorageOperations = ( /** * We update all ES records with data received. */ - const esUpdateItems: BatchWriteItem[] = []; + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); for (const item of esItems) { - esUpdateItems.push( - esEntity.putBatch({ - ...item, - data: await compress(plugins, { - ...item.data, - ...updatedEntryMetaFields, - wbyDeleted: entry.wbyDeleted, - location: entry.location, - binOriginalFolderId: entry.binOriginalFolderId - }) + elasticsearchEntityBatch.put({ + ...item, + data: await compress(plugins, { + ...item.data, + ...updatedEntryMetaFields, + wbyDeleted: entry.wbyDeleted, + location: entry.location, + binOriginalFolderId: entry.binOriginalFolderId }) - ); + }); } /** * We write the records back to the primary DynamoDB Elasticsearch table. */ try { - await batchWriteAll({ - table: esEntity.table, - items: esUpdateItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not restore entry records from DynamoDB Elasticsearch table.", @@ -1158,25 +1129,29 @@ export const createEntriesStorageOperations = ( } }); - const deleteItems = items.map(item => { - return entity.deleteBatch({ - PK: item.PK, - SK: item.SK - }); + const entityBatch = createEntityWriteBatch({ + entity, + delete: items.map(item => { + return { + PK: item.PK, + SK: item.SK + }; + }) }); - const deleteEsItems = esItems.map(item => { - return esEntity.deleteBatch({ - PK: item.PK, - SK: item.SK - }); + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + delete: esItems.map(item => { + return { + PK: item.PK, + SK: item.SK + }; + }) }); try { - await batchWriteAll({ - table: entity.table, - items: deleteItems - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -1192,10 +1167,7 @@ export const createEntriesStorageOperations = ( } try { - await batchWriteAll({ - table: esEntity.table, - items: deleteEsItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not destroy entry records from DynamoDB Elasticsearch table.", @@ -1234,34 +1206,33 @@ export const createEntriesStorageOperations = ( /** * We need to delete all existing records of the given entry revision. */ - const items = [ - /** - * Delete records of given entry revision. - */ - entity.deleteBatch({ - PK: partitionKey, - SK: createRevisionSortKey(entry) - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [ + { + PK: partitionKey, + SK: createRevisionSortKey(entry) + } + ] + }); - const esItems: BatchWriteItem[] = []; + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); /** * If revision we are deleting is the published one as well, we need to delete those records as well. */ if (publishedStorageEntry?.id === entry.id) { - items.push( - entity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ); - esItems.push( - esEntity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ); + entityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); + + elasticsearchEntityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); } if (latestEntry && initialLatestStorageEntry) { @@ -1273,31 +1244,27 @@ export const createEntriesStorageOperations = ( /** * In the end we need to set the new latest entry. */ - items.push( - entity.putBatch({ - ...latestStorageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestRecordType() - }) - ); + entityBatch.put({ + ...latestStorageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestRecordType() + }); /** * Also perform an update on the actual revision. This is needed * because of updates on the entry-level meta fields. */ - items.push( - entity.putBatch({ - ...latestStorageEntry, - PK: createPartitionKey({ - id: initialLatestStorageEntry.id, - locale: model.locale, - tenant: model.tenant - }), - SK: createRevisionSortKey(initialLatestStorageEntry), - TYPE: createRecordType() - }) - ); + entityBatch.put({ + ...latestStorageEntry, + PK: createPartitionKey({ + id: initialLatestStorageEntry.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createRevisionSortKey(initialLatestStorageEntry), + TYPE: createRecordType() + }); const latestTransformer = createTransformer({ plugins, @@ -1307,21 +1274,16 @@ export const createEntriesStorageOperations = ( }); const esLatestData = await latestTransformer.getElasticsearchLatestEntryData(); - esItems.push( - esEntity.putBatch({ - PK: partitionKey, - SK: createLatestSortKey(), - index, - data: esLatestData - }) - ); + elasticsearchEntityBatch.put({ + PK: partitionKey, + SK: createLatestSortKey(), + index, + data: esLatestData + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model @@ -1339,15 +1301,8 @@ export const createEntriesStorageOperations = ( ); } - if (esItems.length === 0) { - return; - } - try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || @@ -1379,82 +1334,74 @@ export const createEntriesStorageOperations = ( /** * Then we need to construct the queries for all the revisions and entries. */ - const items: Record[] = []; - const esItems: Record[] = []; + + const entityBatch = createEntityWriteBatch({ + entity + }); + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); for (const id of entries) { /** * Latest item. */ - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "L" - }) - ); - esItems.push( - esEntity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "L" - }) - ); + entityBatch.delete({ + PK: createPartitionKey({ + id, + locale: model.locale, + tenant: model.tenant + }), + SK: "L" + }); + + elasticsearchEntityBatch.delete({ + PK: createPartitionKey({ + id, + locale: model.locale, + tenant: model.tenant + }), + SK: "L" + }); + /** * Published item. */ - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "P" - }) - ); - esItems.push( - esEntity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "P" - }) - ); + entityBatch.delete({ + PK: createPartitionKey({ + id, + locale: model.locale, + tenant: model.tenant + }), + SK: "P" + }); + + elasticsearchEntityBatch.delete({ + PK: createPartitionKey({ + id, + locale: model.locale, + tenant: model.tenant + }), + SK: "P" + }); } /** * Exact revisions of all the entries */ for (const revision of revisions) { - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id: revision.id, - locale: model.locale, - tenant: model.tenant - }), - SK: createRevisionSortKey({ - version: revision.version - }) + entityBatch.delete({ + PK: createPartitionKey({ + id: revision.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createRevisionSortKey({ + version: revision.version }) - ); + }); } - await batchWriteAll({ - table: entity.table, - items - }); - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await entityBatch.execute(); + await elasticsearchEntityBatch.execute(); }; const list: CmsEntryStorageOperations["list"] = async (initialModel, params) => { @@ -1641,19 +1588,25 @@ export const createEntriesStorageOperations = ( }); // 1. Update REV# and P records with new data. - const items = [ - entity.putBatch({ - ...storageEntry, - ...revisionKeys, - TYPE: createRecordType() - }), - entity.putBatch({ - ...storageEntry, - ...publishedKeys, - TYPE: createPublishedRecordType() - }) - ]; - const esItems: BatchWriteItem[] = []; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...storageEntry, + ...revisionKeys, + TYPE: createRecordType() + }, + { + ...storageEntry, + ...publishedKeys, + TYPE: createPublishedRecordType() + } + ] + }); + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); const { index: esIndex } = configurations.es({ model @@ -1666,12 +1619,10 @@ export const createEntriesStorageOperations = ( if (publishingLatestRevision) { // 2.1 If we're publishing the latest revision, we first need to update the L record. - items.push( - entity.putBatch({ - ...storageEntry, - ...latestKeys - }) - ); + entityBatch.put({ + ...storageEntry, + ...latestKeys + }); // 2.2 Additionally, if we have a previously published entry, we need to mark it as unpublished. // Note that we need to take re-publishing into account (same published revision being @@ -1680,18 +1631,16 @@ export const createEntriesStorageOperations = ( if (publishedStorageEntry) { const isRepublishing = publishedStorageEntry.id === entry.id; if (!isRepublishing) { - items.push( - /** - * Update currently published entry (unpublish it) - */ - entity.putBatch({ - ...publishedStorageEntry, - status: CONTENT_ENTRY_STATUS.UNPUBLISHED, - TYPE: createRecordType(), - PK: createPartitionKey(publishedStorageEntry), - SK: createRevisionSortKey(publishedStorageEntry) - }) - ); + /** + * Update currently published entry (unpublish it) + */ + entityBatch.put({ + ...publishedStorageEntry, + status: CONTENT_ENTRY_STATUS.UNPUBLISHED, + TYPE: createRecordType(), + PK: createPartitionKey(publishedStorageEntry), + SK: createRevisionSortKey(publishedStorageEntry) + }); } } } else { @@ -1716,24 +1665,20 @@ export const createEntriesStorageOperations = ( status: latestRevisionStatus }; - items.push( - entity.putBatch({ - ...latestStorageEntryFields, - PK: createPartitionKey(latestStorageEntry), - SK: createLatestSortKey(), - TYPE: createLatestRecordType() - }) - ); + entityBatch.put({ + ...latestStorageEntryFields, + PK: createPartitionKey(latestStorageEntry), + SK: createLatestSortKey(), + TYPE: createLatestRecordType() + }); // 2.5 Update REV# record. - items.push( - entity.putBatch({ - ...latestStorageEntryFields, - PK: createPartitionKey(latestStorageEntry), - SK: createRevisionSortKey(latestStorageEntry), - TYPE: createRecordType() - }) - ); + entityBatch.put({ + ...latestStorageEntryFields, + PK: createPartitionKey(latestStorageEntry), + SK: createRevisionSortKey(latestStorageEntry), + TYPE: createRecordType() + }); // 2.6 Additionally, if we have a previously published entry, we need to mark it as unpublished. // Note that we need to take re-publishing into account (same published revision being @@ -1745,15 +1690,13 @@ export const createEntriesStorageOperations = ( publishedRevisionId !== latestStorageEntry.id; if (!isRepublishing && publishedRevisionDifferentFromLatest) { - items.push( - entity.putBatch({ - ...publishedStorageEntry, - PK: createPartitionKey(publishedStorageEntry), - SK: createRevisionSortKey(publishedStorageEntry), - TYPE: createRecordType(), - status: CONTENT_ENTRY_STATUS.UNPUBLISHED - }) - ); + entityBatch.put({ + ...publishedStorageEntry, + PK: createPartitionKey(publishedStorageEntry), + SK: createRevisionSortKey(publishedStorageEntry), + TYPE: createRecordType(), + status: CONTENT_ENTRY_STATUS.UNPUBLISHED + }); } } } @@ -1764,13 +1707,11 @@ export const createEntriesStorageOperations = ( * Update the published revision entry in ES. */ const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); - esItems.push( - esEntity.putBatch({ - ...publishedKeys, - index: esIndex, - data: esPublishedData - }) - ); + elasticsearchEntityBatch.put({ + ...publishedKeys, + index: esIndex, + data: esPublishedData + }); /** * Need to decompress the data from Elasticsearch DynamoDB table. @@ -1797,14 +1738,12 @@ export const createEntriesStorageOperations = ( } }); - esItems.push( - esEntity.putBatch({ - index: esIndex, - PK: createPartitionKey(latestEsEntryDataDecompressed), - SK: createLatestSortKey(), - data: await latestTransformer.getElasticsearchLatestEntryData() - }) - ); + elasticsearchEntityBatch.put({ + index: esIndex, + PK: createPartitionKey(latestEsEntryDataDecompressed), + SK: createLatestSortKey(), + data: await latestTransformer.getElasticsearchLatestEntryData() + }); } else { const updatedEntryLevelMetaFields = pickEntryMetaFields( entry, @@ -1836,13 +1775,11 @@ export const createEntriesStorageOperations = ( status: latestRevisionStatus }); - esItems.push( - esEntity.putBatch({ - ...latestKeys, - index: esIndex, - data: updatedLatestEntry - }) - ); + elasticsearchEntityBatch.put({ + ...latestKeys, + index: esIndex, + data: updatedLatestEntry + }); } } @@ -1850,10 +1787,8 @@ export const createEntriesStorageOperations = ( * Finally, execute regular table batch. */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -1873,10 +1808,7 @@ export const createEntriesStorageOperations = ( * And Elasticsearch table batch. */ try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || @@ -1919,25 +1851,34 @@ export const createEntriesStorageOperations = ( tenant: model.tenant }); - const items = [ - entity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }), - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createRevisionSortKey(entry), - TYPE: createRecordType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...storageEntry, + PK: partitionKey, + SK: createRevisionSortKey(entry), + TYPE: createRecordType() + } + ], + delete: [ + { + PK: partitionKey, + SK: createPublishedSortKey() + } + ] + }); + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + delete: [ + { + PK: partitionKey, + SK: createPublishedSortKey() + } + ] + }); - const esItems: BatchWriteItem[] = [ - esEntity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ]; /** * If we are unpublishing the latest revision, let's also update the latest revision entry's status in both DynamoDB tables. */ @@ -1946,34 +1887,28 @@ export const createEntriesStorageOperations = ( model }); - items.push( - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestRecordType() - }) - ); + entityBatch.put({ + ...storageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestRecordType() + }); const esLatestData = await transformer.getElasticsearchLatestEntryData(); - esItems.push( - esEntity.putBatch({ - PK: partitionKey, - SK: createLatestSortKey(), - index, - data: esLatestData - }) - ); + elasticsearchEntityBatch.put({ + PK: partitionKey, + SK: createLatestSortKey(), + index, + data: esLatestData + }); } /** * Finally, execute regular table batch. */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -1991,10 +1926,7 @@ export const createEntriesStorageOperations = ( * And Elasticsearch table batch. */ try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || diff --git a/packages/db-dynamodb/src/index.ts b/packages/db-dynamodb/src/index.ts index 3aa680435aa..e56b1e14553 100644 --- a/packages/db-dynamodb/src/index.ts +++ b/packages/db-dynamodb/src/index.ts @@ -1,6 +1,6 @@ import { default as DynamoDbDriver } from "./DynamoDbDriver"; export * from "./utils"; -export { DbItem } from "./types"; +export type { DbItem } from "./types"; export { DynamoDbDriver }; diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index 88527554195..c6475656f6c 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -20,9 +20,9 @@ export interface IEntityWriteBatchParams { } export class EntityWriteBatch implements IEntityWriteBatch { - public readonly entity: Entity; - public readonly items: BatchWriteItem[] = []; - public readonly builder: IEntityWriteBatchBuilder; + private readonly entity: Entity; + private readonly items: BatchWriteItem[] = []; + private readonly builder: IEntityWriteBatchBuilder; public constructor(params: IEntityWriteBatchParams) { if (!params.entity.name) { diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts index c344f5dfb81..27c5316a1af 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts @@ -7,7 +7,7 @@ import type { } from "./types"; export class EntityWriteBatchBuilder implements IEntityWriteBatchBuilder { - public readonly entity: Entity; + private readonly entity: Entity; public constructor(entity: Entity) { this.entity = entity; diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index f9e0b24e9ff..743bc8e603e 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -16,8 +16,8 @@ export interface ITableWriteBatchParams { } export class TableWriteBatch implements ITableWriteBatch { - public readonly table: TableDef; - public readonly items: BatchWriteItem[] = []; + private readonly table: TableDef; + private readonly items: BatchWriteItem[] = []; private readonly builders: Map = new Map(); public constructor(params: ITableWriteBatchParams) { diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index 6cdf4c1d299..b5965cbdbe4 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -1,5 +1,5 @@ import type { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity, TableDef } from "~/toolbox"; +import { Entity } from "~/toolbox"; export interface BatchWriteResponse { next?: () => Promise; @@ -31,15 +31,15 @@ export interface BatchWriteItem { } export interface IEntityWriteBatchBuilder { - readonly entity: Entity; + // readonly entity: Entity; put>(item: IPutBatchItem): BatchWriteItem; delete(item: IDeleteBatchItem): BatchWriteItem; } export interface IEntityWriteBatch { - readonly entity: Entity; - readonly items: BatchWriteItem[]; - readonly builder: IEntityWriteBatchBuilder; + // readonly entity: Entity; + // readonly items: BatchWriteItem[]; + // readonly builder: IEntityWriteBatchBuilder; put(item: IPutBatchItem): void; delete(item: IDeleteBatchItem): void; @@ -48,8 +48,8 @@ export interface IEntityWriteBatch { } export interface ITableWriteBatch { - readonly table: TableDef; - readonly items: BatchWriteItem[]; + // readonly table: TableDef; + // readonly items: BatchWriteItem[]; put(entity: Entity, item: IPutBatchItem): void; delete(entity: Entity, item: IDeleteBatchItem): void; execute(): Promise; From 1c7c973967e2dbe001baab0f1b5cc06cd22dc814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 09:40:41 +0100 Subject: [PATCH 06/19] fix(api-headless-cms-ddb): implement batch write tools --- .../src/operations/entry/index.ts | 765 +++++++++--------- 1 file changed, 364 insertions(+), 401 deletions(-) diff --git a/packages/api-headless-cms-ddb/src/operations/entry/index.ts b/packages/api-headless-cms-ddb/src/operations/entry/index.ts index 7511895b32f..5600f97a037 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/index.ts @@ -1,15 +1,15 @@ import WebinyError from "@webiny/error"; import { DataLoadersHandler } from "./dataLoaders"; -import { +import type { CmsEntry, CmsEntryListWhere, CmsEntryUniqueValue, CmsModel, CmsStorageEntry, - CONTENT_ENTRY_STATUS, StorageOperationsCmsModel } from "@webiny/api-headless-cms/types"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { CONTENT_ENTRY_STATUS } from "@webiny/api-headless-cms/types"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; import { createGSIPartitionKey, createGSISortKey, @@ -18,24 +18,23 @@ import { createPublishedSortKey, createRevisionSortKey } from "~/operations/entry/keys"; -import { batchWriteAll } from "@webiny/db-dynamodb"; import { - DbItem, + cleanupItem, + cleanupItems, + createEntityWriteBatch, queryAll, QueryAllParams, queryOne, QueryOneParams -} from "@webiny/db-dynamodb/utils/query"; -import { cleanupItem, cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; -import { PluginsContainer } from "@webiny/plugins"; +} from "@webiny/db-dynamodb"; +import type { PluginsContainer } from "@webiny/plugins"; import { decodeCursor, encodeCursor } from "@webiny/utils/cursor"; import { zeroPad } from "@webiny/utils/zeroPad"; import { StorageOperationsCmsModelPlugin, StorageTransformPlugin } from "@webiny/api-headless-cms"; -import { FilterItemFromStorage } from "./filtering/types"; +import type { FilterItemFromStorage } from "./filtering/types"; import { createFields } from "~/operations/entry/filtering/createFields"; import { filter, sort } from "~/operations/entry/filtering"; -import { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import { CmsEntryStorageOperations } from "~/types"; +import type { CmsEntryStorageOperations } from "~/types"; import { isDeletedEntryMetaField, isEntryLevelEntryMetaField, @@ -167,49 +166,47 @@ export const createEntriesStorageOperations = ( * - create new main entry item * - create new or update the latest entry item */ - const items = [ - entity.putBatch({ - ...storageEntry, - locked, - PK: partitionKey, - SK: createRevisionSortKey(entry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(storageEntry) - }), - entity.putBatch({ - ...storageEntry, - locked, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(storageEntry) - }) - ]; - - /** - * We need to create published entry if - */ - if (isPublished) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...storageEntry, locked, PK: partitionKey, - SK: createPublishedSortKey(), + SK: createRevisionSortKey(entry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(storageEntry) + }, + { + ...storageEntry, + locked, + PK: partitionKey, + SK: createLatestSortKey(), TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_PK: createGSIPartitionKey(model, "L"), GSI1_SK: createGSISortKey(storageEntry) - }) - ); + } + ] + }); + + /** + * We need to create published entry if + */ + if (isPublished) { + entityBatch.put({ + ...storageEntry, + locked, + PK: partitionKey, + SK: createPublishedSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_SK: createGSISortKey(storageEntry) + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -253,37 +250,38 @@ export const createEntriesStorageOperations = ( * - update the published entry item to the current one * - unpublish previously published revision (if any) */ - const items = [ - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createRevisionSortKey(storageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(storageEntry) - }), - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(storageEntry) - }) - ]; - - const isPublished = entry.status === "published"; - if (isPublished) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...storageEntry, PK: partitionKey, - SK: createPublishedSortKey(), - TYPE: createPublishedType(), - GSI1_PK: createGSIPartitionKey(model, "P"), + SK: createRevisionSortKey(storageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), GSI1_SK: createGSISortKey(storageEntry) - }) - ); + }, + { + ...storageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(storageEntry) + } + ] + }); + + const isPublished = entry.status === "published"; + if (isPublished) { + entityBatch.put({ + ...storageEntry, + PK: partitionKey, + SK: createPublishedSortKey(), + TYPE: createPublishedType(), + GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_SK: createGSISortKey(storageEntry) + }); // Unpublish previously published revision (if any). const [publishedRevisionStorageEntry] = await dataLoaders.getPublishedRevisionByEntryId( @@ -294,25 +292,20 @@ export const createEntriesStorageOperations = ( ); if (publishedRevisionStorageEntry) { - items.push( - entity.putBatch({ - ...publishedRevisionStorageEntry, - PK: partitionKey, - SK: createRevisionSortKey(publishedRevisionStorageEntry), - TYPE: createType(), - status: CONTENT_ENTRY_STATUS.UNPUBLISHED, - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(publishedRevisionStorageEntry) - }) - ); + entityBatch.put({ + ...publishedRevisionStorageEntry, + PK: partitionKey, + SK: createRevisionSortKey(publishedRevisionStorageEntry), + TYPE: createType(), + status: CONTENT_ENTRY_STATUS.UNPUBLISHED, + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(publishedRevisionStorageEntry) + }); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -346,8 +339,6 @@ export const createEntriesStorageOperations = ( const isPublished = entry.status === "published"; const locked = isPublished ? true : entry.locked; - const items = []; - const storageEntry = convertToStorageEntry({ model, storageEntry: initialStorageEntry @@ -357,30 +348,32 @@ export const createEntriesStorageOperations = ( * - update the current entry * - update the latest entry if the current entry is the latest one */ - items.push( - entity.putBatch({ - ...storageEntry, - locked, - PK: partitionKey, - SK: createRevisionSortKey(storageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(storageEntry) - }) - ); - if (isPublished) { - items.push( - entity.putBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { ...storageEntry, locked, PK: partitionKey, - SK: createPublishedSortKey(), - TYPE: createPublishedType(), - GSI1_PK: createGSIPartitionKey(model, "P"), + SK: createRevisionSortKey(storageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), GSI1_SK: createGSISortKey(storageEntry) - }) - ); + } + ] + }); + + if (isPublished) { + entityBatch.put({ + ...storageEntry, + locked, + PK: partitionKey, + SK: createPublishedSortKey(), + TYPE: createPublishedType(), + GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_SK: createGSISortKey(storageEntry) + }); } /** @@ -391,17 +384,15 @@ export const createEntriesStorageOperations = ( if (latestStorageEntry) { const updatingLatestRevision = latestStorageEntry.id === entry.id; if (updatingLatestRevision) { - items.push( - entity.putBatch({ - ...storageEntry, - locked, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(entry) - }) - ); + entityBatch.put({ + ...storageEntry, + locked, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(entry) + }); } else { /** * If not updating latest revision, we still want to update the latest revision's @@ -417,37 +408,30 @@ export const createEntriesStorageOperations = ( * - one for the actual revision record * - one for the latest record */ - items.push( - entity.putBatch({ - ...latestStorageEntry, - ...updatedEntryLevelMetaFields, - PK: partitionKey, - SK: createRevisionSortKey(latestStorageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + ...updatedEntryLevelMetaFields, + PK: partitionKey, + SK: createRevisionSortKey(latestStorageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); - items.push( - entity.putBatch({ - ...latestStorageEntry, - ...updatedEntryLevelMetaFields, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + ...updatedEntryLevelMetaFields, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -490,23 +474,24 @@ export const createEntriesStorageOperations = ( /** * Then create the batch writes for the DynamoDB, with the updated folderId. */ - const items = records.map(item => { - return entity.putBatch({ - ...item, - location: { - ...item.location, - folderId - } - }); + const entityBatch = createEntityWriteBatch({ + entity, + put: records.map(item => { + return { + ...item, + location: { + ...item.location, + folderId + } + }; + }) }); + /** * And finally write it... */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw WebinyError.from(ex, { message: "Could not move records to a new folder.", @@ -518,7 +503,10 @@ export const createEntriesStorageOperations = ( } }; - const moveToBin: CmsEntryStorageOperations["moveToBin"] = async (initialModel, params) => { + const moveToBin: CmsEntryStorageOperations["moveToBin"] = async ( + initialModel, + params + ): Promise => { const { entry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); @@ -537,9 +525,9 @@ export const createEntriesStorageOperations = ( } }; - let records: DbItem[] = []; + let records: Awaited>> = []; try { - records = await queryAll(queryAllParams); + records = await queryAll(queryAllParams); } catch (ex) { throw new WebinyError( ex.message || "Could not load all records.", @@ -550,6 +538,9 @@ export const createEntriesStorageOperations = ( } ); } + if (records.length === 0) { + return; + } const storageEntry = convertToStorageEntry({ model, @@ -564,23 +555,23 @@ export const createEntriesStorageOperations = ( /** * Then create the batch writes for the DynamoDB, with the updated data. */ - const items = records.map(record => { - return entity.putBatch({ - ...record, - ...updatedDeletedMetaFields, - wbyDeleted: storageEntry.wbyDeleted, - location: storageEntry.location, - binOriginalFolderId: storageEntry.binOriginalFolderId - }); + const entityBatch = createEntityWriteBatch({ + entity, + put: records.map(record => { + return { + ...record, + ...updatedDeletedMetaFields, + wbyDeleted: storageEntry.wbyDeleted, + location: storageEntry.location, + binOriginalFolderId: storageEntry.binOriginalFolderId + }; + }) }); /** * And finally write it... */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not move the entry to the bin.", @@ -611,7 +602,7 @@ export const createEntriesStorageOperations = ( } }; - let records: DbItem[] = []; + let records: Awaited>> = []; try { records = await queryAll(queryAllParams); } catch (ex) { @@ -624,18 +615,19 @@ export const createEntriesStorageOperations = ( } ); } - const items = records.map(item => { - return entity.deleteBatch({ - PK: item.PK, - SK: item.SK - }); + + const entityBatch = createEntityWriteBatch({ + entity, + delete: records.map(item => { + return { + PK: item.PK, + SK: item.SK + }; + }) }); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -655,7 +647,7 @@ export const createEntriesStorageOperations = ( const restoreFromBin: CmsEntryStorageOperations["restoreFromBin"] = async ( initialModel, params - ) => { + ): Promise => { const { entry, storageEntry: initialStorageEntry } = params; const model = getStorageOperationsModel(initialModel); @@ -674,9 +666,9 @@ export const createEntriesStorageOperations = ( } }; - let records: DbItem[] = []; + let records: Awaited>> = []; try { - records = await queryAll(queryAllParams); + records = await queryAll(queryAllParams); } catch (ex) { throw new WebinyError( ex.message || "Could not load all records.", @@ -687,6 +679,9 @@ export const createEntriesStorageOperations = ( } ); } + if (records.length === 0) { + return initialStorageEntry; + } const storageEntry = convertToStorageEntry({ model, @@ -701,23 +696,24 @@ export const createEntriesStorageOperations = ( isRestoredEntryMetaField ); - const items = records.map(record => { - return entity.putBatch({ - ...record, - ...updatedRestoredMetaFields, - wbyDeleted: storageEntry.wbyDeleted, - location: storageEntry.location, - binOriginalFolderId: storageEntry.binOriginalFolderId - }); + const entityBatch = createEntityWriteBatch({ + entity, + put: records.map(record => { + return { + ...record, + ...updatedRestoredMetaFields, + wbyDeleted: storageEntry.wbyDeleted, + location: storageEntry.location, + binOriginalFolderId: storageEntry.binOriginalFolderId + }; + }) }); + /** * And finally write it... */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model @@ -751,12 +747,15 @@ export const createEntriesStorageOperations = ( tenant: model.tenant }); - const items = [ - entity.deleteBatch({ - PK: partitionKey, - SK: createRevisionSortKey(entry) - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [ + { + PK: partitionKey, + SK: createRevisionSortKey(entry) + } + ] + }); const publishedStorageEntry = await getPublishedRevisionByEntryId(model, entry); @@ -764,12 +763,10 @@ export const createEntriesStorageOperations = ( * If revision we are deleting is the published one as well, we need to delete those records as well. */ if (publishedStorageEntry && entry.id === publishedStorageEntry.id) { - items.push( - entity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ); + entityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); } if (initialLatestStorageEntry) { @@ -777,35 +774,29 @@ export const createEntriesStorageOperations = ( storageEntry: initialLatestStorageEntry, model }); - items.push( - entity.putBatch({ - ...latestStorageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); // Do an update on the latest revision. We need to update the latest revision's // entry-level meta fields to match the previous revision's entry-level meta fields. - items.push( - entity.putBatch({ - ...latestStorageEntry, - PK: partitionKey, - SK: createRevisionSortKey(initialLatestStorageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(initialLatestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + PK: partitionKey, + SK: createRevisionSortKey(initialLatestStorageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(initialLatestStorageEntry) + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + entityBatch.execute(); + dataLoaders.clearAll({ model }); @@ -834,57 +825,43 @@ export const createEntriesStorageOperations = ( /** * Then we need to construct the queries for all the revisions and entries. */ - const items: Record[] = []; + + const entityBatch = createEntityWriteBatch({ + entity + }); + for (const id of entries) { - /** - * Latest item. - */ - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "L" - }) - ); - /** - * Published item. - */ - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id, - locale: model.locale, - tenant: model.tenant - }), - SK: "P" - }) - ); + const partitionKey = createPartitionKey({ + id, + locale: model.locale, + tenant: model.tenant + }); + entityBatch.delete({ + PK: partitionKey, + SK: "L" + }); + entityBatch.delete({ + PK: partitionKey, + SK: "P" + }); } /** * Exact revisions of all the entries */ for (const revision of revisions) { - items.push( - entity.deleteBatch({ - PK: createPartitionKey({ - id: revision.id, - locale: model.locale, - tenant: model.tenant - }), - SK: createRevisionSortKey({ - version: revision.version - }) + entityBatch.delete({ + PK: createPartitionKey({ + id: revision.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createRevisionSortKey({ + version: revision.version }) - ); + }); } - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); }; const getLatestRevisionByEntryId: CmsEntryStorageOperations["getLatestRevisionByEntryId"] = @@ -1239,24 +1216,27 @@ export const createEntriesStorageOperations = ( }); // 1. Update REV# and P records with new data. - const items = [ - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createRevisionSortKey(entry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(entry) - }), - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createPublishedSortKey(), - TYPE: createPublishedType(), - GSI1_PK: createGSIPartitionKey(model, "P"), - GSI1_SK: createGSISortKey(entry) - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...storageEntry, + PK: partitionKey, + SK: createRevisionSortKey(entry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(entry) + }, + { + ...storageEntry, + PK: partitionKey, + SK: createPublishedSortKey(), + TYPE: createPublishedType(), + GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_SK: createGSISortKey(entry) + } + ] + }); // 2. When it comes to the latest record, we need to perform a couple of different // updates, based on whether the entry being published is the latest revision or not. @@ -1265,16 +1245,14 @@ export const createEntriesStorageOperations = ( if (publishingLatestRevision) { // 2.1 If we're publishing the latest revision, we first need to update the L record. - items.push( - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(entry) - }) - ); + entityBatch.put({ + ...storageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(entry) + }); // 2.2 Additionally, if we have a previously published entry, we need to mark it as unpublished. if (publishedRevisionId && publishedRevisionId !== entry.id) { @@ -1283,17 +1261,15 @@ export const createEntriesStorageOperations = ( model }); - items.push( - entity.putBatch({ - ...publishedStorageEntry, - PK: partitionKey, - SK: createRevisionSortKey(publishedStorageEntry), - TYPE: createType(), - status: CONTENT_ENTRY_STATUS.UNPUBLISHED, - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(publishedStorageEntry) - }) - ); + entityBatch.put({ + ...publishedStorageEntry, + PK: partitionKey, + SK: createRevisionSortKey(publishedStorageEntry), + TYPE: createType(), + status: CONTENT_ENTRY_STATUS.UNPUBLISHED, + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(publishedStorageEntry) + }); } } else { // 2.3 If the published revision is not the latest one, the situation is a bit @@ -1322,28 +1298,24 @@ export const createEntriesStorageOperations = ( status: latestRevisionStatus }; - items.push( - entity.putBatch({ - ...latestStorageEntryFields, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntryFields, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); // 2.3.2 Update REV# record. - items.push( - entity.putBatch({ - ...latestStorageEntryFields, - PK: partitionKey, - SK: createRevisionSortKey(latestStorageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntryFields, + PK: partitionKey, + SK: createRevisionSortKey(latestStorageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); // 2.3.3 Finally, if we got a published entry, but it wasn't the latest one, we need to take // an extra step and mark it as unpublished. @@ -1355,25 +1327,20 @@ export const createEntriesStorageOperations = ( model }); - items.push( - entity.putBatch({ - ...publishedStorageEntry, - PK: partitionKey, - SK: createRevisionSortKey(publishedStorageEntry), - TYPE: createType(), - status: CONTENT_ENTRY_STATUS.UNPUBLISHED, - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(publishedStorageEntry) - }) - ); + entityBatch.put({ + ...publishedStorageEntry, + PK: partitionKey, + SK: createRevisionSortKey(publishedStorageEntry), + TYPE: createType(), + status: CONTENT_ENTRY_STATUS.UNPUBLISHED, + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(publishedStorageEntry) + }); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); @@ -1411,20 +1378,25 @@ export const createEntriesStorageOperations = ( * - update current entry revision with new data * - update the latest entry status - if entry being unpublished is latest */ - const items = [ - entity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }), - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createRevisionSortKey(entry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(entry) - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [ + { + PK: partitionKey, + SK: createPublishedSortKey() + } + ], + put: [ + { + ...storageEntry, + PK: partitionKey, + SK: createRevisionSortKey(entry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(entry) + } + ] + }); /** * We need the latest entry to see if something needs to be updated alongside the unpublishing one. @@ -1434,16 +1406,14 @@ export const createEntriesStorageOperations = ( if (initialLatestStorageEntry) { const unpublishingLatestRevision = entry.id === initialLatestStorageEntry.id; if (unpublishingLatestRevision) { - items.push( - entity.putBatch({ - ...storageEntry, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(entry) - }) - ); + entityBatch.put({ + ...storageEntry, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(entry) + }); } else { const latestStorageEntry = convertToStorageEntry({ storageEntry: initialLatestStorageEntry, @@ -1458,38 +1428,31 @@ export const createEntriesStorageOperations = ( ); // 1. Update actual revision record. - items.push( - entity.putBatch({ - ...latestStorageEntry, - ...updatedEntryLevelMetaFields, - PK: partitionKey, - SK: createRevisionSortKey(latestStorageEntry), - TYPE: createType(), - GSI1_PK: createGSIPartitionKey(model, "A"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + ...updatedEntryLevelMetaFields, + PK: partitionKey, + SK: createRevisionSortKey(latestStorageEntry), + TYPE: createType(), + GSI1_PK: createGSIPartitionKey(model, "A"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); // 2. Update latest record. - items.push( - entity.putBatch({ - ...latestStorageEntry, - ...updatedEntryLevelMetaFields, - PK: partitionKey, - SK: createLatestSortKey(), - TYPE: createLatestType(), - GSI1_PK: createGSIPartitionKey(model, "L"), - GSI1_SK: createGSISortKey(latestStorageEntry) - }) - ); + entityBatch.put({ + ...latestStorageEntry, + ...updatedEntryLevelMetaFields, + PK: partitionKey, + SK: createLatestSortKey(), + TYPE: createLatestType(), + GSI1_PK: createGSIPartitionKey(model, "L"), + GSI1_SK: createGSISortKey(latestStorageEntry) + }); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); dataLoaders.clearAll({ model }); From e5ca4482e8ed40926be9b7c6998e81c824e8dad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 09:40:56 +0100 Subject: [PATCH 07/19] fix(api-page-builder-so-ddb-es): implement batch write tools --- .../src/operations/pages/index.ts | 517 ++++++++---------- 1 file changed, 242 insertions(+), 275 deletions(-) diff --git a/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts b/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts index b11f059e46c..b523ec37d03 100644 --- a/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts +++ b/packages/api-page-builder-so-ddb-es/src/operations/pages/index.ts @@ -1,4 +1,4 @@ -import { +import type { Page, PageStorageOperations, PageStorageOperationsCreateFromParams, @@ -14,12 +14,12 @@ import { PageStorageOperationsUnpublishParams, PageStorageOperationsUpdateParams } from "@webiny/api-page-builder/types"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; import omit from "lodash/omit"; import WebinyError from "@webiny/error"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; -import { Client } from "@elastic/elasticsearch"; -import { +import type { Client } from "@elastic/elasticsearch"; +import type { ElasticsearchBoolQueryConfig, ElasticsearchSearchResponse } from "@webiny/api-elasticsearch/types"; @@ -28,11 +28,18 @@ import { createLimit, encodeCursor } from "@webiny/api-elasticsearch"; import { createElasticsearchQueryBody } from "./elasticsearchQueryBody"; import { SearchLatestPagesPlugin } from "~/plugins/definitions/SearchLatestPagesPlugin"; import { SearchPublishedPagesPlugin } from "~/plugins/definitions/SearchPublishedPagesPlugin"; -import { DbItem, queryAll, QueryAllParams, queryOne } from "@webiny/db-dynamodb/utils/query"; +import { + createEntityWriteBatch, + getClean, + put, + queryAll, + QueryAllParams, + queryOne, + sortItems +} from "@webiny/db-dynamodb"; import { SearchPagesPlugin } from "~/plugins/definitions/SearchPagesPlugin"; -import { batchWriteAll } from "@webiny/db-dynamodb"; import { getESLatestPageData, getESPublishedPageData } from "./helpers"; -import { PluginsContainer } from "@webiny/plugins"; +import type { PluginsContainer } from "@webiny/plugins"; import { createBasicType, createLatestSortKey, @@ -45,9 +52,7 @@ import { createPublishedType, createSortKey } from "./keys"; -import { sortItems } from "@webiny/db-dynamodb/utils/sort"; import { PageDynamoDbElasticsearchFieldPlugin } from "~/plugins/definitions/PageDynamoDbElasticsearchFieldPlugin"; -import { getClean, put } from "@webiny/db-dynamodb"; import { shouldIgnoreEsResponseError } from "~/operations/pages/shouldIgnoreEsResponseError"; import { logIgnoredEsResponseError } from "~/operations/pages/logIgnoredEsResponseError"; @@ -81,24 +86,26 @@ export const createPageStorageOperations = ( SK: createLatestSortKey() }; - const items = [ - entity.putBatch({ - ...page, - ...versionKeys, - TYPE: createBasicType() - }), - entity.putBatch({ - ...page, - ...latestKeys, - TYPE: createLatestType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + ...versionKeys, + TYPE: createBasicType() + }, + { + ...page, + ...latestKeys, + TYPE: createLatestType() + } + ] + }); + const esData = getESLatestPageData(plugins, page, input); try { - await batchWriteAll({ - table: entity.table, - items: items - }); + await entityBatch.execute(); + await put({ entity: esEntity, item: { @@ -133,26 +140,26 @@ export const createPageStorageOperations = ( SK: createLatestSortKey() }; - const items = [ - entity.putBatch({ - ...page, - TYPE: createBasicType(), - ...versionKeys - }), - entity.putBatch({ - ...page, - TYPE: createLatestType(), - ...latestKeys - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + TYPE: createBasicType(), + ...versionKeys + }, + { + ...page, + TYPE: createLatestType(), + ...latestKeys + } + ] + }); const esData = getESLatestPageData(plugins, page); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); await put({ entity: esEntity, @@ -195,13 +202,16 @@ export const createPageStorageOperations = ( keys: latestKeys }); - const items = [ - entity.putBatch({ - ...page, - TYPE: createBasicType(), - ...keys - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + TYPE: createBasicType(), + ...keys + } + ] + }); const esData = getESLatestPageData(plugins, page, input); @@ -209,22 +219,17 @@ export const createPageStorageOperations = ( /** * We also update the regular record. */ - items.push( - entity.putBatch({ - ...page, - TYPE: createLatestType(), - ...latestKeys - }) - ); + entityBatch.put({ + ...page, + TYPE: createLatestType(), + ...latestKeys + }); } /** * Unfortunately we cannot push regular and es record in the batch write because they are two separate tables. */ try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); await put({ entity: esEntity, @@ -265,32 +270,35 @@ export const createPageStorageOperations = ( const partitionKey = createPartitionKey(page); - const items = [ - entity.deleteBatch({ - PK: partitionKey, - SK: createSortKey(page) - }) - ]; - const esItems = []; - if (publishedPage && publishedPage.id === page.id) { - items.push( - entity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ); - items.push( - entity.deleteBatch({ - PK: createPathPartitionKey(page), - SK: createPathSortKey(page) - }) - ); - esItems.push( - esEntity.deleteBatch({ + const entityBatch = createEntityWriteBatch({ + entity, + delete: [ + { PK: partitionKey, - SK: createPublishedSortKey() - }) - ); + SK: createSortKey(page) + } + ] + }); + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); + + if (publishedPage && publishedPage.id === page.id) { + entityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); + + entityBatch.delete({ + PK: createPathPartitionKey(page), + SK: createPathSortKey(page) + }); + + elasticsearchEntityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); } let previousLatestPage: Page | null = null; if (latestPage && latestPage.id === page.id) { @@ -303,44 +311,34 @@ export const createPageStorageOperations = ( } }); if (previousLatestRecord) { - items.push( - entity.putBatch({ - ...previousLatestRecord, - TYPE: createLatestType(), - PK: partitionKey, - SK: createLatestSortKey() - }) - ); - esItems.push( - esEntity.putBatch({ - PK: partitionKey, - SK: createLatestSortKey(), - index: configurations.es(page).index, - data: getESLatestPageData(plugins, previousLatestRecord) - }) - ); + entityBatch.put({ + ...previousLatestRecord, + TYPE: createLatestType(), + PK: partitionKey, + SK: createLatestSortKey() + }); + + elasticsearchEntityBatch.put({ + PK: partitionKey, + SK: createLatestSortKey(), + index: configurations.es(page).index, + data: getESLatestPageData(plugins, previousLatestRecord) + }); + previousLatestPage = cleanupItem(entity, previousLatestRecord); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not batch write all the page records.", ex.code || "BATCH_WRITE_RECORDS_ERROR" ); } - if (esItems.length === 0) { - return [page, previousLatestPage]; - } + try { - await batchWriteAll({ - table: entity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not batch write all the page Elasticsearch records.", @@ -370,7 +368,7 @@ export const createPageStorageOperations = ( gte: " " } }; - let revisions: DbItem[]; + let revisions: Awaited>>; try { revisions = await queryAll(queryAllParams); } catch (ex) { @@ -387,48 +385,45 @@ export const createPageStorageOperations = ( * We need to go through all possible entries and delete them. * Also, delete the published entry path record. */ - const items = []; + + const entityBatch = createEntityWriteBatch({ + entity + }); + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); + let publishedPathEntryDeleted = false; for (const revision of revisions) { if (revision.status === "published" && !publishedPathEntryDeleted) { publishedPathEntryDeleted = true; - items.push( - entity.deleteBatch({ - PK: createPathPartitionKey(page), - SK: revision.path - }) - ); + entityBatch.delete({ + PK: createPathPartitionKey(page), + SK: revision.path + }); } - items.push( - entity.deleteBatch({ - PK: revision.PK, - SK: revision.SK - }) - ); + entityBatch.delete({ + PK: revision.PK, + SK: revision.SK + }); } - const esItems = [ - esEntity.deleteBatch({ - PK: partitionKey, - SK: createLatestSortKey() - }) - ]; + elasticsearchEntityBatch.delete({ + PK: partitionKey, + SK: createLatestSortKey() + }); + /** * Delete published record if it is published. */ if (publishedPathEntryDeleted) { - esItems.push( - esEntity.deleteBatch({ - PK: partitionKey, - SK: createPublishedSortKey() - }) - ); + elasticsearchEntityBatch.delete({ + PK: partitionKey, + SK: createPublishedSortKey() + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete all the page records.", @@ -436,10 +431,7 @@ export const createPageStorageOperations = ( ); } try { - await batchWriteAll({ - table: entity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete all the page Elasticsearch records.", @@ -457,118 +449,100 @@ export const createPageStorageOperations = ( /** * Update the given revision of the page. */ - const items = [ - entity.putBatch({ - ...page, - TYPE: createBasicType(), - PK: createPartitionKey(page), - SK: createSortKey(page) - }) - ]; - const esItems = []; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + TYPE: createBasicType(), + PK: createPartitionKey(page), + SK: createSortKey(page) + } + ] + }); + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity + }); /** * If we are publishing the latest revision, update the latest revision * status in ES. We also need to update the latest page revision entry in ES. */ if (latestPage.id === page.id) { - items.push( - entity.putBatch({ - ...page, - TYPE: createLatestType(), - PK: createPartitionKey(page), - SK: createLatestSortKey() - }) - ); + entityBatch.put({ + ...page, + TYPE: createLatestType(), + PK: createPartitionKey(page), + SK: createLatestSortKey() + }); - esItems.push( - esEntity.putBatch({ - PK: createPartitionKey(page), - SK: createLatestSortKey(), - index: configurations.es(page).index, - data: getESLatestPageData(plugins, page) - }) - ); + elasticsearchEntityBatch.put({ + PK: createPartitionKey(page), + SK: createLatestSortKey(), + index: configurations.es(page).index, + data: getESLatestPageData(plugins, page) + }); } /** * If we already have a published revision, and it's not the revision being published: * - set the existing published revision to "unpublished" */ if (publishedPage && publishedPage.id !== page.id) { - items.push( - entity.putBatch({ - ...publishedPage, - status: "unpublished", - PK: createPartitionKey(publishedPage), - SK: createSortKey(publishedPage) - }) - ); + entityBatch.put({ + ...publishedPage, + status: "unpublished", + PK: createPartitionKey(publishedPage), + SK: createSortKey(publishedPage) + }); + /** * Remove old published path if required. */ if (publishedPage.path !== page.path) { - items.push( - entity.deleteBatch({ - PK: createPathPartitionKey(page), - SK: publishedPage.path - }) - ); + entityBatch.delete({ + PK: createPathPartitionKey(page), + SK: publishedPage.path + }); } } - esItems.push( - esEntity.putBatch({ - PK: createPartitionKey(page), - SK: createPublishedSortKey(), - index: configurations.es(page).index, - data: getESPublishedPageData(plugins, page) - }) - ); + elasticsearchEntityBatch.put({ + PK: createPartitionKey(page), + SK: createPublishedSortKey(), + index: configurations.es(page).index, + data: getESPublishedPageData(plugins, page) + }); /** * Update or insert published path. */ - items.push( - entity.putBatch({ - ...page, - TYPE: createPublishedPathType(), - PK: createPathPartitionKey(page), - SK: createPathSortKey(page) - }) - ); + entityBatch.put({ + ...page, + TYPE: createPublishedPathType(), + PK: createPathPartitionKey(page), + SK: createPathSortKey(page) + }); + /** * Update or insert published page. */ - items.push( - entity.putBatch({ - ...page, - TYPE: createPublishedType(), - PK: createPartitionKey(page), - SK: createPublishedSortKey() - }) - ); + entityBatch.put({ + ...page, + TYPE: createPublishedType(), + PK: createPartitionKey(page), + SK: createPublishedSortKey() + }); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update all the page records when publishing.", ex.code || "UPDATE_RECORDS_ERROR" ); } - /** - * No point in continuing if there are no items in Elasticsearch data - */ - if (esItems.length === 0) { - return page; - } + try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || @@ -584,74 +558,67 @@ export const createPageStorageOperations = ( page.status = "unpublished"; - const items = [ - entity.deleteBatch({ - PK: createPartitionKey(page), - SK: createPublishedSortKey() - }), - entity.deleteBatch({ - PK: createPathPartitionKey(page), - SK: createPathSortKey(page) - }), - entity.putBatch({ - ...page, - TYPE: createBasicType(), - PK: createPartitionKey(page), - SK: createSortKey(page) - }) - ]; - const esItems = []; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [ + { + PK: createPartitionKey(page), + SK: createPublishedSortKey() + }, + { + PK: createPathPartitionKey(page), + SK: createPathSortKey(page) + } + ], + put: [ + { + ...page, + TYPE: createBasicType(), + PK: createPartitionKey(page), + SK: createSortKey(page) + } + ] + }); + + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: esEntity, + delete: [ + { + PK: createPartitionKey(page), + SK: createPublishedSortKey() + } + ] + }); /* * If we are unpublishing the latest revision, let's also update the latest revision entry's status in ES. */ if (latestPage.id === page.id) { - items.push( - entity.putBatch({ - ...page, - TYPE: createLatestType(), - PK: createPartitionKey(page), - SK: createLatestSortKey() - }) - ); - esItems.push( - esEntity.putBatch({ - PK: createPartitionKey(page), - SK: createLatestSortKey(), - index: configurations.es(page).index, - data: getESLatestPageData(plugins, page) - }) - ); - } + entityBatch.put({ + ...page, + TYPE: createLatestType(), + PK: createPartitionKey(page), + SK: createLatestSortKey() + }); - esItems.push( - esEntity.deleteBatch({ + elasticsearchEntityBatch.put({ PK: createPartitionKey(page), - SK: createPublishedSortKey() - }) - ); + SK: createLatestSortKey(), + index: configurations.es(page).index, + data: getESLatestPageData(plugins, page) + }); + } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update all the page records when unpublishing.", ex.code || "UPDATE_RECORDS_ERROR" ); } - /** - * No need to go further if no Elasticsearch items to be applied. - */ - if (esItems.length === 0) { - return page; - } + try { - await batchWriteAll({ - table: esEntity.table, - items: esItems - }); + await elasticsearchEntityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || From 41d70a5e52529430b841e7683241ed8b926d0b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 09:41:19 +0100 Subject: [PATCH 08/19] fix(db-dynamodb): reusable batch writers --- .../src/utils/batch/EntityWriteBatch.ts | 7 ++- .../src/utils/batch/TableWriteBatch.ts | 7 ++- .../db-dynamodb/src/utils/entity/Entity.ts | 45 +++++++++++++++++++ .../db-dynamodb/src/utils/entity/index.ts | 2 + .../db-dynamodb/src/utils/entity/types.ts | 8 ++++ packages/db-dynamodb/src/utils/get.ts | 10 +++-- packages/db-dynamodb/src/utils/index.ts | 1 + packages/db-dynamodb/src/utils/update.ts | 2 +- 8 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 packages/db-dynamodb/src/utils/entity/Entity.ts create mode 100644 packages/db-dynamodb/src/utils/entity/index.ts create mode 100644 packages/db-dynamodb/src/utils/entity/types.ts diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index c6475656f6c..04d2c17da9d 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -56,8 +56,13 @@ export class EntityWriteBatch implements IEntityWriteBatch { } public async execute(): Promise { + if (this.items.length === 0) { + return []; + } + const items = [...this.items]; + this.items.length = 0; return await batchWriteAll({ - items: this.items, + items, table: this.entity.table }); } diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index 743bc8e603e..bb6942f5779 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -46,8 +46,13 @@ export class TableWriteBatch implements ITableWriteBatch { } public async execute(): Promise { + if (this.items.length === 0) { + return []; + } + const items = [...this.items]; + this.items.length = 0; return await batchWriteAll({ - items: this.items, + items, table: this.table }); } diff --git a/packages/db-dynamodb/src/utils/entity/Entity.ts b/packages/db-dynamodb/src/utils/entity/Entity.ts new file mode 100644 index 00000000000..3593993df80 --- /dev/null +++ b/packages/db-dynamodb/src/utils/entity/Entity.ts @@ -0,0 +1,45 @@ +import type { Entity as BaseEntity } from "dynamodb-toolbox"; +import { createEntityWriteBatch, createTableWriteBatch } from "~/utils/batch"; +import type { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; +import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table"; +import { IEntity } from "./types"; +import { IPutParamsItem, put } from "~/utils/put"; +import { getClean, GetRecordParamsKeys } from "~/utils/get"; + +export class Entity implements IEntity { + public readonly entity: BaseEntity; + + public constructor(entity: BaseEntity) { + this.entity = entity; + } + + public createEntityWriter(): IEntityWriteBatch { + return createEntityWriteBatch({ + entity: this.entity + }); + } + + public createTableWriter(): ITableWriteBatch { + return createTableWriteBatch({ + table: this.entity.table as TableDef + }); + } + + public async put(item: IPutParamsItem): ReturnType { + return put({ + entity: this.entity, + item + }); + } + + public async get(keys: GetRecordParamsKeys): ReturnType> { + return getClean({ + entity: this.entity, + keys + }); + } +} + +export const createEntity = (params: BaseEntity): IEntity => { + return new Entity(params); +}; diff --git a/packages/db-dynamodb/src/utils/entity/index.ts b/packages/db-dynamodb/src/utils/entity/index.ts new file mode 100644 index 00000000000..1f6b52ffc94 --- /dev/null +++ b/packages/db-dynamodb/src/utils/entity/index.ts @@ -0,0 +1,2 @@ +export * from "./Entity"; +export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/entity/types.ts b/packages/db-dynamodb/src/utils/entity/types.ts new file mode 100644 index 00000000000..2d5d8ae106a --- /dev/null +++ b/packages/db-dynamodb/src/utils/entity/types.ts @@ -0,0 +1,8 @@ +import { Entity as BaseEntity } from "dynamodb-toolbox"; +import { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; + +export interface IEntity { + readonly entity: BaseEntity; + createEntityWriter(): IEntityWriteBatch; + createTableWriter(): ITableWriteBatch; +} diff --git a/packages/db-dynamodb/src/utils/get.ts b/packages/db-dynamodb/src/utils/get.ts index b3e9d0f52e0..b3d29980851 100644 --- a/packages/db-dynamodb/src/utils/get.ts +++ b/packages/db-dynamodb/src/utils/get.ts @@ -1,12 +1,14 @@ import { Entity } from "~/toolbox"; import { cleanupItem } from "~/utils/cleanup"; +export interface GetRecordParamsKeys { + PK: string; + SK: string; +} + export interface GetRecordParams { entity: Entity; - keys: { - PK: string; - SK: string; - }; + keys: GetRecordParamsKeys; } /** diff --git a/packages/db-dynamodb/src/utils/index.ts b/packages/db-dynamodb/src/utils/index.ts index 087e51b7f56..e7af65f04e6 100644 --- a/packages/db-dynamodb/src/utils/index.ts +++ b/packages/db-dynamodb/src/utils/index.ts @@ -14,3 +14,4 @@ export * from "./sort"; export * from "./table"; export * from "./update"; export * from "./batch"; +export * from "./entity"; diff --git a/packages/db-dynamodb/src/utils/update.ts b/packages/db-dynamodb/src/utils/update.ts index 5da727ccc48..c4e563972ab 100644 --- a/packages/db-dynamodb/src/utils/update.ts +++ b/packages/db-dynamodb/src/utils/update.ts @@ -5,7 +5,7 @@ interface Params { item: { PK: string; SK: string; - TYPE: string; + TYPE?: string; [key: string]: any; }; } From 86645aac51abef59cd3f87afc1b3bbae9afdacde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 10:14:14 +0100 Subject: [PATCH 09/19] fix(api-prerendering-service-so-ddb): implement batch write tools --- .../src/operations/pages/index.ts | 307 +++++++++--------- .../src/operations/queueJob.ts | 28 +- .../src/utils/batch/EntityWriteBatch.ts | 4 + .../src/utils/batch/TableWriteBatch.ts | 4 + packages/db-dynamodb/src/utils/batch/types.ts | 2 + 5 files changed, 171 insertions(+), 174 deletions(-) diff --git a/packages/api-page-builder-so-ddb/src/operations/pages/index.ts b/packages/api-page-builder-so-ddb/src/operations/pages/index.ts index 7130312a8f3..1938bace62b 100644 --- a/packages/api-page-builder-so-ddb/src/operations/pages/index.ts +++ b/packages/api-page-builder-so-ddb/src/operations/pages/index.ts @@ -1,6 +1,5 @@ import WebinyError from "@webiny/error"; -import lodashGet from "lodash/get"; -import { +import type { Page, PageStorageOperations, PageStorageOperationsCreateFromParams, @@ -18,21 +17,21 @@ import { PageStorageOperationsUpdateParams } from "@webiny/api-page-builder/types"; import { getClean } from "@webiny/db-dynamodb/utils/get"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; import { cleanupItem } from "@webiny/db-dynamodb/utils/cleanup"; +import type { QueryAllParams } from "@webiny/db-dynamodb"; import { - DbItem, + createEntityWriteBatch, + decodeCursor, + encodeCursor, + filterItems, queryAll, - QueryAllParams, queryOne, - queryOneClean -} from "@webiny/db-dynamodb/utils/query"; -import { batchWriteAll } from "@webiny/db-dynamodb"; -import { filterItems } from "@webiny/db-dynamodb/utils/filter"; -import { sortItems } from "@webiny/db-dynamodb/utils/sort"; -import { decodeCursor, encodeCursor } from "@webiny/db-dynamodb/utils/cursor"; + queryOneClean, + sortItems +} from "@webiny/db-dynamodb"; import { PageDynamoDbFieldPlugin } from "~/plugins/definitions/PageDynamoDbFieldPlugin"; -import { PluginsContainer } from "@webiny/plugins"; +import type { PluginsContainer } from "@webiny/plugins"; import { createLatestPartitionKey, createLatestSortKey, @@ -105,25 +104,26 @@ export const createPageStorageOperations = ( * - latest * - revision */ - const items = [ - entity.putBatch({ - ...page, - titleLC, - ...latestKeys, - TYPE: createLatestType() - }), - entity.putBatch({ - ...page, - titleLC, - ...revisionKeys, - TYPE: createRevisionType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + titleLC, + ...latestKeys, + TYPE: createLatestType() + }, + { + ...page, + titleLC, + ...revisionKeys, + TYPE: createRevisionType() + } + ] + }); + try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); return page; } catch (ex) { throw new WebinyError( @@ -154,24 +154,24 @@ export const createPageStorageOperations = ( * - latest * - revision */ - const items = [ - entity.putBatch({ - ...page, - ...latestKeys, - TYPE: createLatestType() - }), - entity.putBatch({ - ...page, - ...revisionKeys, - TYPE: createRevisionType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + ...latestKeys, + TYPE: createLatestType() + }, + { + ...page, + ...revisionKeys, + TYPE: createRevisionType() + } + ] + }); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); return page; } catch (ex) { throw new WebinyError( @@ -211,33 +211,32 @@ export const createPageStorageOperations = ( * - revision * - latest if this is the latest */ - const items = [ - entity.putBatch({ - ...page, - titleLC, - ...revisionKeys, - TYPE: createRevisionType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + titleLC, + ...revisionKeys, + TYPE: createRevisionType() + } + ] + }); + /** * Latest if it is the one. */ if (latestPage && latestPage.id === page.id) { - items.push( - entity.putBatch({ - ...page, - titleLC, - ...latestKeys, - TYPE: createLatestType() - }) - ); + entityBatch.put({ + ...page, + titleLC, + ...latestKeys, + TYPE: createLatestType() + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); return page; } catch (ex) { @@ -280,9 +279,13 @@ export const createPageStorageOperations = ( * We need to update * - latest, if it exists, with previous record */ - const items = [entity.deleteBatch(revisionKeys)]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [revisionKeys] + }); + if (publishedPage && publishedPage.id === page.id) { - items.push(entity.deleteBatch(publishedKeys)); + entityBatch.delete(publishedKeys); } let previousLatestPage: Page | null = null; if (latestPage && latestPage.id === page.id) { @@ -296,21 +299,17 @@ export const createPageStorageOperations = ( } }); if (previousLatestRecord) { - items.push( - entity.putBatch({ - ...previousLatestRecord, - ...latestKeys, - TYPE: createLatestType() - }) - ); + entityBatch.put({ + ...previousLatestRecord, + ...latestKeys, + TYPE: createLatestType() + }); + previousLatestPage = cleanupItem(entity, previousLatestRecord); } } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not batch write all the page records.", @@ -347,11 +346,14 @@ export const createPageStorageOperations = ( SK: createPublishedSortKey(page) }; - const items = [entity.deleteBatch(latestKeys)]; + const entityBatch = createEntityWriteBatch({ + entity, + delete: [latestKeys] + }); - let revisions: DbItem[]; + let revisions: Awaited>> = []; try { - revisions = await queryAll(queryAllParams); + revisions = await queryAll(queryAllParams); } catch (ex) { throw new WebinyError( ex.message || "Could not query for all revisions of the page.", @@ -369,22 +371,17 @@ export const createPageStorageOperations = ( */ for (const revision of revisions) { if (!deletedPublishedRecord && revision.status === "published") { - items.push(entity.deleteBatch(publishedKeys)); + entityBatch.delete(publishedKeys); deletedPublishedRecord = true; } - items.push( - entity.deleteBatch({ - PK: revision.PK, - SK: revision.SK - }) - ); + entityBatch.delete({ + PK: revision.PK, + SK: revision.SK + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not delete all the page records.", @@ -420,22 +417,23 @@ export const createPageStorageOperations = ( /** * Update the given revision of the page. */ - const items = [ - entity.putBatch({ - ...page, - ...revisionKeys, - TYPE: createRevisionType() - }) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + ...revisionKeys, + TYPE: createRevisionType() + } + ] + }); if (latestPage.id === page.id) { - items.push( - entity.putBatch({ - ...page, - ...latestKeys, - TYPE: createLatestType() - }) - ); + entityBatch.put({ + ...page, + ...latestKeys, + TYPE: createLatestType() + }); } /** * If we already have a published revision, and it's not the revision being published: @@ -446,31 +444,24 @@ export const createPageStorageOperations = ( PK: createRevisionPartitionKey(publishedPage), SK: createRevisionSortKey(publishedPage) }; - items.push( - entity.putBatch({ - ...publishedPage, - status: "unpublished", - ...publishedRevisionKeys, - TYPE: createRevisionType() - }) - ); + entityBatch.put({ + ...publishedPage, + status: "unpublished", + ...publishedRevisionKeys, + TYPE: createRevisionType() + }); } - items.push( - entity.putBatch({ - ...page, - ...publishedKeys, - GSI1_PK: createPathPartitionKey(page), - GSI1_SK: page.path, - TYPE: createPublishedType() - }) - ); + entityBatch.put({ + ...page, + ...publishedKeys, + GSI1_PK: createPathPartitionKey(page), + GSI1_SK: page.path, + TYPE: createPublishedType() + }); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update all the page records when publishing.", @@ -504,30 +495,28 @@ export const createPageStorageOperations = ( SK: createPublishedSortKey(page) }; - const items = [ - entity.putBatch({ - ...page, - ...revisionKeys, - TYPE: createRevisionType() - }), - entity.deleteBatch(publishedKeys) - ]; + const entityBatch = createEntityWriteBatch({ + entity, + put: [ + { + ...page, + ...revisionKeys, + TYPE: createRevisionType() + } + ], + delete: [publishedKeys] + }); if (latestPage.id === page.id) { - items.push( - entity.putBatch({ - ...page, - ...latestKeys, - TYPE: createLatestType() - }) - ); + entityBatch.put({ + ...page, + ...latestKeys, + TYPE: createLatestType() + }); } try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); } catch (ex) { throw new WebinyError( ex.message || "Could not update all the page records when unpublishing.", @@ -819,7 +808,7 @@ export const createPageStorageOperations = ( options }; - let pages: DbItem[] = []; + let pages: Awaited>> = []; try { pages = await queryAll(queryAllParams); } catch (ex) { @@ -833,22 +822,20 @@ export const createPageStorageOperations = ( ); } - const tags = pages.reduce((collection, page) => { - let list: string[] = lodashGet(page, "settings.general.tags") as unknown as string[]; - if (!list || list.length === 0) { - return collection; + const tags = new Set(); + for (const page of pages) { + let tagList = page.settings?.general?.tags; + if (!tagList?.length) { + continue; } else if (where.search) { const re = new RegExp(where.search, "i"); - list = list.filter(t => t.match(re) !== null); + tagList = tagList.filter(tag => !!tag && tag.match(re) !== null); } - - for (const t of list) { - collection[t] = undefined; + for (const tag of tagList) { + tags.add(tag); } - return collection; - }, {} as Record); - - return Object.keys(tags); + } + return Array.from(tags); }; return { diff --git a/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts b/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts index 2c170040860..f8371e1c7e5 100644 --- a/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts +++ b/packages/api-prerendering-service-so-ddb/src/operations/queueJob.ts @@ -1,14 +1,14 @@ import WebinyError from "@webiny/error"; -import { +import type { PrerenderingServiceQueueJobStorageOperations, PrerenderingServiceStorageOperationsCreateQueueJobParams, PrerenderingServiceStorageOperationsDeleteQueueJobsParams, QueueJob } from "@webiny/api-prerendering-service/types"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { batchWriteAll } from "@webiny/db-dynamodb"; -import { queryAllClean, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; -import { put } from "@webiny/db-dynamodb"; +import type { Entity } from "@webiny/db-dynamodb/toolbox"; +import { createEntityWriteBatch, put } from "@webiny/db-dynamodb"; +import type { QueryAllParams } from "@webiny/db-dynamodb/utils/query"; +import { queryAllClean } from "@webiny/db-dynamodb/utils/query"; export interface CreateQueueJobStorageOperationsParams { entity: Entity; @@ -89,18 +89,18 @@ export const createQueueJobStorageOperations = ( ) => { const { queueJobs } = params; - const items = queueJobs.map(job => { - return entity.deleteBatch({ - PK: createQueueJobPartitionKey(), - SK: createQueueJobSortKey(job.id) - }); + const entityBatch = createEntityWriteBatch({ + entity, + delete: queueJobs.map(job => { + return { + PK: createQueueJobPartitionKey(), + SK: createQueueJobSortKey(job.id) + }; + }) }); try { - await batchWriteAll({ - table: entity.table, - items - }); + await entityBatch.execute(); return queueJobs; } catch (ex) { throw new WebinyError( diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index 04d2c17da9d..5ead5075342 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -24,6 +24,10 @@ export class EntityWriteBatch implements IEntityWriteBatch { private readonly items: BatchWriteItem[] = []; private readonly builder: IEntityWriteBatchBuilder; + public get total(): number { + return this.items.length; + } + public constructor(params: IEntityWriteBatchParams) { if (!params.entity.name) { throw new Error(`No name provided for entity.`); diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index bb6942f5779..d4d57a1474e 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -20,6 +20,10 @@ export class TableWriteBatch implements ITableWriteBatch { private readonly items: BatchWriteItem[] = []; private readonly builders: Map = new Map(); + public get total(): number { + return this.items.length; + } + public constructor(params: ITableWriteBatchParams) { this.table = params.table; if (!params.items?.length) { diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index b5965cbdbe4..e13111cee32 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -37,6 +37,7 @@ export interface IEntityWriteBatchBuilder { } export interface IEntityWriteBatch { + readonly total: number; // readonly entity: Entity; // readonly items: BatchWriteItem[]; // readonly builder: IEntityWriteBatchBuilder; @@ -48,6 +49,7 @@ export interface IEntityWriteBatch { } export interface ITableWriteBatch { + readonly total: number; // readonly table: TableDef; // readonly items: BatchWriteItem[]; put(entity: Entity, item: IPutBatchItem): void; From 0f6c48b200420a9b102091227cd7375cc508c179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 10:22:04 +0100 Subject: [PATCH 10/19] wip(migrations): implement batch write tools --- .../5.35.0/001/ddb-es/FileDataMigration.ts | 15 ++++--- .../src/migrations/5.35.0/003/index.ts | 42 ++++++++++--------- .../src/migrations/5.35.0/005/index.ts | 36 ++++++++-------- .../src/migrations/5.37.0/002/ddb-es/index.ts | 8 ++-- .../5.37.0/003/ddb-es/AcoFolderMigration.ts | 33 +++++++-------- .../src/migrations/5.40.0/001/ddb/index.ts | 26 ++++++------ .../src/migrations/5.41.0/001/index.ts | 38 +++++++++-------- 7 files changed, 102 insertions(+), 96 deletions(-) diff --git a/packages/migrations/src/migrations/5.35.0/001/ddb-es/FileDataMigration.ts b/packages/migrations/src/migrations/5.35.0/001/ddb-es/FileDataMigration.ts index 0f2750f9257..b4848b89c70 100644 --- a/packages/migrations/src/migrations/5.35.0/001/ddb-es/FileDataMigration.ts +++ b/packages/migrations/src/migrations/5.35.0/001/ddb-es/FileDataMigration.ts @@ -5,12 +5,12 @@ import { PrimitiveValue } from "@webiny/api-elasticsearch/types"; import { executeWithRetry } from "@webiny/utils"; import { DataMigration, DataMigrationContext } from "@webiny/data-migration"; import { - createStandardEntity, - queryOne, - queryAll, batchWriteAll, + createStandardEntity, + esGetIndexName, esQueryAllWithCallback, - esGetIndexName + queryAll, + queryOne } from "~/utils"; import { createFileEntity, getFileData, legacyAttributes } from "../entities/createFileEntity"; import { createLocaleEntity } from "../entities/createLocaleEntity"; @@ -160,8 +160,11 @@ export class FileManager_5_35_0_001_FileData implements DataMigration { return Promise.all( - chunk(items, 200).map(fileChunk => { - return batchWriteAll({ + chunk(items, 200).map(async fileChunk => { + /** + * Leave batch write for now. + */ + return await batchWriteAll({ table: this.newFileEntity.table, items: fileChunk }); diff --git a/packages/migrations/src/migrations/5.35.0/003/index.ts b/packages/migrations/src/migrations/5.35.0/003/index.ts index 562a703c3ba..92d7685ea33 100644 --- a/packages/migrations/src/migrations/5.35.0/003/index.ts +++ b/packages/migrations/src/migrations/5.35.0/003/index.ts @@ -1,10 +1,11 @@ import { Table } from "@webiny/db-dynamodb/toolbox"; import { DataMigrationContext, PrimaryDynamoTableSymbol } from "@webiny/data-migration"; -import { queryOne, queryAll, batchWriteAll } from "~/utils"; +import { queryAll, queryOne } from "~/utils"; import { createTenantEntity } from "./createTenantEntity"; import { createLegacyUserEntity, createUserEntity, getUserData } from "./createUserEntity"; -import { makeInjectable, inject } from "@webiny/ioc"; +import { inject, makeInjectable } from "@webiny/ioc"; import { executeWithRetry } from "@webiny/utils"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; export class AdminUsers_5_35_0_003 { private readonly newUserEntity: ReturnType; @@ -73,24 +74,27 @@ export class AdminUsers_5_35_0_003 { continue; } - const newUsers = users - .filter(user => !user.data) - .map(user => { - return this.newUserEntity.putBatch({ - PK: `T#${tenant.id}#ADMIN_USER#${user.id}`, - SK: "A", - GSI1_PK: `T#${tenant.id}#ADMIN_USERS`, - GSI1_SK: user.email, - TYPE: "adminUsers.user", - ...getUserData(user), - // Move all data to a `data` envelope - data: getUserData(user) - }); - }); + const newUsersEntityBatch = createEntityWriteBatch({ + entity: this.newUserEntity, + put: users + .filter(user => !user.data) + .map(user => { + return { + PK: `T#${tenant.id}#ADMIN_USER#${user.id}`, + SK: "A", + GSI1_PK: `T#${tenant.id}#ADMIN_USERS`, + GSI1_SK: user.email, + TYPE: "adminUsers.user", + ...getUserData(user), + // Move all data to a `data` envelope + data: getUserData(user) + }; + }) + }); - await executeWithRetry(() => - batchWriteAll({ table: this.newUserEntity.table, items: newUsers }) - ); + await executeWithRetry(async () => { + return await newUsersEntityBatch.execute(); + }); } } } diff --git a/packages/migrations/src/migrations/5.35.0/005/index.ts b/packages/migrations/src/migrations/5.35.0/005/index.ts index f02cd1e77c8..6b32acd2168 100644 --- a/packages/migrations/src/migrations/5.35.0/005/index.ts +++ b/packages/migrations/src/migrations/5.35.0/005/index.ts @@ -1,14 +1,15 @@ import { Table } from "@webiny/db-dynamodb/toolbox"; -import { makeInjectable, inject } from "@webiny/ioc"; +import { inject, makeInjectable } from "@webiny/ioc"; import { DataMigrationContext, PrimaryDynamoTableSymbol } from "@webiny/data-migration"; -import { queryAll, batchWriteAll } from "~/utils"; +import { queryAll } from "~/utils"; import { createModelEntity } from "./createModelEntity"; import { createTenantEntity } from "./createTenantEntity"; import { createLocaleEntity } from "./createLocaleEntity"; -import { Tenant, I18NLocale, CmsModel } from "./types"; +import { CmsModel, I18NLocale, Tenant } from "./types"; import pluralize from "pluralize"; import upperFirst from "lodash/upperFirst"; import camelCase from "lodash/camelCase"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; const createSingularApiName = (model: CmsModel) => { return upperFirst(camelCase(model.modelId)); @@ -91,22 +92,23 @@ export class CmsModels_5_35_0_005 { return; } - const items = models.map(model => { - return this.modelEntity.putBatch({ - ...model, - /** - * Add singular and plural API names. - */ - singularApiName: createSingularApiName(model), - pluralApiName: createPluralApiName(model) - }); + const entityBatch = createEntityWriteBatch({ + entity: this.modelEntity, + put: models.map(model => { + return { + ...model, + /** + * Add singular and plural API names. + */ + singularApiName: createSingularApiName(model), + pluralApiName: createPluralApiName(model) + }; + }) }); - logger.info(`Updating total of ${items.length} models.`); - await batchWriteAll({ - table: this.modelEntity.table, - items - }); + logger.info(`Updating total of ${entityBatch.total} models.`); + + await entityBatch.execute(); logger.info("Updated all the models."); } diff --git a/packages/migrations/src/migrations/5.37.0/002/ddb-es/index.ts b/packages/migrations/src/migrations/5.37.0/002/ddb-es/index.ts index f673b58b039..429fc900dee 100644 --- a/packages/migrations/src/migrations/5.37.0/002/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.37.0/002/ddb-es/index.ts @@ -272,15 +272,15 @@ export class CmsEntriesRootFolder_5_37_0_002 ); } - const execute = () => { - return batchWriteAll({ + const execute = async () => { + return await batchWriteAll({ table: this.ddbEntryEntity.table, items: ddbItems }); }; - const executeDdbEs = () => { - return batchWriteAll({ + const executeDdbEs = async () => { + return await batchWriteAll({ table: this.ddbEsEntryEntity.table, items: ddbEsItems }); diff --git a/packages/migrations/src/migrations/5.37.0/003/ddb-es/AcoFolderMigration.ts b/packages/migrations/src/migrations/5.37.0/003/ddb-es/AcoFolderMigration.ts index 9869ada4a64..1cdc644b628 100644 --- a/packages/migrations/src/migrations/5.37.0/003/ddb-es/AcoFolderMigration.ts +++ b/packages/migrations/src/migrations/5.37.0/003/ddb-es/AcoFolderMigration.ts @@ -7,8 +7,6 @@ import { createLocaleEntity } from "../entities/createLocaleEntity"; import { createTenantEntity } from "../entities/createTenantEntity"; import { createDdbEntryEntity, createDdbEsEntryEntity } from "../entities/createEntryEntity"; import { - batchWriteAll, - BatchWriteItem, esFindOne, esGetIndexExist, esGetIndexName, @@ -20,6 +18,7 @@ import { CmsEntryAcoFolder, I18NLocale, ListLocalesParams, Tenant } from "../typ import { ACO_FOLDER_MODEL_ID, ROOT_FOLDER, UPPERCASE_ROOT_FOLDER } from "../constants"; import { getElasticsearchLatestEntryData } from "./latestElasticsearchData"; import { getDecompressedData } from "~/migrations/5.37.0/003/utils/getDecompressedData"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; const isGroupMigrationCompleted = ( status: PrimitiveValue[] | boolean | undefined @@ -236,8 +235,12 @@ export class AcoRecords_5_37_0_003_AcoFolder `Processing batch #${batch} in group ${groupId} (${folders.length} folders).` ); - const ddbItems: BatchWriteItem[] = []; - const ddbEsItems: BatchWriteItem[] = []; + const entityBatch = createEntityWriteBatch({ + entity: this.ddbEntryEntity + }); + const elasticsearchEntityBatch = createEntityWriteBatch({ + entity: this.ddbEsEntryEntity + }); for (const folder of folders) { const folderPk = `T#${tenantId}#L#${localeCode}#CMS#CME#${folder.entryId}`; @@ -278,10 +281,8 @@ export class AcoRecords_5_37_0_003_AcoFolder TYPE: "cms.entry" }; - ddbItems.push( - this.ddbEntryEntity.putBatch(latestDdb), - this.ddbEntryEntity.putBatch(revisionDdb) - ); + entityBatch.put(latestDdb); + entityBatch.put(revisionDdb); const esLatestRecord = await get({ entity: this.ddbEsEntryEntity, @@ -316,21 +317,15 @@ export class AcoRecords_5_37_0_003_AcoFolder index: foldersIndexName }; - ddbEsItems.push(this.ddbEsEntryEntity.putBatch(latestDdbEs)); + elasticsearchEntityBatch.put(latestDdbEs); } - const executeDdb = () => { - return batchWriteAll({ - table: this.ddbEntryEntity.table, - items: ddbItems - }); + const executeDdb = async () => { + return entityBatch.execute(); }; - const executeDdbEs = () => { - return batchWriteAll({ - table: this.ddbEsEntryEntity.table, - items: ddbEsItems - }); + const executeDdbEs = async () => { + return elasticsearchEntityBatch.execute(); }; await executeWithRetry(executeDdb, { diff --git a/packages/migrations/src/migrations/5.40.0/001/ddb/index.ts b/packages/migrations/src/migrations/5.40.0/001/ddb/index.ts index 47166648883..ee0fa247684 100644 --- a/packages/migrations/src/migrations/5.40.0/001/ddb/index.ts +++ b/packages/migrations/src/migrations/5.40.0/001/ddb/index.ts @@ -4,18 +4,13 @@ import { DataMigrationContext, PrimaryDynamoTableSymbol } from "@webiny/data-migration"; -import { - batchWriteAll, - BatchWriteItem, - count, - ddbQueryAllWithCallback, - forEachTenantLocale -} from "~/utils"; +import { count, ddbQueryAllWithCallback, forEachTenantLocale } from "~/utils"; import { inject, makeInjectable } from "@webiny/ioc"; import { executeWithRetry, generateAlphaNumericId } from "@webiny/utils"; import { createBlockEntity } from "~/migrations/5.40.0/001/ddb/createBlockEntity"; import { ContentElement, PageBlock } from "./types"; import { compress, decompress } from "./compression"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; const isGroupMigrationCompleted = (status: boolean | undefined): status is boolean => { return typeof status === "boolean"; @@ -99,6 +94,10 @@ export class PbUniqueBlockElementIds_5_40_0_001 implements DataMigration { `Processing batch #${batch} in group ${groupId} (${blocks.length} blocks).` ); + const entityBatch = createEntityWriteBatch({ + entity: this.blockEntity + }); + const items = await Promise.all( blocks.map(async block => { const newContent = await this.generateElementIds(block); @@ -106,18 +105,17 @@ export class PbUniqueBlockElementIds_5_40_0_001 implements DataMigration { return null; } - return this.blockEntity.putBatch({ + const item = { ...block, content: newContent - }); + }; + entityBatch.put(item); + return item; }) ); - const execute = () => { - return batchWriteAll({ - table: this.blockEntity.table, - items: items.filter(Boolean) as BatchWriteItem[] - }); + const execute = async () => { + return await entityBatch.execute(); }; await executeWithRetry(execute, { diff --git a/packages/migrations/src/migrations/5.41.0/001/index.ts b/packages/migrations/src/migrations/5.41.0/001/index.ts index 6cec0e2917d..070146eb9f6 100644 --- a/packages/migrations/src/migrations/5.41.0/001/index.ts +++ b/packages/migrations/src/migrations/5.41.0/001/index.ts @@ -1,10 +1,11 @@ import { Table } from "@webiny/db-dynamodb/toolbox"; import { DataMigrationContext, PrimaryDynamoTableSymbol } from "@webiny/data-migration"; -import { queryOne, queryAll, batchWriteAll } from "~/utils"; +import { queryAll, queryOne } from "~/utils"; import { createTenantEntity } from "./createTenantEntity"; import { createUserEntity } from "./createUserEntity"; -import { makeInjectable, inject } from "@webiny/ioc"; +import { inject, makeInjectable } from "@webiny/ioc"; import { executeWithRetry } from "@webiny/utils"; +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; export class AdminUsers_5_41_0_001 { private readonly newUserEntity: ReturnType; @@ -71,22 +72,25 @@ export class AdminUsers_5_41_0_001 { continue; } - const newUsers = users - .filter(user => !Array.isArray(user.data.groups)) - .map(user => { - return this.newUserEntity.putBatch({ - ...user, - data: { - ...user.data, - groups: [user.data.group].filter(Boolean), - teams: [user.data.team].filter(Boolean) - } - }); - }); + const newUsersEntityBatch = createEntityWriteBatch({ + entity: this.newUserEntity, + put: users + .filter(user => !Array.isArray(user.data.groups)) + .map(user => { + return { + ...user, + data: { + ...user.data, + groups: [user.data.group].filter(Boolean), + teams: [user.data.team].filter(Boolean) + } + }; + }) + }); - await executeWithRetry(() => - batchWriteAll({ table: this.newUserEntity.table, items: newUsers }) - ); + await executeWithRetry(async () => { + return await newUsersEntityBatch.execute(); + }); } } } From ec79125285452b8047521f6e9d70a1fda4eccda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 11:41:36 +0100 Subject: [PATCH 11/19] fix(api-elasticsearch-tasks): implement entity wrapper --- .../src/definitions/entry.ts | 53 +++++----- .../src/tasks/Manager.ts | 19 ++-- .../tasks/reindexing/ReindexingTaskRunner.ts | 2 +- packages/api-elasticsearch-tasks/src/types.ts | 5 +- .../src/definitions/localeEntity.ts | 74 +++++++------- .../src/definitions/systemEntity.ts | 52 +++++----- .../locales/LocalesStorageOperations.ts | 96 +++++++++---------- .../system/SystemStorageOperations.ts | 32 +++---- packages/api-page-builder-so-ddb/package.json | 3 +- .../src/utils/batch/EntityWriteBatch.ts | 20 ++-- .../src/utils/batch/TableWriteBatch.ts | 22 +++-- packages/db-dynamodb/src/utils/batch/types.ts | 4 +- packages/db-dynamodb/src/utils/delete.ts | 14 +-- .../db-dynamodb/src/utils/entity/Entity.ts | 19 +++- .../db-dynamodb/src/utils/entity/types.ts | 17 +++- .../src/utils/forEachTenantLocale.ts | 12 ++- yarn.lock | 1 - 17 files changed, 243 insertions(+), 202 deletions(-) diff --git a/packages/api-elasticsearch-tasks/src/definitions/entry.ts b/packages/api-elasticsearch-tasks/src/definitions/entry.ts index 71c11511ff9..27b73fbf72b 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/entry.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/entry.ts @@ -1,36 +1,41 @@ /** * TODO If adding GSIs to the Elasticsearch table, add them here. */ -import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; +import type { TableDef } from "@webiny/db-dynamodb/toolbox"; +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import type { IEntity } from "@webiny/db-dynamodb"; +import { createEntity } from "@webiny/db-dynamodb"; interface Params { table: TableDef; entityName: string; } -export const createEntry = (params: Params): Entity => { +export const createEntry = (params: Params): IEntity => { const { table, entityName } = params; - return new Entity({ - name: entityName, - table, - attributes: { - PK: { - type: "string", - partitionKey: true - }, - SK: { - type: "string", - sortKey: true - }, - index: { - type: "string" - }, - data: { - type: "map" - }, - TYPE: { - type: "string" + return createEntity( + new Entity({ + name: entityName, + table, + attributes: { + PK: { + type: "string", + partitionKey: true + }, + SK: { + type: "string", + sortKey: true + }, + index: { + type: "string" + }, + data: { + type: "map" + }, + TYPE: { + type: "string" + } } - } - }); + }) + ); }; diff --git a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts index 7155aa7f042..2d981674e49 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts @@ -1,13 +1,16 @@ import { DynamoDBDocument, getDocumentClient } from "@webiny/aws-sdk/client-dynamodb"; import { Client, createElasticsearchClient } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; -import { Context, IManager } from "~/types"; +import type { Context, IManager } from "~/types"; import { createEntry } from "~/definitions/entry"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { ITaskResponse } from "@webiny/tasks/response/abstractions"; -import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; -import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; -import { ITimer } from "@webiny/handler-aws/utils"; +import type { ITaskResponse } from "@webiny/tasks/response/abstractions"; +import type { + IIsCloseToTimeoutCallable, + ITaskManagerStore +} from "@webiny/tasks/runner/abstractions"; +import type { BatchReadItem, IEntity } from "@webiny/db-dynamodb"; +import { batchReadAll } from "@webiny/db-dynamodb"; +import type { ITimer } from "@webiny/handler-aws/utils"; export interface ManagerParams { context: Context; @@ -31,7 +34,7 @@ export class Manager implements IManager { public readonly store: ITaskManagerStore; public readonly timer: ITimer; - private readonly entities: Record> = {}; + private readonly entities: Record = {}; public constructor(params: ManagerParams) { this.context = params.context; @@ -58,7 +61,7 @@ export class Manager implements IManager { this.timer = params.timer; } - public getEntity(name: string): Entity { + public getEntity(name: string): IEntity { if (this.entities[name]) { return this.entities[name]; } diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts index f38ae06b763..930eb1b8166 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts @@ -113,7 +113,7 @@ export class ReindexingTaskRunner { /** * Reindexing will be triggered by the `putBatch` method. */ - tableWriteBatch.put(entity, { + tableWriteBatch.put(entity.entity, { ...item, TYPE: item.TYPE || "unknown", modified: new Date().toISOString() diff --git a/packages/api-elasticsearch-tasks/src/types.ts b/packages/api-elasticsearch-tasks/src/types.ts index 87a0f5cbd49..8446398e1ee 100644 --- a/packages/api-elasticsearch-tasks/src/types.ts +++ b/packages/api-elasticsearch-tasks/src/types.ts @@ -1,5 +1,4 @@ import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; -import type { Entity } from "@webiny/db-dynamodb/toolbox"; import type { Context as TasksContext, IIsCloseToTimeoutCallable, @@ -10,7 +9,7 @@ import type { import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; import type { Client } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; -import type { BatchReadItem } from "@webiny/db-dynamodb"; +import type { BatchReadItem, IEntity } from "@webiny/db-dynamodb"; import type { ITimer } from "@webiny/handler-aws"; import type { GenericRecord } from "@webiny/api/types"; @@ -72,7 +71,7 @@ export interface IManager< readonly store: ITaskManagerStore; readonly timer: ITimer; - getEntity: (name: string) => Entity; + getEntity: (name: string) => IEntity; read(items: BatchReadItem[]): Promise; } diff --git a/packages/api-i18n-ddb/src/definitions/localeEntity.ts b/packages/api-i18n-ddb/src/definitions/localeEntity.ts index 4513054a2d7..36e9c6ab38b 100644 --- a/packages/api-i18n-ddb/src/definitions/localeEntity.ts +++ b/packages/api-i18n-ddb/src/definitions/localeEntity.ts @@ -1,43 +1,49 @@ import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; -import { I18NContext } from "@webiny/api-i18n/types"; +import type { I18NContext } from "@webiny/api-i18n/types"; import { getExtraAttributes } from "@webiny/db-dynamodb/utils/attributes"; +import type { IEntity } from "@webiny/db-dynamodb"; +import { createEntity } from "@webiny/db-dynamodb"; -export default (params: { +export interface ILocaleEntityParams { context: I18NContext; table: Table; -}): Entity => { +} + +export default (params: ILocaleEntityParams): IEntity => { const { context, table } = params; const entityName = "I18NLocale"; const attributes = getExtraAttributes(context, entityName); - return new Entity({ - name: entityName, - table, - attributes: { - PK: { - partitionKey: true - }, - SK: { - sortKey: true - }, - createdOn: { - type: "string" - }, - createdBy: { - type: "map" - }, - code: { - type: "string" - }, - default: { - type: "boolean" - }, - webinyVersion: { - type: "string" - }, - tenant: { - type: "string" - }, - ...attributes - } - }); + return createEntity( + new Entity({ + name: entityName, + table, + attributes: { + PK: { + partitionKey: true + }, + SK: { + sortKey: true + }, + createdOn: { + type: "string" + }, + createdBy: { + type: "map" + }, + code: { + type: "string" + }, + default: { + type: "boolean" + }, + webinyVersion: { + type: "string" + }, + tenant: { + type: "string" + }, + ...attributes + } + }) + ); }; diff --git a/packages/api-i18n-ddb/src/definitions/systemEntity.ts b/packages/api-i18n-ddb/src/definitions/systemEntity.ts index a2e80306a98..19a77f25b32 100644 --- a/packages/api-i18n-ddb/src/definitions/systemEntity.ts +++ b/packages/api-i18n-ddb/src/definitions/systemEntity.ts @@ -1,31 +1,35 @@ import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; -import { I18NContext } from "@webiny/api-i18n/types"; -import { getExtraAttributes } from "@webiny/db-dynamodb/utils/attributes"; +import type { I18NContext } from "@webiny/api-i18n/types"; +import { getExtraAttributesFromPlugins } from "@webiny/db-dynamodb/utils/attributes"; +import type { IEntity } from "@webiny/db-dynamodb"; +import { createEntity } from "@webiny/db-dynamodb"; export default (params: { - context: I18NContext; + context: Pick; table: Table; -}): Entity => { +}): IEntity => { const { context, table } = params; const entityName = "I18NSystem"; - const attributes = getExtraAttributes(context, entityName); - return new Entity({ - name: entityName, - table, - attributes: { - PK: { - partitionKey: true - }, - SK: { - sortKey: true - }, - version: { - type: "string" - }, - tenant: { - type: "string" - }, - ...attributes - } - }); + const attributes = getExtraAttributesFromPlugins(context.plugins, entityName); + return createEntity( + new Entity({ + name: entityName, + table, + attributes: { + PK: { + partitionKey: true + }, + SK: { + sortKey: true + }, + version: { + type: "string" + }, + tenant: { + type: "string" + }, + ...attributes + } + }) + ); }; diff --git a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts index 96abf44272f..23db6b62431 100644 --- a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts +++ b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts @@ -1,4 +1,4 @@ -import { +import type { I18NContext, I18NLocaleData, I18NLocalesStorageOperations, @@ -11,17 +11,19 @@ import { I18NLocalesStorageOperationsUpdateDefaultParams, I18NLocalesStorageOperationsUpdateParams } from "@webiny/api-i18n/types"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import type { Table } from "@webiny/db-dynamodb/toolbox"; import WebinyError from "@webiny/error"; import defineTable from "~/definitions/table"; import defineLocaleEntity from "~/definitions/localeEntity"; -import { queryAll, QueryAllParams } from "@webiny/db-dynamodb/utils/query"; -import { filterItems } from "@webiny/db-dynamodb/utils/filter"; -import { sortItems } from "@webiny/db-dynamodb/utils/sort"; -import { createListResponse } from "@webiny/db-dynamodb/utils/listResponse"; -import { cleanupItems } from "@webiny/db-dynamodb/utils/cleanup"; +import type { IEntity, QueryAllParams } from "@webiny/db-dynamodb"; +import { + cleanupItems, + createListResponse, + filterItems, + queryAll, + sortItems +} from "@webiny/db-dynamodb"; import { LocaleDynamoDbFieldPlugin } from "~/plugins/LocaleDynamoDbFieldPlugin"; -import { deleteItem, getClean, put } from "@webiny/db-dynamodb"; interface ConstructorParams { context: I18NContext; @@ -32,7 +34,7 @@ const DEFAULT_SORT_KEY = "default"; export class LocalesStorageOperations implements I18NLocalesStorageOperations { private readonly context: I18NContext; private readonly table: Table; - private readonly entity: Entity; + private readonly entity: IEntity; public constructor({ context }: ConstructorParams) { this.context = context; @@ -48,12 +50,9 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { public async getDefault(params: I18NLocalesStorageOperationsGetDefaultParams) { try { - return await getClean({ - entity: this.entity, - keys: { - PK: this.createDefaultPartitionKey(params), - SK: DEFAULT_SORT_KEY - } + return this.entity.getClean({ + PK: this.createDefaultPartitionKey(params), + SK: DEFAULT_SORT_KEY }); } catch (ex) { throw new WebinyError( @@ -65,12 +64,9 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { public async get(params: I18NLocalesStorageOperationsGetParams) { try { - return await getClean({ - entity: this.entity, - keys: { - PK: this.createPartitionKey(params), - SK: params.code - } + return this.entity.getClean({ + PK: this.createPartitionKey(params), + SK: params.code }); } catch (ex) { throw new WebinyError( @@ -89,12 +85,9 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { }; try { - await put({ - entity: this.entity, - item: { - ...locale, - ...keys - } + await this.entity.put({ + ...locale, + ...keys }); return locale; } catch (ex) { @@ -117,12 +110,9 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { SK: this.getSortKey(locale) }; try { - await put({ - entity: this.entity, - item: { - ...locale, - ...keys - } + await this.entity.put({ + ...locale, + ...keys }); return locale; } catch (ex) { @@ -144,23 +134,24 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { /** * Set the locale as the default one. */ - const batch = [ - { - ...locale, - PK: this.createPartitionKey(locale), - SK: this.getSortKey(locale) - }, - { - ...locale, - PK: this.createDefaultPartitionKey(locale), - SK: DEFAULT_SORT_KEY - } - ]; + const entityBatch = this.entity.createEntityWriter(); + + entityBatch.put({ + ...locale, + PK: this.createPartitionKey(locale), + SK: this.getSortKey(locale) + }); + entityBatch.put({ + ...locale, + PK: this.createDefaultPartitionKey(locale), + SK: DEFAULT_SORT_KEY + }); + /** * Set the previous locale not to be default in its data. */ if (previous) { - batch.push({ + entityBatch.put({ ...previous, default: false, PK: this.createPartitionKey(locale), @@ -168,8 +159,10 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { }); } + const batch = entityBatch.items; + try { - await this.table.batchWrite(batch.map(item => this.entity.putBatch(item))); + await entityBatch.execute(); return locale; } catch (ex) { throw new WebinyError( @@ -190,10 +183,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { SK: this.getSortKey(locale) }; try { - await deleteItem({ - entity: this.entity, - keys - }); + await this.entity.delete(keys); } catch (ex) { throw new WebinyError( ex.message || "Cannot delete I18N locale.", @@ -256,7 +246,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { * Use the common db-dynamodb method to create the required response. */ return createListResponse({ - items: cleanupItems(this.entity, sortedFiles), + items: cleanupItems(this.entity.entity, sortedFiles), after, totalCount, limit @@ -295,7 +285,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { delete where.default; } return { - entity: this.entity, + entity: this.entity.entity, partitionKey, options: {} }; diff --git a/packages/api-i18n-ddb/src/operations/system/SystemStorageOperations.ts b/packages/api-i18n-ddb/src/operations/system/SystemStorageOperations.ts index e333f8eef59..b840cf0c0e3 100644 --- a/packages/api-i18n-ddb/src/operations/system/SystemStorageOperations.ts +++ b/packages/api-i18n-ddb/src/operations/system/SystemStorageOperations.ts @@ -1,15 +1,14 @@ -import { +import type { I18NContext, I18NSystem, I18NSystemStorageOperations, I18NSystemStorageOperationsCreate, I18NSystemStorageOperationsUpdate } from "@webiny/api-i18n/types"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; import WebinyError from "@webiny/error"; import defineSystemEntity from "~/definitions/systemEntity"; import defineTable from "~/definitions/table"; -import { getClean, put } from "@webiny/db-dynamodb"; +import type { IEntity } from "@webiny/db-dynamodb"; interface ConstructorParams { context: I18NContext; @@ -19,7 +18,7 @@ const SORT_KEY = "I18N"; export class SystemStorageOperations implements I18NSystemStorageOperations { private readonly _context: I18NContext; - private readonly _entity: Entity; + private readonly entity: IEntity; private get partitionKey(): string { const tenant = this._context.tenancy.getCurrentTenant(); @@ -35,7 +34,7 @@ export class SystemStorageOperations implements I18NSystemStorageOperations { context }); - this._entity = defineSystemEntity({ + this.entity = defineSystemEntity({ context, table }); @@ -48,10 +47,7 @@ export class SystemStorageOperations implements I18NSystemStorageOperations { }; try { - return await getClean({ - entity: this._entity, - keys - }); + return await this.entity.getClean(keys); } catch (ex) { throw new WebinyError( "Could not load system data from the database.", @@ -67,12 +63,9 @@ export class SystemStorageOperations implements I18NSystemStorageOperations { SK: SORT_KEY }; try { - await put({ - entity: this._entity, - item: { - ...system, - ...keys - } + await this.entity.put({ + ...system, + ...keys }); return system; } catch (ex) { @@ -90,12 +83,9 @@ export class SystemStorageOperations implements I18NSystemStorageOperations { SK: SORT_KEY }; try { - await put({ - entity: this._entity, - item: { - ...system, - ...keys - } + await this.entity.put({ + ...system, + ...keys }); return system; } catch (ex) { diff --git a/packages/api-page-builder-so-ddb/package.json b/packages/api-page-builder-so-ddb/package.json index ce0353d12e0..9ceb55a9e92 100644 --- a/packages/api-page-builder-so-ddb/package.json +++ b/packages/api-page-builder-so-ddb/package.json @@ -25,8 +25,7 @@ "@webiny/error": "0.0.0", "@webiny/handler-db": "0.0.0", "@webiny/utils": "0.0.0", - "dataloader": "^2.0.0", - "lodash": "^4.17.21" + "dataloader": "^2.0.0" }, "devDependencies": { "@babel/cli": "^7.23.9", diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index 5ead5075342..fc46aa6305b 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -21,11 +21,15 @@ export interface IEntityWriteBatchParams { export class EntityWriteBatch implements IEntityWriteBatch { private readonly entity: Entity; - private readonly items: BatchWriteItem[] = []; + private readonly _items: BatchWriteItem[] = []; private readonly builder: IEntityWriteBatchBuilder; public get total(): number { - return this.items.length; + return this._items.length; + } + + public get items(): BatchWriteItem[] { + return Array.from(this.items); } public constructor(params: IEntityWriteBatchParams) { @@ -45,26 +49,26 @@ export class EntityWriteBatch implements IEntityWriteBatch { } public put>(item: IPutBatchItem): void { - this.items.push(this.builder.put(item)); + this._items.push(this.builder.put(item)); } public delete(item: IDeleteBatchItem): void { - this.items.push(this.builder.delete(item)); + this._items.push(this.builder.delete(item)); } public combine(items: BatchWriteItem[]): ITableWriteBatch { return createTableWriteBatch({ table: this.entity!.table as TableDef, - items: this.items.concat(items) + items: this._items.concat(items) }); } public async execute(): Promise { - if (this.items.length === 0) { + if (this._items.length === 0) { return []; } - const items = [...this.items]; - this.items.length = 0; + const items = Array.from(this._items); + this._items.length = 0; return await batchWriteAll({ items, table: this.entity.table diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index d4d57a1474e..1fcb000089e 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -17,11 +17,15 @@ export interface ITableWriteBatchParams { export class TableWriteBatch implements ITableWriteBatch { private readonly table: TableDef; - private readonly items: BatchWriteItem[] = []; + private readonly _items: BatchWriteItem[] = []; private readonly builders: Map = new Map(); public get total(): number { - return this.items.length; + return this._items.length; + } + + public get items(): BatchWriteItem[] { + return Array.from(this._items); } public constructor(params: ITableWriteBatchParams) { @@ -29,32 +33,32 @@ export class TableWriteBatch implements ITableWriteBatch { if (!params.items?.length) { return; } - this.items.push(...params.items); + this._items.push(...params.items); } public put(entity: Entity, item: IPutBatchItem): void { const builder = this.getBuilder(entity); - this.items.push(builder.put(item)); + this._items.push(builder.put(item)); } public delete(entity: Entity, item: IDeleteBatchItem): void { const builder = this.getBuilder(entity); - this.items.push(builder.delete(item)); + this._items.push(builder.delete(item)); } public combine(items: BatchWriteItem[]): ITableWriteBatch { return createTableWriteBatch({ table: this.table, - items: this.items.concat(items) + items: this._items.concat(items) }); } public async execute(): Promise { - if (this.items.length === 0) { + if (this._items.length === 0) { return []; } - const items = [...this.items]; - this.items.length = 0; + const items = Array.from(this._items); + this._items.length = 0; return await batchWriteAll({ items, table: this.table diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index e13111cee32..74ecdfa9496 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -39,7 +39,7 @@ export interface IEntityWriteBatchBuilder { export interface IEntityWriteBatch { readonly total: number; // readonly entity: Entity; - // readonly items: BatchWriteItem[]; + readonly items: BatchWriteItem[]; // readonly builder: IEntityWriteBatchBuilder; put(item: IPutBatchItem): void; @@ -51,7 +51,7 @@ export interface IEntityWriteBatch { export interface ITableWriteBatch { readonly total: number; // readonly table: TableDef; - // readonly items: BatchWriteItem[]; + readonly items: BatchWriteItem[]; put(entity: Entity, item: IPutBatchItem): void; delete(entity: Entity, item: IDeleteBatchItem): void; execute(): Promise; diff --git a/packages/db-dynamodb/src/utils/delete.ts b/packages/db-dynamodb/src/utils/delete.ts index 0540e39d57e..b4adc394d08 100644 --- a/packages/db-dynamodb/src/utils/delete.ts +++ b/packages/db-dynamodb/src/utils/delete.ts @@ -1,14 +1,14 @@ import { Entity } from "~/toolbox"; - -interface Params { +export interface IDeleteItemKeys { + PK: string; + SK: string; +} +export interface IDeleteItemParams { entity: Entity; - keys: { - PK: string; - SK: string; - }; + keys: IDeleteItemKeys; } -export const deleteItem = async (params: Params) => { +export const deleteItem = async (params: IDeleteItemParams) => { const { entity, keys } = params; return await entity.delete(keys, { diff --git a/packages/db-dynamodb/src/utils/entity/Entity.ts b/packages/db-dynamodb/src/utils/entity/Entity.ts index 3593993df80..4c74f014ae8 100644 --- a/packages/db-dynamodb/src/utils/entity/Entity.ts +++ b/packages/db-dynamodb/src/utils/entity/Entity.ts @@ -4,7 +4,8 @@ import type { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table"; import { IEntity } from "./types"; import { IPutParamsItem, put } from "~/utils/put"; -import { getClean, GetRecordParamsKeys } from "~/utils/get"; +import { get, getClean, GetRecordParamsKeys } from "~/utils/get"; +import { deleteItem, IDeleteItemKeys } from "~/utils"; export class Entity implements IEntity { public readonly entity: BaseEntity; @@ -32,12 +33,26 @@ export class Entity implements IEntity { }); } - public async get(keys: GetRecordParamsKeys): ReturnType> { + public async get(keys: GetRecordParamsKeys): ReturnType> { + return get({ + entity: this.entity, + keys + }); + } + + public async getClean(keys: GetRecordParamsKeys): ReturnType> { return getClean({ entity: this.entity, keys }); } + + public async delete(keys: IDeleteItemKeys): ReturnType { + return deleteItem({ + entity: this.entity, + keys + }); + } } export const createEntity = (params: BaseEntity): IEntity => { diff --git a/packages/db-dynamodb/src/utils/entity/types.ts b/packages/db-dynamodb/src/utils/entity/types.ts index 2d5d8ae106a..84227553315 100644 --- a/packages/db-dynamodb/src/utils/entity/types.ts +++ b/packages/db-dynamodb/src/utils/entity/types.ts @@ -1,8 +1,21 @@ -import { Entity as BaseEntity } from "dynamodb-toolbox"; -import { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; +import type { Entity as BaseEntity } from "dynamodb-toolbox"; +import type { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; +import type { + deleteItem, + get, + getClean, + GetRecordParamsKeys, + IDeleteItemKeys, + IPutParamsItem, + put +} from "~/utils"; export interface IEntity { readonly entity: BaseEntity; createEntityWriter(): IEntityWriteBatch; createTableWriter(): ITableWriteBatch; + put(item: IPutParamsItem): ReturnType; + get(keys: GetRecordParamsKeys): ReturnType>; + getClean(keys: GetRecordParamsKeys): ReturnType>; + delete(keys: IDeleteItemKeys): ReturnType; } diff --git a/packages/migrations/src/utils/forEachTenantLocale.ts b/packages/migrations/src/utils/forEachTenantLocale.ts index a88284551a9..bd1f7cb13c7 100644 --- a/packages/migrations/src/utils/forEachTenantLocale.ts +++ b/packages/migrations/src/utils/forEachTenantLocale.ts @@ -1,8 +1,18 @@ import { createLocaleEntity, createTenantEntity, queryAll } from "~/utils"; -import { I18NLocale, Tenant } from "~/migrations/5.37.0/003/types"; import { Table } from "@webiny/db-dynamodb/toolbox"; import { Logger } from "@webiny/logger"; +export interface Tenant { + data: { + id: string; + name: string; + }; +} + +export interface I18NLocale { + code: string; +} + type ForEachTenantLocaleCallback = (params: { tenantId: string; localeCode: string; diff --git a/yarn.lock b/yarn.lock index 9474c8c091f..873244d4526 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14716,7 +14716,6 @@ __metadata: dataloader: ^2.0.0 jest: ^29.7.0 jest-dynalite: ^3.2.0 - lodash: ^4.17.21 rimraf: ^5.0.5 ttypescript: ^1.5.12 typescript: 4.9.5 From da623c3e2796a1842e8e7af404a5743e29343a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 13:34:15 +0100 Subject: [PATCH 12/19] fix(db-dynamodb): table wrapper --- .../src/definitions/entry.ts | 47 ++++++------ .../src/utils/batch/EntityReadBatch.ts | 55 ++++++++++++++ .../src/utils/batch/EntityReadBatchBuilder.ts | 29 +++++++ .../src/utils/batch/TableReadBatch.ts | 76 +++++++++++++++++++ .../src/utils/batch/TableWriteBatch.ts | 2 +- packages/db-dynamodb/src/utils/batch/index.ts | 2 + packages/db-dynamodb/src/utils/batch/types.ts | 35 ++++++++- .../db-dynamodb/src/utils/entity/Entity.ts | 51 ++++++++++--- .../db-dynamodb/src/utils/entity/types.ts | 3 +- packages/db-dynamodb/src/utils/index.ts | 1 + packages/db-dynamodb/src/utils/table/Table.ts | 31 ++++++++ packages/db-dynamodb/src/utils/table/index.ts | 2 + packages/db-dynamodb/src/utils/table/types.ts | 8 ++ 13 files changed, 304 insertions(+), 38 deletions(-) create mode 100644 packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts create mode 100644 packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts create mode 100644 packages/db-dynamodb/src/utils/batch/TableReadBatch.ts create mode 100644 packages/db-dynamodb/src/utils/table/Table.ts create mode 100644 packages/db-dynamodb/src/utils/table/index.ts create mode 100644 packages/db-dynamodb/src/utils/table/types.ts diff --git a/packages/api-elasticsearch-tasks/src/definitions/entry.ts b/packages/api-elasticsearch-tasks/src/definitions/entry.ts index 27b73fbf72b..e2559d417f4 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/entry.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/entry.ts @@ -2,7 +2,6 @@ * TODO If adding GSIs to the Elasticsearch table, add them here. */ import type { TableDef } from "@webiny/db-dynamodb/toolbox"; -import { Entity } from "@webiny/db-dynamodb/toolbox"; import type { IEntity } from "@webiny/db-dynamodb"; import { createEntity } from "@webiny/db-dynamodb"; @@ -13,29 +12,27 @@ interface Params { export const createEntry = (params: Params): IEntity => { const { table, entityName } = params; - return createEntity( - new Entity({ - name: entityName, - table, - attributes: { - PK: { - type: "string", - partitionKey: true - }, - SK: { - type: "string", - sortKey: true - }, - index: { - type: "string" - }, - data: { - type: "map" - }, - TYPE: { - type: "string" - } + return createEntity({ + name: entityName, + table, + attributes: { + PK: { + type: "string", + partitionKey: true + }, + SK: { + type: "string", + sortKey: true + }, + index: { + type: "string" + }, + data: { + type: "map" + }, + TYPE: { + type: "string" } - }) - ); + } + }); }; diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts new file mode 100644 index 00000000000..1439693776e --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts @@ -0,0 +1,55 @@ +import type { + IEntityReadBatch, + IEntityReadBatchBuilder, + IEntityReadBatchBuilderGetResponse, + IEntityReadBatchKey, + IPutBatchItem +} from "./types"; +import type { Entity } from "dynamodb-toolbox"; +import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table/types"; +import { batchReadAll } from "./batchRead"; +import { GenericRecord } from "@webiny/api/types"; +import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; + +export interface IEntityReadBatchParams { + entity: Entity; + read?: IPutBatchItem[]; +} + +export class EntityReadBatch implements IEntityReadBatch { + private readonly entity: Entity; + private readonly builder: IEntityReadBatchBuilder; + private readonly items: IEntityReadBatchBuilderGetResponse[] = []; + + public constructor(params: IEntityReadBatchParams) { + if (!params.entity.name) { + throw new Error(`No name provided for entity.`); + } else if (!params.entity.table) { + throw new Error(`No table provided for entity ${params.entity.name}.`); + } + this.entity = params.entity; + this.builder = createEntityReadBatchBuilder(this.entity); + } + public get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void { + if (Array.isArray(input)) { + this.items.push( + ...input.map(item => { + return this.builder.get(item); + }) + ); + return; + } + this.items.push(this.builder.get(input)); + } + + public async execute() { + return await batchReadAll({ + table: this.entity.table as TableDef, + items: this.items + }); + } +} + +export const createEntityReadBatch = (params: IEntityReadBatchParams): IEntityReadBatch => { + return new EntityReadBatch(params); +}; diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts b/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts new file mode 100644 index 00000000000..ca231f32536 --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts @@ -0,0 +1,29 @@ +import type { Entity } from "dynamodb-toolbox"; +import type { + IEntityReadBatchBuilder, + IEntityReadBatchBuilderGetResponse, + IEntityReadBatchKey +} from "./types"; +import { WebinyError } from "@webiny/error"; + +export class EntityReadBatchBuilder implements IEntityReadBatchBuilder { + private readonly entity: Entity; + + public constructor(entity: Entity) { + this.entity = entity; + } + + public get(item: IEntityReadBatchKey): IEntityReadBatchBuilderGetResponse { + const result = this.entity.getBatch(item); + if (!result.Table) { + throw new WebinyError(`No table provided for entity ${this.entity.name}.`); + } else if (!result.Key) { + throw new WebinyError(`No key provided for entity ${this.entity.name}.`); + } + return result as IEntityReadBatchBuilderGetResponse; + } +} + +export const createEntityReadBatchBuilder = (entity: Entity): IEntityReadBatchBuilder => { + return new EntityReadBatchBuilder(entity); +}; diff --git a/packages/db-dynamodb/src/utils/batch/TableReadBatch.ts b/packages/db-dynamodb/src/utils/batch/TableReadBatch.ts new file mode 100644 index 00000000000..f8f6b374013 --- /dev/null +++ b/packages/db-dynamodb/src/utils/batch/TableReadBatch.ts @@ -0,0 +1,76 @@ +import type { Entity, TableDef } from "~/toolbox"; +import type { + IEntityReadBatchBuilder, + IEntityReadBatchBuilderGetResponse, + ITableReadBatch, + ITableReadBatchKey +} from "./types"; +import { batchReadAll } from "./batchRead"; +import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; +import type { GenericRecord } from "@webiny/api/types"; +import { WebinyError } from "@webiny/error"; + +export interface ITableReadBatchParams { + table: TableDef; +} + +export class TableReadBatch implements ITableReadBatch { + private readonly table: TableDef; + + private readonly _items: IEntityReadBatchBuilderGetResponse[] = []; + private readonly builders: Map = new Map(); + + public constructor(params: ITableReadBatchParams) { + this.table = params.table; + } + + public get total(): number { + return this._items.length; + } + + public get items(): IEntityReadBatchBuilderGetResponse[] { + return Array.from(this._items); + } + + public get(entity: Entity, input: ITableReadBatchKey): void { + const builder = this.getBuilder(entity); + + const items = Array.isArray(input) ? input : [input]; + for (const item of items) { + /** + * We cannot read from two tables at the same time, so check for that. + */ + if (this.table.name !== entity.table!.name) { + throw new WebinyError(`Cannot read from two different tables at the same time.`); + } + + this._items.push(builder.get(item)); + } + } + + public async execute(): Promise { + if (this._items.length === 0) { + return []; + } + const items = Array.from(this._items); + this._items.length = 0; + return await batchReadAll({ + items, + table: this.table + }); + } + + private getBuilder(entity: Entity): IEntityReadBatchBuilder { + const builder = this.builders.get(entity.name); + if (builder) { + return builder; + } + const newBuilder = createEntityReadBatchBuilder(entity); + this.builders.set(entity.name, newBuilder); + return newBuilder; + } +} + +export const createTableReadBatch = (params: ITableReadBatchParams): ITableReadBatch => { + return new TableReadBatch(params); +}; diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index 1fcb000089e..123da97b8ab 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -8,7 +8,7 @@ import { ITableWriteBatch } from "./types"; import { batchWriteAll } from "./batchWrite"; -import { createEntityWriteBatchBuilder } from "~/utils/batch/EntityWriteBatchBuilder"; +import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; export interface ITableWriteBatchParams { table: TableDef; diff --git a/packages/db-dynamodb/src/utils/batch/index.ts b/packages/db-dynamodb/src/utils/batch/index.ts index 58cdc76c1ff..84638425a18 100644 --- a/packages/db-dynamodb/src/utils/batch/index.ts +++ b/packages/db-dynamodb/src/utils/batch/index.ts @@ -1,5 +1,7 @@ export * from "./batchRead"; export * from "./batchWrite"; +export * from "./EntityReadBatch"; +export * from "./EntityReadBatchBuilder"; export * from "./EntityWriteBatch"; export * from "./EntityWriteBatchBuilder"; export * from "./TableWriteBatch"; diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index 74ecdfa9496..bec52153c6e 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -1,5 +1,7 @@ import type { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity } from "~/toolbox"; +import type { Entity, TableDef } from "~/toolbox"; +import type { GenericRecord } from "@webiny/api/types"; +import type { batchReadAll } from "~/utils"; export interface BatchWriteResponse { next?: () => Promise; @@ -57,3 +59,34 @@ export interface ITableWriteBatch { execute(): Promise; combine(items: BatchWriteItem[]): ITableWriteBatch; } + +export interface IEntityReadBatchKey { + PK: string; + SK: string; +} + +export interface IEntityReadBatch { + get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void; + execute(): ReturnType>; +} + +export interface IEntityReadBatchBuilderGetResponse { + Table: TableDef; + Key: IEntityReadBatchKey; +} + +export interface IEntityReadBatchBuilder { + get(item: IEntityReadBatchKey): IEntityReadBatchBuilderGetResponse; +} + +export interface ITableReadBatchKey { + PK: string; + SK: string; +} + +export interface ITableReadBatch { + readonly total: number; + readonly items: IEntityReadBatchBuilderGetResponse[]; + get(entity: Entity, input: ITableReadBatchKey | ITableReadBatchKey[]): void; + execute(): Promise; +} diff --git a/packages/db-dynamodb/src/utils/entity/Entity.ts b/packages/db-dynamodb/src/utils/entity/Entity.ts index 4c74f014ae8..3bb400bb37a 100644 --- a/packages/db-dynamodb/src/utils/entity/Entity.ts +++ b/packages/db-dynamodb/src/utils/entity/Entity.ts @@ -1,17 +1,48 @@ -import type { Entity as BaseEntity } from "dynamodb-toolbox"; -import { createEntityWriteBatch, createTableWriteBatch } from "~/utils/batch"; -import type { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; +import { Entity as BaseEntity } from "dynamodb-toolbox"; +import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "../batch/types"; import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table"; -import { IEntity } from "./types"; -import { IPutParamsItem, put } from "~/utils/put"; -import { get, getClean, GetRecordParamsKeys } from "~/utils/get"; -import { deleteItem, IDeleteItemKeys } from "~/utils"; +import type { IEntity } from "./types"; +import type { IPutParamsItem } from "../put"; +import { put } from "../put"; +import type { GetRecordParamsKeys } from "../get"; +import { get, getClean } from "../get"; +import type { IDeleteItemKeys } from "../delete"; +import { deleteItem } from "../delete"; +import type { + AttributeDefinitions, + EntityConstructor as BaseEntityConstructor, + Readonly +} from "dynamodb-toolbox/dist/cjs/classes/Entity/types"; +import { createEntityReadBatch } from "../batch/EntityReadBatch"; +import { createEntityWriteBatch } from "../batch/EntityWriteBatch"; +import { createTableWriteBatch } from "../batch/TableWriteBatch"; + +export type EntityConstructor< + T extends Readonly = Readonly +> = BaseEntityConstructor< + TableDef, + string, + true, + true, + true, + "created", + "modified", + "entity", + false, + T +>; export class Entity implements IEntity { public readonly entity: BaseEntity; - public constructor(entity: BaseEntity) { - this.entity = entity; + public constructor(params: EntityConstructor) { + this.entity = new BaseEntity(params); + } + + public createEntityReader(): IEntityReadBatch { + return createEntityReadBatch({ + entity: this.entity + }); } public createEntityWriter(): IEntityWriteBatch { @@ -55,6 +86,6 @@ export class Entity implements IEntity { } } -export const createEntity = (params: BaseEntity): IEntity => { +export const createEntity = (params: EntityConstructor): IEntity => { return new Entity(params); }; diff --git a/packages/db-dynamodb/src/utils/entity/types.ts b/packages/db-dynamodb/src/utils/entity/types.ts index 84227553315..1eaafc3389c 100644 --- a/packages/db-dynamodb/src/utils/entity/types.ts +++ b/packages/db-dynamodb/src/utils/entity/types.ts @@ -1,5 +1,5 @@ import type { Entity as BaseEntity } from "dynamodb-toolbox"; -import type { IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; +import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; import type { deleteItem, get, @@ -12,6 +12,7 @@ import type { export interface IEntity { readonly entity: BaseEntity; + createEntityReader(): IEntityReadBatch; createEntityWriter(): IEntityWriteBatch; createTableWriter(): ITableWriteBatch; put(item: IPutParamsItem): ReturnType; diff --git a/packages/db-dynamodb/src/utils/index.ts b/packages/db-dynamodb/src/utils/index.ts index e7af65f04e6..58f326db072 100644 --- a/packages/db-dynamodb/src/utils/index.ts +++ b/packages/db-dynamodb/src/utils/index.ts @@ -15,3 +15,4 @@ export * from "./table"; export * from "./update"; export * from "./batch"; export * from "./entity"; +export * from "./table"; diff --git a/packages/db-dynamodb/src/utils/table/Table.ts b/packages/db-dynamodb/src/utils/table/Table.ts new file mode 100644 index 00000000000..c3429a369ff --- /dev/null +++ b/packages/db-dynamodb/src/utils/table/Table.ts @@ -0,0 +1,31 @@ +import type { TableConstructor } from "~/toolbox"; +import { Table as BaseTable } from "~/toolbox"; +import type { ITable } from "./types"; +import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; +import { createTableWriteBatch } from "../batch/TableWriteBatch"; +import { createTableReadBatch } from "../batch/TableReadBatch"; + +export class Table< + Name extends string = string, + PartitionKey extends string = string, + SortKey extends string = string +> implements ITable +{ + public readonly table: BaseTable; + + public constructor(params: TableConstructor) { + this.table = new BaseTable(params); + } + + public createWriter(): ITableWriteBatch { + return createTableWriteBatch({ + table: this.table + }); + } + + public createReader(): ITableReadBatch { + return createTableReadBatch({ + table: this.table + }); + } +} diff --git a/packages/db-dynamodb/src/utils/table/index.ts b/packages/db-dynamodb/src/utils/table/index.ts new file mode 100644 index 00000000000..865db5984b5 --- /dev/null +++ b/packages/db-dynamodb/src/utils/table/index.ts @@ -0,0 +1,2 @@ +export * from "./Table"; +export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/table/types.ts b/packages/db-dynamodb/src/utils/table/types.ts new file mode 100644 index 00000000000..3aa3c780e1a --- /dev/null +++ b/packages/db-dynamodb/src/utils/table/types.ts @@ -0,0 +1,8 @@ +import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table/types"; +import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; + +export interface ITable { + table: TableDef; + createWriter(): ITableWriteBatch; + createReader(): ITableReadBatch; +} From ac14f843fc6fa7aa47686b005bbd58e9c503caf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 13:53:36 +0100 Subject: [PATCH 13/19] fix(db-dynamodb): table scan --- packages/db-dynamodb/src/toolbox.ts | 2 ++ .../src/utils/batch/EntityReadBatch.ts | 3 +- .../src/utils/batch/EntityReadBatchBuilder.ts | 2 +- .../src/utils/batch/EntityWriteBatch.ts | 3 +- .../db-dynamodb/src/utils/entity/Entity.ts | 31 ++++++++++++++----- .../db-dynamodb/src/utils/entity/types.ts | 12 +++++-- packages/db-dynamodb/src/utils/scan.ts | 8 ++--- packages/db-dynamodb/src/utils/table/Table.ts | 10 +++++- packages/db-dynamodb/src/utils/table/types.ts | 6 ++++ 9 files changed, 57 insertions(+), 20 deletions(-) diff --git a/packages/db-dynamodb/src/toolbox.ts b/packages/db-dynamodb/src/toolbox.ts index 22b943165eb..7d90fb1c654 100644 --- a/packages/db-dynamodb/src/toolbox.ts +++ b/packages/db-dynamodb/src/toolbox.ts @@ -6,6 +6,8 @@ export type { TableConstructor } from "dynamodb-toolbox/dist/cjs/classes/Table"; export type { + Readonly, + EntityConstructor, AttributeDefinition, EntityQueryOptions, AttributeDefinitions diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts index 1439693776e..4464a4f1ea3 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts @@ -5,8 +5,7 @@ import type { IEntityReadBatchKey, IPutBatchItem } from "./types"; -import type { Entity } from "dynamodb-toolbox"; -import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table/types"; +import type { Entity, TableDef } from "~/toolbox"; import { batchReadAll } from "./batchRead"; import { GenericRecord } from "@webiny/api/types"; import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts b/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts index ca231f32536..77fcc52d486 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts @@ -1,4 +1,4 @@ -import type { Entity } from "dynamodb-toolbox"; +import type { Entity } from "~/toolbox"; import type { IEntityReadBatchBuilder, IEntityReadBatchBuilderGetResponse, diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index fc46aa6305b..6cc19de71e9 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -1,4 +1,4 @@ -import type { Entity } from "dynamodb-toolbox"; +import type { Entity, TableDef } from "~/toolbox"; import { batchWriteAll } from "./batchWrite"; import type { BatchWriteItem, @@ -10,7 +10,6 @@ import type { ITableWriteBatch } from "./types"; import { createTableWriteBatch } from "./TableWriteBatch"; -import type { TableDef } from "~/toolbox"; import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; export interface IEntityWriteBatchParams { diff --git a/packages/db-dynamodb/src/utils/entity/Entity.ts b/packages/db-dynamodb/src/utils/entity/Entity.ts index 3bb400bb37a..4af6e3a4aa5 100644 --- a/packages/db-dynamodb/src/utils/entity/Entity.ts +++ b/packages/db-dynamodb/src/utils/entity/Entity.ts @@ -1,21 +1,22 @@ -import { Entity as BaseEntity } from "dynamodb-toolbox"; +import type { + AttributeDefinitions, + EntityConstructor as BaseEntityConstructor, + Readonly, + TableDef +} from "~/toolbox"; +import { Entity as BaseEntity } from "~/toolbox"; import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "../batch/types"; -import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table"; -import type { IEntity } from "./types"; +import type { IEntity, IEntityQueryAllParams, IEntityQueryOneParams } from "./types"; import type { IPutParamsItem } from "../put"; import { put } from "../put"; import type { GetRecordParamsKeys } from "../get"; import { get, getClean } from "../get"; import type { IDeleteItemKeys } from "../delete"; import { deleteItem } from "../delete"; -import type { - AttributeDefinitions, - EntityConstructor as BaseEntityConstructor, - Readonly -} from "dynamodb-toolbox/dist/cjs/classes/Entity/types"; import { createEntityReadBatch } from "../batch/EntityReadBatch"; import { createEntityWriteBatch } from "../batch/EntityWriteBatch"; import { createTableWriteBatch } from "../batch/TableWriteBatch"; +import { queryAllClean, queryOneClean } from "../query"; export type EntityConstructor< T extends Readonly = Readonly @@ -84,6 +85,20 @@ export class Entity implements IEntity { keys }); } + + public queryOne(params: IEntityQueryOneParams): Promise { + return queryOneClean({ + ...params, + entity: this.entity + }); + } + + public queryAll(params: IEntityQueryAllParams): Promise { + return queryAllClean({ + ...params, + entity: this.entity + }); + } } export const createEntity = (params: EntityConstructor): IEntity => { diff --git a/packages/db-dynamodb/src/utils/entity/types.ts b/packages/db-dynamodb/src/utils/entity/types.ts index 1eaafc3389c..02aeb5f1c19 100644 --- a/packages/db-dynamodb/src/utils/entity/types.ts +++ b/packages/db-dynamodb/src/utils/entity/types.ts @@ -1,15 +1,21 @@ import type { Entity as BaseEntity } from "dynamodb-toolbox"; import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; -import type { +import { deleteItem, get, getClean, GetRecordParamsKeys, IDeleteItemKeys, IPutParamsItem, - put + put, + QueryAllParams, + QueryOneParams } from "~/utils"; +export type IEntityQueryOneParams = Omit; + +export type IEntityQueryAllParams = Omit; + export interface IEntity { readonly entity: BaseEntity; createEntityReader(): IEntityReadBatch; @@ -19,4 +25,6 @@ export interface IEntity { get(keys: GetRecordParamsKeys): ReturnType>; getClean(keys: GetRecordParamsKeys): ReturnType>; delete(keys: IDeleteItemKeys): ReturnType; + queryOne(params: IEntityQueryOneParams): Promise; + queryAll(params: IEntityQueryAllParams): Promise; } diff --git a/packages/db-dynamodb/src/utils/scan.ts b/packages/db-dynamodb/src/utils/scan.ts index 6feac2268f1..8606f85a41a 100644 --- a/packages/db-dynamodb/src/utils/scan.ts +++ b/packages/db-dynamodb/src/utils/scan.ts @@ -1,8 +1,8 @@ -import { ScanInput, ScanOutput } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity, ScanOptions, Table } from "~/toolbox"; +import type { ScanInput, ScanOutput } from "@webiny/aws-sdk/client-dynamodb"; +import type { Entity, ScanOptions, TableDef } from "~/toolbox"; import { executeWithRetry, ExecuteWithRetryOptions } from "@webiny/utils"; -export type { ScanOptions }; +export type { ScanOptions, ScanInput, ScanOutput }; export interface BaseScanParams { options?: ScanOptions; @@ -10,7 +10,7 @@ export interface BaseScanParams { } export interface ScanWithTable extends BaseScanParams { - table: Table; + table: TableDef; entity?: never; } diff --git a/packages/db-dynamodb/src/utils/table/Table.ts b/packages/db-dynamodb/src/utils/table/Table.ts index c3429a369ff..3c7b6dab936 100644 --- a/packages/db-dynamodb/src/utils/table/Table.ts +++ b/packages/db-dynamodb/src/utils/table/Table.ts @@ -1,9 +1,10 @@ import type { TableConstructor } from "~/toolbox"; import { Table as BaseTable } from "~/toolbox"; -import type { ITable } from "./types"; +import { ITable, ITableScanParams, ITableScanResponse } from "./types"; import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; import { createTableWriteBatch } from "../batch/TableWriteBatch"; import { createTableReadBatch } from "../batch/TableReadBatch"; +import { scan } from "../scan"; export class Table< Name extends string = string, @@ -28,4 +29,11 @@ export class Table< table: this.table }); } + + public async scan(params: ITableScanParams): Promise> { + return scan({ + ...params, + table: this.table + }); + } } diff --git a/packages/db-dynamodb/src/utils/table/types.ts b/packages/db-dynamodb/src/utils/table/types.ts index 3aa3c780e1a..a1a12385914 100644 --- a/packages/db-dynamodb/src/utils/table/types.ts +++ b/packages/db-dynamodb/src/utils/table/types.ts @@ -1,8 +1,14 @@ import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table/types"; import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; +import { BaseScanParams, ScanResponse } from "../scan"; + +export type ITableScanParams = BaseScanParams; + +export type ITableScanResponse = ScanResponse; export interface ITable { table: TableDef; createWriter(): ITableWriteBatch; createReader(): ITableReadBatch; + scan(params: ITableScanParams): Promise>; } From f9de254595c152e28f8397509ff8987cc1ee0ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 14:35:06 +0100 Subject: [PATCH 14/19] fix(api-i18n-ddb): entity --- .../src/definitions/localeEntity.ts | 70 +++++++++---------- .../src/definitions/systemEntity.ts | 42 ++++++----- .../locales/LocalesStorageOperations.ts | 15 ++-- 3 files changed, 58 insertions(+), 69 deletions(-) diff --git a/packages/api-i18n-ddb/src/definitions/localeEntity.ts b/packages/api-i18n-ddb/src/definitions/localeEntity.ts index 36e9c6ab38b..bdbed1a9747 100644 --- a/packages/api-i18n-ddb/src/definitions/localeEntity.ts +++ b/packages/api-i18n-ddb/src/definitions/localeEntity.ts @@ -1,6 +1,6 @@ -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import type { Table } from "@webiny/db-dynamodb/toolbox"; import type { I18NContext } from "@webiny/api-i18n/types"; -import { getExtraAttributes } from "@webiny/db-dynamodb/utils/attributes"; +import { getExtraAttributesFromPlugins } from "@webiny/db-dynamodb/utils/attributes"; import type { IEntity } from "@webiny/db-dynamodb"; import { createEntity } from "@webiny/db-dynamodb"; @@ -12,38 +12,36 @@ export interface ILocaleEntityParams { export default (params: ILocaleEntityParams): IEntity => { const { context, table } = params; const entityName = "I18NLocale"; - const attributes = getExtraAttributes(context, entityName); - return createEntity( - new Entity({ - name: entityName, - table, - attributes: { - PK: { - partitionKey: true - }, - SK: { - sortKey: true - }, - createdOn: { - type: "string" - }, - createdBy: { - type: "map" - }, - code: { - type: "string" - }, - default: { - type: "boolean" - }, - webinyVersion: { - type: "string" - }, - tenant: { - type: "string" - }, - ...attributes - } - }) - ); + const attributes = getExtraAttributesFromPlugins(context.plugins, entityName); + return createEntity({ + name: entityName, + table, + attributes: { + PK: { + partitionKey: true + }, + SK: { + sortKey: true + }, + createdOn: { + type: "string" + }, + createdBy: { + type: "map" + }, + code: { + type: "string" + }, + default: { + type: "boolean" + }, + webinyVersion: { + type: "string" + }, + tenant: { + type: "string" + }, + ...attributes + } + }); }; diff --git a/packages/api-i18n-ddb/src/definitions/systemEntity.ts b/packages/api-i18n-ddb/src/definitions/systemEntity.ts index 19a77f25b32..44cf24d3800 100644 --- a/packages/api-i18n-ddb/src/definitions/systemEntity.ts +++ b/packages/api-i18n-ddb/src/definitions/systemEntity.ts @@ -1,4 +1,4 @@ -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import type { Table } from "@webiny/db-dynamodb/toolbox"; import type { I18NContext } from "@webiny/api-i18n/types"; import { getExtraAttributesFromPlugins } from "@webiny/db-dynamodb/utils/attributes"; import type { IEntity } from "@webiny/db-dynamodb"; @@ -11,25 +11,23 @@ export default (params: { const { context, table } = params; const entityName = "I18NSystem"; const attributes = getExtraAttributesFromPlugins(context.plugins, entityName); - return createEntity( - new Entity({ - name: entityName, - table, - attributes: { - PK: { - partitionKey: true - }, - SK: { - sortKey: true - }, - version: { - type: "string" - }, - tenant: { - type: "string" - }, - ...attributes - } - }) - ); + return createEntity({ + name: entityName, + table, + attributes: { + PK: { + partitionKey: true + }, + SK: { + sortKey: true + }, + version: { + type: "string" + }, + tenant: { + type: "string" + }, + ...attributes + } + }); }; diff --git a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts index 23db6b62431..8a28a945747 100644 --- a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts +++ b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts @@ -15,14 +15,8 @@ import type { Table } from "@webiny/db-dynamodb/toolbox"; import WebinyError from "@webiny/error"; import defineTable from "~/definitions/table"; import defineLocaleEntity from "~/definitions/localeEntity"; -import type { IEntity, QueryAllParams } from "@webiny/db-dynamodb"; -import { - cleanupItems, - createListResponse, - filterItems, - queryAll, - sortItems -} from "@webiny/db-dynamodb"; +import type { IEntity, IEntityQueryAllParams } from "@webiny/db-dynamodb"; +import { cleanupItems, createListResponse, filterItems, sortItems } from "@webiny/db-dynamodb"; import { LocaleDynamoDbFieldPlugin } from "~/plugins/LocaleDynamoDbFieldPlugin"; interface ConstructorParams { @@ -210,7 +204,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { let results: I18NLocaleData[] = []; try { - results = await queryAll(queryAllParams); + results = await this.entity.queryAll(queryAllParams); } catch (ex) { throw new WebinyError( ex.message || "Cannot list I18N locales.", @@ -272,7 +266,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { private createQueryAllParamsOptions( params: I18NLocalesStorageOperationsListParams - ): QueryAllParams { + ): IEntityQueryAllParams { const { where } = params; const tenant = where.tenant; @@ -285,7 +279,6 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { delete where.default; } return { - entity: this.entity.entity, partitionKey, options: {} }; From 313c9017be5b1734a167e49d95b074a980e084ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 15:01:06 +0100 Subject: [PATCH 15/19] fix(db-dynamodb): cloning array by wrong reference --- .../__tests__/__api__/setupFile.js | 3 +++ .../__tests__/converters/convertersDisabled.test.ts | 2 -- .../__tests__/graphql/dummyLocales.ts | 5 ++++- .../src/operations/locales/LocalesStorageOperations.ts | 4 ++-- packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts | 8 ++++---- packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js b/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js index e657c72bbb3..a436a3f75d9 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js +++ b/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js @@ -66,6 +66,9 @@ module.exports = () => { }); }); + createOrRefreshIndexSubscription.name = + "headlessCmsDdbEs.context.createOrRefreshIndexSubscription"; + return { storageOperations: createStorageOperations({ documentClient, diff --git a/packages/api-headless-cms-ddb-es/__tests__/converters/convertersDisabled.test.ts b/packages/api-headless-cms-ddb-es/__tests__/converters/convertersDisabled.test.ts index b3193273d49..dac86634f7c 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/converters/convertersDisabled.test.ts +++ b/packages/api-headless-cms-ddb-es/__tests__/converters/convertersDisabled.test.ts @@ -7,8 +7,6 @@ import { CmsModel } from "@webiny/api-headless-cms/types"; import { get } from "@webiny/db-dynamodb"; import { createPartitionKey } from "~/operations/entry/keys"; -jest.retryTimes(0); - describe("storage field path converters disabled", () => { const { elasticsearch, entryEntity } = useHandler(); diff --git a/packages/api-headless-cms-ddb-es/__tests__/graphql/dummyLocales.ts b/packages/api-headless-cms-ddb-es/__tests__/graphql/dummyLocales.ts index 9424ee10e21..f6b763e697c 100644 --- a/packages/api-headless-cms-ddb-es/__tests__/graphql/dummyLocales.ts +++ b/packages/api-headless-cms-ddb-es/__tests__/graphql/dummyLocales.ts @@ -2,7 +2,7 @@ import { ContextPlugin } from "@webiny/api"; import { CmsContext } from "@webiny/api-headless-cms/types"; export const createDummyLocales = () => { - return new ContextPlugin(async context => { + const plugin = new ContextPlugin(async context => { const { i18n, security } = context; await security.withoutAuthorization(async () => { @@ -23,4 +23,7 @@ export const createDummyLocales = () => { }); }); }); + + plugin.name = "headlessCmsDdbEs.context.createDummyLocales"; + return plugin; }; diff --git a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts index 8a28a945747..18e344863d6 100644 --- a/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts +++ b/packages/api-i18n-ddb/src/operations/locales/LocalesStorageOperations.ts @@ -16,7 +16,7 @@ import WebinyError from "@webiny/error"; import defineTable from "~/definitions/table"; import defineLocaleEntity from "~/definitions/localeEntity"; import type { IEntity, IEntityQueryAllParams } from "@webiny/db-dynamodb"; -import { cleanupItems, createListResponse, filterItems, sortItems } from "@webiny/db-dynamodb"; +import { createListResponse, filterItems, sortItems } from "@webiny/db-dynamodb"; import { LocaleDynamoDbFieldPlugin } from "~/plugins/LocaleDynamoDbFieldPlugin"; interface ConstructorParams { @@ -240,7 +240,7 @@ export class LocalesStorageOperations implements I18NLocalesStorageOperations { * Use the common db-dynamodb method to create the required response. */ return createListResponse({ - items: cleanupItems(this.entity.entity, sortedFiles), + items: sortedFiles, after, totalCount, limit diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts index 4464a4f1ea3..862dcd1d315 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts @@ -18,7 +18,7 @@ export interface IEntityReadBatchParams { export class EntityReadBatch implements IEntityReadBatch { private readonly entity: Entity; private readonly builder: IEntityReadBatchBuilder; - private readonly items: IEntityReadBatchBuilderGetResponse[] = []; + private readonly _items: IEntityReadBatchBuilderGetResponse[] = []; public constructor(params: IEntityReadBatchParams) { if (!params.entity.name) { @@ -31,20 +31,20 @@ export class EntityReadBatch implements IEntityReadBatch { } public get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void { if (Array.isArray(input)) { - this.items.push( + this._items.push( ...input.map(item => { return this.builder.get(item); }) ); return; } - this.items.push(this.builder.get(input)); + this._items.push(this.builder.get(input)); } public async execute() { return await batchReadAll({ table: this.entity.table as TableDef, - items: this.items + items: this._items }); } } diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts index 6cc19de71e9..d98fa8156ea 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts @@ -28,7 +28,7 @@ export class EntityWriteBatch implements IEntityWriteBatch { } public get items(): BatchWriteItem[] { - return Array.from(this.items); + return Array.from(this._items); } public constructor(params: IEntityWriteBatchParams) { From 03d4e7c7ba0139f258e9f70041779dfcd05a7c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 15:40:23 +0100 Subject: [PATCH 16/19] fix(db-dynamodb): add a readme for new classes --- packages/db-dynamodb/README.md | 40 ++++++++++++++++++- .../src/utils/batch/TableWriteBatch.ts | 4 +- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/db-dynamodb/README.md b/packages/db-dynamodb/README.md index 7990665227b..018e3ff4f17 100644 --- a/packages/db-dynamodb/README.md +++ b/packages/db-dynamodb/README.md @@ -14,6 +14,44 @@ For more information, please visit yarn add @webiny/db-dynamodb ``` +### Helper classes +We have some classes that ease the use of dynamodb-toolbox. +#### [Table](./src/utils/table/Table.ts) +```typescript +import { createTable } from "@webiny/db-dynamodb"; + +const table = createTable({...params}); +const writer = table.createWriter(); // see TableWriteBatch +const reader = table.createReader(); // see TableReadBatch + +const result = await table.scan({...params}); // see scan +``` + +#### [Entity](./src/utils/entity/Entity.ts) +```typescript + +import { createEntity } from "@webiny/db-dynamodb"; + +const entity = createEntity({...params}); +const writer = entity.createWriter(); // see EntityWriteBatch +const reader = entity.createReader(); // see EntityReadBatch +const tableWriter = entity.createTableWriter(); // see TableWriteBatch + +const getResult = await entity.get({...params}); // see get +const getCleanResult = await entity.getClean({...params}); // see get +const queryAllResult = await entity.queryAll({...params}); // see queryAllClean +const queryOneResult = await entity.queryOne({...params}); // see queryOneClean +const putResult = await entity.put({...params}); // see put + +``` + +#### [EntityWriteBatch](./src/utils/batch/EntityWriteBatch.ts) +#### [EntityReadBatch](./src/utils/batch/EntityReadBatch.ts) +#### [TableReadBatch](./src/utils/batch/TableReadBatch.ts) +#### [TableWriteBatch](./src/utils/batch/TableWriteBatch.ts) + + + ### Helper functions We have a number [helper](./src/utils) functions that ease the use of either dynamodb-toolbox, filtering, sorting or just creating proper response. @@ -64,4 +102,4 @@ This function accepts items (records) to be filtered, a definition of fields to #### [sort](./src/utils/sort.ts) Sort the DynamoDB records by given sort condition. -This function accepts items (records) to be sorted, sort options (eg. createdBy_ASC, id_DESC, etc.) and a definitions of fields to sort by (not required by default if no field modification is required). \ No newline at end of file +This function accepts items (records) to be sorted, sort options (eg. createdBy_ASC, id_DESC, etc.) and a definitions of fields to sort by (not required by default if no field modification is required). diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts index 123da97b8ab..85f247b8ef5 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts @@ -1,5 +1,5 @@ -import { Entity, TableDef } from "~/toolbox"; -import { +import type { Entity, TableDef } from "~/toolbox"; +import type { BatchWriteItem, BatchWriteResult, IDeleteBatchItem, From a162c297ebf74a9dffaa02016b7101ac988d3fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 20:41:50 +0100 Subject: [PATCH 17/19] fix(db-dynamodb): remove unnecessary export --- packages/db-dynamodb/src/utils/batch/index.ts | 5 -- packages/db-dynamodb/src/utils/batch/types.ts | 62 ------------------ .../db-dynamodb/src/utils/entity/Entity.ts | 16 +++-- .../{batch => entity}/EntityReadBatch.ts | 6 +- .../EntityReadBatchBuilder.ts | 0 .../{batch => entity}/EntityWriteBatch.ts | 13 ++-- .../EntityWriteBatchBuilder.ts | 8 +-- .../db-dynamodb/src/utils/entity/index.ts | 4 ++ .../db-dynamodb/src/utils/entity/types.ts | 63 +++++++++++++++---- packages/db-dynamodb/src/utils/index.ts | 1 - packages/db-dynamodb/src/utils/table.ts | 16 ----- packages/db-dynamodb/src/utils/table/Table.ts | 13 ++-- .../utils/{batch => table}/TableReadBatch.ts | 11 ++-- .../utils/{batch => table}/TableWriteBatch.ts | 12 ++-- packages/db-dynamodb/src/utils/table/index.ts | 2 + packages/db-dynamodb/src/utils/table/types.ts | 43 ++++++++++++- 16 files changed, 140 insertions(+), 135 deletions(-) rename packages/db-dynamodb/src/utils/{batch => entity}/EntityReadBatch.ts (92%) rename packages/db-dynamodb/src/utils/{batch => entity}/EntityReadBatchBuilder.ts (100%) rename packages/db-dynamodb/src/utils/{batch => entity}/EntityWriteBatch.ts (87%) rename packages/db-dynamodb/src/utils/{batch => entity}/EntityWriteBatchBuilder.ts (82%) delete mode 100644 packages/db-dynamodb/src/utils/table.ts rename packages/db-dynamodb/src/utils/{batch => table}/TableReadBatch.ts (88%) rename packages/db-dynamodb/src/utils/{batch => table}/TableWriteBatch.ts (87%) diff --git a/packages/db-dynamodb/src/utils/batch/index.ts b/packages/db-dynamodb/src/utils/batch/index.ts index 84638425a18..cc976f74632 100644 --- a/packages/db-dynamodb/src/utils/batch/index.ts +++ b/packages/db-dynamodb/src/utils/batch/index.ts @@ -1,8 +1,3 @@ export * from "./batchRead"; export * from "./batchWrite"; -export * from "./EntityReadBatch"; -export * from "./EntityReadBatchBuilder"; -export * from "./EntityWriteBatch"; -export * from "./EntityWriteBatchBuilder"; -export * from "./TableWriteBatch"; export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/batch/types.ts b/packages/db-dynamodb/src/utils/batch/types.ts index bec52153c6e..f7b3ad9984b 100644 --- a/packages/db-dynamodb/src/utils/batch/types.ts +++ b/packages/db-dynamodb/src/utils/batch/types.ts @@ -1,7 +1,4 @@ import type { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import type { Entity, TableDef } from "~/toolbox"; -import type { GenericRecord } from "@webiny/api/types"; -import type { batchReadAll } from "~/utils"; export interface BatchWriteResponse { next?: () => Promise; @@ -31,62 +28,3 @@ export type IPutBatchItem = Record> = export interface BatchWriteItem { [key: string]: WriteRequest; } - -export interface IEntityWriteBatchBuilder { - // readonly entity: Entity; - put>(item: IPutBatchItem): BatchWriteItem; - delete(item: IDeleteBatchItem): BatchWriteItem; -} - -export interface IEntityWriteBatch { - readonly total: number; - // readonly entity: Entity; - readonly items: BatchWriteItem[]; - // readonly builder: IEntityWriteBatchBuilder; - - put(item: IPutBatchItem): void; - delete(item: IDeleteBatchItem): void; - execute(): Promise; - combine(items: BatchWriteItem[]): ITableWriteBatch; -} - -export interface ITableWriteBatch { - readonly total: number; - // readonly table: TableDef; - readonly items: BatchWriteItem[]; - put(entity: Entity, item: IPutBatchItem): void; - delete(entity: Entity, item: IDeleteBatchItem): void; - execute(): Promise; - combine(items: BatchWriteItem[]): ITableWriteBatch; -} - -export interface IEntityReadBatchKey { - PK: string; - SK: string; -} - -export interface IEntityReadBatch { - get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void; - execute(): ReturnType>; -} - -export interface IEntityReadBatchBuilderGetResponse { - Table: TableDef; - Key: IEntityReadBatchKey; -} - -export interface IEntityReadBatchBuilder { - get(item: IEntityReadBatchKey): IEntityReadBatchBuilderGetResponse; -} - -export interface ITableReadBatchKey { - PK: string; - SK: string; -} - -export interface ITableReadBatch { - readonly total: number; - readonly items: IEntityReadBatchBuilderGetResponse[]; - get(entity: Entity, input: ITableReadBatchKey | ITableReadBatchKey[]): void; - execute(): Promise; -} diff --git a/packages/db-dynamodb/src/utils/entity/Entity.ts b/packages/db-dynamodb/src/utils/entity/Entity.ts index 4af6e3a4aa5..3e056d2b016 100644 --- a/packages/db-dynamodb/src/utils/entity/Entity.ts +++ b/packages/db-dynamodb/src/utils/entity/Entity.ts @@ -5,17 +5,23 @@ import type { TableDef } from "~/toolbox"; import { Entity as BaseEntity } from "~/toolbox"; -import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "../batch/types"; -import type { IEntity, IEntityQueryAllParams, IEntityQueryOneParams } from "./types"; +import type { ITableWriteBatch } from "../table/types"; +import type { + IEntity, + IEntityQueryAllParams, + IEntityQueryOneParams, + IEntityReadBatch, + IEntityWriteBatch +} from "./types"; import type { IPutParamsItem } from "../put"; import { put } from "../put"; import type { GetRecordParamsKeys } from "../get"; import { get, getClean } from "../get"; import type { IDeleteItemKeys } from "../delete"; import { deleteItem } from "../delete"; -import { createEntityReadBatch } from "../batch/EntityReadBatch"; -import { createEntityWriteBatch } from "../batch/EntityWriteBatch"; -import { createTableWriteBatch } from "../batch/TableWriteBatch"; +import { createEntityReadBatch } from "./EntityReadBatch"; +import { createEntityWriteBatch } from "./EntityWriteBatch"; +import { createTableWriteBatch } from "~/utils/table/TableWriteBatch"; import { queryAllClean, queryOneClean } from "../query"; export type EntityConstructor< diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts b/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts similarity index 92% rename from packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts rename to packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts index 862dcd1d315..499555d7a4f 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityReadBatch.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts @@ -1,12 +1,12 @@ +import type { IPutBatchItem } from "~/utils/batch/types"; import type { IEntityReadBatch, IEntityReadBatchBuilder, IEntityReadBatchBuilderGetResponse, - IEntityReadBatchKey, - IPutBatchItem + IEntityReadBatchKey } from "./types"; import type { Entity, TableDef } from "~/toolbox"; -import { batchReadAll } from "./batchRead"; +import { batchReadAll } from "~/utils/batch/batchRead"; import { GenericRecord } from "@webiny/api/types"; import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; diff --git a/packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts b/packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts similarity index 100% rename from packages/db-dynamodb/src/utils/batch/EntityReadBatchBuilder.ts rename to packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts similarity index 87% rename from packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts rename to packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts index d98fa8156ea..7c66f887c10 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts @@ -1,15 +1,14 @@ import type { Entity, TableDef } from "~/toolbox"; -import { batchWriteAll } from "./batchWrite"; +import { batchWriteAll } from "~/utils/batch/batchWrite"; import type { BatchWriteItem, BatchWriteResult, IDeleteBatchItem, - IEntityWriteBatch, - IEntityWriteBatchBuilder, - IPutBatchItem, - ITableWriteBatch -} from "./types"; -import { createTableWriteBatch } from "./TableWriteBatch"; + IPutBatchItem +} from "~/utils/batch/types"; +import type { IEntityWriteBatch, IEntityWriteBatchBuilder } from "./types"; +import type { ITableWriteBatch } from "~/utils/table/types"; +import { createTableWriteBatch } from "~/utils/table/TableWriteBatch"; import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; export interface IEntityWriteBatchParams { diff --git a/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts b/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts similarity index 82% rename from packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts rename to packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts index 27c5316a1af..a84578810c7 100644 --- a/packages/db-dynamodb/src/utils/batch/EntityWriteBatchBuilder.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts @@ -1,10 +1,6 @@ import type { Entity } from "~/toolbox"; -import type { - BatchWriteItem, - IDeleteBatchItem, - IEntityWriteBatchBuilder, - IPutBatchItem -} from "./types"; +import type { BatchWriteItem, IDeleteBatchItem, IPutBatchItem } from "~/utils/batch/types"; +import type { IEntityWriteBatchBuilder } from "./types"; export class EntityWriteBatchBuilder implements IEntityWriteBatchBuilder { private readonly entity: Entity; diff --git a/packages/db-dynamodb/src/utils/entity/index.ts b/packages/db-dynamodb/src/utils/entity/index.ts index 1f6b52ffc94..980ebd5fb7f 100644 --- a/packages/db-dynamodb/src/utils/entity/index.ts +++ b/packages/db-dynamodb/src/utils/entity/index.ts @@ -1,2 +1,6 @@ export * from "./Entity"; +export * from "./EntityReadBatch"; +export * from "./EntityReadBatchBuilder"; +export * from "./EntityWriteBatch"; +export * from "./EntityWriteBatchBuilder"; export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/entity/types.ts b/packages/db-dynamodb/src/utils/entity/types.ts index 02aeb5f1c19..a3322d22d4e 100644 --- a/packages/db-dynamodb/src/utils/entity/types.ts +++ b/packages/db-dynamodb/src/utils/entity/types.ts @@ -1,16 +1,18 @@ import type { Entity as BaseEntity } from "dynamodb-toolbox"; -import type { IEntityReadBatch, IEntityWriteBatch, ITableWriteBatch } from "~/utils/batch/types"; -import { - deleteItem, - get, - getClean, - GetRecordParamsKeys, - IDeleteItemKeys, - IPutParamsItem, - put, - QueryAllParams, - QueryOneParams -} from "~/utils"; +import type { + BatchWriteItem, + BatchWriteResult, + IDeleteBatchItem, + IPutBatchItem +} from "~/utils/batch/types"; +import type { GenericRecord } from "@webiny/api/types"; +import type { TableDef } from "~/toolbox"; +import type { ITableWriteBatch } from "~/utils/table/types"; +import type { IPutParamsItem, put } from "~/utils/put"; +import type { QueryAllParams, QueryOneParams } from "~/utils/query"; +import type { get, getClean, GetRecordParamsKeys } from "~/utils/get"; +import type { deleteItem, IDeleteItemKeys } from "~/utils/delete"; +import type { batchReadAll } from "~/utils/batch/batchRead"; export type IEntityQueryOneParams = Omit; @@ -28,3 +30,40 @@ export interface IEntity { queryOne(params: IEntityQueryOneParams): Promise; queryAll(params: IEntityQueryAllParams): Promise; } + +export interface IEntityWriteBatchBuilder { + // readonly entity: Entity; + put>(item: IPutBatchItem): BatchWriteItem; + delete(item: IDeleteBatchItem): BatchWriteItem; +} + +export interface IEntityWriteBatch { + readonly total: number; + // readonly entity: Entity; + readonly items: BatchWriteItem[]; + // readonly builder: IEntityWriteBatchBuilder; + + put(item: IPutBatchItem): void; + delete(item: IDeleteBatchItem): void; + execute(): Promise; + combine(items: BatchWriteItem[]): ITableWriteBatch; +} + +export interface IEntityReadBatchKey { + PK: string; + SK: string; +} + +export interface IEntityReadBatch { + get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void; + execute(): ReturnType>; +} + +export interface IEntityReadBatchBuilderGetResponse { + Table: TableDef; + Key: IEntityReadBatchKey; +} + +export interface IEntityReadBatchBuilder { + get(item: IEntityReadBatchKey): IEntityReadBatchBuilderGetResponse; +} diff --git a/packages/db-dynamodb/src/utils/index.ts b/packages/db-dynamodb/src/utils/index.ts index 58f326db072..6e5246f3896 100644 --- a/packages/db-dynamodb/src/utils/index.ts +++ b/packages/db-dynamodb/src/utils/index.ts @@ -11,7 +11,6 @@ export * from "./query"; export * from "./count"; export * from "./scan"; export * from "./sort"; -export * from "./table"; export * from "./update"; export * from "./batch"; export * from "./entity"; diff --git a/packages/db-dynamodb/src/utils/table.ts b/packages/db-dynamodb/src/utils/table.ts deleted file mode 100644 index a435ba992d0..00000000000 --- a/packages/db-dynamodb/src/utils/table.ts +++ /dev/null @@ -1,16 +0,0 @@ -import WebinyError from "@webiny/error"; -import { DbContext } from "@webiny/handler-db/types"; - -/** - * Will be removed in favor of passing the table name directly to the storage operations. - * - * @deprecated - */ -export const getTable = (context: T): string => { - if (!context.db) { - throw new WebinyError("Missing db on context.", "DB_ERROR"); - } else if (!context.db.table) { - throw new WebinyError("Missing table on context.db.", "TABLE_ERROR"); - } - return context.db.table; -}; diff --git a/packages/db-dynamodb/src/utils/table/Table.ts b/packages/db-dynamodb/src/utils/table/Table.ts index 3c7b6dab936..7d7fba286ed 100644 --- a/packages/db-dynamodb/src/utils/table/Table.ts +++ b/packages/db-dynamodb/src/utils/table/Table.ts @@ -1,9 +1,14 @@ import type { TableConstructor } from "~/toolbox"; import { Table as BaseTable } from "~/toolbox"; -import { ITable, ITableScanParams, ITableScanResponse } from "./types"; -import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; -import { createTableWriteBatch } from "../batch/TableWriteBatch"; -import { createTableReadBatch } from "../batch/TableReadBatch"; +import type { + ITable, + ITableReadBatch, + ITableScanParams, + ITableScanResponse, + ITableWriteBatch +} from "./types"; +import { createTableWriteBatch } from "./TableWriteBatch"; +import { createTableReadBatch } from "./TableReadBatch"; import { scan } from "../scan"; export class Table< diff --git a/packages/db-dynamodb/src/utils/batch/TableReadBatch.ts b/packages/db-dynamodb/src/utils/table/TableReadBatch.ts similarity index 88% rename from packages/db-dynamodb/src/utils/batch/TableReadBatch.ts rename to packages/db-dynamodb/src/utils/table/TableReadBatch.ts index f8f6b374013..826bae5f96d 100644 --- a/packages/db-dynamodb/src/utils/batch/TableReadBatch.ts +++ b/packages/db-dynamodb/src/utils/table/TableReadBatch.ts @@ -1,14 +1,13 @@ import type { Entity, TableDef } from "~/toolbox"; import type { IEntityReadBatchBuilder, - IEntityReadBatchBuilderGetResponse, - ITableReadBatch, - ITableReadBatchKey -} from "./types"; -import { batchReadAll } from "./batchRead"; -import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; + IEntityReadBatchBuilderGetResponse +} from "~/utils/entity/types"; +import { batchReadAll } from "~/utils/batch/batchRead"; +import { createEntityReadBatchBuilder } from "~/utils/entity/EntityReadBatchBuilder"; import type { GenericRecord } from "@webiny/api/types"; import { WebinyError } from "@webiny/error"; +import type { ITableReadBatch, ITableReadBatchKey } from "./types"; export interface ITableReadBatchParams { table: TableDef; diff --git a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts b/packages/db-dynamodb/src/utils/table/TableWriteBatch.ts similarity index 87% rename from packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts rename to packages/db-dynamodb/src/utils/table/TableWriteBatch.ts index 85f247b8ef5..a8cbf0c5c9b 100644 --- a/packages/db-dynamodb/src/utils/batch/TableWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/table/TableWriteBatch.ts @@ -3,12 +3,12 @@ import type { BatchWriteItem, BatchWriteResult, IDeleteBatchItem, - IEntityWriteBatchBuilder, - IPutBatchItem, - ITableWriteBatch -} from "./types"; -import { batchWriteAll } from "./batchWrite"; -import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; + IPutBatchItem +} from "~/utils/batch/types"; +import type { IEntityWriteBatchBuilder } from "~/utils/entity/types"; +import { batchWriteAll } from "~/utils/batch/batchWrite"; +import { createEntityWriteBatchBuilder } from "~/utils/entity/EntityWriteBatchBuilder"; +import type { ITableWriteBatch } from "./types"; export interface ITableWriteBatchParams { table: TableDef; diff --git a/packages/db-dynamodb/src/utils/table/index.ts b/packages/db-dynamodb/src/utils/table/index.ts index 865db5984b5..fd8e934067a 100644 --- a/packages/db-dynamodb/src/utils/table/index.ts +++ b/packages/db-dynamodb/src/utils/table/index.ts @@ -1,2 +1,4 @@ export * from "./Table"; +export * from "./TableReadBatch"; +export * from "./TableWriteBatch"; export * from "./types"; diff --git a/packages/db-dynamodb/src/utils/table/types.ts b/packages/db-dynamodb/src/utils/table/types.ts index a1a12385914..c35ce53f1e6 100644 --- a/packages/db-dynamodb/src/utils/table/types.ts +++ b/packages/db-dynamodb/src/utils/table/types.ts @@ -1,6 +1,13 @@ import type { TableDef } from "dynamodb-toolbox/dist/cjs/classes/Table/types"; -import type { ITableReadBatch, ITableWriteBatch } from "../batch/types"; -import { BaseScanParams, ScanResponse } from "../scan"; +import type { + BatchWriteItem, + BatchWriteResult, + IDeleteBatchItem, + IPutBatchItem +} from "~/utils/batch/types"; +import type { BaseScanParams, ScanResponse } from "../scan"; +import type { Entity } from "~/toolbox"; +import type { GenericRecord } from "@webiny/api/types"; export type ITableScanParams = BaseScanParams; @@ -12,3 +19,35 @@ export interface ITable { createReader(): ITableReadBatch; scan(params: ITableScanParams): Promise>; } + +export interface ITableWriteBatch { + readonly total: number; + // readonly table: TableDef; + readonly items: BatchWriteItem[]; + put(entity: Entity, item: IPutBatchItem): void; + delete(entity: Entity, item: IDeleteBatchItem): void; + execute(): Promise; + combine(items: BatchWriteItem[]): ITableWriteBatch; +} + +export interface ITableReadBatchKey { + PK: string; + SK: string; +} + +export interface ITableReadBatchBuilderGetResponse { + Table: TableDef; + Key: ITableReadBatchKey; +} + +export interface ITableReadBatchKey { + PK: string; + SK: string; +} + +export interface ITableReadBatch { + readonly total: number; + readonly items: ITableReadBatchBuilderGetResponse[]; + get(entity: Entity, input: ITableReadBatchKey | ITableReadBatchKey[]): void; + execute(): Promise; +} From 9ee03f9bc4c8d64dcf512270ecf72891d4828687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 16 Dec 2024 21:05:22 +0100 Subject: [PATCH 18/19] fix(db-dynamodb): remove unnecessary package --- packages/db-dynamodb/package.json | 1 - packages/db-dynamodb/tsconfig.build.json | 1 - packages/db-dynamodb/tsconfig.json | 3 --- yarn.lock | 1 - 4 files changed, 6 deletions(-) diff --git a/packages/db-dynamodb/package.json b/packages/db-dynamodb/package.json index 00ab30f0cfb..f61f4d233ca 100644 --- a/packages/db-dynamodb/package.json +++ b/packages/db-dynamodb/package.json @@ -14,7 +14,6 @@ "@webiny/aws-sdk": "0.0.0", "@webiny/db": "0.0.0", "@webiny/error": "0.0.0", - "@webiny/handler-db": "0.0.0", "@webiny/plugins": "0.0.0", "@webiny/utils": "0.0.0", "date-fns": "^2.22.1", diff --git a/packages/db-dynamodb/tsconfig.build.json b/packages/db-dynamodb/tsconfig.build.json index d14ec0b973f..a1e7b5b5b27 100644 --- a/packages/db-dynamodb/tsconfig.build.json +++ b/packages/db-dynamodb/tsconfig.build.json @@ -6,7 +6,6 @@ { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../db/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, - { "path": "../handler-db/tsconfig.build.json" }, { "path": "../plugins/tsconfig.build.json" }, { "path": "../utils/tsconfig.build.json" } ], diff --git a/packages/db-dynamodb/tsconfig.json b/packages/db-dynamodb/tsconfig.json index 13fd84d1241..df4af1353b1 100644 --- a/packages/db-dynamodb/tsconfig.json +++ b/packages/db-dynamodb/tsconfig.json @@ -6,7 +6,6 @@ { "path": "../aws-sdk" }, { "path": "../db" }, { "path": "../error" }, - { "path": "../handler-db" }, { "path": "../plugins" }, { "path": "../utils" } ], @@ -25,8 +24,6 @@ "@webiny/db": ["../db/src"], "@webiny/error/*": ["../error/src/*"], "@webiny/error": ["../error/src"], - "@webiny/handler-db/*": ["../handler-db/src/*"], - "@webiny/handler-db": ["../handler-db/src"], "@webiny/plugins/*": ["../plugins/src/*"], "@webiny/plugins": ["../plugins/src"], "@webiny/utils/*": ["../utils/src/*"], diff --git a/yarn.lock b/yarn.lock index 873244d4526..08a8f9dba39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17301,7 +17301,6 @@ __metadata: "@webiny/cli": 0.0.0 "@webiny/db": 0.0.0 "@webiny/error": 0.0.0 - "@webiny/handler-db": 0.0.0 "@webiny/plugins": 0.0.0 "@webiny/project-utils": 0.0.0 "@webiny/utils": 0.0.0 From ae9a11e555dad5f4c3f053fb568a20cd33a38b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Tue, 17 Dec 2024 08:59:32 +0100 Subject: [PATCH 19/19] fix(db-dynamodb): readme --- packages/db-dynamodb/README.md | 50 +++++++++++++++++-- .../src/utils/entity/EntityReadBatch.ts | 19 +++---- .../utils/entity/EntityReadBatchBuilder.ts | 15 ++++-- .../src/utils/entity/EntityWriteBatch.ts | 16 +++--- .../utils/entity/EntityWriteBatchBuilder.ts | 6 ++- .../db-dynamodb/src/utils/entity/getEntity.ts | 14 ++++++ .../db-dynamodb/src/utils/entity/index.ts | 1 + 7 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 packages/db-dynamodb/src/utils/entity/getEntity.ts diff --git a/packages/db-dynamodb/README.md b/packages/db-dynamodb/README.md index 018e3ff4f17..b6bbb743004 100644 --- a/packages/db-dynamodb/README.md +++ b/packages/db-dynamodb/README.md @@ -45,11 +45,53 @@ const putResult = await entity.put({...params}); // see put ``` -#### [EntityWriteBatch](./src/utils/batch/EntityWriteBatch.ts) -#### [EntityReadBatch](./src/utils/batch/EntityReadBatch.ts) -#### [TableReadBatch](./src/utils/batch/TableReadBatch.ts) -#### [TableWriteBatch](./src/utils/batch/TableWriteBatch.ts) +#### [EntityWriteBatch](./src/utils/entity/EntityWriteBatch.ts) +```typescript +import { createEntityWriteBatch } from "@webiny/db-dynamodb"; + +const writer = createEntityWriteBatch({...params}); + +writer.put({...item}); +writer.delete({...keys}); +writer.delete({...moreKeys}); + +await writer.execute(); +``` + +#### [EntityReadBatch](./src/utils/entity/EntityReadBatch.ts) +```typescript +import { createEntityReadBatch } from "@webiny/db-dynamodb"; + +const reader = createEntityReadBatch({...params}); + +reader.get({...keys}); +reader.get({...moreKeys}); + +const result = await reader.execute(); +``` +#### [TableWriteBatch](./src/utils/table/TableWriteBatch.ts) +```typescript +import { createTableWriteBatch } from "@webiny/db-dynamodb"; + +const writer = createTableWriteBatch({...params}); + +writer.put(entity, {...item}); +writer.delete(entity, {...keys}); +writer.delete(entity, {...moreKeys}); + +await writer.execute(); +``` +#### [TableReadBatch](./src/utils/table/TableReadBatch.ts) +```typescript +import {createTableReadBatch} from "@webiny/db-dynamodb"; + +const reader = createTableReadBatch({...params}); +writer.get(entity, {...keys}); +writer.get(entity, {...moreKeys}); + +const result = await reader.execute(); +``` diff --git a/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts b/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts index 499555d7a4f..c1101bf84b5 100644 --- a/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityReadBatch.ts @@ -5,29 +5,30 @@ import type { IEntityReadBatchBuilderGetResponse, IEntityReadBatchKey } from "./types"; -import type { Entity, TableDef } from "~/toolbox"; +import type { TableDef } from "~/toolbox"; +import type { Entity as ToolboxEntity } from "~/toolbox"; import { batchReadAll } from "~/utils/batch/batchRead"; import { GenericRecord } from "@webiny/api/types"; import { createEntityReadBatchBuilder } from "./EntityReadBatchBuilder"; +import type { EntityOption } from "./getEntity"; +import { getEntity } from "./getEntity"; export interface IEntityReadBatchParams { - entity: Entity; + entity: EntityOption; read?: IPutBatchItem[]; } export class EntityReadBatch implements IEntityReadBatch { - private readonly entity: Entity; + private readonly entity: ToolboxEntity; private readonly builder: IEntityReadBatchBuilder; private readonly _items: IEntityReadBatchBuilderGetResponse[] = []; public constructor(params: IEntityReadBatchParams) { - if (!params.entity.name) { - throw new Error(`No name provided for entity.`); - } else if (!params.entity.table) { - throw new Error(`No table provided for entity ${params.entity.name}.`); - } - this.entity = params.entity; + this.entity = getEntity(params.entity); this.builder = createEntityReadBatchBuilder(this.entity); + for (const item of params.read || []) { + this.get(item); + } } public get(input: IEntityReadBatchKey | IEntityReadBatchKey[]): void { if (Array.isArray(input)) { diff --git a/packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts b/packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts index 77fcc52d486..9b72ca8ee8c 100644 --- a/packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityReadBatchBuilder.ts @@ -1,16 +1,19 @@ -import type { Entity } from "~/toolbox"; +import type { Entity as ToolboxEntity } from "~/toolbox"; import type { IEntityReadBatchBuilder, IEntityReadBatchBuilderGetResponse, IEntityReadBatchKey } from "./types"; import { WebinyError } from "@webiny/error"; +import { Entity } from "./Entity"; +import type { EntityOption } from "./getEntity"; +import { getEntity } from "./getEntity"; export class EntityReadBatchBuilder implements IEntityReadBatchBuilder { - private readonly entity: Entity; + private readonly entity: ToolboxEntity; - public constructor(entity: Entity) { - this.entity = entity; + public constructor(entity: EntityOption) { + this.entity = getEntity(entity); } public get(item: IEntityReadBatchKey): IEntityReadBatchBuilderGetResponse { @@ -24,6 +27,8 @@ export class EntityReadBatchBuilder implements IEntityReadBatchBuilder { } } -export const createEntityReadBatchBuilder = (entity: Entity): IEntityReadBatchBuilder => { +export const createEntityReadBatchBuilder = ( + entity: ToolboxEntity | Entity +): IEntityReadBatchBuilder => { return new EntityReadBatchBuilder(entity); }; diff --git a/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts b/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts index 7c66f887c10..49eef6beaa3 100644 --- a/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityWriteBatch.ts @@ -1,4 +1,5 @@ -import type { Entity, TableDef } from "~/toolbox"; +import type { TableDef } from "~/toolbox"; +import type { Entity as ToolboxEntity } from "~/toolbox"; import { batchWriteAll } from "~/utils/batch/batchWrite"; import type { BatchWriteItem, @@ -10,15 +11,17 @@ import type { IEntityWriteBatch, IEntityWriteBatchBuilder } from "./types"; import type { ITableWriteBatch } from "~/utils/table/types"; import { createTableWriteBatch } from "~/utils/table/TableWriteBatch"; import { createEntityWriteBatchBuilder } from "./EntityWriteBatchBuilder"; +import type { EntityOption } from "./getEntity"; +import { getEntity } from "./getEntity"; export interface IEntityWriteBatchParams { - entity: Entity; + entity: EntityOption; put?: IPutBatchItem[]; delete?: IDeleteBatchItem[]; } export class EntityWriteBatch implements IEntityWriteBatch { - private readonly entity: Entity; + private readonly entity: ToolboxEntity; private readonly _items: BatchWriteItem[] = []; private readonly builder: IEntityWriteBatchBuilder; @@ -31,12 +34,7 @@ export class EntityWriteBatch implements IEntityWriteBatch { } public constructor(params: IEntityWriteBatchParams) { - if (!params.entity.name) { - throw new Error(`No name provided for entity.`); - } else if (!params.entity.table) { - throw new Error(`No table provided for entity ${params.entity.name}.`); - } - this.entity = params.entity; + this.entity = getEntity(params.entity); this.builder = createEntityWriteBatchBuilder(this.entity); for (const item of params.put || []) { this.put(item); diff --git a/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts b/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts index a84578810c7..73cb5cff13e 100644 --- a/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts +++ b/packages/db-dynamodb/src/utils/entity/EntityWriteBatchBuilder.ts @@ -1,12 +1,14 @@ import type { Entity } from "~/toolbox"; import type { BatchWriteItem, IDeleteBatchItem, IPutBatchItem } from "~/utils/batch/types"; import type { IEntityWriteBatchBuilder } from "./types"; +import type { EntityOption } from "./getEntity"; +import { getEntity } from "./getEntity"; export class EntityWriteBatchBuilder implements IEntityWriteBatchBuilder { private readonly entity: Entity; - public constructor(entity: Entity) { - this.entity = entity; + public constructor(entity: EntityOption) { + this.entity = getEntity(entity); } public put>(item: IPutBatchItem): BatchWriteItem { diff --git a/packages/db-dynamodb/src/utils/entity/getEntity.ts b/packages/db-dynamodb/src/utils/entity/getEntity.ts new file mode 100644 index 00000000000..5634f6d3b7d --- /dev/null +++ b/packages/db-dynamodb/src/utils/entity/getEntity.ts @@ -0,0 +1,14 @@ +import { Entity as ToolboxEntity } from "~/toolbox"; +import { Entity } from "./Entity"; + +export type EntityOption = ToolboxEntity | Entity; + +export const getEntity = (entity: EntityOption): ToolboxEntity => { + const result = entity instanceof ToolboxEntity ? entity : entity.entity; + if (!result.name) { + throw new Error(`No name provided for entity.`); + } else if (!result.table) { + throw new Error(`No table provided for entity ${result.name}.`); + } + return result; +}; diff --git a/packages/db-dynamodb/src/utils/entity/index.ts b/packages/db-dynamodb/src/utils/entity/index.ts index 980ebd5fb7f..cab547e858b 100644 --- a/packages/db-dynamodb/src/utils/entity/index.ts +++ b/packages/db-dynamodb/src/utils/entity/index.ts @@ -3,4 +3,5 @@ export * from "./EntityReadBatch"; export * from "./EntityReadBatchBuilder"; export * from "./EntityWriteBatch"; export * from "./EntityWriteBatchBuilder"; +export * from "./getEntity"; export * from "./types";