Feature/3966 database access layer refactoring agregation (#4048)

* fix: get cache from base64 * fix: devide cases responce bufer and base64 in cache * feat: move aggregation filters to bd helper[3966] * feat: move aggregation filters to bd helper for logger service[3966] * feat: move aggregation filters to bd helper for policy service[3966] * feat: move aggregation filters to bd helper for queue service[3966] * feat: move aggregation filters to bd helper and server[3966] --------- Co-authored-by: Ihar <[email protected]>
hashgraph · Aug 20, 2024 · 9543b3c · 9543b3c
1 parent e0a5adc
commit 9543b3c
Show file tree

Hide file tree

Showing 7 changed files with 505 additions and 213 deletions.
diff --git a/analytics-service/src/analytics/report.service.ts b/analytics-service/src/analytics/report.service.ts
@@ -1,4 +1,4 @@
-import { DatabaseServer, MessageAction, PinoLogger } from '@guardian/common';
+import { DatabaseServer, MAP_REPORT_ANALYTICS_AGGREGATION_FILTERS, MessageAction, PinoLogger } from '@guardian/common';
 import { GenerateUUIDv4 } from '@guardian/interfaces';
 import JSZip from 'jszip';
 import xl from 'excel4node';
@@ -218,36 +218,14 @@ export class ReportService {
         const schemaCount = await databaseServer.find(Schema, { uuid, action: MessageAction.PublishSchema });
         const systemSchemaCount = await databaseServer.find(Schema, { uuid, action: MessageAction.PublishSystemSchema });
 
-        const docByPolicy = await databaseServer.aggregate(Document, [
-            { $match: { uuid, } },
-            {
-                $group: {
-                    _id: {
-                        policyTopicId: '$policyTopicId',
-                        type: '$type',
-                        action: '$action'
-                    }, count: { $sum: 1 }
-                }
-            }
-        ] as FilterObject<any>[]);
-
-        const docByInstance = await databaseServer.aggregate(Document, [
-            { $match: { uuid } },
-            {
-                $group: {
-                    _id: {
-                        instanceTopicId: '$instanceTopicId',
-                        type: '$type',
-                        action: '$action'
-                    }, count: { $sum: 1 }
-                }
-            }
-        ] as FilterObject<any>[]);
+        const docByPolicy =
+            await databaseServer.aggregate(Document, databaseServer.getAnalyticsDocAggregationFilters(MAP_REPORT_ANALYTICS_AGGREGATION_FILTERS.DOC_BY_POLICY, uuid) as FilterObject<any>[])
+
+        const docByInstance =
+            await databaseServer.aggregate(Document, databaseServer.getAnalyticsDocAggregationFilters(MAP_REPORT_ANALYTICS_AGGREGATION_FILTERS.DOC_BY_INSTANCE, uuid) as FilterObject<any>[])
 
-        const docsGroups = await databaseServer.aggregate(Document, [
-            { $match: { uuid } },
-            { $group: { _id: { type: '$type', action: '$action' }, count: { $sum: 1 } } }
-        ] as FilterObject<any>[]);
+        const docsGroups =
+            await databaseServer.aggregate(Document, databaseServer.getAnalyticsDocAggregationFilters(MAP_REPORT_ANALYTICS_AGGREGATION_FILTERS.DOCS_GROUPS, uuid) as FilterObject<any>[])
 
         const didCount = docsGroups
             .filter(g => g._id.type === DocumentType.DID && g._id.action !== MessageAction.RevokeDocument)
@@ -482,18 +460,9 @@ export class ReportService {
             }
         }), size);
 
-        const schemasByName = await new DatabaseServer().aggregate(Document, [
-            { $match: { uuid: report.uuid } },
-            {
-                $group: {
-                    _id: {
-                        name: '$name',
-                        action: '$action',
-                    }, count: { $sum: 1 }
-                }
-            },
-            { $sort: { count: -1 } }
-        ] as FilterObject<any>[]);
+        const databaseServer = new DatabaseServer();
+
+        const schemasByName = await databaseServer.aggregate(Document, databaseServer.getAnalyticsDocAggregationFilters(MAP_REPORT_ANALYTICS_AGGREGATION_FILTERS.SCHEMA_BY_NAME, report.uuid) as FilterObject<any>[])
 
         const topAllSchemasByName = [];
         const topSystemSchemasByName = [];

diff --git a/api-gateway/src/helpers/interceptors/cache.ts b/api-gateway/src/helpers/interceptors/cache.ts
@@ -48,8 +48,10 @@ export class CacheInterceptor implements NestInterceptor {
                 if (cachedResponse) {
                     let result = JSON.parse(cachedResponse);
 
-                    if (typeof result === 'string') {
-                        result = Buffer.from(result, 'base64');
+                    if (result.type === 'buffer') {
+                        result = Buffer.from(result.data, 'base64');
+                    } else  {
+                        result = result.data;
                     }
 
                     return result;
@@ -73,7 +75,11 @@ export class CacheInterceptor implements NestInterceptor {
                         }
 
                         if (Buffer.isBuffer(result)) {
-                            result = result.toString('base64');
+                            result = { type: 'buffer', data: result.toString('base64') };
+                        } else if (typeof response === 'object') {
+                            result = { type: 'json', data: result };
+                        } else {
+                            result = { type: 'string', data: result };
                         }
 
                         await this.cacheService.set(cacheKey, JSON.stringify(result), ttl, cacheTag);

diff --git a/common/src/database-modules/database-server.ts b/common/src/database-modules/database-server.ts
@@ -50,7 +50,7 @@ import {
     TopicType,
 } from '@guardian/interfaces';
 import { BaseEntity } from '../models/index.js';
-import { DataBaseHelper } from '../helpers/index.js';
+import { DataBaseHelper, IGetDocumentAggregationFilters, MAP_TRANSACTION_SERIALS_AGGREGATION_FILTERS } from '../helpers/index.js';
 import { Theme } from '../entity/theme.js';
 import { GetConditionsPoliciesByCategories } from '../helpers/policy-category.js';
 import { PolicyTool } from '../entity/tool.js';
@@ -281,20 +281,9 @@ export class DatabaseServer {
      */
     public async aggregate<T extends BaseEntity>(entityClass: new () => T, aggregation: FilterObject<T>[]): Promise<T[]> {
         if (this.dryRun) {
-            if (Array.isArray(aggregation)) {
-                aggregation.unshift({
-                    $match: {
-                        dryRunId: this.dryRun,
-                        dryRunClass: this.classMap.get(entityClass),
-                    },
-                } as FilterObject<unknown> & {
-                    $match?: {
-                        dryRunId?: string;
-                        dryRunClass?: string;
-                    }
-                });
-            }
-            return await new DataBaseHelper(DryRun).aggregate(aggregation) as unknown as T[];
+            const dryRunClass =  this.classMap.get(entityClass)
+
+            return await new DataBaseHelper(DryRun).aggregateDryRan(aggregation, this.dryRun, dryRunClass) as unknown as T[];
         } else {
             return await new DataBaseHelper(entityClass).aggregate(aggregation);
         }
@@ -1070,6 +1059,50 @@ export class DatabaseServer {
         return await this.aggregate(ApprovalDocumentCollection, aggregation) as ApprovalDocumentCollection[];
     }
 
+    /**
+     * get document aggregation filters
+     * @param props
+     *
+     * @returns Result
+     */
+    public getDocumentAggregationFilters(props: IGetDocumentAggregationFilters): void {
+        return DataBaseHelper.getDocumentAggregationFilters(props);
+    }
+
+    /**
+     * get document aggregation filters for analytics
+     * @param nameFilter
+     * @param uuid
+     *
+     * @returns Result
+     */
+    public getAnalyticsDocAggregationFilters(nameFilter: string, uuid: string): unknown[] {
+        return DataBaseHelper.getAnalyticsDocAggregationFilters(nameFilter, uuid);
+    }
+
+    /**
+     * get document aggregation filters for analytics
+     * @param nameFilterMap
+     * @param nameFilterAttributes
+     * @param existingAttributes
+     *
+     * @returns Result
+     */
+    public getAttributesAggregationFilters(nameFilterMap: string, nameFilterAttributes: string, existingAttributes: string[] | []): unknown[] {
+        return DataBaseHelper.getAttributesAggregationFilters(nameFilterMap, nameFilterAttributes, existingAttributes);
+    }
+
+    /**
+     * get tasks aggregation filters
+     * @param nameFilter
+     * @param processTimeout
+     *
+     * @returns Result
+     */
+    public getTasksAggregationFilters(nameFilter: string, processTimeout: number): unknown[] {
+        return DataBaseHelper.getTasksAggregationFilters(nameFilter, processTimeout);
+    }
+
     /**
      * Get Vc Documents
      * @param filters
@@ -1879,21 +1912,25 @@ export class DatabaseServer {
     /**
      * Get transactions serials count
      * @param mintRequestId Mint request identifier
+     * @param transferStatus Transfer status
+     *
      * @returns Serials count
      */
     public async getTransactionsSerialsCount(
         mintRequestId: string,
         transferStatus?: MintTransactionStatus | any
     ): Promise<number> {
-        const aggregation = this._getTransactionsSerialsAggregation(
+        const aggregation = DataBaseHelper._getTransactionsSerialsAggregation(
             mintRequestId,
             transferStatus
         );
-        aggregation.push({
-            $project: {
-                serials: { $size: '$serials' },
-            },
+
+        DataBaseHelper.getTransactionsSerialsAggregationFilters({
+            aggregation,
+            aggregateMethod: 'push',
+            nameFilter: MAP_TRANSACTION_SERIALS_AGGREGATION_FILTERS.COUNT
         });
+
         const result: any = await this.aggregate(MintTransaction, aggregation);
         return result[0]?.serials || 0;
     }
@@ -2050,61 +2087,18 @@ export class DatabaseServer {
         ];
     }
 
-    /**
-     * Get aggregation filter for transactions serials
-     * @param mintRequestId Mint request identifier
-     * @returns Aggregation filter
-     */
-    private _getTransactionsSerialsAggregation(
-        mintRequestId: string,
-        transferStatus?: MintTransactionStatus | any
-    ): any[] {
-        const match: any = {
-            mintRequestId,
-        };
-        if (transferStatus) {
-            match.transferStatus = transferStatus;
-        }
-        const aggregation: any[] = [
-            {
-                $match: match,
-            },
-            {
-                $group: {
-                    _id: 1,
-                    serials: {
-                        $push: '$serials',
-                    },
-                },
-            },
-            {
-                $project: {
-                    serials: {
-                        $reduce: {
-                            input: '$serials',
-                            initialValue: [],
-                            in: {
-                                $concatArrays: ['$$value', '$$this'],
-                            },
-                        },
-                    },
-                },
-            },
-        ];
-
-        return aggregation;
-    }
-
     /**
      * Get transactions serials
      * @param mintRequestId Mint request identifier
+     * @param transferStatus Transfer status
+     *
      * @returns Serials
      */
     public async getTransactionsSerials(
         mintRequestId: string,
         transferStatus?: MintTransactionStatus | any
     ): Promise<number[]> {
-        const aggregation = this._getTransactionsSerialsAggregation(
+        const aggregation = DataBaseHelper._getTransactionsSerialsAggregation(
             mintRequestId,
             transferStatus
         );