Memory consumption enhancements (#55)

* Stream file in and out of mongo, add pubsub api, add wip cloudant version
* Fix compare logic

Signed-off-by: Sean Sundberg <[email protected]>

.gitignore

@@ -13,3 +13,4 @@ test-report.xml
 **/charts/*.tgz
 Chart.lock
 .env
+output.log

package.json

@@ -24,6 +24,7 @@
   "dependencies": {
     "@apollo/server": "^4.9.2",
     "@fast-csv/format": "^5.0.0",
+    "@ibm-cloud/cloudant": "^0.10.0",
     "@nestjs/apollo": "^12.0.7",
     "@nestjs/common": "^10.0.0",
     "@nestjs/core": "^10.0.0",
@@ -39,6 +40,7 @@
     "mongodb": "^6.4.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
+    "stream-transform": "^3.3.2",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
   },
   "devDependencies": {

schema.gql

@@ -2,6 +2,24 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED (DO NOT MODIFY)
 # ------------------------------------------------------
 
+"""AI Model input"""
+type AIModelInput {
+  aliases: [String!]
+  formatterName: String
+  name: String!
+}
+
+"""AI Model"""
+type AiModel {
+  default: Boolean
+  deploymentId: String!
+  description: String
+  id: ID!
+  inputs: [AIModelInput!]!
+  label: String!
+  name: String!
+}
+
 """CSV Document"""
 type CsvDocument {
   description: String
@@ -141,6 +159,7 @@ type PerformanceSummary {
 type Query {
   getCsvDocument(id: ID!): CsvDocument!
   getCsvPrediction(id: ID!): [CsvPrediction!]!
+  listAiModels: [AiModel!]!
   listCsvDocumentRecords(id: ID!, pagination: PaginationInput): PaginatedCsvDocumentRecords!
   listCsvDocuments(pagination: PaginationInput, status: CsvDocumentStatusFilter): PaginatedCsvDocuments!
   listCsvPredictionRecords(id: ID!, options: CsvPredictionRecordOptions, pagination: PaginationInput): PaginatedCsvPredictionResults!

src/backends/cloudant.backend.ts

@@ -0,0 +1,30 @@
+import { CloudantV1 } from "@ibm-cloud/cloudant";
+import {IamAuthenticator} from "ibm-cloud-sdk-core";
+
+export interface CloudantBackendConfig {
+    url: string;
+    apikey: string;
+}
+
+export const cloudantBackend = (): Partial<CloudantBackendConfig> => {
+    return {
+        apikey: process.env.CLOUDANT_API_KEY,
+        url: process.env.CLOUDANT_URL,
+    }
+}
+
+let _instance: CloudantV1;
+export const cloudantClient = ({apikey, url}: CloudantBackendConfig): CloudantV1 => {
+    if (_instance) {
+        return _instance;
+    }
+
+    const authenticator = new IamAuthenticator({apikey})
+
+    const instance = new CloudantV1({
+        authenticator,
+    });
+    instance.setServiceUrl(url);
+
+    return _instance = instance;
+}

src/backends/index.ts

@@ -1,2 +1,3 @@
+export * from './cloudant.backend'
 export * from './mongodb.config'
 export * from './watsonx.config'

src/graphql-types/ai-model.graphql.ts

@@ -0,0 +1,30 @@
+import {AIModelInputFormatter, AIModelInputModel, AIModelModel, InputField} from "../models";
+import {Field, ID, ObjectType} from "@nestjs/graphql";
+
+@ObjectType({description: 'AI Model'})
+export class AiModel implements AIModelModel {
+    @Field(() => ID)
+    id: string;
+    @Field()
+    name: string;
+    @Field()
+    deploymentId: string;
+    @Field({nullable: true})
+    description?: string;
+    @Field({nullable: true})
+    default?: boolean;
+    @Field()
+    label: string;
+    @Field(() => [AIModelInput])
+    inputs: AIModelInputModel[];
+}
+
+@ObjectType({description: 'AI Model input'})
+export class AIModelInput implements AIModelInputModel {
+    @Field()
+    name: string;
+    @Field(() => [String], {nullable: true})
+    aliases?: string[];
+    @Field({nullable: true})
+    formatterName?: string;
+}

src/graphql-types/index.ts

@@ -1 +1,2 @@
 export * from './csv-document.graphql'
+export * from './ai-model.graphql'

src/models/ai-model.model.ts

@@ -7,16 +7,18 @@ export interface AIModelModel {
     name: string;
     deploymentId: string;
     description?: string;
+    default?: boolean;
     inputs: InputField[]
     label: string
 }
 
 export type AIModelInputFormatter = <T> (data: T, fields: InputField[], currentField: InputField) => string
 
 export interface AIModelInputModel {
-    name: string;
-    aliases?: string[],
+    name: string
+    aliases?: string[]
     formatter?: AIModelInputFormatter
+    formatterName?: string
 }
 
 export const isAIModelInputModel = (value: unknown): value is AIModelInputModel => {

src/resolvers/ai-models/ai-models.resolver.ts

@@ -0,0 +1,34 @@
+import {Query, Resolver} from "@nestjs/graphql";
+
+import {AiModel} from "../../graphql-types";
+import {AiModelApi} from "../../services";
+import {AIModelInputFormatter, AIModelInputModel, AIModelModel, InputField} from "../../models";
+
+@Resolver(of => AiModel)
+export class AiModelsResolver {
+
+    constructor(private readonly service: AiModelApi) {}
+
+    @Query(returns => [AiModel])
+    async listAiModels(): Promise<AIModelModel[]> {
+        const models = await this.service.listAIModels()
+
+        return models.map(mapAiModel)
+    }
+}
+
+const mapAiModel = (model: AIModelModel): AIModelModel => {
+    return Object.assign({}, model, {inputs: mapInputs(model.inputs)})
+}
+
+const mapInputs = (inputs: InputField[]): AIModelInputModel[] => {
+    return inputs.map(mapInput)
+}
+
+const mapInput = (input: InputField): AIModelInputModel => {
+    if (typeof input === 'string') {
+        return {name: input}
+    }
+
+    return input
+}

src/resolvers/ai-models/index.ts

@@ -0,0 +1 @@
+export * from './ai-models.resolver'

src/resolvers/providers.ts

@@ -1,9 +1,11 @@
 import {Provider} from "@nestjs/common";
 
 import {CsvDocumentResolver} from "./csv-document";
+import {AiModelsResolver} from "./ai-models";
 
 export * from './csv-document'
 
 export const providers: Provider[] = [
-    CsvDocumentResolver
+    CsvDocumentResolver,
+    AiModelsResolver,
 ]

src/services/ai-model/ai-model.api.ts

@@ -6,4 +6,5 @@ export abstract class AiModelApi {
     abstract listAIModels(): Promise<AIModelModel[]>
     abstract findAIModel(name: string): Promise<AIModelModel>
     abstract getAIModel(id: string): Promise<AIModelModel>
+    abstract getDefaultModel(): Promise<AIModelModel>
 }

src/services/ai-model/ai-model.mock.ts

@@ -1,11 +1,13 @@
 import {AiModelApi} from "./ai-model.api";
 import {AIModelModel, getAIModelInputValue, InputField, isMatchingAIModelInputModel} from "../../models";
 import {first} from "../../util";
+import {fullDescriptionUniqueFormatter} from "./ai-model.support";
 
 const _models: AIModelModel[] = [{
-    id: '1',
-    name: 'tax_withholding_v5',
-    deploymentId: 'tax_withholding_v5',
+    id: '2',
+    name: 'tax_withholding_v6a',
+    deploymentId: 'tax_withholding_v6a',
+    default: true,
     inputs: [
         "MCO_NO",
         "MCO_CMP_NO",
@@ -17,29 +19,11 @@ const _models: AIModelModel[] = [{
         {name: "PERFORMING_LEGAL_ENTITY_NAME", aliases: ["TP_NAME"]},
         {
             name: "Full_Description_Unique",
-            formatter: <T> (data: T, fields: InputField[], currentField: InputField): string => {
-                return fields.filter(field => !isMatchingAIModelInputModel(field, currentField))
-                    .map(getAIModelInputValue(data))
-                    .join(' ')
-            }
+            formatter: fullDescriptionUniqueFormatter,
+            formatterName: 'fullDescriptionUniqueFormatter'
         }
     ],
     label: "WHT_PER"
-// }, {
-//     id: '2',
-//     name: 'tax_withholding_v2',
-//     deploymentId: 'tax_withholding_v2',
-//     inputs: [
-//         "MCO_NO",
-//         "MCO_CMP_NO",
-//         "CTRY_NO",
-//         "SERVICE_PERFORMED_IN",
-//         {name: "NEC_DESCRIPTION_Cleaned", aliases: ["NEC_DESCRIPTION"]},
-//         "NEC_CODE",
-//         "MARKETING_LEGAL_ENTITY_NAME",
-//         "PERFORMING_LEGAL_ENTITY_NAME"
-//     ],
-//     label: "WHT_PER"
 }]
 
 let _id: number = _models.length
@@ -50,6 +34,12 @@ const nextId = (): string => {
 }
 
 export class AiModelMock implements AiModelApi {
+    async getDefaultModel(): Promise<AIModelModel> {
+        return first(_models.filter(val => val.default))
+            .or(() => first(_models))
+            .orElseThrow(() => new Error('No default AI model found'));
+    }
+
     async addAIModels(inputs: Omit<AIModelModel, "id">[]): Promise<AIModelModel[]> {
         return Promise.all(
             inputs.map(model => this.addAIModel(model))