Merge pull request #15 from knovator/review

Review API

apps/api/package.json

@@ -26,6 +26,9 @@
     "@nestjs/core": "^9.1.2",
     "@nestjs/platform-express": "^9.1.2",
     "@nestjs/swagger": "^6.1.2",
+    "ajv": "^8.11.0",
+    "ajv-formats": "^2.1.1",
+    "ajv-keywords": "^5.1.0",
     "body-parser": "^1.20.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",

apps/api/src/app.module.ts

@@ -7,6 +7,7 @@ import { TemplateModule } from './app/template/template.module';
 import { ColumnModule } from './app/column/column.module';
 import { UploadModule } from './app/upload/upload.module';
 import { MappingModule } from './app/mapping/mapping.module';
+import { ReviewModule } from './app/review/review.module';
 
 const modules: Array<Type | DynamicModule | Promise<DynamicModule> | ForwardReference> = [
   ProjectModule,
@@ -15,6 +16,7 @@ const modules: Array<Type | DynamicModule | Promise<DynamicModule> | ForwardRefe
   ColumnModule,
   UploadModule,
   MappingModule,
+  ReviewModule,
 ];
 
 const providers = [];

apps/api/src/app/column/column.controller.ts

@@ -21,7 +21,7 @@ export class ColumnController {
   })
   @ApiBody({ type: [UpdateColumnRequestDto] })
   async updateTemplateColumns(
-    @Param('templateId', ValidateMongoId) templateId: string,
+    @Param('templateId', ValidateMongoId) _templateId: string,
     @Body(new ParseArrayPipe({ items: UpdateColumnRequestDto })) body: UpdateColumnRequestDto[]
   ): Promise<ColumnResponseDto[]> {
     return this.updateColumns.execute(
@@ -36,19 +36,19 @@ export class ColumnController {
           regexDescription: columnData.regexDescription,
           selectValues: columnData.selectValues,
           sequence: columnData.sequence,
-          templateId: templateId,
+          _templateId,
           type: columnData.type,
         })
       ),
-      templateId
+      _templateId
     );
   }
 
   @Get(':templateId')
   @ApiOperation({
     summary: 'Get template columns',
   })
-  async getTemplateColumns(@Param('templateId') templateId: string): Promise<ColumnResponseDto[]> {
-    return this.getColumns.execute(templateId);
+  async getTemplateColumns(@Param('templateId') _templateId: string): Promise<ColumnResponseDto[]> {
+    return this.getColumns.execute(_templateId);
   }
 }

apps/api/src/app/column/usecases/get-columns/get-columns.usecase.ts

@@ -5,7 +5,7 @@ import { ColumnRepository } from '@impler/dal';
 export class GetColumns {
   constructor(private columnRepository: ColumnRepository) {}
 
-  async execute(templateId: string) {
-    return this.columnRepository.find({ templateId });
+  async execute(_templateId: string) {
+    return this.columnRepository.find({ _templateId });
   }
 }

apps/api/src/app/column/usecases/update-columns/update-columns.command.ts

@@ -47,5 +47,5 @@ export class UpdateColumnCommand extends BaseCommand {
 
   @IsDefined()
   @IsMongoId()
-  templateId: string;
+  _templateId: string;
 }

apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts

@@ -2,23 +2,21 @@ import { Injectable } from '@nestjs/common';
 import { SupportedFileMimeTypesEnum } from '@impler/shared';
 import { ColumnRepository, TemplateRepository } from '@impler/dal';
 import { UpdateColumnCommand } from './update-columns.command';
-import { CSVFileService } from '../../../shared/file/file.service';
 import { StorageService } from '../../../shared/storage/storage.service';
 import { FileNameService } from '../../../shared/file/name.service';
 
 @Injectable()
 export class UpdateColumns {
   constructor(
     private columnRepository: ColumnRepository,
-    private csvFileService: CSVFileService,
     private storageService: StorageService,
     private fileNameService: FileNameService,
     private templateRepository: TemplateRepository
   ) {}
 
-  async execute(command: UpdateColumnCommand[], templateId: string) {
-    await this.columnRepository.delete({ templateId });
-    this.saveSampleFile(command, templateId);
+  async execute(command: UpdateColumnCommand[], _templateId: string) {
+    await this.columnRepository.deleteMany({ _templateId });
+    this.saveSampleFile(command, _templateId);
 
     return this.columnRepository.createMany(command);
   }

apps/api/src/app/mapping/usecases/do-mapping/do-mapping.usecase.ts

@@ -14,7 +14,7 @@ export class DoMapping {
   async execute(command: DoMappingCommand) {
     const columns = await this.columnRepository.find(
       {
-        templateId: command._templateId,
+        _templateId: command._templateId,
       },
       'key alternateKeys sequence',
       {

apps/api/src/app/mapping/usecases/update-mappings/update-mappings.usecase.ts

@@ -7,7 +7,7 @@ export class UpdateMappings {
   constructor(private mappingRepository: MappingRepository) {}
 
   async execute(command: UpdateMappingCommand[], _uploadId: string) {
-    await this.mappingRepository.delete({ _uploadId });
+    await this.mappingRepository.deleteMany({ _uploadId });
 
     return this.mappingRepository.createMany(command);
   }

apps/api/src/app/review/review.controller.ts

@@ -0,0 +1,20 @@
+import { Controller, Get, Param, UseGuards } from '@nestjs/common';
+import { ApiOperation, ApiTags, ApiSecurity } from '@nestjs/swagger';
+import { APIKeyGuard } from '../shared/framework/auth.gaurd';
+import { DoReview } from './usecases/do-review/do-review.usecase';
+
+@Controller('/review')
+@ApiTags('Review')
+@ApiSecurity('ACCESS_KEY')
+@UseGuards(APIKeyGuard)
+export class ReviewController {
+  constructor(private doReview: DoReview) {}
+
+  @Get(':uploadId')
+  @ApiOperation({
+    summary: 'Get Review data for uploaded file',
+  })
+  async getReview(@Param('uploadId') uploadId: string) {
+    return await this.doReview.execute(uploadId);
+  }
+}

apps/api/src/app/review/review.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { USE_CASES } from './usecases';
+import { ReviewController } from './review.controller';
+import { SharedModule } from '../shared/shared.module';
+import { AJVService } from './service/AJV.service';
+
+@Module({
+  imports: [SharedModule],
+  providers: [...USE_CASES, AJVService],
+  controllers: [ReviewController],
+})
+export class ReviewModule {}

apps/api/src/app/review/service/AJV.service.ts

@@ -0,0 +1,166 @@
+import { Injectable } from '@nestjs/common';
+import { ColumnTypesEnum } from '@impler/shared';
+import { ColumnEntity, MappingEntity } from '@impler/dal';
+import Ajv, { ErrorObject } from 'ajv';
+import addFormats from 'ajv-formats';
+import addKeywords from 'ajv-keywords';
+
+const ajv = new Ajv({
+  allErrors: true,
+  coerceTypes: true,
+  allowUnionTypes: true,
+  removeAdditional: true,
+  verbose: true,
+});
+addFormats(ajv, ['email']);
+addKeywords(ajv);
+ajv.addFormat('custom-date-time', function (dateTimeString) {
+  if (typeof dateTimeString === 'object') {
+    dateTimeString = (dateTimeString as Date).toISOString();
+  }
+
+  return !isNaN(Date.parse(dateTimeString)); // any test that returns true/false
+});
+
+@Injectable()
+export class AJVService {
+  async validate(columns: ColumnEntity[], mappings: MappingEntity[], data: any) {
+    const schema = this.buildAJVSchema(columns, mappings);
+    const validator = ajv.compile(schema);
+
+    const valid = validator(data);
+    const returnData = {
+      invalid: [],
+      valid: [],
+    };
+    if (!valid) {
+      const errors: Record<number, any> = this.buildErrorRecords(validator.errors, data);
+
+      returnData.invalid = Object.values(errors);
+      Object.keys(errors).forEach((index) => (data as any).splice(index as unknown as number, 1));
+    }
+    returnData.valid = data as any;
+
+    return returnData;
+  }
+  private buildAJVSchema(columns: ColumnEntity[], mappings: MappingEntity[]) {
+    const formattedColumns: Record<string, ColumnEntity> = columns.reduce((acc, column) => {
+      acc[column._id] = { ...column };
+
+      return acc;
+    }, {});
+    const properties: Record<string, unknown> = mappings.reduce((acc, mapping) => {
+      acc[mapping.columnHeading] = this.getProperty(formattedColumns[mapping._columnId]);
+
+      return acc;
+    }, {});
+    const requiredProperties: string[] = mappings.reduce((acc, mapping) => {
+      if (formattedColumns[mapping._columnId].isRequired) acc.push(mapping.columnHeading);
+
+      return acc;
+    }, []);
+    const uniqueItemProperties = mappings.reduce((acc, mapping) => {
+      if (formattedColumns[mapping._columnId].isUnique) acc.push(mapping.columnHeading);
+
+      return acc;
+    }, []);
+    const objectSchema = {
+      type: 'object',
+      properties,
+      required: requiredProperties,
+      additionalProperties: false,
+    };
+
+    return {
+      type: 'array',
+      items: objectSchema,
+      uniqueItemProperties,
+    };
+  }
+  private getProperty(column: ColumnEntity) {
+    /*
+     * ${1} is record object
+     * ${1#} is record index
+     * ${0#} is key
+     * ${0} is value
+     */
+    switch (column.type) {
+      case ColumnTypesEnum.STRING:
+        return {
+          type: 'string',
+          // errorMessage: { type: `\${1#}${this.indexSeperator}${heading} is not valid string` },
+        };
+      case ColumnTypesEnum.NUMBER:
+        return {
+          type: 'number',
+          // errorMessage: { type: `\${1#}${this.indexSeperator}${heading} is not valid number` },
+        };
+      case ColumnTypesEnum.SELECT:
+        return {
+          type: 'string',
+          enum: column.selectValues || [],
+          /*
+           * errorMessage: {
+           *   enum: `\${1#}${this.indexSeperator}${heading} value should be from [${column.selectValues.join(',')}]`,
+           *   type: `\${1#}${this.indexSeperator}${heading} is not valid string value`,
+           * },
+           */
+        };
+      case ColumnTypesEnum.REGEX:
+        const [full, pattern, flags] = column.regex.match(/\/(.*)\/(.*)|(.*)/);
+
+        return { type: 'string', regexp: { pattern: pattern || full, flags: flags || '' } };
+      case ColumnTypesEnum.EMAIL:
+        return { type: 'string', format: 'email' };
+      case ColumnTypesEnum.DATE:
+        return { type: 'string', format: 'custom-date-time' };
+      case ColumnTypesEnum.ANY:
+        return { type: ['string', 'number', 'object'] };
+      default:
+        throw new Error(`Column type ${column.type} is not supported`);
+    }
+  }
+  private buildErrorRecords(errors: ErrorObject[], data?: any[]) {
+    let index: string, field: string, message: string;
+
+    return errors.reduce((acc, error) => {
+      [, index, field] = error.instancePath.split('/');
+      message = this.getMessage(error, field);
+
+      if (acc[index]) {
+        acc[index].message += `${message}`;
+      } else
+        acc[index] = {
+          index,
+          message,
+          ...data[index],
+        };
+
+      return acc;
+    }, {});
+  }
+  private getMessage(error: ErrorObject, field: string): string {
+    let message = '';
+    switch (error.keyword) {
+      case 'type':
+        message = `${field} ${error.message}`;
+        break;
+      case 'enum':
+        message = `${field} must be from [${error.params.allowedValues}]`;
+        break;
+      case 'regexp':
+        message = `${field} must match pattern ${new RegExp(
+          error.parentSchema?.regexp?.pattern,
+          error.parentSchema?.regexp?.flags || ''
+        ).toString()}`;
+        break;
+      case 'pattern':
+        message = `${field} must match pattern ${error.params.pattern}`;
+        break;
+      default:
+        return `${field} contains invalid data`;
+    }
+
+    return `<li>${message}</li>`;
+  }
+}

apps/api/src/app/review/usecases/do-review/do-review.usecase.ts

@@ -0,0 +1,50 @@
+import { UploadStatusEnum } from '@impler/shared';
+import { Injectable, BadRequestException } from '@nestjs/common';
+import { ColumnRepository, UploadRepository, MappingRepository, FileEntity } from '@impler/dal';
+import { StorageService } from '../../../shared/storage/storage.service';
+import { AJVService } from '../../service/AJV.service';
+import { APIMessages } from '../../../shared/constants';
+import { FileNotExistError } from '../../../shared/errors/file-not-exist.error';
+
+@Injectable()
+export class DoReview {
+  constructor(
+    private uploadRepository: UploadRepository,
+    private storageService: StorageService,
+    private columnRepository: ColumnRepository,
+    private mappingRepository: MappingRepository,
+    private ajvService: AJVService
+  ) {}
+
+  async execute(uploadId: string) {
+    const uploadInfo = await this.uploadRepository.getUploadInformation(uploadId);
+    if (!uploadInfo) {
+      throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
+    }
+    if (uploadInfo.status !== UploadStatusEnum.MAPPED) {
+      throw new BadRequestException(APIMessages.FILE_MAPPING_REMAINING);
+    }
+    const allDataFileInfo = uploadInfo._allDataFileId as unknown as FileEntity;
+    const dataContent = await this.getFileContent(allDataFileInfo.path);
+    const mappings = await this.mappingRepository.find({ _uploadId: uploadId }, '_columnId columnHeading');
+    const columns = await this.columnRepository.find(
+      { _templateId: uploadInfo._templateId },
+      'isRequired isUnique selectValues type regex'
+    );
+
+    return this.ajvService.validate(columns, mappings, dataContent);
+  }
+
+  async getFileContent(path): Promise<string> {
+    try {
+      const dataContent = await this.storageService.getFileContent(path, 'utf8');
+
+      return JSON.parse(dataContent);
+    } catch (error) {
+      if (error instanceof FileNotExistError) {
+        throw new BadRequestException(APIMessages.FILE_NOT_FOUND_IN_STORAGE);
+      }
+      throw error;
+    }
+  }
+}

apps/api/src/app/review/usecases/index.ts

@@ -0,0 +1,6 @@
+import { DoReview } from './do-review/do-review.usecase';
+
+export const USE_CASES = [
+  DoReview,
+  //
+];

apps/api/src/app/shared/constants.ts

@@ -3,4 +3,8 @@ export const APIMessages = {
   FILE_IS_EMPTY: 'File is empty',
   EMPTY_HEADING_PREFIX: 'Empty Heading',
   INVALID_TEMPLATE_ID_CODE_SUFFIX: 'is not valid TemplateId or CODE.',
+  FILE_MAPPING_REMAINING: 'File mapping is not yet done, please finalize mapping before.',
+  UPLOAD_NOT_FOUND: 'Upload information not found with specified uploadId.',
+  FILE_NOT_FOUND_IN_STORAGE:
+    "File not found, make sure you're using the same storage provider, that you were using before.",
 };

apps/api/src/app/shared/file/name.service.ts

@@ -18,4 +18,10 @@ export class FileNameService {
   getUploadedFileName(fileName: string): string {
     return `uploaded.${this.getFileExtension(fileName)}`;
   }
+  getAllJsonDataFileName(): string {
+    return `all-data.json`;
+  }
+  getAllJsonDataFilePath(uploadId: string): string {
+    return `${uploadId}/all-data.json`;
+  }
 }