Merge pull request #16 from knovator/valid-invalid-data_save

Valid Invalid data save

apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { SupportedFileMimeTypesEnum } from '@impler/shared';
+import { FileMimeTypesEnum } from '@impler/shared';
 import { ColumnRepository, TemplateRepository } from '@impler/dal';
 import { UpdateColumnCommand } from './update-columns.command';
 import { StorageService } from '../../../shared/storage/storage.service';
@@ -25,7 +25,7 @@ export class UpdateColumns {
     const csvContent = this.createCSVFileHeadingContent(data);
     const fileName = this.fileNameService.getSampleFileName(templateId);
     const sampleFileUrl = this.fileNameService.getSampleFileUrl(templateId);
-    await this.storageService.uploadFile(fileName, csvContent, SupportedFileMimeTypesEnum.CSV, true);
+    await this.storageService.uploadFile(fileName, csvContent, FileMimeTypesEnum.CSV, true);
     await this.templateRepository.update({ _id: templateId }, { sampleFileUrl });
   }
 

apps/api/src/app/review/review.controller.ts

@@ -1,20 +1,48 @@
-import { Controller, Get, Param, UseGuards } from '@nestjs/common';
+import { BadRequestException, Controller, Get, Param, UseGuards } from '@nestjs/common';
 import { ApiOperation, ApiTags, ApiSecurity } from '@nestjs/swagger';
+import { FileEntity } from '@impler/dal';
+import { UploadStatusEnum } from '@impler/shared';
+import { APIMessages } from '../shared/constants';
 import { APIKeyGuard } from '../shared/framework/auth.gaurd';
+import { validateUploadStatus } from '../shared/helpers/upload.helpers';
 import { DoReview } from './usecases/do-review/do-review.usecase';
+import { GetUploadInvalidData } from './usecases/get-upload-invalid-data/get-upload-invalid-data.usecase';
+import { SaveReviewData } from './usecases/save-review-data/save-review-data.usecase';
+import { GetFileInvalidData } from './usecases/get-file-invalid-data/get-file-invalid-data.usecase';
 
 @Controller('/review')
 @ApiTags('Review')
 @ApiSecurity('ACCESS_KEY')
 @UseGuards(APIKeyGuard)
 export class ReviewController {
-  constructor(private doReview: DoReview) {}
+  constructor(
+    private doReview: DoReview,
+    private saveReviewData: SaveReviewData,
+    private getFileInvalidData: GetFileInvalidData,
+    private getUploadInvalidData: GetUploadInvalidData
+  ) {}
 
   @Get(':uploadId')
   @ApiOperation({
     summary: 'Get Review data for uploaded file',
   })
-  async getReview(@Param('uploadId') uploadId: string) {
-    return await this.doReview.execute(uploadId);
+  async getReview(@Param('uploadId') _uploadId: string) {
+    const uploadData = await this.getUploadInvalidData.execute(_uploadId);
+    if (!uploadData) throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
+
+    // Only Mapped & Reviewing status are allowed
+    validateUploadStatus(uploadData.status as UploadStatusEnum, [UploadStatusEnum.MAPPED, UploadStatusEnum.REVIEWING]);
+
+    if (uploadData.status === UploadStatusEnum.MAPPED) {
+      // uploaded file is mapped, do review
+      const reviewData = await this.doReview.execute(_uploadId);
+      // save invalid data to storage
+      this.saveReviewData.execute(_uploadId, reviewData.invalid, reviewData.valid);
+
+      return reviewData.invalid;
+    } else {
+      // Uploaded file is already reviewed, return reviewed data
+      return this.getFileInvalidData.execute((uploadData._invalidDataFileId as unknown as FileEntity).path);
+    }
   }
 }

apps/api/src/app/review/service/AJV.service.ts

@@ -24,7 +24,7 @@ ajv.addFormat('custom-date-time', function (dateTimeString) {
 
 @Injectable()
 export class AJVService {
-  async validate(columns: ColumnEntity[], mappings: MappingEntity[], data: any) {
+  validate(columns: ColumnEntity[], mappings: MappingEntity[], data: any) {
     const schema = this.buildAJVSchema(columns, mappings);
     const validator = ajv.compile(schema);
 

apps/api/src/app/review/usecases/do-review/do-review.usecase.ts

@@ -1,4 +1,4 @@
-import { UploadStatusEnum } from '@impler/shared';
+import { FileEncodingsEnum, UploadStatusEnum } from '@impler/shared';
 import { Injectable, BadRequestException } from '@nestjs/common';
 import { ColumnRepository, UploadRepository, MappingRepository, FileEntity } from '@impler/dal';
 import { StorageService } from '../../../shared/storage/storage.service';
@@ -31,13 +31,15 @@ export class DoReview {
       { _templateId: uploadInfo._templateId },
       'isRequired isUnique selectValues type regex'
     );
+    const reviewData = this.ajvService.validate(columns, mappings, dataContent);
+    this.uploadRepository.update({ _id: uploadId }, { status: UploadStatusEnum.REVIEWING });
 
-    return this.ajvService.validate(columns, mappings, dataContent);
+    return reviewData;
   }
 
   async getFileContent(path): Promise<string> {
     try {
-      const dataContent = await this.storageService.getFileContent(path, 'utf8');
+      const dataContent = await this.storageService.getFileContent(path, FileEncodingsEnum.JSON);
 
       return JSON.parse(dataContent);
     } catch (error) {

apps/api/src/app/review/usecases/get-file-invalid-data/get-file-invalid-data.usecase.ts

@@ -0,0 +1,14 @@
+import { Injectable } from '@nestjs/common';
+import { FileEncodingsEnum } from '@impler/shared';
+import { StorageService } from '../../../shared/storage/storage.service';
+
+@Injectable()
+export class GetFileInvalidData {
+  constructor(private storageService: StorageService) {}
+
+  async execute(path: string) {
+    const stringContent = await this.storageService.getFileContent(path, FileEncodingsEnum.JSON);
+
+    return JSON.parse(stringContent);
+  }
+}

apps/api/src/app/review/usecases/get-upload-invalid-data/get-upload-invalid-data.usecase.ts

@@ -0,0 +1,11 @@
+import { Injectable } from '@nestjs/common';
+import { UploadRepository } from '@impler/dal';
+
+@Injectable()
+export class GetUploadInvalidData {
+  constructor(private uploadRepository: UploadRepository) {}
+
+  async execute(_uploadId: string) {
+    return this.uploadRepository.getUploadInvalidDataInformation(_uploadId);
+  }
+}

apps/api/src/app/review/usecases/index.ts

@@ -1,6 +1,12 @@
 import { DoReview } from './do-review/do-review.usecase';
+import { SaveReviewData } from './save-review-data/save-review-data.usecase';
+import { GetUploadInvalidData } from './get-upload-invalid-data/get-upload-invalid-data.usecase';
+import { GetFileInvalidData } from './get-file-invalid-data/get-file-invalid-data.usecase';
 
 export const USE_CASES = [
   DoReview,
+  SaveReviewData,
+  GetFileInvalidData,
+  GetUploadInvalidData,
   //
 ];

apps/api/src/app/review/usecases/save-review-data/save-review-data.usecase.ts

@@ -0,0 +1,58 @@
+import { Injectable } from '@nestjs/common';
+import { FileRepository, UploadRepository } from '@impler/dal';
+import { FileMimeTypesEnum } from '@impler/shared';
+import { FileNameService } from '../../../shared/file/name.service';
+import { StorageService } from '../../../shared/storage/storage.service';
+
+@Injectable()
+export class SaveReviewData {
+  constructor(
+    private fileNameService: FileNameService,
+    private storageService: StorageService,
+    private fileRepository: FileRepository,
+    private uploadRepository: UploadRepository
+  ) {}
+
+  async execute(_uploadId: string, invalidData: any[], validData: any[]) {
+    const _invalidDataFileId = await this.storeInvalidFile(_uploadId, invalidData);
+    const _validDataFileId = await this.storeValidFile(_uploadId, validData);
+    await this.uploadRepository.update({ _id: _uploadId }, { _invalidDataFileId, _validDataFileId });
+  }
+
+  private async storeValidFile(_uploadId: string, validData: any[]): Promise<string> {
+    if (validData.length < 1) return null;
+
+    const strinValidData = JSON.stringify(validData);
+    const validFilePath = this.fileNameService.getValidDataFilePath(_uploadId);
+    await this.storeFile(validFilePath, strinValidData);
+    const validFileName = this.fileNameService.getValidDataFileName();
+    const entry = await this.makeFileEntry(validFileName, validFilePath);
+
+    return entry._id;
+  }
+
+  private async storeInvalidFile(_uploadId: string, invalidData: any[]): Promise<string> {
+    if (invalidData.length < 1) return null;
+
+    const stringInvalidData = JSON.stringify(invalidData);
+    const invalidFilePath = this.fileNameService.getInvalidDataFilePath(_uploadId);
+    await this.storeFile(invalidFilePath, stringInvalidData);
+    const invalidFileName = this.fileNameService.getInvalidDataFileName();
+    const entry = await this.makeFileEntry(invalidFileName, invalidFilePath);
+
+    return entry._id;
+  }
+
+  private async storeFile(invalidFilePath: string, data: string) {
+    await this.storageService.uploadFile(invalidFilePath, data, FileMimeTypesEnum.JSON);
+  }
+
+  private async makeFileEntry(fileName: string, filePath: string) {
+    return await this.fileRepository.create({
+      mimeType: FileMimeTypesEnum.JSON,
+      name: fileName,
+      originalName: fileName,
+      path: filePath,
+    });
+  }
+}

apps/api/src/app/shared/constants.ts

@@ -7,4 +7,8 @@ export const APIMessages = {
   UPLOAD_NOT_FOUND: 'Upload information not found with specified uploadId.',
   FILE_NOT_FOUND_IN_STORAGE:
     "File not found, make sure you're using the same storage provider, that you were using before.",
+  DO_MAPPING_FIRST: 'You may landed to wrong place, Please finalize mapping and proceed ahead.',
+  DO_REVIEW_FIRST: 'You may landed to wrong place, Please review data and proceed ahead.',
+  DO_CONFIRM_FIRST: 'You may landed to wrong place, Please confirm data and proceed ahead.',
+  ALREADY_CONFIRMED: '`You may landed to wrong place, This upload file is confirmed already.',
 };

apps/api/src/app/shared/file/name.service.ts

@@ -13,7 +13,7 @@ export class FileNameService {
     return fileName.split('.').pop();
   }
   getUploadedFilePath(uploadId: string, fileName: string): string {
-    return `${uploadId}/uploaded.${this.getFileExtension(fileName)}`;
+    return `${uploadId}/${this.getUploadedFileName(fileName)}`;
   }
   getUploadedFileName(fileName: string): string {
     return `uploaded.${this.getFileExtension(fileName)}`;
@@ -22,6 +22,18 @@ export class FileNameService {
     return `all-data.json`;
   }
   getAllJsonDataFilePath(uploadId: string): string {
-    return `${uploadId}/all-data.json`;
+    return `${uploadId}/${this.getAllJsonDataFileName()}`;
+  }
+  getInvalidDataFileName(): string {
+    return `invalid-data.json`;
+  }
+  getInvalidDataFilePath(uploadId: string): string {
+    return `${uploadId}/${this.getInvalidDataFileName()}`;
+  }
+  getValidDataFileName(): string {
+    return `valid-data.json`;
+  }
+  getValidDataFilePath(uploadId: string): string {
+    return `${uploadId}/${this.getValidDataFileName()}`;
   }
 }

apps/api/src/app/shared/helpers/file.helper.ts

@@ -1,11 +1,11 @@
-import { SupportedFileMimeTypesEnum } from '@impler/shared';
+import { FileMimeTypesEnum } from '@impler/shared';
 import { APIMessages } from '../constants';
 import { FileService, CSVFileService, ExcelFileService } from '../file/file.service';
 
 export const getFileService = (mimeType: string): FileService => {
-  if (mimeType === SupportedFileMimeTypesEnum.CSV) {
+  if (mimeType === FileMimeTypesEnum.CSV) {
     return new CSVFileService();
-  } else if (mimeType === SupportedFileMimeTypesEnum.EXCEL || mimeType === SupportedFileMimeTypesEnum.EXCELX) {
+  } else if (mimeType === FileMimeTypesEnum.EXCEL || mimeType === FileMimeTypesEnum.EXCELX) {
     return new ExcelFileService();
   }
 

apps/api/src/app/shared/helpers/upload.helpers.ts

@@ -0,0 +1,18 @@
+import { UploadStatusEnum } from '@impler/shared';
+import { BadRequestException } from '@nestjs/common';
+import { APIMessages } from '../constants';
+
+export function validateUploadStatus(currentStatus: UploadStatusEnum, expectedStatus: UploadStatusEnum[]): boolean {
+  if (expectedStatus.includes(currentStatus)) return true;
+  else {
+    if (currentStatus === UploadStatusEnum.UPLOADED) {
+      throw new BadRequestException(APIMessages.DO_MAPPING_FIRST);
+    } else if (currentStatus === UploadStatusEnum.MAPPED) {
+      throw new BadRequestException(APIMessages.DO_REVIEW_FIRST);
+    } else if (currentStatus === UploadStatusEnum.REVIEWED) {
+      throw new BadRequestException(APIMessages.DO_CONFIRM_FIRST);
+    } else if (currentStatus === UploadStatusEnum.CONFIRMED) {
+      throw new BadRequestException(APIMessages.ALREADY_CONFIRMED);
+    }
+  }
+}

apps/api/src/app/shared/validations/valid-import-file.validation.ts

@@ -1,13 +1,13 @@
 // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
 import _whatever from 'multer';
-import { SupportedFileMimeTypesEnum } from '@impler/shared';
+import { SupportedFileMimeTypes } from '@impler/shared';
 import { Injectable, PipeTransform } from '@nestjs/common';
 import { FileNotValidError } from '../exceptions/file-not-valid.exception';
 
 @Injectable()
 export class ValidImportFile implements PipeTransform<Express.Multer.File> {
   transform(value: Express.Multer.File) {
-    if (!(Object.values(SupportedFileMimeTypesEnum) as string[]).includes(value.mimetype)) {
+    if (!SupportedFileMimeTypes.includes(value.mimetype)) {
       throw new FileNotValidError();
     }
 

apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { UploadStatusEnum } from '@impler/shared';
+import { FileMimeTypesEnum, UploadStatusEnum } from '@impler/shared';
 import { CommonRepository, FileEntity, FileRepository, UploadRepository } from '@impler/dal';
 import { MakeUploadEntryCommand } from './make-upload-entry.command';
 import { FileNameService } from '../../../shared/file/name.service';
@@ -79,10 +79,10 @@ export class MakeUploadEntry {
   private async addAllDataEntry(uploadId: string, data: Record<string, unknown>[]): Promise<FileEntity> {
     const allDataFileName = this.fileNameService.getAllJsonDataFileName();
     const allDataFilePath = this.fileNameService.getAllJsonDataFilePath(uploadId);
-    await this.storageService.uploadFile(allDataFilePath, JSON.stringify(data), 'application/json');
+    await this.storageService.uploadFile(allDataFilePath, JSON.stringify(data), FileMimeTypesEnum.JSON);
 
     return await this.fileRepository.create({
-      mimeType: 'application/json',
+      mimeType: FileMimeTypesEnum.JSON,
       path: allDataFilePath,
       name: allDataFileName,
       originalName: allDataFileName,

libs/dal/src/repositories/upload/upload.repository.ts

@@ -10,4 +10,7 @@ export class UploadRepository extends BaseRepository<UploadEntity> {
   async getUploadInformation(uploadId: string): Promise<UploadEntity> {
     return await Upload.findById(uploadId).populate('_allDataFileId', 'path name');
   }
+  async getUploadInvalidDataInformation(uploadId: string): Promise<UploadEntity> {
+    return await Upload.findById(uploadId).populate('_invalidDataFileId', 'path name');
+  }
 }

libs/shared/src/types/upload/upload.types.ts

@@ -9,16 +9,24 @@ export enum UploadStatusEnum {
   'COMPLETED' = 'Completed',
 }
 
-export enum SupportedFileMimeTypesEnum {
+export const SupportedFileMimeTypes = [
+  'text/csv', // csv
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // EXCELX
+  'application/vnd.ms-excel', // EXCEL
+];
+
+export enum FileMimeTypesEnum {
   'CSV' = 'text/csv',
   'EXCEL' = 'application/vnd.ms-excel',
   'EXCELX' = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  'JSON' = 'application/json',
 }
 
 export enum FileEncodingsEnum {
   'CSV' = 'utf8',
   'EXCEL' = 'base64',
   'EXCELX' = 'base64',
+  'JSON' = 'utf8',
 }
 
 export interface IFileInformation {