From 53c7dff72c835a0d86e6f4320d09816f3433ecd8 Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 11:38:33 +0530 Subject: [PATCH 1/6] feat: Added file upload API --- apps/api/package.json | 1 + apps/api/src/app.module.ts | 2 + .../src/app/ops/dtos/upload-request.dto.ts | 34 +++++++++++ apps/api/src/app/ops/ops.controller.ts | 41 +++++++++++++ apps/api/src/app/ops/ops.module.ts | 11 ++++ apps/api/src/app/ops/usecases/index.ts | 6 ++ .../make-upload-entry.command.ts | 19 +++++++ .../make-upload-entry.usecase.ts | 57 +++++++++++++++++++ apps/api/src/app/shared/constants.ts | 3 + .../app/shared/errors/file-not-valid.error.ts | 8 +++ apps/api/src/app/shared/file/name.service.ts | 9 +++ .../framework/is-valid-template.validator.ts | 23 ++++++++ apps/api/src/app/shared/shared.module.ts | 21 ++++++- .../valid-import-file.validation.ts | 16 ++++++ apps/api/tsconfig.json | 2 +- .../repositories/common/common.repository.ts | 4 ++ .../src/repositories/upload/upload.schema.ts | 6 ++ libs/shared/src/types/upload/upload.types.ts | 7 +++ pnpm-lock.yaml | 8 +++ 19 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/app/ops/dtos/upload-request.dto.ts create mode 100644 apps/api/src/app/ops/ops.controller.ts create mode 100644 apps/api/src/app/ops/ops.module.ts create mode 100644 apps/api/src/app/ops/usecases/index.ts create mode 100644 apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.command.ts create mode 100644 apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts create mode 100644 apps/api/src/app/shared/constants.ts create mode 100644 apps/api/src/app/shared/errors/file-not-valid.error.ts create mode 100644 apps/api/src/app/shared/framework/is-valid-template.validator.ts create mode 100644 apps/api/src/app/shared/validations/valid-import-file.validation.ts diff --git a/apps/api/package.json b/apps/api/package.json index 080e75399..7c266bfaf 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -37,6 +37,7 @@ "devDependencies": { "@types/express": "^4.17.14", "@types/json2csv": "^5.0.3", + "@types/multer": "^1.4.7", "@types/node": "^18.7.18", "nodemon": "^2.0.20", "ts-loader": "^9.4.1", diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 804669b22..d63e5b671 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -5,12 +5,14 @@ import { SharedModule } from './app/shared/shared.module'; import { ProjectModule } from './app/project/project.module'; import { TemplateModule } from './app/template/template.module'; import { ColumnModule } from './app/column/column.module'; +import { OpsModule } from './app/ops/ops.module'; const modules: Array | ForwardReference> = [ ProjectModule, SharedModule, TemplateModule, ColumnModule, + OpsModule, ]; const providers = []; diff --git a/apps/api/src/app/ops/dtos/upload-request.dto.ts b/apps/api/src/app/ops/dtos/upload-request.dto.ts new file mode 100644 index 000000000..218b6a7de --- /dev/null +++ b/apps/api/src/app/ops/dtos/upload-request.dto.ts @@ -0,0 +1,34 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsJSON, IsOptional, IsString, Validate } from 'class-validator'; +import { IsValidTemplateValidator } from '../../shared/framework/is-valid-template.validator'; + +export class UploadRequestDto { + @ApiProperty({ + description: 'Id or CODE of the template', + }) + @IsString() + @Validate(IsValidTemplateValidator) + template: string; + + @ApiProperty({ + type: 'file', + required: true, + }) + file: Express.Multer.File; + + @ApiProperty({ + description: 'Auth header value to send during webhook call', + required: false, + }) + @IsOptional() + @IsString() + authHeaderValue: string; + + @ApiProperty({ + description: 'Payload to send during webhook call', + required: false, + }) + @IsOptional() + @IsJSON() + extra: string; +} diff --git a/apps/api/src/app/ops/ops.controller.ts b/apps/api/src/app/ops/ops.controller.ts new file mode 100644 index 000000000..f06ad2342 --- /dev/null +++ b/apps/api/src/app/ops/ops.controller.ts @@ -0,0 +1,41 @@ +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +import _whatever from 'multer'; +import { Body, Controller, Post, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { ApiTags, ApiSecurity, ApiConsumes } from '@nestjs/swagger'; +import { APIKeyGuard } from '../shared/framework/auth.gaurd'; +import { UploadRequestDto } from './dtos/upload-request.dto'; +import { ValidImportFile } from '../shared/validations/valid-import-file.validation'; +import { MakeUploadEntry } from './usecases/make-upload-entry/make-upload-entry.usecase'; +import { MakeUploadEntryCommand } from './usecases/make-upload-entry/make-upload-entry.command'; + +@Controller('/ops') +@ApiTags('Operations') +@ApiSecurity('ACCESS_KEY') +@UseGuards(APIKeyGuard) +export class OpsController { + constructor(private makeUploadEntry: MakeUploadEntry) {} + + @Post('upload') + @ApiConsumes('multipart/form-data') + @UseInterceptors(FileInterceptor('file')) + async uploadFile(@UploadedFile('file', ValidImportFile) file: Express.Multer.File, @Body() body: UploadRequestDto) { + return await this.makeUploadEntry.execute( + MakeUploadEntryCommand.create({ + file: file, + templateId: body.template, + extra: body.extra, + authHeaderValue: body.authHeaderValue, + }) + ); + + /* + * validate template (done) + * upload file (done) + * make entry to file (done) + * make entry to uploads (done) + * Get headings + * do mapping + */ + } +} diff --git a/apps/api/src/app/ops/ops.module.ts b/apps/api/src/app/ops/ops.module.ts new file mode 100644 index 000000000..23533aec0 --- /dev/null +++ b/apps/api/src/app/ops/ops.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { USE_CASES } from './usecases'; +import { OpsController } from './ops.controller'; +import { SharedModule } from '../shared/shared.module'; + +@Module({ + imports: [SharedModule], + providers: [...USE_CASES], + controllers: [OpsController], +}) +export class OpsModule {} diff --git a/apps/api/src/app/ops/usecases/index.ts b/apps/api/src/app/ops/usecases/index.ts new file mode 100644 index 000000000..3d686050f --- /dev/null +++ b/apps/api/src/app/ops/usecases/index.ts @@ -0,0 +1,6 @@ +import { MakeUploadEntry } from './make-upload-entry/make-upload-entry.usecase'; + +export const USE_CASES = [ + MakeUploadEntry, + // +]; diff --git a/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.command.ts b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.command.ts new file mode 100644 index 000000000..92a8183f8 --- /dev/null +++ b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.command.ts @@ -0,0 +1,19 @@ +import { IsDefined, IsString, IsOptional, IsJSON } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class MakeUploadEntryCommand extends BaseCommand { + @IsDefined() + file: Express.Multer.File; + + @IsDefined() + @IsString() + templateId: string; + + @IsOptional() + @IsJSON() + extra?: string; + + @IsOptional() + @IsString() + authHeaderValue?: string; +} diff --git a/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts new file mode 100644 index 000000000..932581ced --- /dev/null +++ b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts @@ -0,0 +1,57 @@ +import { Injectable } from '@nestjs/common'; +import { UploadStatusEnum } from '@impler/shared'; +import { CommonRepository, FileRepository, UploadRepository } from '@impler/dal'; +import { MakeUploadEntryCommand } from './make-upload-entry.command'; +import { FileNameService } from '../../../shared/file/name.service'; +import { StorageService } from '../../../shared/storage/storage.service'; + +@Injectable() +export class MakeUploadEntry { + constructor( + private uploadRepository: UploadRepository, + private commonRepository: CommonRepository, + private fileRepository: FileRepository, + private storageService: StorageService, + private fileNameService: FileNameService + ) {} + + async execute({ file, templateId, extra, authHeaderValue }: MakeUploadEntryCommand) { + const uploadId = this.commonRepository.generateMongoId(); + const fileId = await this.makeFileEntry(uploadId, file); + + return this.makeUploadEntry(templateId, fileId, uploadId, extra, authHeaderValue); + } + + private async makeFileEntry(uploadId: string, file: Express.Multer.File): Promise { + const uploadedFileName = this.fileNameService.getUploadedFileName(file.originalname); + const uploadedFilePath = this.fileNameService.getUploadedFilePath(uploadId, file.originalname); + this.storageService.uploadFile(uploadedFilePath, file.buffer, file.mimetype); + const fileEntry = await this.fileRepository.create({ + mimeType: file.mimetype, + name: uploadedFileName, + originalName: file.originalname, + path: uploadedFilePath, + }); + + return fileEntry._id; + } + + private async makeUploadEntry( + templateId: string, + fileId: string, + uploadId: string, + extra?: string, + authHeaderValue?: string + ) { + return this.uploadRepository.create({ + _id: uploadId, + _uploadedFileId: fileId, + _templateId: templateId, + extra: extra, + headings: [''], + status: UploadStatusEnum.MAPPING, + authHeaderValue: authHeaderValue, + totalRecords: 0, + }); + } +} diff --git a/apps/api/src/app/shared/constants.ts b/apps/api/src/app/shared/constants.ts new file mode 100644 index 000000000..f1b59594a --- /dev/null +++ b/apps/api/src/app/shared/constants.ts @@ -0,0 +1,3 @@ +export const APIMessages = { + FILE_TYPE_NOT_VALID: 'File type is not valid', +}; diff --git a/apps/api/src/app/shared/errors/file-not-valid.error.ts b/apps/api/src/app/shared/errors/file-not-valid.error.ts new file mode 100644 index 000000000..3c826a3a8 --- /dev/null +++ b/apps/api/src/app/shared/errors/file-not-valid.error.ts @@ -0,0 +1,8 @@ +import { UnprocessableEntityException } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +export class FileNotValidError extends UnprocessableEntityException { + constructor() { + super(APIMessages.FILE_TYPE_NOT_VALID); + } +} diff --git a/apps/api/src/app/shared/file/name.service.ts b/apps/api/src/app/shared/file/name.service.ts index 70f98d1fc..ae4b25e3d 100644 --- a/apps/api/src/app/shared/file/name.service.ts +++ b/apps/api/src/app/shared/file/name.service.ts @@ -9,4 +9,13 @@ export class FileNameService { return path.join(process.env.S3_LOCAL_STACK, process.env.S3_BUCKET_NAME, fileName); } + getFileExtension(fileName: string): string { + return fileName.split('.').pop(); + } + getUploadedFilePath(uploadId: string, fileName: string): string { + return `${uploadId}/uploaded.${this.getFileExtension(fileName)}`; + } + getUploadedFileName(fileName: string): string { + return `uploaded.${this.getFileExtension(fileName)}`; + } } diff --git a/apps/api/src/app/shared/framework/is-valid-template.validator.ts b/apps/api/src/app/shared/framework/is-valid-template.validator.ts new file mode 100644 index 000000000..241cc99bc --- /dev/null +++ b/apps/api/src/app/shared/framework/is-valid-template.validator.ts @@ -0,0 +1,23 @@ +import { ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface } from 'class-validator'; +import { TemplateRepository, CommonRepository } from '@impler/dal'; + +@ValidatorConstraint({ name: 'IsValidTemplate', async: true }) +export class IsValidTemplateValidator implements ValidatorConstraintInterface { + private templateRepository: TemplateRepository; + private commonRepository: CommonRepository; + constructor() { + this.templateRepository = new TemplateRepository(); + this.commonRepository = new CommonRepository(); + } + + async validate(value: string) { + const isMongoId = this.commonRepository.validMongoId(value); + const count = await this.templateRepository.count(isMongoId ? { _id: value } : { code: value }); + + return !!count; + } + + defaultMessage(args: ValidationArguments) { + return `${args.value} is not valid TemplateID or CODE`; + } +} diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts index 8a1898891..2374284e5 100644 --- a/apps/api/src/app/shared/shared.module.ts +++ b/apps/api/src/app/shared/shared.module.ts @@ -1,10 +1,27 @@ import { Module } from '@nestjs/common'; -import { ColumnRepository, DalService, ProjectRepository, TemplateRepository } from '@impler/dal'; +import { + ColumnRepository, + CommonRepository, + DalService, + FileRepository, + MappingRepository, + ProjectRepository, + TemplateRepository, + UploadRepository, +} from '@impler/dal'; import { S3StorageService, StorageService } from './storage/storage.service'; import { CSVFileService } from './file/file.service'; import { FileNameService } from './file/name.service'; -const DAL_MODELS = [ProjectRepository, TemplateRepository, ColumnRepository]; +const DAL_MODELS = [ + ProjectRepository, + TemplateRepository, + ColumnRepository, + FileRepository, + UploadRepository, + MappingRepository, + CommonRepository, +]; const FILE_SERVICES = [CSVFileService, FileNameService]; const dalService = new DalService(); diff --git a/apps/api/src/app/shared/validations/valid-import-file.validation.ts b/apps/api/src/app/shared/validations/valid-import-file.validation.ts new file mode 100644 index 000000000..53a3aecfe --- /dev/null +++ b/apps/api/src/app/shared/validations/valid-import-file.validation.ts @@ -0,0 +1,16 @@ +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +import _whatever from 'multer'; +import { SupportedFileMimeTypesEnum } from '@impler/shared'; +import { Injectable, PipeTransform } from '@nestjs/common'; +import { FileNotValidError } from '../errors/file-not-valid.error'; + +@Injectable() +export class ValidImportFile implements PipeTransform { + transform(value: Express.Multer.File) { + if (!(Object.values(SupportedFileMimeTypesEnum) as string[]).includes(value.mimetype)) { + throw new FileNotValidError(); + } + + return value; + } +} diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json index 7f2ec97a4..81cb9aa35 100644 --- a/apps/api/tsconfig.json +++ b/apps/api/tsconfig.json @@ -4,7 +4,7 @@ "module": "commonjs", "allowSyntheticDefaultImports": true, "emitDecoratorMetadata": true, - "types": ["node"], + "types": ["node", "multer"], "target": "es2017", "allowJs": false, "esModuleInterop": false, diff --git a/libs/dal/src/repositories/common/common.repository.ts b/libs/dal/src/repositories/common/common.repository.ts index 2acbf46e0..f5782632d 100644 --- a/libs/dal/src/repositories/common/common.repository.ts +++ b/libs/dal/src/repositories/common/common.repository.ts @@ -15,4 +15,8 @@ export class CommonRepository { validMongoId(id: string): boolean { return Types.ObjectId.isValid(id); } + + generateMongoId(): string { + return new Types.ObjectId().toString(); + } } diff --git a/libs/dal/src/repositories/upload/upload.schema.ts b/libs/dal/src/repositories/upload/upload.schema.ts index 94d63fa32..f8ac2c494 100644 --- a/libs/dal/src/repositories/upload/upload.schema.ts +++ b/libs/dal/src/repositories/upload/upload.schema.ts @@ -7,6 +7,7 @@ const uploadSchema = new Schema( _templateId: { type: Schema.Types.String, ref: 'Template', + index: true, }, _uploadedFileId: { type: Schema.Types.String, @@ -20,6 +21,11 @@ const uploadSchema = new Schema( type: Schema.Types.String, ref: 'File', }, + uploadedDate: { + type: Schema.Types.Date, + default: Date.now, + index: true, + }, headings: [String], uploadDate: Date, totalRecords: Number, diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index bac2feb8e..115ec2434 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -6,3 +6,10 @@ export enum UploadStatusEnum { 'PROCESSING' = 'Processing', 'COMPLETED' = 'Completed', } + +export enum SupportedFileMimeTypesEnum { + 'CSV' = 'text/csv', + 'XML' = 'application/xml', + 'EXCEL' = 'application/vnd.ms-excel', + 'EXCELX' = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae7f045ee..25d3aead3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,6 +63,7 @@ importers: '@nestjs/swagger': ^6.1.2 '@types/express': ^4.17.14 '@types/json2csv': ^5.0.3 + '@types/multer': ^1.4.7 '@types/node': ^18.7.18 body-parser: ^1.20.0 class-transformer: ^0.5.1 @@ -96,6 +97,7 @@ importers: devDependencies: '@types/express': 4.17.14 '@types/json2csv': 5.0.3 + '@types/multer': 1.4.7 '@types/node': 18.7.18 nodemon: 2.0.20 ts-loader: 9.4.1_kb3egcnme7w23lfa5xodfjfhge @@ -2284,6 +2286,12 @@ packages: resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} dev: true + /@types/multer/1.4.7: + resolution: {integrity: sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==} + dependencies: + '@types/express': 4.17.14 + dev: true + /@types/node/14.18.29: resolution: {integrity: sha512-LhF+9fbIX4iPzhsRLpK5H7iPdvW8L4IwGciXQIOEcuF62+9nw/VQVsOViAOOGxY3OlOKGLFv0sWwJXdwQeTn6A==} dev: true From 0e653ccf4228c7e81c8e818bd298af920b7b3c57 Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 13:29:45 +0530 Subject: [PATCH 2/6] feat: Completed CSV File Upload API --- apps/api/package.json | 1 + .../add-upload-entry.command.ts | 32 +++++++++ .../make-upload-entry.usecase.ts | 49 +++++++++----- apps/api/src/app/shared/constants.ts | 2 +- apps/api/src/app/shared/file/file.service.ts | 26 ++++++++ .../api/src/app/shared/helpers/file.helper.ts | 11 ++++ .../src/app/shared/storage/storage.service.ts | 9 ++- libs/shared/src/types/upload/upload.types.ts | 6 ++ pnpm-lock.yaml | 66 ++++++++++++++++++- 9 files changed, 179 insertions(+), 23 deletions(-) create mode 100644 apps/api/src/app/ops/usecases/make-upload-entry/add-upload-entry.command.ts create mode 100644 apps/api/src/app/shared/helpers/file.helper.ts diff --git a/apps/api/package.json b/apps/api/package.json index 7c266bfaf..0463ee6ba 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -31,6 +31,7 @@ "compression": "^1.7.4", "dotenv": "^16.0.2", "envalid": "^7.3.1", + "fast-csv": "^4.3.6", "json2csv": "^5.0.7", "rimraf": "^3.0.2" }, diff --git a/apps/api/src/app/ops/usecases/make-upload-entry/add-upload-entry.command.ts b/apps/api/src/app/ops/usecases/make-upload-entry/add-upload-entry.command.ts new file mode 100644 index 000000000..40126b145 --- /dev/null +++ b/apps/api/src/app/ops/usecases/make-upload-entry/add-upload-entry.command.ts @@ -0,0 +1,32 @@ +import { IsDefined, IsString, IsOptional, IsJSON, IsArray, IsNumber } from 'class-validator'; +import { BaseCommand } from '../../../shared/commands/base.command'; + +export class AddUploadEntryCommand extends BaseCommand { + @IsDefined() + @IsString() + templateId: string; + + @IsDefined() + @IsString() + fileId: string; + + @IsDefined() + @IsString() + uploadId: string; + + @IsOptional() + @IsJSON() + extra?: string; + + @IsOptional() + @IsString() + authHeaderValue?: string; + + @IsOptional() + @IsArray() + headings?: string[]; + + @IsOptional() + @IsNumber() + totalRecords?: number; +} diff --git a/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts index 932581ced..cb4088063 100644 --- a/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts +++ b/apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts @@ -1,9 +1,12 @@ import { Injectable } from '@nestjs/common'; import { UploadStatusEnum } from '@impler/shared'; -import { CommonRepository, FileRepository, UploadRepository } from '@impler/dal'; +import { CommonRepository, FileEntity, FileRepository, UploadRepository } from '@impler/dal'; import { MakeUploadEntryCommand } from './make-upload-entry.command'; import { FileNameService } from '../../../shared/file/name.service'; import { StorageService } from '../../../shared/storage/storage.service'; +import { FileService } from '../../../shared/file/file.service'; +import { getFileService } from '../../../shared/helpers/file.helper'; +import { AddUploadEntryCommand } from './add-upload-entry.command'; @Injectable() export class MakeUploadEntry { @@ -17,15 +20,27 @@ export class MakeUploadEntry { async execute({ file, templateId, extra, authHeaderValue }: MakeUploadEntryCommand) { const uploadId = this.commonRepository.generateMongoId(); - const fileId = await this.makeFileEntry(uploadId, file); + const fileEntity = await this.makeFileEntry(uploadId, file); + const fileService: FileService = getFileService(file.mimetype); + const fileInformation = await fileService.getFileInformation(this.storageService, fileEntity.path); - return this.makeUploadEntry(templateId, fileId, uploadId, extra, authHeaderValue); + return this.addUploadEntry( + AddUploadEntryCommand.create({ + templateId: templateId, + fileId: fileEntity._id, + uploadId, + extra, + authHeaderValue, + headings: fileInformation.headings, + totalRecords: fileInformation.totalRecords, + }) + ); } - private async makeFileEntry(uploadId: string, file: Express.Multer.File): Promise { - const uploadedFileName = this.fileNameService.getUploadedFileName(file.originalname); + private async makeFileEntry(uploadId: string, file: Express.Multer.File): Promise { const uploadedFilePath = this.fileNameService.getUploadedFilePath(uploadId, file.originalname); - this.storageService.uploadFile(uploadedFilePath, file.buffer, file.mimetype); + await this.storageService.uploadFile(uploadedFilePath, file.buffer, file.mimetype); + const uploadedFileName = this.fileNameService.getUploadedFileName(file.originalname); const fileEntry = await this.fileRepository.create({ mimeType: file.mimetype, name: uploadedFileName, @@ -33,25 +48,27 @@ export class MakeUploadEntry { path: uploadedFilePath, }); - return fileEntry._id; + return fileEntry; } - private async makeUploadEntry( - templateId: string, - fileId: string, - uploadId: string, - extra?: string, - authHeaderValue?: string - ) { + private async addUploadEntry({ + templateId, + fileId, + uploadId, + extra, + authHeaderValue, + headings, + totalRecords, + }: AddUploadEntryCommand) { return this.uploadRepository.create({ _id: uploadId, _uploadedFileId: fileId, _templateId: templateId, extra: extra, - headings: [''], + headings: Array.isArray(headings) ? headings : [], status: UploadStatusEnum.MAPPING, authHeaderValue: authHeaderValue, - totalRecords: 0, + totalRecords: totalRecords || 0, }); } } diff --git a/apps/api/src/app/shared/constants.ts b/apps/api/src/app/shared/constants.ts index f1b59594a..d5f6a33c3 100644 --- a/apps/api/src/app/shared/constants.ts +++ b/apps/api/src/app/shared/constants.ts @@ -1,3 +1,3 @@ export const APIMessages = { - FILE_TYPE_NOT_VALID: 'File type is not valid', + FILE_TYPE_NOT_VALID: 'File type is not supported.', }; diff --git a/apps/api/src/app/shared/file/file.service.ts b/apps/api/src/app/shared/file/file.service.ts index f7325581d..010c19bf8 100644 --- a/apps/api/src/app/shared/file/file.service.ts +++ b/apps/api/src/app/shared/file/file.service.ts @@ -1,10 +1,13 @@ import { parse } from 'json2csv'; +import { IFileInformation } from '@impler/shared'; +import { ParserOptionsArgs, parseString } from 'fast-csv'; import { StorageService } from '../storage/storage.service'; export abstract class FileService { // abstract convertToJson(data: any, fields: string[]): string; abstract convertFromJson(data: any, fields: string[]): string; abstract saveFile(storageService: StorageService, data: any, key: string): Promise; + abstract getFileInformation(storageService: StorageService, storageKey: string): Promise; } export class CSVFileService extends FileService { @@ -14,4 +17,27 @@ export class CSVFileService extends FileService { async saveFile(storageService: StorageService, data: any, key: string, isPublic = false): Promise { await storageService.uploadFile(key, data, 'application/csv', isPublic); } + async getFileInformation(storageService: StorageService, storageKey: string): Promise { + const fileContent = await storageService.getFileContent(storageKey); + + return await this.getCSVInformation(fileContent, { headers: true }); + } + private async getCSVInformation(fileContent: string, options?: ParserOptionsArgs): Promise { + return new Promise((resolve, reject) => { + const information: IFileInformation = { + data: [], + headings: [], + totalRecords: 0, + }; + + parseString(fileContent, options) + .on('error', reject) + .on('headers', (headers) => information.headings.push(...headers)) + .on('data', (row) => information.data.push(row)) + .on('end', () => { + information.totalRecords = information.data.length; + resolve(information); + }); + }); + } } diff --git a/apps/api/src/app/shared/helpers/file.helper.ts b/apps/api/src/app/shared/helpers/file.helper.ts new file mode 100644 index 000000000..ce18d0665 --- /dev/null +++ b/apps/api/src/app/shared/helpers/file.helper.ts @@ -0,0 +1,11 @@ +import { SupportedFileMimeTypesEnum } from '@impler/shared'; +import { APIMessages } from '../constants'; +import { FileService, CSVFileService } from '../file/file.service'; + +export const getFileService = (mimeType: string): FileService => { + if (mimeType === SupportedFileMimeTypesEnum.CSV) { + return new CSVFileService(); + } + + throw new Error(APIMessages.FILE_TYPE_NOT_VALID); +}; diff --git a/apps/api/src/app/shared/storage/storage.service.ts b/apps/api/src/app/shared/storage/storage.service.ts index 52f0205b3..047e81823 100644 --- a/apps/api/src/app/shared/storage/storage.service.ts +++ b/apps/api/src/app/shared/storage/storage.service.ts @@ -20,7 +20,7 @@ export abstract class StorageService { contentType: string, isPublic?: boolean ): Promise; - abstract getFile(key: string): Promise; + abstract getFileContent(key: string): Promise; abstract deleteFile(key: string): Promise; } @@ -29,7 +29,7 @@ async function streamToString(stream: Readable): Promise { const chunks: Uint8Array[] = []; stream.on('data', (chunk) => chunks.push(chunk)); stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(chunks).toString('base64'))); + stream.on('end', () => resolve(Buffer.concat(chunks).toString())); }); } @@ -52,16 +52,15 @@ export class S3StorageService implements StorageService { return await this.s3.send(command); } - async getFile(key: string): Promise { + async getFileContent(key: string): Promise { try { const command = new GetObjectCommand({ Bucket: process.env.S3_BUCKET_NAME, Key: key, }); const data = await this.s3.send(command); - const bodyContents = await streamToString(data.Body as Readable); - return bodyContents as unknown as Buffer; + return await streamToString(data.Body as Readable); } catch (error) { if (error.code === 404 || error.message === 'The specified key does not exist.') { throw new FileNotExistError(); diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index 115ec2434..c1c1e1d22 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -13,3 +13,9 @@ export enum SupportedFileMimeTypesEnum { 'EXCEL' = 'application/vnd.ms-excel', 'EXCELX' = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', } + +export interface IFileInformation { + headings: string[]; + data: Record[]; + totalRecords: number; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25d3aead3..501b249d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,7 @@ importers: compression: ^1.7.4 dotenv: ^16.0.2 envalid: ^7.3.1 + fast-csv: ^4.3.6 json2csv: ^5.0.7 nodemon: ^2.0.20 rimraf: ^3.0.2 @@ -92,6 +93,7 @@ importers: compression: 1.7.4 dotenv: 16.0.2 envalid: 7.3.1 + fast-csv: 4.3.6 json2csv: 5.0.7 rimraf: 3.0.2 devDependencies: @@ -1549,6 +1551,29 @@ packages: - supports-color dev: true + /@fast-csv/format/4.3.5: + resolution: {integrity: sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==} + dependencies: + '@types/node': 14.18.29 + lodash.escaperegexp: 4.1.2 + lodash.isboolean: 3.0.3 + lodash.isequal: 4.5.0 + lodash.isfunction: 3.0.9 + lodash.isnil: 4.0.0 + dev: false + + /@fast-csv/parse/4.3.6: + resolution: {integrity: sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==} + dependencies: + '@types/node': 14.18.29 + lodash.escaperegexp: 4.1.2 + lodash.groupby: 4.6.0 + lodash.isfunction: 3.0.9 + lodash.isnil: 4.0.0 + lodash.isundefined: 3.0.1 + lodash.uniq: 4.5.0 + dev: false + /@humanwhocodes/config-array/0.10.5: resolution: {integrity: sha512-XVVDtp+dVvRxMoxSiSfasYaG02VEe1qH5cKgMQJWhol6HwzbcqoCMJi8dAGoYAO57jhUyhI6cWuRiTcRaDaYug==} engines: {node: '>=10.10.0'} @@ -2294,7 +2319,6 @@ packages: /@types/node/14.18.29: resolution: {integrity: sha512-LhF+9fbIX4iPzhsRLpK5H7iPdvW8L4IwGciXQIOEcuF62+9nw/VQVsOViAOOGxY3OlOKGLFv0sWwJXdwQeTn6A==} - dev: true /@types/node/18.7.18: resolution: {integrity: sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg==} @@ -4176,6 +4200,14 @@ packages: - supports-color dev: false + /fast-csv/4.3.6: + resolution: {integrity: sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==} + engines: {node: '>=10.0.0'} + dependencies: + '@fast-csv/format': 4.3.5 + '@fast-csv/parse': 4.3.6 + dev: false + /fast-deep-equal/3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true @@ -5482,14 +5514,46 @@ packages: p-locate: 5.0.0 dev: true + /lodash.escaperegexp/4.1.2: + resolution: {integrity: sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==} + dev: false + /lodash.get/4.4.2: resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} dev: false + /lodash.groupby/4.6.0: + resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} + dev: false + + /lodash.isboolean/3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + dev: false + + /lodash.isequal/4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + dev: false + + /lodash.isfunction/3.0.9: + resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==} + dev: false + + /lodash.isnil/4.0.0: + resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==} + dev: false + + /lodash.isundefined/3.0.1: + resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true + /lodash.uniq/4.5.0: + resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} + dev: false + /lodash/4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} From 45118fe069b3cbf6780d3c4a209008e8ce120381 Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 13:53:45 +0530 Subject: [PATCH 3/6] feat: removed json2csv dependency and build csv by logic --- apps/api/package.json | 1 - .../update-columns/update-columns.usecase.ts | 12 +++++++--- apps/api/src/app/shared/file/file.service.ts | 10 --------- .../src/app/shared/storage/storage.service.ts | 2 +- pnpm-lock.yaml | 22 +------------------ 5 files changed, 11 insertions(+), 36 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index 0463ee6ba..3aef56aa2 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -32,7 +32,6 @@ "dotenv": "^16.0.2", "envalid": "^7.3.1", "fast-csv": "^4.3.6", - "json2csv": "^5.0.7", "rimraf": "^3.0.2" }, "devDependencies": { diff --git a/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts b/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts index 9da662f50..4d0f35af1 100644 --- a/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts +++ b/apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { SupportedFileMimeTypesEnum } from '@impler/shared'; import { ColumnRepository, TemplateRepository } from '@impler/dal'; import { UpdateColumnCommand } from './update-columns.command'; import { CSVFileService } from '../../../shared/file/file.service'; @@ -23,11 +24,16 @@ export class UpdateColumns { } async saveSampleFile(data: UpdateColumnCommand[], templateId: string) { - const fields = data.map((column) => column.columnKeys[0]); - const csvContent = this.csvFileService.convertFromJson({}, fields); + const csvContent = this.createCSVFileHeadingContent(data); const fileName = this.fileNameService.getSampleFileName(templateId); const sampleFileUrl = this.fileNameService.getSampleFileUrl(templateId); - await this.csvFileService.saveFile(this.storageService, csvContent, fileName, true); + this.storageService.uploadFile(fileName, csvContent, SupportedFileMimeTypesEnum.CSV, true); await this.templateRepository.update({ _id: templateId }, { sampleFileUrl }); } + + createCSVFileHeadingContent(data: UpdateColumnCommand[]): string { + const headings = data.map((column) => column.columnKeys[0]); + + return headings.join(','); + } } diff --git a/apps/api/src/app/shared/file/file.service.ts b/apps/api/src/app/shared/file/file.service.ts index 010c19bf8..c003e69c8 100644 --- a/apps/api/src/app/shared/file/file.service.ts +++ b/apps/api/src/app/shared/file/file.service.ts @@ -1,22 +1,12 @@ -import { parse } from 'json2csv'; import { IFileInformation } from '@impler/shared'; import { ParserOptionsArgs, parseString } from 'fast-csv'; import { StorageService } from '../storage/storage.service'; export abstract class FileService { - // abstract convertToJson(data: any, fields: string[]): string; - abstract convertFromJson(data: any, fields: string[]): string; - abstract saveFile(storageService: StorageService, data: any, key: string): Promise; abstract getFileInformation(storageService: StorageService, storageKey: string): Promise; } export class CSVFileService extends FileService { - convertFromJson(data: any, fields: string[]): string { - return parse(data, { fields, header: true }); - } - async saveFile(storageService: StorageService, data: any, key: string, isPublic = false): Promise { - await storageService.uploadFile(key, data, 'application/csv', isPublic); - } async getFileInformation(storageService: StorageService, storageKey: string): Promise { const fileContent = await storageService.getFileContent(storageKey); diff --git a/apps/api/src/app/shared/storage/storage.service.ts b/apps/api/src/app/shared/storage/storage.service.ts index 047e81823..8fc58a9d7 100644 --- a/apps/api/src/app/shared/storage/storage.service.ts +++ b/apps/api/src/app/shared/storage/storage.service.ts @@ -16,7 +16,7 @@ export interface IFilePath { export abstract class StorageService { abstract uploadFile( key: string, - file: Buffer, + file: Buffer | string, contentType: string, isPublic?: boolean ): Promise; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 501b249d4..af68a5528 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,7 +72,6 @@ importers: dotenv: ^16.0.2 envalid: ^7.3.1 fast-csv: ^4.3.6 - json2csv: ^5.0.7 nodemon: ^2.0.20 rimraf: ^3.0.2 ts-loader: ^9.4.1 @@ -94,7 +93,6 @@ importers: dotenv: 16.0.2 envalid: 7.3.1 fast-csv: 4.3.6 - json2csv: 5.0.7 rimraf: 3.0.2 devDependencies: '@types/express': 4.17.14 @@ -3240,11 +3238,6 @@ packages: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} dev: true - /commander/6.2.1: - resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==} - engines: {node: '>= 6'} - dev: false - /commander/9.4.0: resolution: {integrity: sha512-sRPT+umqkz90UA8M1yqYfnHlZA7fF6nSphDtxeywPZ49ysjxDQybzk13CL+mXekDRG92skbcqCLVovuCusNmFw==} engines: {node: ^12.20.0 || >=14} @@ -5361,16 +5354,6 @@ packages: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} dev: true - /json2csv/5.0.7: - resolution: {integrity: sha512-YRZbUnyaJZLZUJSRi2G/MqahCyRv9n/ds+4oIetjDF3jWQA7AG7iSeKTiZiCNqtMZM7HDyt0e/W6lEnoGEmMGA==} - engines: {node: '>= 10', npm: '>= 6.13.0'} - hasBin: true - dependencies: - commander: 6.2.1 - jsonparse: 1.3.1 - lodash.get: 4.4.2 - dev: false - /json5/1.0.1: resolution: {integrity: sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==} hasBin: true @@ -5399,6 +5382,7 @@ packages: /jsonparse/1.3.1: resolution: {integrity: sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==} engines: {'0': node >= 0.2.0} + dev: true /jsx-ast-utils/3.3.3: resolution: {integrity: sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==} @@ -5518,10 +5502,6 @@ packages: resolution: {integrity: sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==} dev: false - /lodash.get/4.4.2: - resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} - dev: false - /lodash.groupby/4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} dev: false From cb5e7ea8ad9ef9cd34f3d9340360527cc7611e22 Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 17:17:33 +0530 Subject: [PATCH 4/6] feat: Updated file information on upload --- apps/api/package.json | 3 +- apps/api/src/app/shared/constants.ts | 2 + .../shared/exceptions/empty-file.exception.ts | 8 +++ apps/api/src/app/shared/file/file.service.ts | 46 +++++++++++++- .../api/src/app/shared/helpers/file.helper.ts | 4 +- .../src/app/shared/storage/storage.service.ts | 10 +-- libs/shared/src/types/upload/upload.types.ts | 7 ++- pnpm-lock.yaml | 62 +++++++++++++++++++ 8 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 apps/api/src/app/shared/exceptions/empty-file.exception.ts diff --git a/apps/api/package.json b/apps/api/package.json index 3aef56aa2..0da380d94 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -32,7 +32,8 @@ "dotenv": "^16.0.2", "envalid": "^7.3.1", "fast-csv": "^4.3.6", - "rimraf": "^3.0.2" + "rimraf": "^3.0.2", + "xlsx": "^0.18.5" }, "devDependencies": { "@types/express": "^4.17.14", diff --git a/apps/api/src/app/shared/constants.ts b/apps/api/src/app/shared/constants.ts index d5f6a33c3..7cdd6434f 100644 --- a/apps/api/src/app/shared/constants.ts +++ b/apps/api/src/app/shared/constants.ts @@ -1,3 +1,5 @@ export const APIMessages = { FILE_TYPE_NOT_VALID: 'File type is not supported.', + FILE_IS_EMPTY: 'File is empty', + EMPTY_HEADING_PREFIX: 'Empty Heading', }; diff --git a/apps/api/src/app/shared/exceptions/empty-file.exception.ts b/apps/api/src/app/shared/exceptions/empty-file.exception.ts new file mode 100644 index 000000000..82153630f --- /dev/null +++ b/apps/api/src/app/shared/exceptions/empty-file.exception.ts @@ -0,0 +1,8 @@ +import { BadRequestException } from '@nestjs/common'; +import { APIMessages } from '../constants'; + +export class EmptyFileException extends BadRequestException { + constructor() { + super(APIMessages.FILE_IS_EMPTY); + } +} diff --git a/apps/api/src/app/shared/file/file.service.ts b/apps/api/src/app/shared/file/file.service.ts index c003e69c8..934152876 100644 --- a/apps/api/src/app/shared/file/file.service.ts +++ b/apps/api/src/app/shared/file/file.service.ts @@ -1,6 +1,9 @@ -import { IFileInformation } from '@impler/shared'; +import * as XLSX from 'xlsx'; +import { FileEncodingsEnum, IFileInformation } from '@impler/shared'; import { ParserOptionsArgs, parseString } from 'fast-csv'; import { StorageService } from '../storage/storage.service'; +import { EmptyFileException } from '../exceptions/empty-file.exception'; +import { APIMessages } from '../constants'; export abstract class FileService { abstract getFileInformation(storageService: StorageService, storageKey: string): Promise; @@ -8,7 +11,7 @@ export abstract class FileService { export class CSVFileService extends FileService { async getFileInformation(storageService: StorageService, storageKey: string): Promise { - const fileContent = await storageService.getFileContent(storageKey); + const fileContent = await storageService.getFileContent(storageKey, FileEncodingsEnum.CSV); return await this.getCSVInformation(fileContent, { headers: true }); } @@ -31,3 +34,42 @@ export class CSVFileService extends FileService { }); } } +export class ExcelFileService extends FileService { + async getFileInformation(storageService: StorageService, storageKey: string): Promise { + const fileContent = await storageService.getFileContent(storageKey, FileEncodingsEnum.EXCEL); + + return this.getExcelInformation(fileContent); + } + async getExcelInformation(fileContent: string): Promise { + const workbookHeaders = XLSX.read(fileContent); + // Throw empty error if file do not have any sheets + if (workbookHeaders.SheetNames.length < 1) throw new EmptyFileException(); + + // get file headings + const headings = XLSX.utils.sheet_to_json(workbookHeaders.Sheets[workbookHeaders.SheetNames[0]], { + header: 1, + })[0] as string[]; + // throw error if sheet is empty + if (!headings || headings.length < 1) throw new EmptyFileException(); + + // Refine headings by replacing empty heading + let emptyHeadingCount = 0; + const updatedHeading = []; + for (const headingItem of headings) { + if (!headingItem) { + emptyHeadingCount += 1; + updatedHeading.push(`${APIMessages.EMPTY_HEADING_PREFIX} ${emptyHeadingCount}`); + } else updatedHeading.push(headingItem); + } + + const data: Record[] = XLSX.utils.sheet_to_json( + workbookHeaders.Sheets[workbookHeaders.SheetNames[0]] + ); + + return { + data, + headings: updatedHeading, + totalRecords: data.length, + }; + } +} diff --git a/apps/api/src/app/shared/helpers/file.helper.ts b/apps/api/src/app/shared/helpers/file.helper.ts index ce18d0665..a4b0907e5 100644 --- a/apps/api/src/app/shared/helpers/file.helper.ts +++ b/apps/api/src/app/shared/helpers/file.helper.ts @@ -1,10 +1,12 @@ import { SupportedFileMimeTypesEnum } from '@impler/shared'; import { APIMessages } from '../constants'; -import { FileService, CSVFileService } from '../file/file.service'; +import { FileService, CSVFileService, ExcelFileService } from '../file/file.service'; export const getFileService = (mimeType: string): FileService => { if (mimeType === SupportedFileMimeTypesEnum.CSV) { return new CSVFileService(); + } else if (mimeType === SupportedFileMimeTypesEnum.EXCEL || mimeType === SupportedFileMimeTypesEnum.EXCELX) { + return new ExcelFileService(); } throw new Error(APIMessages.FILE_TYPE_NOT_VALID); diff --git a/apps/api/src/app/shared/storage/storage.service.ts b/apps/api/src/app/shared/storage/storage.service.ts index 8fc58a9d7..ed9f81883 100644 --- a/apps/api/src/app/shared/storage/storage.service.ts +++ b/apps/api/src/app/shared/storage/storage.service.ts @@ -20,16 +20,16 @@ export abstract class StorageService { contentType: string, isPublic?: boolean ): Promise; - abstract getFileContent(key: string): Promise; + abstract getFileContent(key: string, encoding: string): Promise; abstract deleteFile(key: string): Promise; } -async function streamToString(stream: Readable): Promise { +async function streamToString(stream: Readable, encoding: BufferEncoding): Promise { return await new Promise((resolve, reject) => { const chunks: Uint8Array[] = []; stream.on('data', (chunk) => chunks.push(chunk)); stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(chunks).toString())); + stream.on('end', () => resolve(Buffer.concat(chunks).toString(encoding))); }); } @@ -52,7 +52,7 @@ export class S3StorageService implements StorageService { return await this.s3.send(command); } - async getFileContent(key: string): Promise { + async getFileContent(key: string, encoding: BufferEncoding = 'utf8'): Promise { try { const command = new GetObjectCommand({ Bucket: process.env.S3_BUCKET_NAME, @@ -60,7 +60,7 @@ export class S3StorageService implements StorageService { }); const data = await this.s3.send(command); - return await streamToString(data.Body as Readable); + return await streamToString(data.Body as Readable, encoding); } catch (error) { if (error.code === 404 || error.message === 'The specified key does not exist.') { throw new FileNotExistError(); diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index c1c1e1d22..6ef1c9dad 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -9,11 +9,16 @@ export enum UploadStatusEnum { export enum SupportedFileMimeTypesEnum { 'CSV' = 'text/csv', - 'XML' = 'application/xml', 'EXCEL' = 'application/vnd.ms-excel', 'EXCELX' = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', } +export enum FileEncodingsEnum { + 'CSV' = 'utf8', + 'EXCEL' = 'base64', + 'EXCELX' = 'base64', +} + export interface IFileInformation { headings: string[]; data: Record[]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af68a5528..08ef85126 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,7 @@ importers: ts-node: ^10.9.1 tsconfig-paths: ^4.1.0 typescript: ^4.8.3 + xlsx: ^0.18.5 dependencies: '@aws-sdk/client-s3': 3.185.0 '@impler/dal': link:../../libs/dal @@ -94,6 +95,7 @@ importers: envalid: 7.3.1 fast-csv: 4.3.6 rimraf: 3.0.2 + xlsx: 0.18.5 devDependencies: '@types/express': 4.17.14 '@types/json2csv': 5.0.3 @@ -2679,6 +2681,11 @@ packages: hasBin: true dev: true + /adler-32/1.3.1: + resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==} + engines: {node: '>=0.8'} + dev: false + /aggregate-error/3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -3088,6 +3095,14 @@ packages: resolution: {integrity: sha512-QoblBnuE+rG0lc3Ur9ltP5q47lbguipa/ncNMyyGuqPk44FxbScWAeEO+k5fSQ8WekdAK4mWqNs1rADDAiN5xQ==} dev: true + /cfb/1.2.2: + resolution: {integrity: sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==} + engines: {node: '>=0.8'} + dependencies: + adler-32: 1.3.1 + crc-32: 1.2.2 + dev: false + /chalk/2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} @@ -3207,6 +3222,11 @@ packages: engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} dev: true + /codepage/1.15.0: + resolution: {integrity: sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==} + engines: {node: '>=0.8'} + dev: false + /collect-v8-coverage/1.0.1: resolution: {integrity: sha512-iBPtljfCNcTKNAto0KEtDfZ3qzjJvqE3aTGZsbhjSBlorqpXJlaWWtPO35D+ZImoC3KWejX64o+yPGxhWSTzfg==} dev: true @@ -3393,6 +3413,12 @@ packages: yaml: 1.10.2 dev: true + /crc-32/1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + dev: false + /create-require/1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} dev: true @@ -4350,6 +4376,11 @@ packages: engines: {node: '>= 0.6'} dev: false + /frac/1.1.2: + resolution: {integrity: sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==} + engines: {node: '>=0.8'} + dev: false + /fresh/0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} @@ -6708,6 +6739,13 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true + /ssf/0.11.2: + resolution: {integrity: sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==} + engines: {node: '>=0.8'} + dependencies: + frac: 1.1.2 + dev: false + /stack-utils/2.0.5: resolution: {integrity: sha512-xrQcmYhOsn/1kX+Vraq+7j4oE2j/6BFscZ0etmYg81xuM8Gq0022Pxb8+IqgOFUIaxHs0KaSb7T1+OegiNrNFA==} engines: {node: '>=10'} @@ -7403,11 +7441,21 @@ packages: isexe: 2.0.0 dev: true + /wmf/1.0.2: + resolution: {integrity: sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==} + engines: {node: '>=0.8'} + dev: false + /word-wrap/1.2.3: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} engines: {node: '>=0.10.0'} dev: true + /word/0.3.0: + resolution: {integrity: sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==} + engines: {node: '>=0.8'} + dev: false + /wrap-ansi/6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -7437,6 +7485,20 @@ packages: signal-exit: 3.0.7 dev: true + /xlsx/0.18.5: + resolution: {integrity: sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==} + engines: {node: '>=0.8'} + hasBin: true + dependencies: + adler-32: 1.3.1 + cfb: 1.2.2 + codepage: 1.15.0 + crc-32: 1.2.2 + ssf: 0.11.2 + wmf: 1.0.2 + word: 0.3.0 + dev: false + /xtend/4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} From 556b23a949ec8458622acd258ded8f4504b71eed Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 17:22:58 +0530 Subject: [PATCH 5/6] feat: Updated file names to match others --- apps/api/src/app/column/dtos/update-column-request.dto.ts | 2 +- apps/api/src/app/project/dtos/create-project-request.dto.ts | 2 +- apps/api/src/app/project/project.module.ts | 2 +- .../file-not-valid.exception.ts} | 0 .../framework/{IsUniqueValidator.ts => is-unique.validator.ts} | 2 +- .../{IsValidRegexValidator.ts => is-valid-regex.validator.ts} | 0 .../src/app/shared/validations/valid-import-file.validation.ts | 2 +- apps/api/src/app/template/dtos/create-template-request.dto.ts | 2 +- 8 files changed, 6 insertions(+), 6 deletions(-) rename apps/api/src/app/shared/{errors/file-not-valid.error.ts => exceptions/file-not-valid.exception.ts} (100%) rename apps/api/src/app/shared/framework/{IsUniqueValidator.ts => is-unique.validator.ts} (92%) rename apps/api/src/app/shared/framework/{IsValidRegexValidator.ts => is-valid-regex.validator.ts} (100%) diff --git a/apps/api/src/app/column/dtos/update-column-request.dto.ts b/apps/api/src/app/column/dtos/update-column-request.dto.ts index ed50547c2..d0a48d16e 100644 --- a/apps/api/src/app/column/dtos/update-column-request.dto.ts +++ b/apps/api/src/app/column/dtos/update-column-request.dto.ts @@ -14,7 +14,7 @@ import { } from 'class-validator'; import { Type } from 'class-transformer'; import { ColumnTypesEnum } from '@impler/shared'; -import { IsValidRegex } from '../../shared/framework/IsValidRegexValidator'; +import { IsValidRegex } from '../../shared/framework/is-valid-regex.validator'; export class UpdateColumnRequestDto { @ApiProperty({ diff --git a/apps/api/src/app/project/dtos/create-project-request.dto.ts b/apps/api/src/app/project/dtos/create-project-request.dto.ts index dd807c58b..4cafa3ed0 100644 --- a/apps/api/src/app/project/dtos/create-project-request.dto.ts +++ b/apps/api/src/app/project/dtos/create-project-request.dto.ts @@ -2,7 +2,7 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { IsDefined, IsOptional, IsString, Validate } from 'class-validator'; import { Transform } from 'class-transformer'; import { changeToCode } from '@impler/shared'; -import { UniqueValidator } from '../../shared/framework/IsUniqueValidator'; +import { UniqueValidator } from '../../shared/framework/is-unique.validator'; export class CreateProjectRequestDto { @ApiProperty({ diff --git a/apps/api/src/app/project/project.module.ts b/apps/api/src/app/project/project.module.ts index 53402a1e2..6b404050f 100644 --- a/apps/api/src/app/project/project.module.ts +++ b/apps/api/src/app/project/project.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { USE_CASES } from './usecases'; import { SharedModule } from '../shared/shared.module'; import { ProjectController } from './project.controller'; -import { UniqueValidator } from '../shared/framework/IsUniqueValidator'; +import { UniqueValidator } from '../shared/framework/is-unique.validator'; @Module({ imports: [SharedModule, UniqueValidator], diff --git a/apps/api/src/app/shared/errors/file-not-valid.error.ts b/apps/api/src/app/shared/exceptions/file-not-valid.exception.ts similarity index 100% rename from apps/api/src/app/shared/errors/file-not-valid.error.ts rename to apps/api/src/app/shared/exceptions/file-not-valid.exception.ts diff --git a/apps/api/src/app/shared/framework/IsUniqueValidator.ts b/apps/api/src/app/shared/framework/is-unique.validator.ts similarity index 92% rename from apps/api/src/app/shared/framework/IsUniqueValidator.ts rename to apps/api/src/app/shared/framework/is-unique.validator.ts index e8d645f94..448e0f657 100644 --- a/apps/api/src/app/shared/framework/IsUniqueValidator.ts +++ b/apps/api/src/app/shared/framework/is-unique.validator.ts @@ -1,7 +1,7 @@ import { ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface } from 'class-validator'; import { CommonRepository, ProjectEntity } from '@impler/dal'; -@ValidatorConstraint({ name: 'IsUniqueUser', async: true }) +@ValidatorConstraint({ name: 'IsUnique', async: true }) export class UniqueValidator implements ValidatorConstraintInterface { private commonRepository: CommonRepository; constructor() { diff --git a/apps/api/src/app/shared/framework/IsValidRegexValidator.ts b/apps/api/src/app/shared/framework/is-valid-regex.validator.ts similarity index 100% rename from apps/api/src/app/shared/framework/IsValidRegexValidator.ts rename to apps/api/src/app/shared/framework/is-valid-regex.validator.ts diff --git a/apps/api/src/app/shared/validations/valid-import-file.validation.ts b/apps/api/src/app/shared/validations/valid-import-file.validation.ts index 53a3aecfe..9f47a4cbf 100644 --- a/apps/api/src/app/shared/validations/valid-import-file.validation.ts +++ b/apps/api/src/app/shared/validations/valid-import-file.validation.ts @@ -2,7 +2,7 @@ import _whatever from 'multer'; import { SupportedFileMimeTypesEnum } from '@impler/shared'; import { Injectable, PipeTransform } from '@nestjs/common'; -import { FileNotValidError } from '../errors/file-not-valid.error'; +import { FileNotValidError } from '../exceptions/file-not-valid.exception'; @Injectable() export class ValidImportFile implements PipeTransform { diff --git a/apps/api/src/app/template/dtos/create-template-request.dto.ts b/apps/api/src/app/template/dtos/create-template-request.dto.ts index 15187254f..d4100c3ea 100644 --- a/apps/api/src/app/template/dtos/create-template-request.dto.ts +++ b/apps/api/src/app/template/dtos/create-template-request.dto.ts @@ -2,7 +2,7 @@ import { ApiProperty } from '@nestjs/swagger'; import { Transform } from 'class-transformer'; import { changeToCode } from '@impler/shared'; import { IsDefined, IsString, Validate, IsNumber, IsUrl } from 'class-validator'; -import { UniqueValidator } from '../../shared/framework/IsUniqueValidator'; +import { UniqueValidator } from '../../shared/framework/is-unique.validator'; export class CreateTemplateRequestDto { @ApiProperty({ From 6a4867d88716f3fbe0df2b8e6a435c2560d27a9a Mon Sep 17 00:00:00 2001 From: chavda-bhavik Date: Wed, 12 Oct 2022 17:26:31 +0530 Subject: [PATCH 6/6] fix: removed comment --- apps/api/src/app/ops/ops.controller.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/api/src/app/ops/ops.controller.ts b/apps/api/src/app/ops/ops.controller.ts index f06ad2342..860f769aa 100644 --- a/apps/api/src/app/ops/ops.controller.ts +++ b/apps/api/src/app/ops/ops.controller.ts @@ -28,14 +28,5 @@ export class OpsController { authHeaderValue: body.authHeaderValue, }) ); - - /* - * validate template (done) - * upload file (done) - * make entry to file (done) - * make entry to uploads (done) - * Get headings - * do mapping - */ } }