File Upload API

apps/api/package.json

@@ -31,12 +31,14 @@
     "compression": "^1.7.4",
     "dotenv": "^16.0.2",
     "envalid": "^7.3.1",
-    "json2csv": "^5.0.7",
-    "rimraf": "^3.0.2"
+    "fast-csv": "^4.3.6",
+    "rimraf": "^3.0.2",
+    "xlsx": "^0.18.5"
   },
   "devDependencies": {
     "@types/express": "^4.17.14",
     "@types/json2csv": "^5.0.3",
+    "@types/multer": "^1.4.7",
     "@types/node": "^18.7.18",
     "nodemon": "^2.0.20",
     "ts-loader": "^9.4.1",

apps/api/src/app.module.ts

@@ -5,12 +5,14 @@ import { SharedModule } from './app/shared/shared.module';
 import { ProjectModule } from './app/project/project.module';
 import { TemplateModule } from './app/template/template.module';
 import { ColumnModule } from './app/column/column.module';
+import { OpsModule } from './app/ops/ops.module';
 
 const modules: Array<Type | DynamicModule | Promise<DynamicModule> | ForwardReference> = [
   ProjectModule,
   SharedModule,
   TemplateModule,
   ColumnModule,
+  OpsModule,
 ];
 
 const providers = [];

apps/api/src/app/column/dtos/update-column-request.dto.ts

@@ -14,7 +14,7 @@ import {
 } from 'class-validator';
 import { Type } from 'class-transformer';
 import { ColumnTypesEnum } from '@impler/shared';
-import { IsValidRegex } from '../../shared/framework/IsValidRegexValidator';
+import { IsValidRegex } from '../../shared/framework/is-valid-regex.validator';
 
 export class UpdateColumnRequestDto {
   @ApiProperty({

apps/api/src/app/column/usecases/update-columns/update-columns.usecase.ts

@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { SupportedFileMimeTypesEnum } from '@impler/shared';
 import { ColumnRepository, TemplateRepository } from '@impler/dal';
 import { UpdateColumnCommand } from './update-columns.command';
 import { CSVFileService } from '../../../shared/file/file.service';
@@ -23,11 +24,16 @@ export class UpdateColumns {
   }
 
   async saveSampleFile(data: UpdateColumnCommand[], templateId: string) {
-    const fields = data.map((column) => column.columnKeys[0]);
-    const csvContent = this.csvFileService.convertFromJson({}, fields);
+    const csvContent = this.createCSVFileHeadingContent(data);
     const fileName = this.fileNameService.getSampleFileName(templateId);
     const sampleFileUrl = this.fileNameService.getSampleFileUrl(templateId);
-    await this.csvFileService.saveFile(this.storageService, csvContent, fileName, true);
+    this.storageService.uploadFile(fileName, csvContent, SupportedFileMimeTypesEnum.CSV, true);
     await this.templateRepository.update({ _id: templateId }, { sampleFileUrl });
   }
+
+  createCSVFileHeadingContent(data: UpdateColumnCommand[]): string {
+    const headings = data.map((column) => column.columnKeys[0]);
+
+    return headings.join(',');
+  }
 }

apps/api/src/app/ops/dtos/upload-request.dto.ts

@@ -0,0 +1,34 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsJSON, IsOptional, IsString, Validate } from 'class-validator';
+import { IsValidTemplateValidator } from '../../shared/framework/is-valid-template.validator';
+
+export class UploadRequestDto {
+  @ApiProperty({
+    description: 'Id or CODE of the template',
+  })
+  @IsString()
+  @Validate(IsValidTemplateValidator)
+  template: string;
+
+  @ApiProperty({
+    type: 'file',
+    required: true,
+  })
+  file: Express.Multer.File;
+
+  @ApiProperty({
+    description: 'Auth header value to send during webhook call',
+    required: false,
+  })
+  @IsOptional()
+  @IsString()
+  authHeaderValue: string;
+
+  @ApiProperty({
+    description: 'Payload to send during webhook call',
+    required: false,
+  })
+  @IsOptional()
+  @IsJSON()
+  extra: string;
+}

apps/api/src/app/ops/ops.controller.ts

@@ -0,0 +1,32 @@
+// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
+import _whatever from 'multer';
+import { Body, Controller, Post, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { ApiTags, ApiSecurity, ApiConsumes } from '@nestjs/swagger';
+import { APIKeyGuard } from '../shared/framework/auth.gaurd';
+import { UploadRequestDto } from './dtos/upload-request.dto';
+import { ValidImportFile } from '../shared/validations/valid-import-file.validation';
+import { MakeUploadEntry } from './usecases/make-upload-entry/make-upload-entry.usecase';
+import { MakeUploadEntryCommand } from './usecases/make-upload-entry/make-upload-entry.command';
+
+@Controller('/ops')
+@ApiTags('Operations')
+@ApiSecurity('ACCESS_KEY')
+@UseGuards(APIKeyGuard)
+export class OpsController {
+  constructor(private makeUploadEntry: MakeUploadEntry) {}
+
+  @Post('upload')
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async uploadFile(@UploadedFile('file', ValidImportFile) file: Express.Multer.File, @Body() body: UploadRequestDto) {
+    return await this.makeUploadEntry.execute(
+      MakeUploadEntryCommand.create({
+        file: file,
+        templateId: body.template,
+        extra: body.extra,
+        authHeaderValue: body.authHeaderValue,
+      })
+    );
+  }
+}

apps/api/src/app/ops/ops.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { USE_CASES } from './usecases';
+import { OpsController } from './ops.controller';
+import { SharedModule } from '../shared/shared.module';
+
+@Module({
+  imports: [SharedModule],
+  providers: [...USE_CASES],
+  controllers: [OpsController],
+})
+export class OpsModule {}

apps/api/src/app/ops/usecases/index.ts

@@ -0,0 +1,6 @@
+import { MakeUploadEntry } from './make-upload-entry/make-upload-entry.usecase';
+
+export const USE_CASES = [
+  MakeUploadEntry,
+  //
+];

apps/api/src/app/ops/usecases/make-upload-entry/add-upload-entry.command.ts

@@ -0,0 +1,32 @@
+import { IsDefined, IsString, IsOptional, IsJSON, IsArray, IsNumber } from 'class-validator';
+import { BaseCommand } from '../../../shared/commands/base.command';
+
+export class AddUploadEntryCommand extends BaseCommand {
+  @IsDefined()
+  @IsString()
+  templateId: string;
+
+  @IsDefined()
+  @IsString()
+  fileId: string;
+
+  @IsDefined()
+  @IsString()
+  uploadId: string;
+
+  @IsOptional()
+  @IsJSON()
+  extra?: string;
+
+  @IsOptional()
+  @IsString()
+  authHeaderValue?: string;
+
+  @IsOptional()
+  @IsArray()
+  headings?: string[];
+
+  @IsOptional()
+  @IsNumber()
+  totalRecords?: number;
+}

apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.command.ts

@@ -0,0 +1,19 @@
+import { IsDefined, IsString, IsOptional, IsJSON } from 'class-validator';
+import { BaseCommand } from '../../../shared/commands/base.command';
+
+export class MakeUploadEntryCommand extends BaseCommand {
+  @IsDefined()
+  file: Express.Multer.File;
+
+  @IsDefined()
+  @IsString()
+  templateId: string;
+
+  @IsOptional()
+  @IsJSON()
+  extra?: string;
+
+  @IsOptional()
+  @IsString()
+  authHeaderValue?: string;
+}

apps/api/src/app/ops/usecases/make-upload-entry/make-upload-entry.usecase.ts

@@ -0,0 +1,74 @@
+import { Injectable } from '@nestjs/common';
+import { UploadStatusEnum } from '@impler/shared';
+import { CommonRepository, FileEntity, FileRepository, UploadRepository } from '@impler/dal';
+import { MakeUploadEntryCommand } from './make-upload-entry.command';
+import { FileNameService } from '../../../shared/file/name.service';
+import { StorageService } from '../../../shared/storage/storage.service';
+import { FileService } from '../../../shared/file/file.service';
+import { getFileService } from '../../../shared/helpers/file.helper';
+import { AddUploadEntryCommand } from './add-upload-entry.command';
+
+@Injectable()
+export class MakeUploadEntry {
+  constructor(
+    private uploadRepository: UploadRepository,
+    private commonRepository: CommonRepository,
+    private fileRepository: FileRepository,
+    private storageService: StorageService,
+    private fileNameService: FileNameService
+  ) {}
+
+  async execute({ file, templateId, extra, authHeaderValue }: MakeUploadEntryCommand) {
+    const uploadId = this.commonRepository.generateMongoId();
+    const fileEntity = await this.makeFileEntry(uploadId, file);
+    const fileService: FileService = getFileService(file.mimetype);
+    const fileInformation = await fileService.getFileInformation(this.storageService, fileEntity.path);
+
+    return this.addUploadEntry(
+      AddUploadEntryCommand.create({
+        templateId: templateId,
+        fileId: fileEntity._id,
+        uploadId,
+        extra,
+        authHeaderValue,
+        headings: fileInformation.headings,
+        totalRecords: fileInformation.totalRecords,
+      })
+    );
+  }
+
+  private async makeFileEntry(uploadId: string, file: Express.Multer.File): Promise<FileEntity> {
+    const uploadedFilePath = this.fileNameService.getUploadedFilePath(uploadId, file.originalname);
+    await this.storageService.uploadFile(uploadedFilePath, file.buffer, file.mimetype);
+    const uploadedFileName = this.fileNameService.getUploadedFileName(file.originalname);
+    const fileEntry = await this.fileRepository.create({
+      mimeType: file.mimetype,
+      name: uploadedFileName,
+      originalName: file.originalname,
+      path: uploadedFilePath,
+    });
+
+    return fileEntry;
+  }
+
+  private async addUploadEntry({
+    templateId,
+    fileId,
+    uploadId,
+    extra,
+    authHeaderValue,
+    headings,
+    totalRecords,
+  }: AddUploadEntryCommand) {
+    return this.uploadRepository.create({
+      _id: uploadId,
+      _uploadedFileId: fileId,
+      _templateId: templateId,
+      extra: extra,
+      headings: Array.isArray(headings) ? headings : [],
+      status: UploadStatusEnum.MAPPING,
+      authHeaderValue: authHeaderValue,
+      totalRecords: totalRecords || 0,
+    });
+  }
+}

apps/api/src/app/project/dtos/create-project-request.dto.ts

@@ -2,7 +2,7 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import { IsDefined, IsOptional, IsString, Validate } from 'class-validator';
 import { Transform } from 'class-transformer';
 import { changeToCode } from '@impler/shared';
-import { UniqueValidator } from '../../shared/framework/IsUniqueValidator';
+import { UniqueValidator } from '../../shared/framework/is-unique.validator';
 
 export class CreateProjectRequestDto {
   @ApiProperty({

apps/api/src/app/project/project.module.ts

@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
 import { USE_CASES } from './usecases';
 import { SharedModule } from '../shared/shared.module';
 import { ProjectController } from './project.controller';
-import { UniqueValidator } from '../shared/framework/IsUniqueValidator';
+import { UniqueValidator } from '../shared/framework/is-unique.validator';
 
 @Module({
   imports: [SharedModule, UniqueValidator],

apps/api/src/app/shared/constants.ts

@@ -0,0 +1,5 @@
+export const APIMessages = {
+  FILE_TYPE_NOT_VALID: 'File type is not supported.',
+  FILE_IS_EMPTY: 'File is empty',
+  EMPTY_HEADING_PREFIX: 'Empty Heading',
+};

apps/api/src/app/shared/exceptions/empty-file.exception.ts

@@ -0,0 +1,8 @@
+import { BadRequestException } from '@nestjs/common';
+import { APIMessages } from '../constants';
+
+export class EmptyFileException extends BadRequestException {
+  constructor() {
+    super(APIMessages.FILE_IS_EMPTY);
+  }
+}

apps/api/src/app/shared/exceptions/file-not-valid.exception.ts

@@ -0,0 +1,8 @@
+import { UnprocessableEntityException } from '@nestjs/common';
+import { APIMessages } from '../constants';
+
+export class FileNotValidError extends UnprocessableEntityException {
+  constructor() {
+    super(APIMessages.FILE_TYPE_NOT_VALID);
+  }
+}