SIMSBIOHUB-653: Observation CSV Import (#1474)

- Observation CSV Import using new CSVConfig structure
- New cell validators
- Taxon row validator
- Sampling information row validator
- Shared qualitative quantitative value validators
- Environment validators + measurement validators

api/src/openapi/schemas/observation.ts

@@ -84,6 +84,7 @@ export const findObservationsSchema: OpenAPIV3.SchemaObject = {
             observation_subcount_sign_id: {
               type: 'integer',
               minimum: 1,
+              nullable: true,
               description:
                 'The observation subcount sign ID, indicating whether the subcount was a direct sighting, footprints, scat, etc.'
             },

api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.test.ts

@@ -0,0 +1,82 @@
+import chai, { expect } from 'chai';
+import { describe } from 'mocha';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import * as db from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
+import { ImportObservationsService } from '../../../../../../services/import-services/observation/import-observations-service';
+import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
+import { importObservationCSV } from './import';
+
+chai.use(sinonChai);
+
+describe('importObservationCSV', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('status 204 when successful', async () => {
+    const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() });
+    const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    const importCSVWorksheetStub = sinon.stub(ImportObservationsService.prototype, 'importCSVWorksheet');
+
+    importCSVWorksheetStub.resolves([]);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importObservationCSV();
+
+    await requestHandler(mockReq, mockRes, mockNext);
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(getDBConnectionStub).to.have.been.calledOnce;
+
+    expect(importCSVWorksheetStub).to.have.been.calledOnce;
+
+    expect(mockRes.status).to.have.been.calledOnceWithExactly(204);
+    expect(mockRes.send).to.have.been.calledOnceWithExactly();
+
+    expect(mockDBConnection.commit).to.have.been.calledOnce;
+    expect(mockDBConnection.release).to.have.been.calledOnce;
+  });
+
+  it('status 422 when CSV validation errors', async () => {
+    const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() });
+    const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    const importCSVWorksheetStub = sinon.stub(ImportObservationsService.prototype, 'importCSVWorksheet');
+
+    importCSVWorksheetStub.resolves([{ error: 'error', solution: 'solution' }]);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importObservationCSV();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+      expect.fail('Expected an 422 error to be thrown');
+    } catch (err) {
+      expect(mockDBConnection.open).to.have.been.calledOnce;
+
+      expect(getDBConnectionStub).to.have.been.calledOnce;
+
+      expect(importCSVWorksheetStub).to.have.been.calledOnce;
+
+      expect(mockDBConnection.commit).to.have.not.been.called;
+      expect(mockDBConnection.release).to.have.been.calledOnce;
+      expect(err).to.be.instanceOf(HTTP422CSVValidationError);
+    }
+  });
+});

api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.ts → api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.ts

@@ -2,14 +2,18 @@ import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
+import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv';
 import { csvFileSchema } from '../../../../../../openapi/schemas/file';
 import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
-import { ObservationService } from '../../../../../../services/observation-services/observation-service';
-import { uploadFileToS3 } from '../../../../../../utils/file-utils';
+import { ImportObservationsService } from '../../../../../../services/import-services/observation/import-observations-service';
+import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface';
 import { getLogger } from '../../../../../../utils/logger';
+import { parseMulterFile } from '../../../../../../utils/media/media-utils';
 import { getFileFromRequest } from '../../../../../../utils/request';
+import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../utils/xlsx-utils/worksheet-utils';
 
-const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/upload');
+const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/import');
 
 export const POST: Operation = [
   authorizeRequestHandler((req) => {
@@ -27,11 +31,11 @@ export const POST: Operation = [
       ]
     };
   }),
-  uploadMedia()
+  importObservationCSV()
 ];
 
 POST.apiDoc = {
-  description: 'Upload survey observation submission file.',
+  description: 'Import survey observation CSV file.',
   tags: ['observations'],
   security: [
     {
@@ -42,16 +46,24 @@ POST.apiDoc = {
     {
       in: 'path',
       name: 'projectId',
-      required: true
+      required: true,
+      schema: {
+        type: 'integer',
+        minimum: 1
+      }
     },
     {
       in: 'path',
       name: 'surveyId',
-      required: true
+      required: true,
+      schema: {
+        type: 'integer',
+        minimum: 1
+      }
     }
   ],
   requestBody: {
-    description: 'Survey observation submission file to upload',
+    description: 'Survey observation CSV file to import',
     required: true,
     content: {
       'multipart/form-data': {
@@ -61,33 +73,25 @@ POST.apiDoc = {
           required: ['media'],
           properties: {
             media: {
-              description: 'A survey observation submission file.',
+              description: 'A survey observation CSV file.',
               type: 'array',
               minItems: 1,
               maxItems: 1,
               items: csvFileSchema
+            },
+            surveySamplePeriodId: {
+              description: 'The sample period id to associate the observations with.',
+              type: 'integer',
+              minimum: 1
             }
           }
         }
       }
     }
   },
   responses: {
-    200: {
-      description: 'Upload OK',
-      content: {
-        'application/json': {
-          schema: {
-            type: 'object',
-            additionalProperties: false,
-            properties: {
-              submissionId: {
-                type: 'number'
-              }
-            }
-          }
-        }
-      }
+    204: {
+      description: 'Observation import success.'
     },
     400: {
       $ref: '#/components/responses/400'
@@ -98,6 +102,7 @@ POST.apiDoc = {
     403: {
       $ref: '#/components/responses/403'
     },
+    422: CSVValidationErrorResponse,
     500: {
       $ref: '#/components/responses/500'
     },
@@ -108,43 +113,43 @@ POST.apiDoc = {
 };
 
 /**
- * Uploads a media file to S3 and inserts a matching record in the `survey_observation_submission` table.
+ * Imports a `Observation CSV` which bulk creates observations in SIMS.
  *
  * @return {*}  {RequestHandler}
  */
-export function uploadMedia(): RequestHandler {
+export function importObservationCSV(): RequestHandler {
   return async (req, res) => {
-    const rawMediaFile = getFileFromRequest(req);
+    const surveyId = Number(req.params.surveyId);
+    const surveySamplePeriodId = Number(req.body.surveySamplePeriodId) || undefined;
+
+    const rawFile = getFileFromRequest(req);
+    const mediaFile = parseMulterFile(rawFile);
+    const workbook = constructXLSXWorkbook(mediaFile);
+    const worksheet = getDefaultWorksheet(workbook);
 
     const connection = getDBConnection(req.keycloak_token);
 
     try {
       await connection.open();
 
-      // Insert a new record in the `survey_observation_submission` table
-      const observationService = new ObservationService(connection);
-      const { submission_id: submissionId, key } = await observationService.insertSurveyObservationSubmission(
-        rawMediaFile,
-        Number(req.params.projectId),
-        Number(req.params.surveyId)
+      const importObserservations = new ImportObservationsService(
+        connection,
+        worksheet,
+        surveyId,
+        surveySamplePeriodId
       );
 
-      // Upload file to S3
-      const metadata = {
-        filename: rawMediaFile.originalname,
-        username: req.keycloak_token?.preferred_username ?? '',
-        email: req.keycloak_token?.email ?? ''
-      };
+      const errors = await importObserservations.importCSVWorksheet();
 
-      const result = await uploadFileToS3(rawMediaFile, key, metadata);
-
-      defaultLog.debug({ label: 'uploadMedia', message: 'result', result });
+      if (errors.length) {
+        throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      }
 
       await connection.commit();
 
-      return res.status(200).json({ submissionId });
+      return res.status(204).send();
     } catch (error) {
-      defaultLog.error({ label: 'uploadMedia', message: 'error', error });
+      defaultLog.error({ label: 'importObservationsCSV', message: 'error', error });
       await connection.rollback();
       throw error;
     } finally {