From 2e2516f51c15d97473b23e59505b0112295a7f6b Mon Sep 17 00:00:00 2001 From: Mac Deluca <99926243+MacQSL@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:00:19 -0800 Subject: [PATCH] SIMSBIOHUB-653: Observation CSV Import (#1474) - Observation CSV Import using new CSVConfig structure - New cell validators - Taxon row validator - Sampling information row validator - Shared qualitative quantitative value validators - Environment validators + measurement validators --- api/src/openapi/schemas/observation.ts | 1 + .../{surveyId}/observations/import.test.ts | 82 +++ .../observations/{upload.ts => import.ts} | 93 +-- .../{surveyId}/observations/process.test.ts | 67 -- .../survey/{surveyId}/observations/process.ts | 128 ---- .../{surveyId}/observations/upload.test.ts | 93 --- api/src/services/critterbase-service.ts | 20 + .../import-measurement-service.test.ts | 31 +- .../import-measurements-service.ts | 48 +- .../measurement-header-configs.test.ts | 270 -------- .../measurement/measurement-header-configs.ts | 187 ------ ...measurement-dynamic-headers-config.test.ts | 220 ++++++ .../measurement-dynamic-headers-config.ts | 150 +++++ .../utils/measurement-utils.test.ts | 13 + .../measurement/utils/measurement-utils.ts | 22 + .../import-observations-service.test.ts | 335 ++++++++++ .../import-observations-service.ts | 331 +++++++++ ...environment-dynamic-headers-config.test.ts | 202 ++++++ .../environment-dynamic-headers-config.ts | 139 ++++ .../observation-dynamic-header-config.ts | 55 ++ .../utils/observation-header-configs.ts | 43 ++ ...bservation-sampling-row-validator.test.ts} | 631 ++++++++++-------- .../observation-sampling-row-validator.ts | 475 +++++++++++++ .../import-services/utils/environment.test.ts | 58 ++ .../import-services/utils/environment.ts | 119 ++++ .../import-services/utils/measurement.test.ts | 68 +- .../import-services/utils/measurement.ts | 20 +- .../import-services/utils/qualitative.test.ts | 65 ++ .../import-services/utils/qualitative.ts | 70 ++ .../utils/quantitative.test.ts | 44 ++ .../import-services/utils/quantitative.ts | 69 ++ .../import-services/utils/row-state.test.ts | 48 ++ .../import-services/utils/row-state.ts | 108 +++ .../import-services/utils/taxon.test.ts | 38 ++ .../services/import-services/utils/taxon.ts | 54 ++ .../observation-service.ts | 364 +--------- .../services/observation-services/utils.ts | 499 -------------- api/src/services/platform-service.test.ts | 85 +++ api/src/services/platform-service.ts | 63 +- api/src/utils/case-insensitive-map.test.ts | 34 + api/src/utils/case-insensitive-map.ts | 56 ++ .../utils/csv-utils/csv-config-utils.test.ts | 6 +- api/src/utils/csv-utils/csv-config-utils.ts | 28 +- .../csv-config-validation.interface.ts | 2 +- .../csv-utils/csv-header-configs.test.ts | 127 +++- api/src/utils/csv-utils/csv-header-configs.ts | 101 ++- .../taxon-row-validator.test.ts | 105 +++ .../row-validators/taxon-row-validator.ts | 80 +++ api/src/utils/date-time-utils.test.ts | 78 ++- api/src/utils/date-time-utils.ts | 54 +- api/src/utils/xlsx-utils/cell-utils.test.ts | 85 +-- api/src/utils/xlsx-utils/cell-utils.ts | 63 +- .../utils/xlsx-utils/worksheet-utils.test.ts | 4 + api/src/utils/xlsx-utils/worksheet-utils.ts | 11 +- .../components/csv/CSVSingleImportDialog.tsx | 6 +- .../import-captures/utils/templates.ts | 9 + .../observations/SurveyObservationPage.tsx | 2 +- .../components/ImportObservationsButton.tsx | 124 ++++ .../SurveyObservationHeader.tsx | 0 .../ObservationsTableContainer.tsx | 2 +- .../ImportObservationsButton.tsx | 136 ---- .../components/ImportObservationsButton.tsx | 154 ----- .../period/SamplingSiteListPeriod.tsx | 11 +- app/src/hooks/api/useObservationApi.test.ts | 49 +- app/src/hooks/api/useObservationApi.ts | 77 +-- 65 files changed, 4275 insertions(+), 2537 deletions(-) create mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.test.ts rename api/src/paths/project/{projectId}/survey/{surveyId}/observations/{upload.ts => import.ts} (52%) delete mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.test.ts delete mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts delete mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.test.ts delete mode 100644 api/src/services/import-services/measurement/measurement-header-configs.test.ts delete mode 100644 api/src/services/import-services/measurement/measurement-header-configs.ts create mode 100644 api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.test.ts create mode 100644 api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.ts create mode 100644 api/src/services/import-services/measurement/utils/measurement-utils.test.ts create mode 100644 api/src/services/import-services/measurement/utils/measurement-utils.ts create mode 100644 api/src/services/import-services/observation/import-observations-service.test.ts create mode 100644 api/src/services/import-services/observation/import-observations-service.ts create mode 100644 api/src/services/import-services/observation/utils/environment-dynamic-headers-config.test.ts create mode 100644 api/src/services/import-services/observation/utils/environment-dynamic-headers-config.ts create mode 100644 api/src/services/import-services/observation/utils/observation-dynamic-header-config.ts create mode 100644 api/src/services/import-services/observation/utils/observation-header-configs.ts rename api/src/services/{observation-services/utils.test.ts => import-services/observation/utils/observation-sampling-row-validator.test.ts} (68%) create mode 100644 api/src/services/import-services/observation/utils/observation-sampling-row-validator.ts create mode 100644 api/src/services/import-services/utils/environment.test.ts create mode 100644 api/src/services/import-services/utils/environment.ts create mode 100644 api/src/services/import-services/utils/qualitative.test.ts create mode 100644 api/src/services/import-services/utils/qualitative.ts create mode 100644 api/src/services/import-services/utils/quantitative.test.ts create mode 100644 api/src/services/import-services/utils/quantitative.ts create mode 100644 api/src/services/import-services/utils/row-state.test.ts create mode 100644 api/src/services/import-services/utils/row-state.ts create mode 100644 api/src/services/import-services/utils/taxon.test.ts create mode 100644 api/src/services/import-services/utils/taxon.ts delete mode 100644 api/src/services/observation-services/utils.ts create mode 100644 api/src/utils/case-insensitive-map.test.ts create mode 100644 api/src/utils/case-insensitive-map.ts create mode 100644 api/src/utils/csv-utils/row-validators/taxon-row-validator.test.ts create mode 100644 api/src/utils/csv-utils/row-validators/taxon-row-validator.ts create mode 100644 app/src/features/surveys/observations/components/ImportObservationsButton.tsx rename app/src/features/surveys/observations/{ => components}/SurveyObservationHeader.tsx (100%) delete mode 100644 app/src/features/surveys/observations/observations-table/import-obsevations/ImportObservationsButton.tsx delete mode 100644 app/src/features/surveys/observations/sampling-sites/components/ImportObservationsButton.tsx diff --git a/api/src/openapi/schemas/observation.ts b/api/src/openapi/schemas/observation.ts index de96532500..db0ca1e603 100644 --- a/api/src/openapi/schemas/observation.ts +++ b/api/src/openapi/schemas/observation.ts @@ -84,6 +84,7 @@ export const findObservationsSchema: OpenAPIV3.SchemaObject = { observation_subcount_sign_id: { type: 'integer', minimum: 1, + nullable: true, description: 'The observation subcount sign ID, indicating whether the subcount was a direct sighting, footprints, scat, etc.' }, diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.test.ts new file mode 100644 index 0000000000..a70c3cd7a5 --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.test.ts @@ -0,0 +1,82 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import * as db from '../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../errors/http-error'; +import { ImportObservationsService } from '../../../../../../services/import-services/observation/import-observations-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db'; +import { importObservationCSV } from './import'; + +chai.use(sinonChai); + +describe('importObservationCSV', () => { + afterEach(() => { + sinon.restore(); + }); + + it('status 204 when successful', async () => { + const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() }); + const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + + const importCSVWorksheetStub = sinon.stub(ImportObservationsService.prototype, 'importCSVWorksheet'); + + importCSVWorksheetStub.resolves([]); + + const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importObservationCSV(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(getDBConnectionStub).to.have.been.calledOnce; + + expect(importCSVWorksheetStub).to.have.been.calledOnce; + + expect(mockRes.status).to.have.been.calledOnceWithExactly(204); + expect(mockRes.send).to.have.been.calledOnceWithExactly(); + + expect(mockDBConnection.commit).to.have.been.calledOnce; + expect(mockDBConnection.release).to.have.been.calledOnce; + }); + + it('status 422 when CSV validation errors', async () => { + const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() }); + const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + + const importCSVWorksheetStub = sinon.stub(ImportObservationsService.prototype, 'importCSVWorksheet'); + + importCSVWorksheetStub.resolves([{ error: 'error', solution: 'solution' }]); + + const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importObservationCSV(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail('Expected an 422 error to be thrown'); + } catch (err) { + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(getDBConnectionStub).to.have.been.calledOnce; + + expect(importCSVWorksheetStub).to.have.been.calledOnce; + + expect(mockDBConnection.commit).to.have.not.been.called; + expect(mockDBConnection.release).to.have.been.calledOnce; + expect(err).to.be.instanceOf(HTTP422CSVValidationError); + } + }); +}); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.ts similarity index 52% rename from api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.ts rename to api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.ts index 3d9113ce40..e552e4784c 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/import.ts @@ -2,14 +2,18 @@ import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; import { getDBConnection } from '../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../errors/http-error'; +import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv'; import { csvFileSchema } from '../../../../../../openapi/schemas/file'; import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; -import { ObservationService } from '../../../../../../services/observation-services/observation-service'; -import { uploadFileToS3 } from '../../../../../../utils/file-utils'; +import { ImportObservationsService } from '../../../../../../services/import-services/observation/import-observations-service'; +import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface'; import { getLogger } from '../../../../../../utils/logger'; +import { parseMulterFile } from '../../../../../../utils/media/media-utils'; import { getFileFromRequest } from '../../../../../../utils/request'; +import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../utils/xlsx-utils/worksheet-utils'; -const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/upload'); +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/import'); export const POST: Operation = [ authorizeRequestHandler((req) => { @@ -27,11 +31,11 @@ export const POST: Operation = [ ] }; }), - uploadMedia() + importObservationCSV() ]; POST.apiDoc = { - description: 'Upload survey observation submission file.', + description: 'Import survey observation CSV file.', tags: ['observations'], security: [ { @@ -42,16 +46,24 @@ POST.apiDoc = { { in: 'path', name: 'projectId', - required: true + required: true, + schema: { + type: 'integer', + minimum: 1 + } }, { in: 'path', name: 'surveyId', - required: true + required: true, + schema: { + type: 'integer', + minimum: 1 + } } ], requestBody: { - description: 'Survey observation submission file to upload', + description: 'Survey observation CSV file to import', required: true, content: { 'multipart/form-data': { @@ -61,11 +73,16 @@ POST.apiDoc = { required: ['media'], properties: { media: { - description: 'A survey observation submission file.', + description: 'A survey observation CSV file.', type: 'array', minItems: 1, maxItems: 1, items: csvFileSchema + }, + surveySamplePeriodId: { + description: 'The sample period id to associate the observations with.', + type: 'integer', + minimum: 1 } } } @@ -73,21 +90,8 @@ POST.apiDoc = { } }, responses: { - 200: { - description: 'Upload OK', - content: { - 'application/json': { - schema: { - type: 'object', - additionalProperties: false, - properties: { - submissionId: { - type: 'number' - } - } - } - } - } + 204: { + description: 'Observation import success.' }, 400: { $ref: '#/components/responses/400' @@ -98,6 +102,7 @@ POST.apiDoc = { 403: { $ref: '#/components/responses/403' }, + 422: CSVValidationErrorResponse, 500: { $ref: '#/components/responses/500' }, @@ -108,43 +113,43 @@ POST.apiDoc = { }; /** - * Uploads a media file to S3 and inserts a matching record in the `survey_observation_submission` table. + * Imports a `Observation CSV` which bulk creates observations in SIMS. * * @return {*} {RequestHandler} */ -export function uploadMedia(): RequestHandler { +export function importObservationCSV(): RequestHandler { return async (req, res) => { - const rawMediaFile = getFileFromRequest(req); + const surveyId = Number(req.params.surveyId); + const surveySamplePeriodId = Number(req.body.surveySamplePeriodId) || undefined; + + const rawFile = getFileFromRequest(req); + const mediaFile = parseMulterFile(rawFile); + const workbook = constructXLSXWorkbook(mediaFile); + const worksheet = getDefaultWorksheet(workbook); const connection = getDBConnection(req.keycloak_token); try { await connection.open(); - // Insert a new record in the `survey_observation_submission` table - const observationService = new ObservationService(connection); - const { submission_id: submissionId, key } = await observationService.insertSurveyObservationSubmission( - rawMediaFile, - Number(req.params.projectId), - Number(req.params.surveyId) + const importObserservations = new ImportObservationsService( + connection, + worksheet, + surveyId, + surveySamplePeriodId ); - // Upload file to S3 - const metadata = { - filename: rawMediaFile.originalname, - username: req.keycloak_token?.preferred_username ?? '', - email: req.keycloak_token?.email ?? '' - }; + const errors = await importObserservations.importCSVWorksheet(); - const result = await uploadFileToS3(rawMediaFile, key, metadata); - - defaultLog.debug({ label: 'uploadMedia', message: 'result', result }); + if (errors.length) { + throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + } await connection.commit(); - return res.status(200).json({ submissionId }); + return res.status(204).send(); } catch (error) { - defaultLog.error({ label: 'uploadMedia', message: 'error', error }); + defaultLog.error({ label: 'importObservationsCSV', message: 'error', error }); await connection.rollback(); throw error; } finally { diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.test.ts deleted file mode 100644 index 49c413deba..0000000000 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import chai, { expect } from 'chai'; -import { describe } from 'mocha'; -import sinon from 'sinon'; -import sinonChai from 'sinon-chai'; -import * as db from '../../../../../../database/db'; -import { HTTPError } from '../../../../../../errors/http-error'; -import { ObservationService } from '../../../../../../services/observation-services/observation-service'; -import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db'; -import * as process from './process'; - -chai.use(sinonChai); - -describe('processFile', () => { - afterEach(() => { - sinon.restore(); - }); - - const dbConnectionObj = getMockDBConnection(); - - const mockReq = { - keycloak_token: {}, - params: { - projectId: 1, - surveyId: 2 - }, - body: {} - } as any; - - it('should throw an error if failure occurs', async () => { - sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - } - }); - - const expectedError = new Error('Error'); - sinon.stub(ObservationService.prototype, 'processObservationCsvSubmission').rejects(expectedError); - - try { - const result = process.processFile(); - - await result(mockReq, null as unknown as any, null as unknown as any); - expect.fail(); - } catch (actualError) { - expect((actualError as HTTPError).message).to.equal(expectedError.message); - } - }); - - it('should succeed with valid params', async () => { - const getDBConnectionStub = sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - } - }); - sinon.stub(ObservationService.prototype, 'processObservationCsvSubmission').resolves({} as any); - const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); - const requestHandler = process.processFile(); - - await requestHandler(mockReq, mockRes, mockNext); - - expect(getDBConnectionStub).to.have.been.calledOnce; - expect(mockRes.status).to.be.calledWith(200); - expect(mockRes.json).not.to.have.been.called; - }); -}); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts deleted file mode 100644 index c86fd000a9..0000000000 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { RequestHandler } from 'express'; -import { Operation } from 'express-openapi'; -import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; -import { getDBConnection } from '../../../../../../database/db'; -import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; -import { ObservationService } from '../../../../../../services/observation-services/observation-service'; -import { getLogger } from '../../../../../../utils/logger'; - -const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/process'); - -export const POST: Operation = [ - authorizeRequestHandler((req) => { - return { - or: [ - { - validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR], - surveyId: Number(req.params.surveyId), - discriminator: 'ProjectPermission' - }, - { - validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], - discriminator: 'SystemRole' - } - ] - }; - }), - processFile() -]; - -POST.apiDoc = { - description: 'Processes and validates observation CSV submission', - tags: ['survey', 'observation', 'csv'], - security: [ - { - Bearer: [] - } - ], - parameters: [ - { - in: 'path', - name: 'projectId', - required: true - }, - { - in: 'path', - name: 'surveyId', - required: true - } - ], - requestBody: { - description: 'Request body', - required: true, - content: { - 'application/json': { - schema: { - type: 'object', - additionalProperties: false, - required: ['observation_submission_id'], - properties: { - observation_submission_id: { - description: 'The ID of the submission to validate', - type: 'integer' - }, - options: { - type: 'object', - additionalProperties: false, - properties: { - surveySamplePeriodId: { - type: 'integer', - description: - 'The optional ID of a survey sample period to associate the parsed observation records with. This is used when uploading all observations to a specific sample period, not when each record is for a different sample period.' - } - } - } - } - } - } - } - }, - responses: { - 200: { - description: 'Process Observation File OK' - }, - 400: { - $ref: '#/components/responses/400' - }, - 401: { - $ref: '#/components/responses/401' - }, - 403: { - $ref: '#/components/responses/403' - }, - 500: { - $ref: '#/components/responses/500' - }, - default: { - $ref: '#/components/responses/default' - } - } -}; - -export function processFile(): RequestHandler { - return async (req, res) => { - const surveyId = Number(req.params.surveyId); - const submissionId = req.body.observation_submission_id; - - const connection = getDBConnection(req.keycloak_token); - try { - await connection.open(); - - const options = req.body.options || undefined; - - const observationService = new ObservationService(connection); - - await observationService.processObservationCsvSubmission(surveyId, submissionId, options); - - await connection.commit(); - - res.status(200).send(); - } catch (error) { - defaultLog.error({ label: 'processFile', message: 'error', error }); - await connection.rollback(); - throw error; - } finally { - connection.release(); - } - }; -} diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.test.ts deleted file mode 100644 index ed94dcfc2c..0000000000 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/upload.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import chai, { expect } from 'chai'; -import { describe } from 'mocha'; -import sinon from 'sinon'; -import sinonChai from 'sinon-chai'; -import * as db from '../../../../../../database/db'; -import { HTTPError } from '../../../../../../errors/http-error'; -import { ObservationService } from '../../../../../../services/observation-services/observation-service'; -import * as file_utils from '../../../../../../utils/file-utils'; -import { getMockDBConnection } from '../../../../../../__mocks__/db'; -import * as upload from './upload'; - -chai.use(sinonChai); - -describe('uploadMedia', () => { - afterEach(() => { - sinon.restore(); - }); - - const dbConnectionObj = getMockDBConnection(); - - const mockReq = { - keycloak_token: {}, - params: { - projectId: 1, - surveyId: 2 - }, - files: [ - { - fieldname: 'media', - originalname: 'test.csv', - encoding: '7bit', - mimetype: 'text/plain', - size: 340 - } - ], - body: {} - } as any; - - it('should throw an error if failure occurs', async () => { - sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - } - }); - - const expectedError = new Error('cannot process request'); - sinon.stub(ObservationService.prototype, 'insertSurveyObservationSubmission').rejects(expectedError); - - try { - const result = upload.uploadMedia(); - - await result(mockReq, null as unknown as any, null as unknown as any); - expect.fail(); - } catch (actualError) { - expect((actualError as HTTPError).message).to.equal(expectedError.message); - } - }); - - it('should succeed with valid params', async () => { - sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - } - }); - - sinon.stub(file_utils, 'uploadFileToS3').resolves(); - - const expectedResponse = { submissionId: 1 }; - - let actualResult: any = null; - const sampleRes = { - status: () => { - return { - json: (response: any) => { - actualResult = response; - } - }; - } - }; - - const upsertSurveyAttachmentStub = sinon - .stub(ObservationService.prototype, 'insertSurveyObservationSubmission') - .resolves({ submission_id: 1, key: 'string' }); - - const result = upload.uploadMedia(); - - await result(mockReq, sampleRes as unknown as any, null as unknown as any); - expect(actualResult).to.eql(expectedResponse); - expect(upsertSurveyAttachmentStub).to.be.calledOnce; - }); -}); diff --git a/api/src/services/critterbase-service.ts b/api/src/services/critterbase-service.ts index 72dc3c2cae..7f43384049 100644 --- a/api/src/services/critterbase-service.ts +++ b/api/src/services/critterbase-service.ts @@ -2,6 +2,7 @@ import axios, { AxiosError, AxiosInstance, AxiosResponse } from 'axios'; import { Request } from 'express'; import qs from 'qs'; import { z } from 'zod'; +import { IDBConnection } from '../database/db'; import { ApiError, ApiErrorType } from '../errors/api-error'; import { ISex } from '../models/animal-view'; import { getLogger } from '../utils/logger'; @@ -14,11 +15,30 @@ export interface ICritterbaseUser { keycloak_guid: string; } +/** + * Get Critterbase user from a request + * + * TODO: Rename to `getCritterbaseUserFromRequest` + * + * @param {Request} req + * @returns {ICritterbaseUser} + */ export const getCritterbaseUser = (req: Request): ICritterbaseUser => ({ keycloak_guid: req.system_user?.user_guid ?? '', username: req.system_user?.user_identifier ?? '' }); +/** + * Get Critterbase user from connection + * + * @param {IDBConnection} connection + * @returns {ICritterbaseUser} + */ +export const getCritterbaseUserFromConnection = (connection: IDBConnection) => ({ + keycloak_guid: connection.systemUserGUID(), + username: connection.systemUserIdentifier() +}); + export interface QueryParam { key: string; value: string; diff --git a/api/src/services/import-services/measurement/import-measurement-service.test.ts b/api/src/services/import-services/measurement/import-measurement-service.test.ts index 7fc1618768..48d19d5e31 100644 --- a/api/src/services/import-services/measurement/import-measurement-service.test.ts +++ b/api/src/services/import-services/measurement/import-measurement-service.test.ts @@ -1,5 +1,6 @@ import { expect } from 'chai'; import sinon from 'sinon'; +import { v4 } from 'uuid'; import * as csv from '../../../utils/csv-utils/csv-config-validation'; import { CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; import { NestedRecord } from '../../../utils/nested-record'; @@ -42,16 +43,24 @@ describe('import-measurements-service', () => { const connection = getMockDBConnection(); const service = new ImportMeasurementsService(connection, {}, 1); + const critterIdMock = v4(); + const captureIdMock = v4(); + const taxonMeasurementIdMock = v4(); + const qualitativeOptionIdMock = v4(); + sinon.stub(service, 'getCSVConfig').returns({} as any); sinon.stub(csv, 'validateCSVWorksheet').returns({ errors: [], rows: [ { [CSVRowState]: { - critter_id: 'critter_id', - capture_id: 'capture_id', - qualHeader: { taxon_measurement_id: 1, qualitative_option_id: 1 }, - quantHeader: { taxon_measurement_id: 1, value: 1 } + critter_id: critterIdMock, + capture_id: captureIdMock, + qualHeader: { + taxon_measurement_id: taxonMeasurementIdMock, + qualitative_option_id: qualitativeOptionIdMock + }, + quantHeader: { taxon_measurement_id: taxonMeasurementIdMock, value: 1 } } } ] @@ -66,17 +75,17 @@ describe('import-measurements-service', () => { expect(bulkCreateStub).to.have.been.calledOnceWithExactly({ qualitative_measurements: [ { - critter_id: 'critter_id', - capture_id: 'capture_id', - taxon_measurement_id: 1, - qualitative_option_id: 1 + critter_id: critterIdMock, + capture_id: captureIdMock, + taxon_measurement_id: taxonMeasurementIdMock, + qualitative_option_id: qualitativeOptionIdMock } ], quantitative_measurements: [ { - critter_id: 'critter_id', - capture_id: 'capture_id', - taxon_measurement_id: 1, + critter_id: critterIdMock, + capture_id: captureIdMock, + taxon_measurement_id: taxonMeasurementIdMock, value: 1 } ] diff --git a/api/src/services/import-services/measurement/import-measurements-service.ts b/api/src/services/import-services/measurement/import-measurements-service.ts index 4aba2b447e..60a2246695 100644 --- a/api/src/services/import-services/measurement/import-measurements-service.ts +++ b/api/src/services/import-services/measurement/import-measurements-service.ts @@ -16,10 +16,16 @@ import { DBService } from '../../db-service'; import { SurveyCritterService } from '../../survey-critter-service'; import { getTsnMeasurementDictionary, - isCBQualitativeMeasurement, - isCBQuantitativeMeasurement + isCBQualitativeMeasurementStub, + isCBQuantitativeMeasurementStub } from '../utils/measurement'; -import { getDynamicMeasurementCellValidator } from './measurement-header-configs'; +import { + getCritterCaptureFromRowState, + getQualitativeMeasurementFromRowState, + getQuantitativeMeasurementFromRowState +} from '../utils/row-state'; +import { getDynamicMeasurementCellValidator } from './utils/measurement-dynamic-headers-config'; +import { getTsnFromMeasurementRow } from './utils/measurement-utils'; const defaultLog = getLogger('services/import/import-measurement-service'); @@ -87,25 +93,32 @@ export class ImportMeasurementsService extends DBService { for (const row of rows) { this.utils.worksheetDynamicHeaders.forEach((header) => { - const state = row[CSVRowState]; - const stateMeasurement = state?.[header]; + const stateMeasurement = row[CSVRowState]?.[header]; // Grab the qualitative measurement from the row - if (isCBQualitativeMeasurement(stateMeasurement)) { + if (isCBQualitativeMeasurementStub(stateMeasurement)) { + // Get the critter and qualitative measurement meta from the row state + const critter = getCritterCaptureFromRowState(row); + const qualitativeMeasurement = getQualitativeMeasurementFromRowState(stateMeasurement); + qualitativeMeasurements.push({ - critter_id: state?.critter_id, - capture_id: state?.capture_id, - taxon_measurement_id: stateMeasurement.taxon_measurement_id, - qualitative_option_id: stateMeasurement.qualitative_option_id + critter_id: critter.critter_id, + capture_id: critter.capture_id, + taxon_measurement_id: qualitativeMeasurement.taxon_measurement_id, + qualitative_option_id: qualitativeMeasurement.qualitative_option_id }); } // Grab the quantitative measurement from the row - else if (isCBQuantitativeMeasurement(stateMeasurement)) { + else if (isCBQuantitativeMeasurementStub(stateMeasurement)) { + // Get the critter and quantitative measurement meta from the row state + const critter = getCritterCaptureFromRowState(row); + const quantitativeMeasurement = getQuantitativeMeasurementFromRowState(stateMeasurement); + quantitativeMeasurements.push({ - critter_id: state?.critter_id, - capture_id: state?.capture_id, - taxon_measurement_id: stateMeasurement.taxon_measurement_id, - value: stateMeasurement.value + critter_id: critter.critter_id, + capture_id: critter.capture_id, + taxon_measurement_id: quantitativeMeasurement.taxon_measurement_id, + value: quantitativeMeasurement.value }); } }); @@ -145,7 +158,10 @@ export class ImportMeasurementsService extends DBService { // Inject dynamic header config - handles measurement validation config.dynamicHeadersConfig = { - validateCell: getDynamicMeasurementCellValidator(measurementDictionary, surveyAliasMap, this.utils) + validateCell: getDynamicMeasurementCellValidator( + measurementDictionary, + getTsnFromMeasurementRow(surveyAliasMap, this.utils) + ) }; // Return the final CSV config diff --git a/api/src/services/import-services/measurement/measurement-header-configs.test.ts b/api/src/services/import-services/measurement/measurement-header-configs.test.ts deleted file mode 100644 index 3a255a7577..0000000000 --- a/api/src/services/import-services/measurement/measurement-header-configs.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { expect } from 'chai'; -import sinon from 'sinon'; -import { CSVParams, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; -import { NestedRecord } from '../../../utils/nested-record'; -import * as measurementUtils from '../utils/measurement'; -import * as measurementConfigs from './measurement-header-configs'; -import { - getDynamicMeasurementCellValidator, - getQualitativeMeasurementCellValidator, - getQuantitativeMeasurementCellValidator -} from './measurement-header-configs'; - -describe('measurement-header-configs', () => { - beforeEach(() => { - sinon.restore(); - }); - - describe('getDynamicMeasurementCellValidator', () => { - it('should return no errors if cell is undefined', () => { - const cellValidator = getDynamicMeasurementCellValidator(new NestedRecord(), new Map(), {} as any); - - const result = cellValidator({ cell: undefined, row: {} } as CSVParams); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should return an error if taxon has no measurements', () => { - const cellValidator = getDynamicMeasurementCellValidator(new NestedRecord(), new Map(), { - getCellValue: () => 'ALIAS' - } as any); - - const result = cellValidator({ cell: 'value', row: {} } as CSVParams); - - expect(result[0].error).to.contain('no reference measurements'); - }); - - it('should return an error if the measurement does not exist', () => { - const surveyAliasMap = new Map(); - const tsnMeasurementDictionary = new NestedRecord(); - - surveyAliasMap.set('alias', { itis_tsn: 123 }); - tsnMeasurementDictionary.set({ path: ['123', 'HEADER'], value: 'measurement' }); - - const cellValidator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, surveyAliasMap, { - getCellValue: () => 'ALIAS' - } as any); - - const result = cellValidator({ cell: 'value', row: {}, header: 'BAD_HEADER' } as CSVParams); - - expect(result[0].error).to.contain("'BAD_HEADER' does not exist"); - }); - - it('should return no errors when the cell is a valid qualitative measurement', () => { - const surveyAliasMap = new Map(); - const tsnMeasurementDictionary = new NestedRecord(); - - surveyAliasMap.set('alias', { itis_tsn: 123 }); - tsnMeasurementDictionary.set({ path: ['123', 'HEADER'], value: { options: [], taxon_measurement_id: true } }); - - const cellValidator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, surveyAliasMap, { - getCellValue: () => 'ALIAS' - } as any); - - sinon.stub(measurementUtils, 'isCBQualitativeMeasurementTypeDefinition').returns(true); - sinon.stub(measurementConfigs, 'getQualitativeMeasurementCellValidator').returns(() => []); - - const result = cellValidator({ cell: 'value', row: {}, header: 'HEADER' } as CSVParams); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should return no errors when the cell is a valid qualitative measurement', () => { - const surveyAliasMap = new Map(); - const tsnMeasurementDictionary = new NestedRecord(); - - surveyAliasMap.set('alias', { itis_tsn: 123 }); - tsnMeasurementDictionary.set({ path: ['123', 'HEADER'], value: { options: [], taxon_measurement_id: true } }); - - const cellValidator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, surveyAliasMap, { - getCellValue: () => 'ALIAS' - } as any); - - sinon.stub(measurementUtils, 'isCBQualitativeMeasurementTypeDefinition').returns(false); - sinon.stub(measurementUtils, 'isCBQuantitativeMeasurementTypeDefinition').returns(true); - sinon.stub(measurementConfigs, 'getQuantitativeMeasurementCellValidator').returns(() => []); - - const result = cellValidator({ cell: 'value', row: {}, header: 'HEADER' } as CSVParams); - - expect(result).to.be.an('array').that.is.empty; - }); - }); - - describe('getQuantitativeMeasurementCellValidator', () => { - it('should return an empty array if the cell is valid', () => { - const cellValidator = getQuantitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - min_value: 0, - max_value: 100, - unit: 'millimeter' - }); - - const result = cellValidator({ cell: 1, row: {} } as CSVParams); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should update the state with the taxon measurement id and value', () => { - const cellValidator = getQuantitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - min_value: 0, - max_value: 100, - unit: 'millimeter' - }); - - const params = { cell: 1, row: {}, header: 'MEASUREMENT' } as CSVParams; - - const result = cellValidator(params); - - expect(params.row[CSVRowState]?.MEASUREMENT.taxon_measurement_id).to.equal('taxonID'); - expect(params.row[CSVRowState]?.MEASUREMENT.value).to.equal(1); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should return an error when not a number', () => { - const cellValidator = getQuantitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - min_value: 0, - max_value: 100, - unit: 'millimeter' - }); - - const result = cellValidator({ cell: 'invalid' } as CSVParams); - - expect(result[0].error).to.contain('number'); - }); - - it('should return an error when the value is too large', () => { - const cellValidator = getQuantitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - min_value: 0, - max_value: 100, - unit: 'millimeter' - }); - - const result = cellValidator({ cell: 101, row: {} } as CSVParams); - - expect(result[0].error).to.contain('large'); - }); - - it('should return an error when the value is too small', () => { - const cellValidator = getQuantitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - min_value: 0, - max_value: 100, - unit: 'millimeter' - }); - - const result = cellValidator({ cell: -1, row: {} } as CSVParams); - - expect(result[0].error).to.contain('small'); - }); - }); - - describe('getQualitativeMeasurementCellValidator', () => { - it('should return an empty array if the cell is valid', () => { - const cellValidator = getQualitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - options: [ - { - qualitative_option_id: 'optionID', - option_label: 'label', - option_value: 1, - option_desc: 'desc' - } - ] - }); - - const result = cellValidator({ cell: 'label', row: {} } as CSVParams); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should update the state with the taxon measurement id and value', () => { - const cellValidator = getQualitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - options: [ - { - qualitative_option_id: 'optionID', - option_label: 'label', - option_value: 1, - option_desc: 'desc' - } - ] - }); - - const params = { cell: 'label', row: {}, header: 'MEASUREMENT' } as CSVParams; - - const result = cellValidator(params); - - expect(params.row[CSVRowState]?.MEASUREMENT.taxon_measurement_id).to.equal('taxonID'); - expect(params.row[CSVRowState]?.MEASUREMENT.qualitative_option_id).to.equal('optionID'); - - expect(result).to.be.an('array').that.is.empty; - }); - - it('should return an error when not a string', () => { - const cellValidator = getQualitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - options: [ - { - qualitative_option_id: 'optionID', - option_label: 'label', - option_value: 1, - option_desc: 'desc' - } - ] - }); - - const result = cellValidator({ cell: 123 } as CSVParams); - - expect(result[0].error).to.contain('string'); - }); - - it('should return an error when the option is not valid', () => { - const cellValidator = getQualitativeMeasurementCellValidator({ - itis_tsn: 123, - taxon_measurement_id: 'taxonID', - measurement_name: 'name', - measurement_desc: 'desc', - options: [ - { - qualitative_option_id: 'optionID', - option_label: 'label', - option_value: 1, - option_desc: 'desc' - } - ] - }); - - const result = cellValidator({ cell: 'invalid' } as CSVParams); - - expect(result[0].error).to.contain('option'); - }); - }); -}); diff --git a/api/src/services/import-services/measurement/measurement-header-configs.ts b/api/src/services/import-services/measurement/measurement-header-configs.ts deleted file mode 100644 index 7839f4bb42..0000000000 --- a/api/src/services/import-services/measurement/measurement-header-configs.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; -import { CSVCellValidator, CSVError } from '../../../utils/csv-utils/csv-config-validation.interface'; -import { updateCSVRowState } from '../../../utils/csv-utils/csv-header-configs'; -import { NestedRecord } from '../../../utils/nested-record'; -import { - CBQualitativeMeasurement, - CBQualitativeMeasurementTypeDefinition, - CBQuantitativeMeasurement, - CBQuantitativeMeasurementTypeDefinition, - ICritterDetailed -} from '../../critterbase-service'; -import { - isCBQualitativeMeasurementTypeDefinition, - isCBQuantitativeMeasurementTypeDefinition -} from '../utils/measurement'; -import { MeasurementCSVStaticHeader } from './import-measurements-service'; - -export type TSNMeasurementDictionary = NestedRecord< - CBQualitativeMeasurementTypeDefinition | CBQuantitativeMeasurementTypeDefinition ->; - -/** - * Get the dynamic measurement cell validator. - * - * @param {TSNMeasurementDictionary} tsnMeasurementDictionary The TSN measurement dictionary - * @param {Map} surveyAliasMap The survey alias map - * @param {CSVConfigUtils} utils The CSV config utils - * @returns {*} {CSVCellValidator} The validate cell callback - */ -export const getDynamicMeasurementCellValidator = ( - tsnMeasurementDictionary: TSNMeasurementDictionary, - surveyAliasMap: Map, - utils: CSVConfigUtils -): CSVCellValidator => { - return (params) => { - if (params.cell === undefined) { - return []; - } - - const alias = String(utils.getCellValue('ALIAS', params.row)).toLowerCase(); - const critter = surveyAliasMap.get(alias); - const critterTsn = Number(critter?.itis_tsn); - - const taxonMeasurements = tsnMeasurementDictionary.get(critterTsn); - - if (!taxonMeasurements) { - return [ - { - error: `Taxon has no reference measurements`, - solution: 'Make sure the taxon has reference measurements' - } - ]; - } - - const measurement = tsnMeasurementDictionary.get(critterTsn, params.header); - - if (!measurement) { - return [ - { - error: `Column header '${params.header}' does not exist`, - solution: 'Use a valid taxon measurement as the header', - values: Object.keys(taxonMeasurements) - } - ]; - } - - // Validate the cell based on the measurement type from the header - if (isCBQualitativeMeasurementTypeDefinition(measurement)) { - return getQualitativeMeasurementCellValidator(measurement)(params); - } - - if (isCBQuantitativeMeasurementTypeDefinition(measurement)) { - return getQuantitativeMeasurementCellValidator(measurement)(params); - } - - // Can this path ever be reached? - return [ - { - error: 'Invalid measurement type', - solution: 'Use a supported measurement type' - } - ]; - }; -}; - -/** - * Get the quantitative measurement cell validator. - * - * @param {CBQuantitativeMeasurementTypeDefinition} measurement The quantitative measurement definition - * @returns {*} {CSVCellValidator} The validate cell callback - */ -export const getQuantitativeMeasurementCellValidator = ( - measurement: CBQuantitativeMeasurementTypeDefinition -): CSVCellValidator => { - return (params) => { - const cellErrors: CSVError[] = []; - - // Qualitative measurements are numbers ie: antler count: 2 - if (typeof params.cell !== 'number') { - return [ - { - error: 'Quantitative measurement must be a number', - solution: 'Update the cell value to match the expected type' - } - ]; - } - - // Validate cell value is withing the measurement min max bounds - if (measurement.max_value != null && params.cell > measurement.max_value) { - cellErrors.push({ - error: 'Quantitative measurement too large', - solution: `Value must be less than or equal to ${measurement.max_value}` - }); - } - - // Validate cell value is withing the measurement min max bounds - if (measurement.min_value != null && params.cell < measurement.min_value) { - cellErrors.push({ - error: 'Quantitative measurement too small', - solution: `Value must be greater than or equal to ${measurement.min_value}` - }); - } - - // Update the row state with the taxon measurement id and value - updateCSVRowState(params.row, { - // Using header to prevent overwriting other measurements - // ie: This function will be called once for each dynamic header - [params.header]: { - taxon_measurement_id: measurement.taxon_measurement_id, - value: params.cell - } satisfies Partial - }); - - return cellErrors; - }; -}; - -/** - * Get the qualitative measurement cell validator. - * - * Note: This function will mutate the cell value to the qualitative option id - * - * @param {CBQualitativeMeasurementTypeDefinition} measurement The qualitative measurement definition - * @returns {*} {CSVCellValidator} The validate cell callback - */ -export const getQualitativeMeasurementCellValidator = ( - measurement: CBQualitativeMeasurementTypeDefinition -): CSVCellValidator => { - return (params) => { - if (typeof params.cell !== 'string') { - return [ - { - error: 'Qualitative measurement must be a string', - solution: 'Update the cell value to match the expected type' - } - ]; - } - - const matchingOptionValue = measurement.options.find( - // Not sure why I need to cast params.cell to string here after checking above... - (option) => option.option_label.toLowerCase() === String(params.cell).toLowerCase() - ); - - // Validate cell value is an alowed qualitative measurement option - if (!matchingOptionValue) { - return [ - { - error: `Invalid qualitative measurement option`, - solution: `Use a valid qualitative measurement option`, - values: measurement.options.map((option) => option.option_label) - } - ]; - } - - // Update the row state with the taxon measurement id and qualitative option id - updateCSVRowState(params.row, { - // Using header to prevent overwriting other measurements - // ie: This function will be called once for each dynamic header - [params.header]: { - taxon_measurement_id: measurement.taxon_measurement_id, - qualitative_option_id: matchingOptionValue.qualitative_option_id - } satisfies Partial - }); - - return []; - }; -}; diff --git a/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.test.ts b/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.test.ts new file mode 100644 index 0000000000..284d87d5c2 --- /dev/null +++ b/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.test.ts @@ -0,0 +1,220 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { v4 } from 'uuid'; +import { CSVParams, CSVRowState } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { NestedRecord } from '../../../../utils/nested-record'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition +} from '../../../critterbase-service'; +import { getQualitativeMeasurementFromRowState } from '../../utils/row-state'; +import { + getDynamicMeasurementCellValidator, + TSNMeasurementDictionary, + validateQualitativeMeasurementCell, + validateQuantitativeMeasurementCell +} from './measurement-dynamic-headers-config'; + +import * as measurement from './measurement-dynamic-headers-config'; + +chai.use(sinonChai); + +describe('measurement-dynamic-headers-config', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('getDynamicMeasurementCellValidator', () => { + it('should return an empty array when the cell is undefined', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: undefined } as CSVParams); + + expect(result).to.be.deep.equal([]); + }); + + it('should return an error when the taxon has no reference measurements', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: 'test' } as CSVParams); + + expect(result[0].error).to.contain('no reference measurements'); + }); + + it('should return an error when the column header does not exist', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + tsnMeasurementDictionary.set({ path: [1, 'header'], value: { itis_tsn: 1 } as any }); + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: 'test', header: 'bad' } as CSVParams); + + expect(result[0].error).to.contain("'bad' does not exist"); + }); + + it('should call the qualitative measurement cell validator when the measurement is qualitative', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + const measurementDefinition = { + taxon_measurement_id: v4(), + options: [ + { + qualitative_option_id: v4(), + option_label: 'test' + } + ] + } as CBQualitativeMeasurementTypeDefinition; + + tsnMeasurementDictionary.set({ path: [1, 'QUALITATIVE'], value: measurementDefinition }); + + const validateQualitativeStub = sinon.stub(measurement, 'validateQualitativeMeasurementCell').returns([]); + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: 'test', header: 'QUALITATIVE', row: {} } as CSVParams); + + expect(validateQualitativeStub).to.have.been.calledOnce; + + expect(result).to.be.deep.equal([]); + }); + + it('should call the quantitative measurement cell validator when the measurement is quantitative', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + const measurementDefinition = { + taxon_measurement_id: v4(), + unit: 'kg' + } as unknown as CBQuantitativeMeasurementTypeDefinition; + + tsnMeasurementDictionary.set({ path: [1, 'QUANTITATIVE'], value: measurementDefinition }); + + const validateQuantitativeStub = sinon.stub(measurement, 'validateQuantitativeMeasurementCell').returns([]); + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: 1, header: 'QUANTITATIVE', row: {} } as CSVParams); + + expect(validateQuantitativeStub).to.have.been.calledOnce; + + expect(result).to.be.deep.equal([]); + }); + + it('should return an error when the measurement type is invalid', () => { + const tsnMeasurementDictionary: TSNMeasurementDictionary = new NestedRecord(); + const getCritterTsn = () => 1; + + tsnMeasurementDictionary.set({ path: [1, 'INVALID'], value: {} as any }); + + const validator = getDynamicMeasurementCellValidator(tsnMeasurementDictionary, getCritterTsn); + + const result = validator({ cell: 'test', header: 'INVALID', row: {} } as CSVParams); + + expect(result[0].error).to.contain('Invalid measurement type'); + }); + }); + + describe('validateQualitativeMeasurementCell', () => { + it('should validate the qualitative measurement cell value successfully', () => { + const params = { + cell: 'test', + row: {}, + header: 'QUALITATIVE' + } as CSVParams; + + const measurementDefinition = { + taxon_measurement_id: v4(), + options: [ + { + qualitative_option_id: v4(), + option_label: 'test' + } + ] + } as CBQualitativeMeasurementTypeDefinition; + + const result = validateQualitativeMeasurementCell(params, measurementDefinition); + + expect(result).to.be.deep.equal([]); + + expect(getQualitativeMeasurementFromRowState(params.row[CSVRowState]?.QUALITATIVE)).to.deep.equal({ + taxon_measurement_id: measurementDefinition.taxon_measurement_id, + qualitative_option_id: measurementDefinition.options[0].qualitative_option_id + }); + }); + + it('should return an error when the cell value is not a valid option', () => { + const params = { + cell: 'invalid', + row: {}, + header: 'QUALITATIVE' + } as CSVParams; + + const measurementDefinition = { + taxon_measurement_id: v4(), + options: [ + { + qualitative_option_id: v4(), + option_label: 'test' + } + ] + } as CBQualitativeMeasurementTypeDefinition; + + const result = validateQualitativeMeasurementCell(params, measurementDefinition); + + expect(result).to.be.an('array').with.length(1); + }); + }); + + describe('validateQuantitativeMeasurementCell', () => { + it('should validate the quantitative measurement cell value successfully', () => { + const params = { + cell: 1, + row: {}, + header: 'QUANTITATIVE' + } as CSVParams; + + const measurementDefinition = { + taxon_measurement_id: v4(), + min_value: 0, + max_value: 10 + } as CBQuantitativeMeasurementTypeDefinition; + + const result = validateQuantitativeMeasurementCell(params, measurementDefinition); + + expect(result).to.be.deep.equal([]); + + expect(params.row[CSVRowState]?.QUANTITATIVE).to.deep.equal({ + taxon_measurement_id: measurementDefinition.taxon_measurement_id, + value: params.cell + }); + }); + + it('should return an error when the cell value is not valid', () => { + const params = { + cell: 11, + row: {}, + header: 'QUANTITATIVE' + } as CSVParams; + + const measurementDefinition = { + taxon_measurement_id: v4(), + min_value: 0, + max_value: 10 + } as CBQuantitativeMeasurementTypeDefinition; + + const result = validateQuantitativeMeasurementCell(params, measurementDefinition); + + expect(result).to.be.an('array').with.length(1); + }); + }); +}); diff --git a/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.ts b/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.ts new file mode 100644 index 0000000000..8cfdbae782 --- /dev/null +++ b/api/src/services/import-services/measurement/utils/measurement-dynamic-headers-config.ts @@ -0,0 +1,150 @@ +import { CSVCellValidator, CSVError, CSVParams } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { updateCSVRowState } from '../../../../utils/csv-utils/csv-header-configs'; +import { NestedRecord } from '../../../../utils/nested-record'; +import { + CBQualitativeMeasurement, + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurement, + CBQuantitativeMeasurementTypeDefinition +} from '../../../critterbase-service'; +import { + isCBQualitativeMeasurementTypeDefinition, + isCBQuantitativeMeasurementTypeDefinition +} from '../../utils/measurement'; +import { validateQualitativeValue } from '../../utils/qualitative'; +import { validateQuantitativeValue } from '../../utils/quantitative'; + +export type TSNMeasurementDictionary = NestedRecord< + CBQualitativeMeasurementTypeDefinition | CBQuantitativeMeasurementTypeDefinition +>; + +/** + * Get the dynamic measurement cell validator. + * + * @param {TSNMeasurementDictionary} tsnMeasurementDictionary The TSN measurement dictionary + * @param {(params: CSVParams) => number} getCritterTsn The callback to get the TSN from the row/params + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getDynamicMeasurementCellValidator = ( + tsnMeasurementDictionary: TSNMeasurementDictionary, + getCritterTsn: (params: CSVParams) => number +): CSVCellValidator => { + return (params) => { + if (params.cell === undefined) { + return []; + } + + const tsn = getCritterTsn(params); + + const taxonMeasurements = tsnMeasurementDictionary.get(tsn); + + if (!taxonMeasurements) { + return [ + { + error: `Taxon has no reference measurements`, + solution: 'Make sure the taxon has reference measurements' + } + ]; + } + + const measurement = tsnMeasurementDictionary.get(tsn, params.header); + + if (!measurement) { + return [ + { + error: `Column header '${params.header}' does not exist`, + solution: 'Use a valid taxon measurement as the header', + values: Object.keys(taxonMeasurements) + } + ]; + } + + if (isCBQualitativeMeasurementTypeDefinition(measurement)) { + return validateQualitativeMeasurementCell(params, measurement); + } + + if (isCBQuantitativeMeasurementTypeDefinition(measurement)) { + return validateQuantitativeMeasurementCell(params, measurement); + } + + // Can this path ever be reached? + return [ + { + error: 'Invalid measurement type', + solution: 'Use a supported measurement type' + } + ]; + }; +}; + +/** + * Validate the qualitative measurement cell value. + * + * @param {CSVParams} params The CSV params + * @param {CBQualitativeMeasurementTypeDefinition} measurement The qualitative measurement definition + * @returns {CSVError[]} The list of errors + */ +export const validateQualitativeMeasurementCell = ( + params: CSVParams, + measurement: CBQualitativeMeasurementTypeDefinition +): CSVError[] => { + const options = measurement.options.map((option) => ({ + option_id: option.qualitative_option_id, + option_name: option.option_label + })); + + // Normalize the measurement type definition and validate + const result = validateQualitativeValue(params.cell, { options: options }, 'measurement'); + + // If the result is list of CSV errors + if (typeof result !== 'string') { + return result; + } + + // Update the row state with the taxon measurement id and qualitative option id + updateCSVRowState(params.row, { + [params.header]: { + taxon_measurement_id: measurement.taxon_measurement_id, + qualitative_option_id: result + } satisfies Partial + }); + + return []; +}; + +/** + * Validate the quantitative measurement cell value. + * + * @param {CSVParams} params The CSV params + * @param {CBQuantitativeMeasurementTypeDefinition} measurement The quantitative measurement definition + * @returns {CSVError[]} The list of errors + */ +export const validateQuantitativeMeasurementCell = ( + params: CSVParams, + measurement: CBQuantitativeMeasurementTypeDefinition +): CSVError[] => { + // Normalize the measurement type definition and validate + const result = validateQuantitativeValue( + params.cell, + { + min: measurement.min_value, + max: measurement.max_value + }, + 'measurement' + ); + + // If the result is list of CSV errors + if (typeof result !== 'number') { + return result; + } + + // Update the row state with the taxon measurement id and value + updateCSVRowState(params.row, { + [params.header]: { + taxon_measurement_id: measurement.taxon_measurement_id, + value: result + } satisfies Partial + }); + + return []; +}; diff --git a/api/src/services/import-services/measurement/utils/measurement-utils.test.ts b/api/src/services/import-services/measurement/utils/measurement-utils.test.ts new file mode 100644 index 0000000000..a2df160d0c --- /dev/null +++ b/api/src/services/import-services/measurement/utils/measurement-utils.test.ts @@ -0,0 +1,13 @@ +import { expect } from 'chai'; +import { getTsnFromMeasurementRow } from './measurement-utils'; + +describe('getTsnFromMeasurementRow', () => { + it('should return a function that returns the ITIS TSN from the row', () => { + const surveyAliasMap = new Map(); + surveyAliasMap.set('carl', { itis_tsn: 1234 }); + + const getTsn = getTsnFromMeasurementRow(surveyAliasMap, { getCellValue: () => 'Carl' } as any); + + expect(getTsn({} as any)).to.be.equal(1234); + }); +}); diff --git a/api/src/services/import-services/measurement/utils/measurement-utils.ts b/api/src/services/import-services/measurement/utils/measurement-utils.ts new file mode 100644 index 0000000000..0f4e3aef4e --- /dev/null +++ b/api/src/services/import-services/measurement/utils/measurement-utils.ts @@ -0,0 +1,22 @@ +import { CSVConfigUtils } from '../../../../utils/csv-utils/csv-config-utils'; +import { CSVParams } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { ICritterDetailed } from '../../../critterbase-service'; +import { MeasurementCSVStaticHeader } from '../import-measurements-service'; + +/** + * Get the measurement row TSN getter - ie: Get the ITIS TSN from the row + * + * @param {Map} surveyAliasMap The survey alias map + * @param {CSVConfigUtils} utils The CSV config utils + * @returns {*} {(params: CSVParams) => number} The get measurement row TSN callback + */ +export const getTsnFromMeasurementRow = ( + surveyAliasMap: Map, + utils: CSVConfigUtils +) => { + return (params: CSVParams): number => { + const alias = String(utils.getCellValue('ALIAS', params.row)).toLowerCase(); + const critter = surveyAliasMap.get(alias); + return Number(critter?.itis_tsn); + }; +}; diff --git a/api/src/services/import-services/observation/import-observations-service.test.ts b/api/src/services/import-services/observation/import-observations-service.test.ts new file mode 100644 index 0000000000..366b018f6e --- /dev/null +++ b/api/src/services/import-services/observation/import-observations-service.test.ts @@ -0,0 +1,335 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { v4 } from 'uuid'; +import { CaseInsensitiveMap } from '../../../utils/case-insensitive-map'; +import * as validate from '../../../utils/csv-utils/csv-config-validation'; +import { CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; +import * as taxonRowValidator from '../../../utils/csv-utils/row-validators/taxon-row-validator'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { ObservationService } from '../../observation-services/observation-service'; +import { IItisSearchResult, PlatformService } from '../../platform-service'; +import * as taxonMap from '../utils/taxon'; +import { ImportObservationsService } from './import-observations-service'; +import * as observationSamplingRowValidator from './utils/observation-sampling-row-validator'; + +chai.use(sinonChai); + +describe('import-observations-service', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('_setObservationStaticHeaderConfigs', () => { + it('should set the static headers', async () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const mockCodeRepository = { + getObservationSubcountSigns: sinon.stub().resolves([{ id: 'CODE', name: 'NAME' }]) + }; + + await service._setObservationStaticHeaderConfigs(mockCodeRepository as any); + + expect(mockCodeRepository.getObservationSubcountSigns).to.have.been.calledOnce; + + expect(service.utils.config.staticHeadersConfig).to.have.keys([ + 'SPECIES', + 'COUNT', + 'SUBCOUNT_SIGN', + 'DATE', + 'TIME', + 'LATITUDE', + 'LONGITUDE', + 'SAMPLING_PERIOD', + 'SAMPLING_SITE', + 'METHOD_TECHNIQUE', + 'COMMENT' + ]); + }); + }); + + describe('importCSVWorksheet', () => { + it('should return errors early', async () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const getCSVConfigStub = sinon.stub(service, 'getCSVConfig').resolves({} as any); + const validateCSVWorksheetStub = sinon.stub(validate, 'validateCSVWorksheet').returns({ + errors: [ + { + row: 1, + header: 'header', + error: 'error', + solution: 'solution', + values: null, + cell: 'cell' + } + ], + rows: [] + }); + + const result = await service.importCSVWorksheet(); + + expect(getCSVConfigStub).to.have.been.calledOnce; + expect(validateCSVWorksheetStub).to.have.been.calledOnce; + + expect(result).to.be.an('array'); + expect(result.length).to.be.equal(1); + }); + + it('should grab the observations from the row', async () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const getCSVConfigStub = sinon.stub(service, 'getCSVConfig').resolves({} as any); + const insertObservationStub = sinon.stub(ObservationService.prototype, 'insertUpdateManualSurveyObservations'); + const getRowSubcountsStub = sinon.stub(service, '_getRowSubcounts').returns([]); + const validateCSVWorksheetStub = sinon.stub(validate, 'validateCSVWorksheet').returns({ + errors: [], + rows: [ + { + LATITUDE: 1, + LONGITUDE: 2, + DATE: '2021-01-01', + TIME: '12:00:00', + COUNT: 3, + [CSVRowState]: { + itis_tsn: 4, + itis_scientific_name: 'alces', + sample_period_id: 5 + } + } + ] + }); + + const result = await service.importCSVWorksheet(); + + expect(getCSVConfigStub).to.have.been.calledOnce; + expect(validateCSVWorksheetStub).to.have.been.calledOnce; + expect(getRowSubcountsStub).to.have.been.calledOnce; + + expect(insertObservationStub).to.have.been.calledOnceWithExactly(1, [ + { + standardColumns: { + survey_id: 1, + itis_tsn: 4, + itis_scientific_name: 'alces', + survey_sample_period_id: 5, + latitude: 1, + longitude: 2, + count: 3, + observation_date: '2021-01-01', + observation_time: '12:00:00' + }, + subcounts: [] + } + ]); + + expect(result).to.be.an('array'); + expect(result.length).to.be.equal(0); + }); + }); + + describe('getCSVConfig', () => { + it('should return the CSV config', async () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const getUniqueCellValues = sinon.stub(service.utils, 'getUniqueCellValues'); + const getConfigStub = sinon.stub(service.utils, 'getConfig'); + const setObservationStaticHeaderConfigsStub = sinon.stub(service, '_setObservationStaticHeaderConfigs'); + const setObservationRowValidatorsStub = sinon.stub(service, '_setObservationRowValidators'); + const setObservationDynamicHeadersConfigStub = sinon.stub(service, '_setObservationDynamicHeadersConfig'); + const getTaxonMapStub = sinon.stub(taxonMap, 'getTaxonMap'); + + const taxonMapMock = new CaseInsensitiveMap(); + + setObservationStaticHeaderConfigsStub.resolves(); + setObservationRowValidatorsStub.resolves(); + setObservationDynamicHeadersConfigStub.resolves(); + + getUniqueCellValues.returns([1, 'alces']); + getTaxonMapStub.resolves(taxonMapMock); + + getConfigStub.returns({} as any); + + const result = await service.getCSVConfig(); + + expect(getUniqueCellValues).to.have.been.calledOnce; + expect(getTaxonMapStub.getCall(0).args[0]).to.be.deep.equal([1, 'alces']); + expect(getTaxonMapStub.getCall(0).args[1]).to.be.an.instanceof(PlatformService); + + expect(setObservationStaticHeaderConfigsStub).to.have.been.calledOnce; + expect(setObservationRowValidatorsStub).to.have.been.calledOnce; + expect(setObservationDynamicHeadersConfigStub).to.have.been.calledOnce; + + expect(result).to.be.deep.equal({}); + }); + }); + + describe('_setObservationRowValidators', () => { + it('should set the row validators', async () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const taxonRowValidatorStub = sinon.stub(taxonRowValidator, 'getTaxonRowValidator').returns(() => []); + const observationSamplingRowValidatorStub = sinon + .stub(observationSamplingRowValidator, 'getObservationSamplingInformationRowValidator') + .returns(() => []); + const taxonMap = new CaseInsensitiveMap(); + const samplePeriodService: any = { + getSamplePeriodsForSurvey: sinon.stub().resolves([]) + }; + + await service._setObservationRowValidators(taxonMap, samplePeriodService); + + expect(samplePeriodService.getSamplePeriodsForSurvey).to.have.been.calledOnce; + expect(taxonRowValidatorStub).to.have.been.calledOnce; + expect(observationSamplingRowValidatorStub).to.have.been.calledOnce; + + expect(service.utils.config.rowValidators).to.be.an('array').and.to.have.length(2); + }); + }); + + describe('_getRowSubcounts', () => { + it('should return an array of subcounts', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const row = {}; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + }); + + it('should handle qualitative measurements', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + sinon.stub(service.utils, 'worksheetHeaders').get(() => ['QUALITATIVE_MEASUREMENT']); + + const row = { + [CSVRowState]: { + QUALITATIVE_MEASUREMENT: { + taxon_measurement_id: v4(), + qualitative_option_id: v4() + } + } + }; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].qualitative_measurements[0]).to.deep.equal({ + measurement_id: row[CSVRowState].QUALITATIVE_MEASUREMENT.taxon_measurement_id, + measurement_option_id: row[CSVRowState].QUALITATIVE_MEASUREMENT.qualitative_option_id + }); + }); + + it('should handle quantitative measurements', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + sinon.stub(service.utils, 'worksheetHeaders').get(() => ['QUANTITATIVE_MEASUREMENT']); + + const row = { + [CSVRowState]: { + QUANTITATIVE_MEASUREMENT: { + taxon_measurement_id: v4(), + value: 10 + } + } + }; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].quantitative_measurements[0]).to.deep.equal({ + measurement_id: row[CSVRowState].QUANTITATIVE_MEASUREMENT.taxon_measurement_id, + measurement_value: row[CSVRowState].QUANTITATIVE_MEASUREMENT.value + }); + }); + + it('should handle qualitative environments', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + sinon.stub(service.utils, 'worksheetHeaders').get(() => ['QUALITATIVE_ENVIRONMENT']); + + const row = { + [CSVRowState]: { + QUALITATIVE_ENVIRONMENT: { + environment_qualitative_id: v4(), + environment_qualitative_option_id: v4() + } + } + }; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].qualitative_environments[0]).to.deep.equal({ + environment_qualitative_id: row[CSVRowState].QUALITATIVE_ENVIRONMENT.environment_qualitative_id, + environment_qualitative_option_id: row[CSVRowState].QUALITATIVE_ENVIRONMENT.environment_qualitative_option_id + }); + }); + + it('should handle quantitative environments', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + sinon.stub(service.utils, 'worksheetHeaders').get(() => ['QUANTITATIVE_ENVIRONMENT']); + + const row = { + [CSVRowState]: { + QUANTITATIVE_ENVIRONMENT: { + environment_quantitative_id: v4(), + value: 10 + } + } + }; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].quantitative_environments[0]).to.deep.equal({ + environment_quantitative_id: row[CSVRowState].QUANTITATIVE_ENVIRONMENT.environment_quantitative_id, + value: row[CSVRowState].QUANTITATIVE_ENVIRONMENT.value + }); + }); + + it('should handle the other row properties', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const row = { + COUNT: 10, + SUBCOUNT_SIGN: 1, + COMMENT: 'test' + }; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].subcount).to.equal(row.COUNT); + expect(result[0].observation_subcount_sign_id).to.equal(row.SUBCOUNT_SIGN); + expect(result[0].comment).to.equal(row.COMMENT); + }); + + it('should handle undefined values -> null', () => { + const mockConnection = getMockDBConnection(); + const service = new ImportObservationsService(mockConnection, {}, 1); + + const row = {}; + + const result = service._getRowSubcounts(row); + + expect(result).to.be.an('array').and.to.have.length(1); + expect(result[0].subcount).to.equal(null); + expect(result[0].observation_subcount_sign_id).to.equal(null); + expect(result[0].comment).to.equal(null); + }); + }); +}); diff --git a/api/src/services/import-services/observation/import-observations-service.ts b/api/src/services/import-services/observation/import-observations-service.ts new file mode 100644 index 0000000000..3f6f1482b5 --- /dev/null +++ b/api/src/services/import-services/observation/import-observations-service.ts @@ -0,0 +1,331 @@ +import { WorkSheet } from 'xlsx'; +import { z } from 'zod'; +import { IDBConnection } from '../../../database/db'; +import { CodeRepository } from '../../../repositories/code-repository'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation'; +import { CSVConfig, CSVError, CSVRow, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { + getDateCellValidator, + getDateRangeCellValidator, + getDescriptionCellValidator, + getLatitudeCellValidator, + getLongitudeCellValidator, + getNonEmptyStringCellValidator, + getPositiveNumberCellValidator, + getTimeCellSetter, + getTimeCellValidator, + validateZodCell +} from '../../../utils/csv-utils/csv-header-configs'; +import { getTaxonRowValidator } from '../../../utils/csv-utils/row-validators/taxon-row-validator'; +import { getLogger } from '../../../utils/logger'; +import { CritterbaseService, getCritterbaseUserFromConnection } from '../../critterbase-service'; +import { DBService } from '../../db-service'; +import { + InsertSubCount, + InsertUpdateObservations, + ObservationService +} from '../../observation-services/observation-service'; +import { ObservationSubCountEnvironmentService } from '../../observation-subcount-environment-service'; +import { PlatformService } from '../../platform-service'; +import { SamplePeriodService } from '../../sample-period-service'; +import { + getEnvironmentNameTypeDefinitionMap, + isQualitativeEnvironmentStub, + isQuantitativeEnvironmentStub +} from '../utils/environment'; +import { + getTsnMeasurementDictionary, + isCBQualitativeMeasurementStub, + isCBQuantitativeMeasurementStub +} from '../utils/measurement'; +import { + getQualitativeEnvironmentFromRowState, + getQualitativeMeasurementFromRowState, + getQuantitativeEnvironmentFromRowState, + getQuantitativeMeasurementFromRowState, + getSamplePeriodIdFromRowState, + getTaxonFromRowState +} from '../utils/row-state'; +import { getTaxonMap, getTsnsFromTaxonMap, TaxonMap } from '../utils/taxon'; +import { getObservationDynamicHeaderCellValidator } from './utils/observation-dynamic-header-config'; +import { getObservationSubcountSignCellValidator } from './utils/observation-header-configs'; +import { getObservationSamplingInformationRowValidator } from './utils/observation-sampling-row-validator'; + +const defaultLog = getLogger('services/import/import-observations-service'); + +const SUBCOUNT_SIGN_ALIASES: Uppercase[] = ['OBSERVATION_SUBCOUNT_SIGN', 'OBSERVATION SUBCOUNT SIGN', 'SIGN']; + +export type ObservationCSVStaticHeader = + | 'SPECIES' + | 'COUNT' + | 'SUBCOUNT_SIGN' + | 'DATE' + | 'TIME' + | 'LATITUDE' + | 'LONGITUDE' + | 'SAMPLING_SITE' + | 'SAMPLING_PERIOD' + | 'METHOD_TECHNIQUE' + | 'COMMENT'; + +/** + * ImportObservationsService - A service for importing Observations from a CSV into SIMS. + * + * @class ImportObservationsService + * @extends DBService + */ +export class ImportObservationsService extends DBService { + worksheet: WorkSheet; + surveyId: number; + samplePeriodId?: number; + + utils: CSVConfigUtils; + + /** + * Construct an instance of ImportObservationsService. + * + * @param {IDBConnection} connection - DB connection + * @param {string} surveyId + */ + constructor(connection: IDBConnection, worksheet: WorkSheet, surveyId: number, samplePeriodId?: number) { + super(connection); + + const initialConfig: CSVConfig = { + staticHeadersConfig: { + SPECIES: { aliases: ['ITIS_TSN', 'ITIS TSN', 'TSN', 'TAXON'] }, + COUNT: { aliases: [] }, + SUBCOUNT_SIGN: { aliases: SUBCOUNT_SIGN_ALIASES, optional: true }, + DATE: { aliases: [], optional: true }, + TIME: { aliases: [], optional: true }, + LATITUDE: { aliases: ['LAT'], optional: true }, + LONGITUDE: { aliases: ['LON', 'LONG', 'LNG'], optional: true }, + SAMPLING_PERIOD: { aliases: ['PERIOD', 'TIME PERIOD', 'SESSION'], optional: true }, + SAMPLING_SITE: { aliases: ['SITE', 'SITE ID', 'LOCATION', 'SAMPLING SITE', 'STATION'], optional: true }, + METHOD_TECHNIQUE: { aliases: ['METHOD', 'TECHNIQUE'], optional: true }, + COMMENT: { aliases: ['COMMENTS', 'NOTE', 'NOTES'], optional: true } + }, + ignoreDynamicHeaders: false + }; + + this.worksheet = worksheet; + this.surveyId = surveyId; + this.samplePeriodId = samplePeriodId; + + this.utils = new CSVConfigUtils(worksheet, initialConfig); + } + + /** + * Import a Observation CSV worksheet into SIMS. + * + * @async + * @throws {ApiGeneralError} - If unable to fully insert records into Critterbase + * @returns {*} {Promise} List of CSV errors encountered during import + */ + async importCSVWorksheet(): Promise { + const config = await this.getCSVConfig(); + + const { errors, rows } = validateCSVWorksheet(this.worksheet, config); + + if (errors.length) { + return errors; + } + + const observations: InsertUpdateObservations[] = []; + + for (const row of rows) { + observations.push({ + standardColumns: { + survey_id: this.surveyId, + itis_tsn: getTaxonFromRowState(row).itis_tsn, + itis_scientific_name: getTaxonFromRowState(row).itis_scientific_name, + survey_sample_period_id: getSamplePeriodIdFromRowState(row).sample_period_id ?? null, + latitude: row.LATITUDE, + longitude: row.LONGITUDE, + count: row.COUNT, // deprecated - each subcount will eventually have its own count + observation_date: row.DATE, + observation_time: row.TIME + }, + subcounts: this._getRowSubcounts(row) + }); + } + + defaultLog.debug({ label: 'importCSVWorksheet', observations }); + + const observationService = new ObservationService(this.connection); + + await observationService.insertUpdateManualSurveyObservations(this.surveyId, observations); + + return []; + } + + /** + * Get the CSV configuration for Observations. + * + * @returns {Promise>} The CSV configuration + */ + async getCSVConfig(): Promise> { + // Initialize the required services + const platformService = new PlatformService(this.connection); + const samplePeriodService = new SamplePeriodService(this.connection); + const critterbaseService = new CritterbaseService(getCritterbaseUserFromConnection(this.connection)); + const environmentService = new ObservationSubCountEnvironmentService(this.connection); + const codeRepository = new CodeRepository(this.connection); + + // Generate shared dependencies + const taxonIdentifiers = this.utils.getUniqueCellValues('SPECIES').filter(Boolean) as string[]; + const taxonMap = await getTaxonMap(taxonIdentifiers, platformService); + + // Inject the dependencies and set the static headers, row validators, and dynamic headers + await Promise.all([ + this._setObservationStaticHeaderConfigs(codeRepository), + this._setObservationRowValidators(taxonMap, samplePeriodService), + this._setObservationDynamicHeadersConfig(taxonMap, critterbaseService, environmentService) + ]); + + // Return the final CSV config + return this.utils.getConfig(); + } + + /** + * Set the static headers for the Observation CSV. + * + * @param {CodeRepository} codeRepository - The code repository + * @returns {*} {Promise} + */ + async _setObservationStaticHeaderConfigs(codeRepository: CodeRepository) { + const subcountSignCodes = await codeRepository.getObservationSubcountSigns(); + + this.utils.setAllStaticHeaderConfigs({ + // Species is pre-validated by the taxon row validator + SPECIES: { validateCell: (params) => validateZodCell(params.cell, z.string().or(z.number())) }, + COUNT: { validateCell: getPositiveNumberCellValidator() }, + // Subcount sign must be a valid code value + SUBCOUNT_SIGN: { validateCell: getObservationSubcountSignCellValidator(subcountSignCodes) }, + DATE: { validateCell: getDateCellValidator({ optional: true }) }, + TIME: { validateCell: getTimeCellValidator(), setCellValue: getTimeCellSetter() }, + LATITUDE: { validateCell: getLatitudeCellValidator({ optional: true }) }, + LONGITUDE: { validateCell: getLongitudeCellValidator({ optional: true }) }, + // Sampling period is pre-validated by the sampling information row validator + SAMPLING_PERIOD: { validateCell: getDateRangeCellValidator({ optional: true }) }, + // Sampling site is pre-validated by the sampling information row validator + SAMPLING_SITE: { validateCell: getNonEmptyStringCellValidator({ optional: true }) }, + // Method technique is pre-validated by the sampling information row validator + METHOD_TECHNIQUE: { validateCell: getNonEmptyStringCellValidator({ optional: true }) }, + COMMENT: { validateCell: getDescriptionCellValidator() } + }); + } + + /** + * Sets the taxon row validator, sampling information and location row validators for the Observation CSV. + * + * Note: Row validators run before static and dynamic header validators. + * + * @param {TaxonMap} taxonMap - The taxon map + * @param {SamplePeriodService} samplePeriodService - The sample period service + * @returns {*} {Promise} + */ + async _setObservationRowValidators(taxonMap: TaxonMap, samplePeriodService: SamplePeriodService) { + const samplePeriods = await samplePeriodService.getSamplePeriodsForSurvey(this.surveyId); + + // Inject the row validators - handles taxon, sampling information and location validation + this.utils.config.rowValidators = [ + getTaxonRowValidator(taxonMap, this.utils, 'SPECIES'), + getObservationSamplingInformationRowValidator(samplePeriods, this.utils) + ]; + } + + /** + * Sets the environment and measurement dynamic header validator for the Observation CSV. + * + * @param {TaxonMap} taxonMap - The taxon map + * @param {CritterbaseService} critterbaseService - The critterbase service + * @param {ObservationSubCountEnvironmentService} environmentService - The environment service + * @returns {*} {Promise} + */ + async _setObservationDynamicHeadersConfig( + taxonMap: TaxonMap, + critterbaseService: CritterbaseService, + environmentService: ObservationSubCountEnvironmentService + ) { + // Generate the measurement dictionary and environment map + const measurementDictionary = await getTsnMeasurementDictionary(getTsnsFromTaxonMap(taxonMap), critterbaseService); + const environmentMap = await getEnvironmentNameTypeDefinitionMap( + // Note: Passing ALL dynamic headers intentionally to detect conflicts + // with measurement and environment headers in the dynamic header cell validator + this.utils.worksheetDynamicHeaders, + environmentService + ); + + // Inject dynamic header config - handles measurement and environment validation + this.utils.config.dynamicHeadersConfig = { + validateCell: getObservationDynamicHeaderCellValidator(measurementDictionary, environmentMap) + }; + } + + /** + * Get the subcounts from a row. + * + * @param {CSVRow} row - The row to extract subcounts from + * @returns {*} {InsertSubCount[]} The subcounts + */ + _getRowSubcounts(row: CSVRow): InsertSubCount[] { + const newSubcount: InsertSubCount = { + observation_subcount_id: null, + subcount: row.COUNT ?? null, + observation_subcount_sign_id: row.SUBCOUNT_SIGN ?? null, + comment: row.COMMENT ?? null, + qualitative_measurements: [], + quantitative_measurements: [], + qualitative_environments: [], + quantitative_environments: [] + }; + + // Loop through the dynamic headers to extract measurements and environments + for (const dynamicHeader of this.utils.worksheetDynamicHeaders) { + // Nested state used to prevent conflicts with other CSV headers + const nestedState = row[CSVRowState]?.[dynamicHeader]; + + // Grab the qualitative measurement from the row + if (isCBQualitativeMeasurementStub(nestedState)) { + const qualitativeMeasurement = getQualitativeMeasurementFromRowState(nestedState); + + newSubcount.qualitative_measurements.push({ + measurement_id: qualitativeMeasurement.taxon_measurement_id, + measurement_option_id: qualitativeMeasurement.qualitative_option_id + }); + } + // Grab the quantitative measurement from the row + else if (isCBQuantitativeMeasurementStub(nestedState)) { + const quantitativeMeasurement = getQuantitativeMeasurementFromRowState(nestedState); + + newSubcount.quantitative_measurements.push({ + measurement_id: quantitativeMeasurement.taxon_measurement_id, + measurement_value: quantitativeMeasurement.value + }); + } + // Grab the qualitative environment from the row + else if (isQualitativeEnvironmentStub(nestedState)) { + const qualitativeEnvironment = getQualitativeEnvironmentFromRowState(nestedState); + + newSubcount.qualitative_environments.push({ + environment_qualitative_id: qualitativeEnvironment.environment_qualitative_id, + environment_qualitative_option_id: qualitativeEnvironment.environment_qualitative_option_id + }); + } + // Grab the quantitative environment from the row + else if (isQuantitativeEnvironmentStub(nestedState)) { + const quantitativeEnvironment = getQuantitativeEnvironmentFromRowState(nestedState); + + newSubcount.quantitative_environments.push({ + environment_quantitative_id: quantitativeEnvironment.environment_quantitative_id, + value: quantitativeEnvironment.value + }); + } else { + // NOTE: Should this else path throw an error? + } + } + + return [newSubcount]; + } +} diff --git a/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.test.ts b/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.test.ts new file mode 100644 index 0000000000..457c88bdec --- /dev/null +++ b/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.test.ts @@ -0,0 +1,202 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { CSVRowState } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import * as environment from './environment-dynamic-headers-config'; + +chai.use(sinonChai); + +describe('environment-dynamic-headers-config', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('getDynamicEnvironmentCellValidator', () => { + it('should return an empty array when the cell is undefined', () => { + const environmentMap = new Map(); + const validator = environment.getDynamicEnvironmentCellValidator(environmentMap); + + const result = validator({ cell: undefined } as any); + expect(result).to.be.deep.equal([]); + }); + + it('should return an error when the column header does not exist', () => { + const environmentMap = new Map(); + const validator = environment.getDynamicEnvironmentCellValidator(environmentMap); + + const result = validator({ cell: 'test', header: 'bad' } as any); + expect(result[0].error).to.contain("'bad' does not exist"); + }); + + it('should call the qualitative environment cell validator when the environment is qualitative', () => { + const environmentMap = new Map(); + const environmentData = { + environment_qualitative_id: true, + options: [] + }; + + environmentMap.set('header', environmentData); + + const validateQualitativeStub = sinon.stub(environment, 'validateQualitativeEnvironmentCell').returns([]); + + const validator = environment.getDynamicEnvironmentCellValidator(environmentMap); + + expect(validateQualitativeStub).to.not.have.been.calledOnce; + + const result = validator({ cell: 'test', header: 'header' } as any); + expect(result).to.be.an('array').that.is.empty; + }); + + it('should call the quantitative environment cell validator when the environment is quantitative', () => { + const environmentMap = new Map(); + const environmentData = { + environment_quantitative_id: true, + unit: true + }; + + environmentMap.set('header', environmentData); + + const validateQuantitativeStub = sinon.stub(environment, 'validateQuantitativeEnvironmentCell').returns([]); + + const validator = environment.getDynamicEnvironmentCellValidator(environmentMap); + + expect(validateQuantitativeStub).to.not.have.been.calledOnce; + + const result = validator({ cell: 'test', header: 'header' } as any); + expect(result).to.be.an('array').that.is.empty; + }); + + it('should return an error when the environment type is invalid', () => { + const environmentMap = new Map(); + const environmentData = { + invalid: true + }; + + environmentMap.set('header', environmentData); + + const validator = environment.getDynamicEnvironmentCellValidator(environmentMap); + + const result = validator({ cell: 'test', header: 'header' } as any); + expect(result[0].error.toLowerCase()).to.contain('invalid environment type'); + }); + }); + + describe('validateQualitativeEnvironmentCell', () => { + it('should validate the qualitative environment cell value', () => { + const params = { + cell: 'good', + header: 'header', + row: {} + } as any; + + const environmentData = { + options: [ + { + environment_qualitative_option_id: 'id', + name: 'good' + } + ] + } as any; + + const result = environment.validateQualitativeEnvironmentCell(params, environmentData); + expect(result).to.be.an('array').that.is.empty; + }); + + it('should return an error when the qualitative value is invalid', () => { + const params = { + cell: 'bad', + header: 'header', + row: {} + } as any; + + const environmentData = { + options: [ + { + environment_qualitative_option_id: 'id', + name: 'good' + } + ] + } as any; + + const result = environment.validateQualitativeEnvironmentCell(params, environmentData); + expect(result[0].error.length).to.be.greaterThan(0); + }); + + it('should update the row state with the environment qualitative option id', () => { + const params = { + cell: 'good', + header: 'header', + row: {} + } as any; + + const environmentData = { + environment_qualitative_id: 'id', + options: [ + { + environment_qualitative_option_id: 'id', + name: 'good' + } + ] + } as any; + + environment.validateQualitativeEnvironmentCell(params, environmentData); + expect(params.row[CSVRowState].header).to.deep.equal({ + environment_qualitative_id: environmentData.environment_qualitative_id, + environment_qualitative_option_id: environmentData.options[0].environment_qualitative_option_id + }); + }); + }); + + describe('validateQuantitativeEnvironmentCell', () => { + it('should validate the quantitative environment cell value', () => { + const params = { + cell: 1, + header: 'header', + row: {} + } as any; + + const environmentData = { + environment_quantitative_id: 'id', + unit: 'unit' + } as any; + + const result = environment.validateQuantitativeEnvironmentCell(params, environmentData); + expect(result).to.be.an('array').that.is.empty; + }); + + it('should return an error when the quantitative value is invalid', () => { + const params = { + cell: 'bad', + header: 'header', + row: {} + } as any; + + const environmentData = { + environment_quantitative_id: 'id', + unit: 'unit' + } as any; + + const result = environment.validateQuantitativeEnvironmentCell(params, environmentData); + expect(result[0].error.length).to.be.greaterThan(0); + }); + + it('should update the row state with the environment quantitative value', () => { + const params = { + cell: 1, + header: 'header', + row: {} + } as any; + + const environmentData = { + environment_quantitative_id: 'id', + unit: 'unit' + } as any; + + environment.validateQuantitativeEnvironmentCell(params, environmentData); + expect(params.row[CSVRowState].header).to.deep.equal({ + environment_quantitative_id: environmentData.environment_quantitative_id, + value: params.cell + }); + }); + }); +}); diff --git a/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.ts b/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.ts new file mode 100644 index 0000000000..e9dc222465 --- /dev/null +++ b/api/src/services/import-services/observation/utils/environment-dynamic-headers-config.ts @@ -0,0 +1,139 @@ +import { + ObservationSubCountQualitativeEnvironmentRecord, + ObservationSubCountQuantitativeEnvironmentRecord, + QualitativeEnvironmentTypeDefinition, + QuantitativeEnvironmentTypeDefinition +} from '../../../../repositories/observation-subcount-environment-repository'; +import { CSVCellValidator, CSVError, CSVParams } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { updateCSVRowState } from '../../../../utils/csv-utils/csv-header-configs'; +import { EnvironmentNameTypeDefinitionMap } from '../../../../utils/observation-xlsx-utils/environment-column-utils'; +import { + isQualitativeEnvironmentTypeDefinition, + isQuantitativeEnvironmentTypeDefinition +} from '../../utils/environment'; +import { validateQualitativeValue } from '../../utils/qualitative'; +import { validateQuantitativeValue } from '../../utils/quantitative'; + +/** + * Get the dynamic environment cell validator. + * + * Rules: + * 1. The header must be a valid SIMS environment (qualitative or quantitative) or undefined + * + * @param {EnvironmentNameTypeDefinitionMap} environmentMap The environment map + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getDynamicEnvironmentCellValidator = ( + environmentMap: EnvironmentNameTypeDefinitionMap +): CSVCellValidator => { + return (params) => { + if (params.cell === undefined) { + return []; + } + + const environment = environmentMap.get(params.header); + + if (!environment) { + return [ + { + error: `Column header '${params.header}' does not exist`, + solution: 'Use a valid environment as the header', + values: Object.keys(environmentMap) + } + ]; + } + + // Environment type is qualitative + if (isQualitativeEnvironmentTypeDefinition(environment)) { + return validateQualitativeEnvironmentCell(params, environment); + } + + // Environment type is quantitative + if (isQuantitativeEnvironmentTypeDefinition(environment)) { + return validateQuantitativeEnvironmentCell(params, environment); + } + + // Can this path ever be reached? + return [ + { + error: 'Invalid environment type', + solution: 'Use a supported environment type' + } + ]; + }; +}; + +/** + * Validate the qualitative environment cell value. + * + * @param {CSVParams} params The CSV params + * @param {QualitativeEnvironmentTypeDefinition} environment The qualitative environment definition + * @returns {CSVError[]} The list of errors + */ +export const validateQualitativeEnvironmentCell = ( + params: CSVParams, + environment: QualitativeEnvironmentTypeDefinition +): CSVError[] => { + const options = environment.options.map((option) => ({ + option_id: option.environment_qualitative_option_id, + option_name: option.name + })); + + // Normalize the environment type definition and validate the cell + const result = validateQualitativeValue(params.cell, { options: options }, 'environment'); + + // If the result is not a qualitative value it is a list of CSV errors + if (typeof result !== 'string') { + return result; + } + + // Update the row state with the taxon environment id and qualitative option id + updateCSVRowState(params.row, { + [params.header]: { + environment_qualitative_id: environment.environment_qualitative_id, + environment_qualitative_option_id: result + } satisfies Pick< + ObservationSubCountQualitativeEnvironmentRecord, + 'environment_qualitative_id' | 'environment_qualitative_option_id' + > + }); + + return []; +}; + +/** + * Validate the quantitative environment cell value. + * + * @param {CSVParams} params The CSV params + * @param {QuantitativeEnvironmentTypeDefinition} environment The quantitative environment definition + * @returns {CSVError[]} The list of errors + */ +export const validateQuantitativeEnvironmentCell = ( + params: CSVParams, + environment: QuantitativeEnvironmentTypeDefinition +): CSVError[] => { + // Normalize the environment type definition and validate the cell + const result = validateQuantitativeValue( + params.cell, + { + min: environment.min, + max: environment.max + }, + 'environment' + ); + + // If the result is not a quantitative value it is a list of CSV errors + if (typeof result !== 'number') { + return result; + } + + // Update the row state with the taxon environment id and value + updateCSVRowState(params.row, { + [params.header]: { + environment_quantitative_id: environment.environment_quantitative_id, + value: result + } satisfies Pick + }); + + return []; +}; diff --git a/api/src/services/import-services/observation/utils/observation-dynamic-header-config.ts b/api/src/services/import-services/observation/utils/observation-dynamic-header-config.ts new file mode 100644 index 0000000000..a6c632f65a --- /dev/null +++ b/api/src/services/import-services/observation/utils/observation-dynamic-header-config.ts @@ -0,0 +1,55 @@ +import { CSVCellValidator } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { getDynamicMeasurementCellValidator } from '../../measurement/utils/measurement-dynamic-headers-config'; +import { EnvironmentNameTypeDefinitionMap } from '../../utils/environment'; +import { TSNMeasurementDictionary } from '../../utils/measurement'; +import { getTaxonFromRowState } from '../../utils/row-state'; +import { getDynamicEnvironmentCellValidator } from './environment-dynamic-headers-config'; + +/** + * Get the observation dynamic header config. + * + * Rules: + * 1. The header must be a valid Critterbase measurement or SIMS environment + * + * @param {TSNMeasurementDictionary} tsnMeasurementDictionary The TSN measurement dictionary + * @param {EnvironmentNameTypeDefinitionMap} environmentDictionary The environment dictionary + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getObservationDynamicHeaderCellValidator = ( + tsnMeasurementDictionary: TSNMeasurementDictionary, + environmentDictionary: EnvironmentNameTypeDefinitionMap +): CSVCellValidator => { + return (params) => { + const critterTsn = getTaxonFromRowState(params.row).itis_tsn; + + // Check that the header is not a measurement AND an environment + if (tsnMeasurementDictionary.has(critterTsn, params.header) && environmentDictionary.has(params.header)) { + // Note: This is an edge case that should not happen + return [ + { + error: `Dynamic header conflict`, + solution: `Column header '${params.header}' is both a measurement and an environment`, + cell: null // The cell value is not relevant + } + ]; + } + + // Check if the header is a measurement header and validate the cell + if (tsnMeasurementDictionary.has(critterTsn, params.header)) { + return getDynamicMeasurementCellValidator(tsnMeasurementDictionary, () => critterTsn)(params); + } + + // Check if the header is an environment header and validate the cell + if (environmentDictionary.has(params.header)) { + return getDynamicEnvironmentCellValidator(environmentDictionary)(params); + } + + return [ + { + error: `Invalid dynamic header`, + solution: `Expecting measurement or environment column header`, + cell: null // The cell value is not relevant + } + ]; + }; +}; diff --git a/api/src/services/import-services/observation/utils/observation-header-configs.ts b/api/src/services/import-services/observation/utils/observation-header-configs.ts new file mode 100644 index 0000000000..f435a09ff6 --- /dev/null +++ b/api/src/services/import-services/observation/utils/observation-header-configs.ts @@ -0,0 +1,43 @@ +import { ICode } from '../../../../repositories/code-repository'; +import { CaseInsensitiveMap } from '../../../../utils/case-insensitive-map'; +import { CSVCellValidator } from '../../../../utils/csv-utils/csv-config-validation.interface'; + +/** + * Get the observation subcount sign cell validator + * + * Rules: + * 1. The cell must be a valid subcount sign or undefined + * 2. The cell value will be mutated to the subcount sign ID + * + * @param {ICode[]} subcountSigns - The subcount signs + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getObservationSubcountSignCellValidator = (subcountSigns: ICode[]): CSVCellValidator => { + const subcountSignMap = new CaseInsensitiveMap(subcountSigns.map((sign) => [sign.name, sign])); + + return (params) => { + // Undefined values are allowed, return no errors + if (!params.cell) { + return []; + } + + // Attempt to get the subcount sign from the map + const subcountSign = subcountSignMap.get(String(params.cell)); + + // Value is not a subcount sign, return an error + if (!subcountSign) { + return [ + { + error: `Invalid subcount sign`, + solution: `Use a valid subcount sign`, + values: subcountSigns.map((sign) => sign.name) + } + ]; + } + + // Mutate the cell value to be the subcount sign ID + params.mutateCell = subcountSign.id; + + return []; + }; +}; diff --git a/api/src/services/observation-services/utils.test.ts b/api/src/services/import-services/observation/utils/observation-sampling-row-validator.test.ts similarity index 68% rename from api/src/services/observation-services/utils.test.ts rename to api/src/services/import-services/observation/utils/observation-sampling-row-validator.test.ts index 6f39f17d3a..5dc4189376 100644 --- a/api/src/services/observation-services/utils.test.ts +++ b/api/src/services/import-services/observation/utils/observation-sampling-row-validator.test.ts @@ -1,107 +1,113 @@ import chai, { expect } from 'chai'; +import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { SurveySamplePeriodDetails } from '../../repositories/sample-period-repository'; -import { CSV_COLUMN_ALIASES } from '../../utils/xlsx-utils/column-aliases'; +import { SurveySamplePeriodDetails } from '../../../../repositories/sample-period-repository'; +import { getSamplePeriodIdFromRowState } from '../../utils/row-state'; import { + findMatchingPeriodsWithObservationDateTime, + findMatchingPeriodsWithSamplingInformation, + getObservationSamplingInformationRowValidator, matchSamplePeriodDateToWorksheetPeriodDateTime, - matchSamplePeriodsToObservationDateTime, matchSamplePeriodTimeToWorksheetPeriodDateTime, - matchSamplePeriodToWorksheetPeriod, - pullSamplingDataFromWorksheetRowObject -} from './utils'; + matchSamplePeriodToWorksheetPeriod +} from './observation-sampling-row-validator'; chai.use(sinonChai); describe('Worksheet sampling util functions', () => { - describe('pullSamplingDataFromWorksheetRowObject', () => { - describe('scenario 1 - all periods partially overlap', () => { - const samplingPeriods: SurveySamplePeriodDetails[] = [ + beforeEach(() => { + sinon.restore(); + }); + + describe('findMatchingPeriodsWithSamplingInformation', () => { + it('should return the matching period when only site name provided', () => { + const samplePeriods = [ { - survey_sample_period_id: 11, - survey_id: 21, - survey_sample_site_id: 31, survey_sample_site: { - survey_sample_site_id: 31, name: 'SampleSiteOne' - }, - method_technique_id: 51, + } + } + ] as any[]; + + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: 'samplesiteone', + techniqueName: null, + period: null + }); + + expect(result.length).to.equal(1); + }); + + it('should return the matching period when only technique name provided', () => { + const samplePeriods = [ + { method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 - }, + name: 'MethodTechniqueOne' + } + } + ] as any[]; + + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: null, + techniqueName: 'methodtechniqueone', + period: null + }); + + expect(result.length).to.equal(1); + }); + + it('should return the matching period when only period provided', () => { + const samplePeriods = [ + { start_date: '2021-01-01', start_time: '11:00:00', end_date: '2021-01-02', end_time: '12:00:00' - }, + } + ] as any[]; + + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: null, + techniqueName: null, + period: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' + }); + + expect(result.length).to.equal(1); + }); + + it('should return the matching period when all information provided', () => { + const samplePeriods = [ { - survey_sample_period_id: 12, - survey_id: 21, - survey_sample_site_id: 32, survey_sample_site: { - survey_sample_site_id: 32, - name: 'SampleSiteTwo' + name: 'SampleSiteOne' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 - }, - start_date: '2021-01-02', - start_time: '12:00:00', - end_date: '2021-01-03', - end_time: '13:00:00' - }, - { - survey_sample_period_id: 13, - survey_id: 21, - survey_sample_site_id: 31, - survey_sample_site: { - survey_sample_site_id: 32, - name: 'SampleSiteTwo' + name: 'MethodTechniqueOne' }, - method_technique_id: null, - method_technique: null, start_date: '2021-01-01', start_time: '11:00:00', end_date: '2021-01-02', end_time: '12:00:00' } - ]; - - it('matches on site, technique, period', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' - }; - - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + ] as any[]; - expect(result?.samplePeriodId).to.equal(11); + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: 'samplesiteone', + techniqueName: 'methodtechniqueone', + period: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' }); + + expect(result.length).to.equal(1); }); - describe('scenario 2 - all periods differ only by date', () => { - const samplingPeriods: SurveySamplePeriodDetails[] = [ + it('should return the matching period when all information provided and multiple periods', () => { + const samplePeriods = [ { - survey_sample_period_id: 11, - survey_id: 21, - survey_sample_site_id: 31, survey_sample_site: { - survey_sample_site_id: 31, name: 'SampleSiteOne' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 + name: 'MethodTechniqueOne' }, start_date: '2021-01-01', start_time: '11:00:00', @@ -109,294 +115,357 @@ describe('Worksheet sampling util functions', () => { end_time: '12:00:00' }, { - survey_sample_period_id: 12, - survey_id: 21, - survey_sample_site_id: 31, survey_sample_site: { - survey_sample_site_id: 31, - name: 'SampleSiteOne' + name: 'SampleSiteTwo' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 + name: 'MethodTechniqueTwo' }, - start_date: '2022-01-01', - start_time: '11:00:00', - end_date: '2022-01-02', - end_time: '12:00:00' - }, + start_date: '2021-01-02', + start_time: '12:00:00', + end_date: '2021-01-03', + end_time: '13:00:00' + } + ] as any[]; + + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: 'samplesitetwo', + techniqueName: 'methodtechniquetwo', + period: '2021-01-02 12:00:00 - 2021-01-03 13:00:00' + }); + + expect(result.length).to.equal(1); + }); + + it('should return no matching periods when information incorrect', () => { + const samplePeriods = [ { - survey_sample_period_id: 13, - survey_id: 21, - survey_sample_site_id: 31, survey_sample_site: { - survey_sample_site_id: 31, name: 'SampleSiteOne' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 + name: 'MethodTechniqueOne' }, - start_date: '2023-01-01', + start_date: '2021-01-01', start_time: '11:00:00', - end_date: '2023-01-02', + end_date: '2021-01-02', end_time: '12:00:00' } - ]; + ] as any[]; - it('does not match on site, technique', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null - }; + const result = findMatchingPeriodsWithSamplingInformation(samplePeriods, { + siteName: 'samplesiteone', + techniqueName: 'methodtechniquetwo', + period: '2021-01-02 12:00:00 - 2021-01-03 13:00:00' + }); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + expect(result.length).to.equal(0); + }); + }); - expect(result).to.be.null; - }); + describe('getObservationSamplingInformationRowValidator', () => { + it('should return no errors when no sampling information and observation date provided with lat/lon', () => { + const getCellValueStub = sinon.stub(); - it('Matches on observation date and time', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: null, - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: null, - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null, - DATE: '2022-01-01', - TIME: '18:00:00' - }; + getCellValueStub.onCall(3).returns('2021-01-01'); + getCellValueStub.onCall(4).returns('11:00:00'); + getCellValueStub.onCall(5).returns(1); + getCellValueStub.onCall(6).returns(1); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + const validator = getObservationSamplingInformationRowValidator([], { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); - expect(result?.samplePeriodId).to.equal(12); - }); + const result = validator({ row: {} } as any); - it('matches on site, technique, period', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' - }; + expect(result.length).to.be.equal(0); + }); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + it('should return an error when no sampling information and no latitude but date provided', () => { + const getCellValueStub = sinon.stub(); - expect(result?.samplePeriodId).to.equal(11); - }); + getCellValueStub.onCall(3).returns('2021-01-01'); + getCellValueStub.onCall(4).returns('11:00:00'); + getCellValueStub.onCall(5).returns(null); + getCellValueStub.onCall(6).returns(1); - it('matches on period', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' - }; + const validator = getObservationSamplingInformationRowValidator([], { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + const result = validator({ row: {} } as any); - expect(result?.samplePeriodId).to.equal(11); - }); + expect(result[0].error).to.contain('Latitude is required'); }); - describe('scenario 2 - all periods differ only by site and technique', () => { - const samplingPeriods: SurveySamplePeriodDetails[] = [ + it('should return an error when no sampling information and no longitude but date provided', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(3).returns('2021-01-01'); + getCellValueStub.onCall(4).returns('11:00:00'); + getCellValueStub.onCall(5).returns(1); + getCellValueStub.onCall(6).returns(null); + + const validator = getObservationSamplingInformationRowValidator([], { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); + + const result = validator({ row: {} } as any); + + expect(result[0].error).to.contain('Longitude is required'); + }); + + it('should return date, lat and lon errors when no sampling information and no latitude and longitude provided', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(3).returns(null); + getCellValueStub.onCall(4).returns('11:00:00'); + getCellValueStub.onCall(5).returns(null); + getCellValueStub.onCall(6).returns(null); + + const validator = getObservationSamplingInformationRowValidator([], { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); + + const result = validator({ row: {} } as any); + + expect(result.length).to.equal(3); + + expect(result[0].error).to.contain('date is required'); + expect(result[1].error).to.contain('Latitude is required'); + expect(result[2].error).to.contain('Longitude is required'); + }); + + it('should return an error when no sampling information and no observation date provided', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(3).returns(null); + getCellValueStub.onCall(4).returns(null); + getCellValueStub.onCall(5).returns(1); + getCellValueStub.onCall(6).returns(1); + + const validator = getObservationSamplingInformationRowValidator([], { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); + + const result = validator({ row: {} } as any); + + expect(result[0].error).to.contain('date is required'); + }); + + it('should return an error when no matching period found', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('BAD'); + getCellValueStub.onCall(1).returns('MethodTechniqueOne'); + getCellValueStub.onCall(2).returns('2021-01-01 - 2021-01-03'); + + const samplePeriods = [ { - survey_sample_period_id: 11, - survey_id: 21, - survey_sample_site_id: 31, survey_sample_site: { - survey_sample_site_id: 31, name: 'SampleSiteOne' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueTwo', - description: 'MethodTechniqueTwo Description', - method_response_metric_id: 61 + name: 'MethodTechniqueOne' + }, + start_date: '2021-01-02', + end_date: '2021-01-03' + } + ] as any[]; + + const validator = getObservationSamplingInformationRowValidator(samplePeriods, { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); + + const result = validator({ row: {} } as any); + + expect(result[0].error).to.contain('match observation with sampling'); + }); + + it('should return no errors and update state when matching period found', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('SampleSiteOne'); + getCellValueStub.onCall(1).returns('MethodTechniqueOne'); + getCellValueStub.onCall(2).returns('2021-01-01 11:00:00 - 2021-01-02 12:00:00'); + + const samplePeriods = [ + { + survey_sample_period_id: 1, + survey_sample_site: { + name: 'SampleSiteOne' + }, + method_technique: { + name: 'MethodTechniqueOne' }, start_date: '2021-01-01', start_time: '11:00:00', - end_date: '2022-02-02', + end_date: '2021-01-02', end_time: '12:00:00' - }, + } + ] as any[]; + + const validator = getObservationSamplingInformationRowValidator(samplePeriods, { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); + + const params = { row: {} } as any; + + const result = validator(params); + + expect(result.length).to.equal(0); + expect(getSamplePeriodIdFromRowState(params.row).sample_period_id).to.equal(1); + }); + + it('should return an error when unable to uniquely match period with date and time', () => { + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns(null); + getCellValueStub.onCall(1).returns(null); + getCellValueStub.onCall(2).returns('2021-01-01 11:00:00 - 2021-01-02 12:00:00'); + getCellValueStub.onCall(3).returns('2025-01-01'); + getCellValueStub.onCall(4).returns('11:00:00'); + + const samplePeriods = [ { - survey_sample_period_id: 12, - survey_id: 21, - survey_sample_site_id: 32, + survey_sample_period_id: 1, survey_sample_site: { - survey_sample_site_id: 32, - name: 'SampleSiteTwo' + name: 'SampleSiteOne' }, - method_technique_id: 52, method_technique: { - method_technique_id: 52, - name: 'MethodTechniqueTwo', - description: 'MethodTechniqueTwo Description', - method_response_metric_id: 62 + name: 'MethodTechniqueOne' }, start_date: '2021-01-01', start_time: '11:00:00', - end_date: '2022-02-02', + end_date: '2021-01-02', end_time: '12:00:00' }, { - survey_sample_period_id: 13, - survey_id: 21, - survey_sample_site_id: 33, + survey_sample_period_id: 2, survey_sample_site: { - survey_sample_site_id: 33, - name: 'SampleSiteThree' + name: 'SampleSiteOne' }, - method_technique_id: 53, method_technique: { - method_technique_id: 53, - name: 'MethodTechniqueThree', - description: 'MethodTechniqueThree Description', - method_response_metric_id: 63 + name: 'MethodTechniqueOne' }, start_date: '2021-01-01', start_time: '11:00:00', - end_date: '2022-02-02', + end_date: '2021-01-02', end_time: '12:00:00' } - ]; - - it('does not match on site, technique', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null - }; - - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); - - expect(result).to.be.null; - }); + ] as any[]; - it('Matches non-unique period on observation date and time', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: null, - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: null, - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null, - DATE: '2021-05-15', - TIME: '18:00:00' - }; + const validator = getObservationSamplingInformationRowValidator(samplePeriods, { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + const result = validator({ row: {} } as any); - // Matches multiple periods on observation date/time, therefore the first match is returned - expect(result?.samplePeriodId).to.equal(11); - }); - - it('does not match non-unique period on site, technique, period', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: '2021-01-01 11:00:00 - 2022-02-02 12:00:00' - }; + expect(result.length).to.equal(2); + expect(result[0].error).to.contain('period is ambiguous'); + }); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + it('should return no errors and update state when uniquely matching period with date and time', () => { + const getCellValueStub = sinon.stub(); - // Matches multiple periods on sampling information, therefore null is returned - expect(result).to.be.null; - }); - }); + getCellValueStub.onCall(0).returns(null); + getCellValueStub.onCall(1).returns(null); + getCellValueStub.onCall(2).returns('2021-01-01 - 2021-01-02'); + getCellValueStub.onCall(3).returns('2021-01-01'); + getCellValueStub.onCall(4).returns('11:30:00'); - describe('scenario 3 - all periods are unique', () => { - const samplingPeriods: SurveySamplePeriodDetails[] = [ + const samplePeriods = [ { - survey_sample_period_id: 11, - survey_id: 21, - survey_sample_site_id: 31, + survey_sample_period_id: 1, survey_sample_site: { - survey_sample_site_id: 31, name: 'SampleSiteOne' }, - method_technique_id: 51, method_technique: { - method_technique_id: 51, - name: 'MethodTechniqueOne', - description: 'MethodTechniqueOne Description', - method_response_metric_id: 61 + name: 'MethodTechniqueOne' }, start_date: '2021-01-01', - start_time: '11:00:00', - end_date: '2021-01-02', - end_time: '12:00:00' + end_date: '2021-01-02' }, { - survey_sample_period_id: 12, - survey_id: 21, - survey_sample_site_id: 32, + survey_sample_period_id: 2, survey_sample_site: { - survey_sample_site_id: 32, - name: 'SampleSiteTwo' + name: 'SampleSiteOne' }, - method_technique_id: 52, method_technique: { - method_technique_id: 52, - name: 'MethodTechniqueTwo', - description: 'MethodTechniqueTwo Description', - method_response_metric_id: 62 + name: 'MethodTechniqueOne' }, - start_date: '2021-01-02', - start_time: '12:00:00', - end_date: '2021-01-03', - end_time: '13:00:00' - }, - { - survey_sample_period_id: 13, - survey_id: 21, - survey_sample_site_id: null, - survey_sample_site: null, - method_technique_id: null, - method_technique: null, - start_date: '2021-01-03', - start_time: null, - end_date: '2021-01-04', - end_time: null + start_date: '2021-01-01', + start_time: '11:00:00', + end_date: '2021-01-02', + end_time: '12:00:00' } - ]; + ] as any[]; - it('matches on site', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: 'SampleSiteOne', - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: null, - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null - }; + const validator = getObservationSamplingInformationRowValidator(samplePeriods, { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + const params = { row: {} } as any; - expect(result?.samplePeriodId).to.equal(11); - }); + const result = validator(params); - it('matches on technique', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: null, - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: 'MethodTechniqueOne', - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: null - }; + expect(result.length).to.equal(0); + expect(getSamplePeriodIdFromRowState(params.row).sample_period_id).to.equal(1); + }); - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + it('should return errors when unable to uniquely match period with date and time', () => { + const getCellValueStub = sinon.stub(); - expect(result?.samplePeriodId).to.equal(11); - }); + getCellValueStub.onCall(0).returns(null); + getCellValueStub.onCall(1).returns(null); + getCellValueStub.onCall(2).returns('2021-01-01 - 2021-01-02'); + getCellValueStub.onCall(3).returns('2021-01-01'); + getCellValueStub.onCall(4).returns('11:30:00'); - it('matches on period', () => { - const worksheetRow = { - [CSV_COLUMN_ALIASES['SAMPLING_SITE'][0]]: null, - [CSV_COLUMN_ALIASES['METHOD_TECHNIQUE'][0]]: null, - [CSV_COLUMN_ALIASES['SAMPLING_PERIOD'][0]]: '2021-01-01 11:00:00 - 2021-01-02 12:00:00' - }; + const samplePeriods = [ + { + survey_sample_period_id: 1, + survey_sample_site: { + name: 'SampleSiteOne' + }, + method_technique: { + name: 'MethodTechniqueOne' + }, + start_date: '2021-01-01', + end_date: '2021-01-02' + }, + { + survey_sample_period_id: 2, + survey_sample_site: { + name: 'SampleSiteOne' + }, + method_technique: { + name: 'MethodTechniqueOne' + }, + start_date: '2021-01-01', + end_date: '2021-01-02' + } + ] as any[]; - const result = pullSamplingDataFromWorksheetRowObject(worksheetRow, samplingPeriods); + const validator = getObservationSamplingInformationRowValidator(samplePeriods, { + getCellValue: getCellValueStub, + getWorksheetHeader: () => 'HEADER' + } as any); - expect(result?.samplePeriodId).to.equal(11); - }); + const params = { row: {} } as any; + + const result = validator(params); + + expect(result.length).to.equal(2); }); }); @@ -789,13 +858,13 @@ describe('Worksheet sampling util functions', () => { }); }); - describe('matchSamplePeriodsToObservationDateTime', () => { + describe('findMatchingPeriodsWithObservationDateTime', () => { it('matches no sampling period records when no sampling period records provided', () => { const observationDate = '2021-01-01'; const observationTime = '11:00:00'; const samplingPeriods: SurveySamplePeriodDetails[] = []; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result).to.eql([]); }); @@ -861,7 +930,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result.length).to.equal(2); expect(result[0].survey_sample_period_id).to.equal(11); @@ -929,7 +998,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result.length).to.equal(2); expect(result[0].survey_sample_period_id).to.equal(11); @@ -997,7 +1066,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result.length).to.equal(2); expect(result[0].survey_sample_period_id).to.equal(11); @@ -1065,7 +1134,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result.length).to.equal(2); expect(result[0].survey_sample_period_id).to.equal(11); @@ -1133,7 +1202,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result).to.eql([]); }); @@ -1199,7 +1268,7 @@ describe('Worksheet sampling util functions', () => { } ]; - const result = matchSamplePeriodsToObservationDateTime(observationDate, observationTime, samplingPeriods); + const result = findMatchingPeriodsWithObservationDateTime(observationDate, observationTime, samplingPeriods); expect(result).to.eql([]); }); diff --git a/api/src/services/import-services/observation/utils/observation-sampling-row-validator.ts b/api/src/services/import-services/observation/utils/observation-sampling-row-validator.ts new file mode 100644 index 0000000000..779843c4d5 --- /dev/null +++ b/api/src/services/import-services/observation/utils/observation-sampling-row-validator.ts @@ -0,0 +1,475 @@ +import dayjs from 'dayjs'; +import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'; +import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'; +import { DefaultTimeFormat, DefaultTimeFormatNoSeconds } from '../../../../constants/dates'; +import { SurveySamplePeriodDetails } from '../../../../repositories/sample-period-repository'; +import { CSVConfigUtils } from '../../../../utils/csv-utils/csv-config-utils'; +import { CSVRowError, CSVRowValidator } from '../../../../utils/csv-utils/csv-config-validation.interface'; +import { updateCSVRowState } from '../../../../utils/csv-utils/csv-header-configs'; +import { formatDateString, isDateString, isDateTimeString, isTimeString } from '../../../../utils/date-time-utils'; +import { ObservationCSVStaticHeader } from '../import-observations-service'; + +dayjs.extend(isSameOrAfter); +dayjs.extend(isSameOrBefore); + +/** + * Observation Sampling Information Row Validator - This function will validate a row of observation data and ensure + * that the sampling information provided in the row matches a valid sampling period from the provided list of + * sampling periods. + * + * Successfull paths: + * 1. No sampling information provided, but observation date, latitude and longitude is provided + * 2. Exact period match found using site, technique, and period + * 3. Multiple periods match the site, technique, and period, but only one matches the observation date and time + * + * @param {SurveySamplePeriodDetails[]} samplingPeriods All available sampling periods for the survey. + * @param {CSVConfigUtils} utils CSV Config Utils for the observation CSV. + * @return {*} {CSVRowValidator} + */ +export function getObservationSamplingInformationRowValidator( + samplingPeriods: SurveySamplePeriodDetails[], + utils: CSVConfigUtils +): CSVRowValidator { + return (params) => { + // Extract site, technique, and period data from the row + const worksheetSiteName = utils.getCellValue('SAMPLING_SITE', params.row) as string | null; + const worksheetTechniqueName = utils.getCellValue('METHOD_TECHNIQUE', params.row) as string | null; + const worksheetPeriod = utils.getCellValue('SAMPLING_PERIOD', params.row) as string | null; + + // Extract observation date and time from the row + const worksheetObservationDate = utils.getCellValue('DATE', params.row) as string | null; + const worksheetObservationTime = utils.getCellValue('TIME', params.row) as string | null; + + // Extract latitude and longitude from the row + const worksheetLatitude = utils.getCellValue('LATITUDE', params.row) as string | null; + const worksheetLongitude = utils.getCellValue('LONGITUDE', params.row) as string | null; + + // Determine if the worksheet contains any sampling information + const worksheetHasSamplingInformation = Boolean(worksheetSiteName || worksheetTechniqueName || worksheetPeriod); + + // VALID: No sampling information provided, but observation date / time is provided with lat / lon + if (!worksheetHasSamplingInformation && worksheetObservationDate && worksheetLatitude && worksheetLongitude) { + return []; + } + + const errors: CSVRowError[] = []; + + // INVALID: Observation date is required when sampling information is not provided + if (!worksheetHasSamplingInformation && !worksheetObservationDate) { + errors.push({ + error: 'Observation date is required when sampling information is not provided', + solution: 'Please provide sampling information or an observation date and time', + header: utils.getWorksheetHeader('DATE', params.row), + cell: null + }); + } + + // INVALID: Latitude is required when sampling information is not provided + if (!worksheetHasSamplingInformation && !worksheetLatitude) { + errors.push({ + error: 'Latitude is required when sampling information is not provided', + solution: 'Please provide sampling information or a valid latitude', + header: utils.getWorksheetHeader('LATITUDE', params.row), + cell: null + }); + } + + // INVALID: Longitude is required when sampling information is not provided + if (!worksheetHasSamplingInformation && !worksheetLongitude) { + errors.push({ + error: 'Longitude is required when sampling information is not provided', + solution: 'Please provide sampling information or a valid longitude', + header: utils.getWorksheetHeader('LONGITUDE', params.row), + cell: null + }); + } + + // Return early if any errors are found + if (errors.length) { + return errors; + } + + // Filter the sampling periods by the provided sampling information + // + // Rules: + // 0: No matching periods found for site, technique, and period (ERROR) + // 1: Exact period match found using site, technique, and period (SUCCESS) + // >1: Multiple periods match the site, technique, and period (Filter by observation date/time) + const matchingPeriodsBySamplingInformation = findMatchingPeriodsWithSamplingInformation(samplingPeriods, { + siteName: worksheetSiteName, + techniqueName: worksheetTechniqueName, + period: worksheetPeriod + }); + + // INVALID: No matching periods found for site, technique, and period + if (matchingPeriodsBySamplingInformation.length === 0) { + return [ + { + error: 'Unable to match observation with sampling information', + solution: 'Please provide more specific sampling information (site, technique, period)', + header: null, + cell: null + } + ]; + } + + // VALID: Exact period match found using site, technique, and period + if (matchingPeriodsBySamplingInformation.length === 1) { + updateCSVRowState(params.row, { + sample_period_id: matchingPeriodsBySamplingInformation[0].survey_sample_period_id + }); + + return []; + } + + // Filter the matching periods by the observation date and time + // + // Rules: + // 0 Matches: No matching periods found for observation date/time (ERROR) + // 1 Match: Exact period match found using observation date and time (SUCCESS) + // >1 Match: Multiple periods match the observation date and time (ERROR) + const matchingPeriodsByObservationDateTime = findMatchingPeriodsWithObservationDateTime( + worksheetObservationDate, + worksheetObservationTime, + matchingPeriodsBySamplingInformation + ); + + // INVALID: Unable to match the observation date/time to any existing period uniquely + if (matchingPeriodsByObservationDateTime.length === 0) { + return [ + { + error: 'Sampling period is ambiguous, unable to uniquely identify period using observation date', + solution: + 'Use an observation date that falls within a single period start and end date, or explicitly add a period', + header: utils.getWorksheetHeader('DATE', params.row), + cell: worksheetObservationDate + }, + { + error: 'Sampling period is ambiguous, unable to unquely identify period using observation date and time', + solution: + 'Use an observation date and time that falls within a single period start and end date, or explicitly add a period', + header: utils.getWorksheetHeader('TIME', params.row), + cell: worksheetObservationTime + } + ]; + } + + // VALID: Exact period match found using sampling information and observation date and time + if (matchingPeriodsByObservationDateTime.length === 1) { + updateCSVRowState(params.row, { + sample_period_id: matchingPeriodsByObservationDateTime[0].survey_sample_period_id + }); + + return []; + } + + // INVALID: Multiple periods match the observation date and time + return [ + { + error: 'More than one period matches the observation date and time', + solution: 'Use a observation date and time that falls within the period start and end date of a single period', + header: utils.getWorksheetHeader('DATE', params.row), + cell: worksheetObservationDate + }, + { + error: 'More than one period matches the observation date and time', + solution: 'Use a observation date and time that falls within the period start and end date of a single period', + header: utils.getWorksheetHeader('TIME', params.row), + cell: worksheetObservationTime + } + ]; + }; +} + +/** + * Find Matching Periods with Sampling Information - This function will filter a list of sampling periods by the provided + * sampling information. It will return all periods that match the provided site, technique, and period. + * + * @param {SurveySamplePeriodDetails[]} samplingPeriods All available sampling periods for the survey. + * @param {{ siteName: string | null; techniqueName: string | null; period: string | null; }} samplingInformation + * @return {*} {SurveySamplePeriodDetails[]} + */ +export function findMatchingPeriodsWithSamplingInformation( + samplingPeriods: SurveySamplePeriodDetails[], + samplingInformation: { + siteName: string | null; + techniqueName: string | null; + period: string | null; + } +): SurveySamplePeriodDetails[] { + // Find all periods that match the provided site, technique, and period + // Periods must match all non-null worksheet values to be considered a match + return samplingPeriods.filter((period) => { + if (samplingInformation.siteName) { + const isMatch = matchSamplePeriodToWorksheetSiteName(samplingInformation.siteName, period); + + if (!isMatch) { + // If the worksheet site name is provided but does not match, then this period is not a match + return false; + } + } + + if (samplingInformation.techniqueName) { + const isMatch = matchSamplePeriodToWorksheetTechniqueName(samplingInformation.techniqueName, period); + + if (!isMatch) { + // If the worksheet technique name is provided but does not match, then this period is not a match + return false; + } + } + + if (samplingInformation.period) { + const isMatch = matchSamplePeriodToWorksheetPeriod(samplingInformation.period, period); + + if (!isMatch) { + // If the worksheet period is provided but does not match, then this period is not a match + return false; + } + } + + // If all provided (non-null) values match, then consider this period a match + return true; + }); +} + +/** + * Match Sample Period to Worksheet Site Name - This function will compare a worksheet site name to a sampling site object + * and return true if the names match. + * + * It will compare a worksheet site name to a sampling site object and return true if the names match. + * + * @param {string} worksheetSiteName + * @param {SurveySamplePeriodDetails} samplingPeriod + * @return {*} {boolean} + */ +export function matchSamplePeriodToWorksheetSiteName( + worksheetSiteName: string, + samplingPeriod: SurveySamplePeriodDetails +): boolean { + return samplingPeriod.survey_sample_site?.name.toLowerCase() === worksheetSiteName.toLowerCase(); +} + +/** + * Match Sample Period to Worksheet Technique Name - This function will compare a worksheet technique name to a sampling + * technique object and return true if the names match. + * + * It will compare a worksheet technique name to a sampling technique object and return true if the names match. + * + * @param {string} worksheetTechniqueName + * @param {SurveySamplePeriodDetails} samplingPeriod + * @return {*} {boolean} + */ +export function matchSamplePeriodToWorksheetTechniqueName( + worksheetTechniqueName: string, + samplingPeriod: SurveySamplePeriodDetails +): boolean { + return samplingPeriod.method_technique?.name.toLowerCase() === worksheetTechniqueName.toLowerCase(); +} + +/** + * Match Sample Period to Worksheet Period - This function will compare a worksheet period string to a sampling period + * object and return true if they match. + * + * It will compare a worksheet period string to a sampling period object and return true if they match. + * + * They must match on start_date and end_date. And optionally match on start_time and end_time if they are present + * in the worksheet period string. + * + * Note: This function relies on the incoming period string to separate the start and end dates with a " - " delimiter. + * + * @param {string} worksheetPeriod A string in the format "YYYY-MM-DDTHH:mm:ss - YYYY-MM-DDTHH:mm:ss", or a valid subset or + * superset. (Ex: "2024-07-28 - 2024-07-29", "2024-07-28T00:00:00 - 2024-07-29T23:59:59", etc) + * @param {SurveySamplePeriodDetails} samplingPeriod + * @return {*} {SurveySamplePeriodDetails} + */ +export function matchSamplePeriodToWorksheetPeriod( + worksheetPeriod: string, + samplingPeriod: SurveySamplePeriodDetails +): boolean { + const [worksheetStartDateTime, worksheetEndDateTime] = worksheetPeriod.split(' - '); + + if (!worksheetStartDateTime || !worksheetEndDateTime) { + // Failed to split the period string into expected start and end date strings + return false; + } + + if (!matchSamplePeriodDateToWorksheetPeriodDateTime(worksheetStartDateTime, samplingPeriod.start_date ?? '')) { + // Failed to match the start date + return false; + } + + if (!matchSamplePeriodDateToWorksheetPeriodDateTime(worksheetEndDateTime, samplingPeriod.end_date ?? '')) { + // Failed to match the end date + return false; + } + + if (!matchSamplePeriodTimeToWorksheetPeriodDateTime(worksheetStartDateTime, samplingPeriod.start_time ?? '')) { + // Failed to match the start time + return false; + } + + if (!matchSamplePeriodTimeToWorksheetPeriodDateTime(worksheetEndDateTime, samplingPeriod.end_time ?? '')) { + // Failed to match the end time + return false; + } + + // Successfully matched the period + return true; +} + +/** + * This function is a helper method for the `matchSamplePeriodToWorksheetPeriod` function. + * + * It will compare a worksheet date-time string to a sampling period date string and return true if they match. + * + * @example + * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01 11:00:00', '2021-01-01') // true + * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '2021-01-01') // true + * + * matchSamplePeriodDateToWorksheetPeriodDateTime('', '') // true + * + * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01 11:00:00', '2022-02-02') // false + * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '2022-02-02') // false + * + * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '') // false + * matchSamplePeriodDateToWorksheetPeriodDateTime('', '2021-01-01') // false + * + * @export + * @param {string} worksheetDateTimeString + * @param {string} samplingPeriodDateString + * @return {*} {boolean} + */ +export function matchSamplePeriodDateToWorksheetPeriodDateTime( + worksheetDateTimeString: string, + samplingPeriodDateString: string +): boolean { + const isWorksheetDateTimeString = isDateString(worksheetDateTimeString); + const isSamplingPeriodDateString = isDateString(samplingPeriodDateString); + + if (isWorksheetDateTimeString !== isSamplingPeriodDateString) { + // If either string contains date information, and the other does not, then they cannot possibly match + return false; + } + + if (!isWorksheetDateTimeString && !isSamplingPeriodDateString) { + // If neither string contains date information, then they are considered to match + return true; + } + + const formattedWorksheetDateString = formatDateString(worksheetDateTimeString); + const formattedSamplingPeriodDateString = formatDateString(samplingPeriodDateString); + + if (formattedWorksheetDateString !== formattedSamplingPeriodDateString) { + // Failed to match the date strings + return false; + } + + return true; +} + +/** + * This function is a helper method for the `matchSamplePeriodToWorksheetPeriod` function. + * + * It will compare a worksheet date-time string to a sampling period time string and return true if they match. + * + * @example + * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '11:00:00') // true + * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '11:00') // true + * + * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01', '') // true + * matchSamplePeriodTimeToWorksheetPeriodDateTime('', '') // true + * matchSamplePeriodTimeToWorksheetPeriodDateTime('not_a_time', 'invalid_time') // true + * + * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '12:00:00') // false + * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01', '11:00:00') // false + * + * matchSamplePeriodTimeToWorksheetPeriodDateTime('11:00:00', '') // false + * matchSamplePeriodTimeToWorksheetPeriodDateTime('', '11:00:00') // false + * + * @export + * @param {string} worksheetDateTimeString + * @param {string} samplingPeriodTimeString + * @return {*} {boolean} + */ +export function matchSamplePeriodTimeToWorksheetPeriodDateTime( + worksheetDateTimeString: string, + samplingPeriodTimeString: string +): boolean { + const isWorksheetDateTimeString = isDateTimeString(worksheetDateTimeString); + const isSamplingPeriodTimeString = isTimeString(samplingPeriodTimeString); + + if (isWorksheetDateTimeString !== isSamplingPeriodTimeString) { + // If either string contains time information, and the other does not, then they cannot possibly match + return false; + } + + if (!isWorksheetDateTimeString && !isSamplingPeriodTimeString) { + // If neither string contains time information, then they are considered to match + return true; + } + + const formattedWorksheetTimeString = dayjs(worksheetDateTimeString).format(DefaultTimeFormat); + const formattedSamplingPeriodTimeString = dayjs(samplingPeriodTimeString, [ + DefaultTimeFormat, + DefaultTimeFormatNoSeconds + ]).format(DefaultTimeFormat); + + if (formattedWorksheetTimeString !== formattedSamplingPeriodTimeString) { + // Failed to match the time strings + return false; + } + + return true; +} + +/** + * Find Matching Periods with Observation Date Time - This function will take an observation date and time and find all + * matching sampling periods from the provided samplingPeriods. + * + * @param {(string | null)} observationDate + * @param {(string | null)} observationTime + * @param {SurveySamplePeriodDetails[]} samplingPeriods + * @return {*} + */ +export function findMatchingPeriodsWithObservationDateTime( + observationDate: string | null, + observationTime: string | null, + samplingPeriods: SurveySamplePeriodDetails[] +) { + if (!observationDate) { + // If no observation date is provided, then no periods can be matched + return []; + } + + if (!samplingPeriods.length) { + // If no sampling periods are provided, then no periods can be matched + return []; + } + + const formattedObservationDateTime = observationTime + ? dayjs(`${observationDate} ${observationTime}`) + : dayjs(observationDate); + + const suitablePeriods = samplingPeriods.filter((samplingPeriod) => { + if (!samplingPeriod.start_date || !samplingPeriod.end_date) { + // If the sampling period does not have a start or end date, then it cannot be matched + return false; + } + + const formattedSamplingPeriodStartDateTime = samplingPeriod.start_time + ? dayjs(`${samplingPeriod.start_date} ${samplingPeriod.start_time}`) + : dayjs(samplingPeriod.start_date); + + const formattedSamplingPeriodEndDateTime = samplingPeriod.end_time + ? dayjs(`${samplingPeriod.end_date} ${samplingPeriod.end_time}`) + : dayjs(samplingPeriod.end_date); + + // The observation date time must be within the start and end date time of the sampling period + return ( + formattedObservationDateTime.isSameOrAfter(formattedSamplingPeriodStartDateTime) && + formattedObservationDateTime.isSameOrBefore(formattedSamplingPeriodEndDateTime) + ); + }); + + return suitablePeriods; +} diff --git a/api/src/services/import-services/utils/environment.test.ts b/api/src/services/import-services/utils/environment.test.ts new file mode 100644 index 0000000000..b8223cff13 --- /dev/null +++ b/api/src/services/import-services/utils/environment.test.ts @@ -0,0 +1,58 @@ +import { expect } from 'chai'; +import { + isQualitativeEnvironmentStub, + isQualitativeEnvironmentTypeDefinition, + isQuantitativeEnvironmentStub, + isQuantitativeEnvironmentTypeDefinition +} from './environment'; + +describe('environment', () => { + describe('isQualitativeEnvironmentTypeDefinition', () => { + it('should return true if the object is a QualitativeEnvironmentTypeDefinition', () => { + expect(isQuantitativeEnvironmentTypeDefinition({ unit: 'a', environment_quantitative_id: '1' })).to.be.true; + }); + + it('should return false if the object is not a QualitativeEnvironmentTypeDefinition', () => { + expect(isQuantitativeEnvironmentTypeDefinition({})).to.be.false; + expect(isQuantitativeEnvironmentTypeDefinition({ unit: 'a' })).to.be.false; + expect(isQuantitativeEnvironmentTypeDefinition({ environment_quantitative_id: '1' })).to.be.false; + }); + }); + + describe('isQualitativeEnvironmentTypeDefinition', () => { + it('should return true if the object is a QualitativeEnvironmentTypeDefinition', () => { + expect(isQualitativeEnvironmentTypeDefinition({ options: [], environment_qualitative_id: '1' })).to.be.true; + }); + + it('should return false if the object is not a QualitativeEnvironmentTypeDefinition', () => { + expect(isQualitativeEnvironmentTypeDefinition({})).to.be.false; + expect(isQualitativeEnvironmentTypeDefinition({ options: [] })).to.be.false; + expect(isQualitativeEnvironmentTypeDefinition({ environment_qualitative_id: '1' })).to.be.false; + }); + }); + + describe('isQualitativeEnvironmentStub', () => { + it('should return true if the object is a qualitative environment stub', () => { + expect(isQualitativeEnvironmentStub({ environment_qualitative_option_id: '1', environment_qualitative_id: '1' })) + .to.be.true; + }); + + it('should return false if the object is not a qualitative environment stub', () => { + expect(isQualitativeEnvironmentStub({})).to.be.false; + expect(isQualitativeEnvironmentStub({ environment_qualitative_option_id: '1' })).to.be.false; + expect(isQualitativeEnvironmentStub({ environment_qualitative_id: '1' })).to.be.false; + }); + }); + + describe('isQuantitativeEnvironmentStub', () => { + it('should return true if the object is a quantitative environment stub', () => { + expect(isQuantitativeEnvironmentStub({ environment_quantitative_id: '1', value: 1 })).to.be.true; + }); + + it('should return false if the object is not a quantitative environment stub', () => { + expect(isQuantitativeEnvironmentStub({})).to.be.false; + expect(isQuantitativeEnvironmentStub({ environment_quantitative_id: '1' })).to.be.false; + expect(isQuantitativeEnvironmentStub({ value: 1 })).to.be.false; + }); + }); +}); diff --git a/api/src/services/import-services/utils/environment.ts b/api/src/services/import-services/utils/environment.ts new file mode 100644 index 0000000000..c5f858cbd4 --- /dev/null +++ b/api/src/services/import-services/utils/environment.ts @@ -0,0 +1,119 @@ +import { + QualitativeEnvironmentTypeDefinition, + QuantitativeEnvironmentTypeDefinition +} from '../../../repositories/observation-subcount-environment-repository'; +import { CaseInsensitiveMap } from '../../../utils/case-insensitive-map'; +import { ObservationSubCountEnvironmentService } from '../../observation-subcount-environment-service'; + +export type EnvironmentNameTypeDefinitionMap = CaseInsensitiveMap< + string, // Environment name + QualitativeEnvironmentTypeDefinition | QuantitativeEnvironmentTypeDefinition +>; + +/** + * Check if an object is a `QuantitativeEnvironmentTypeDefinition` + * + * Returns true if the object has the properties `unit` and `environment_quantitative_id` + * + * @param {any} environment - The object to check + * @returns {boolean} True if the object is a QuantitativeEnvironmentTypeDefinition + */ +export const isQuantitativeEnvironmentTypeDefinition = ( + environment: unknown +): environment is QuantitativeEnvironmentTypeDefinition => { + return ( + typeof environment === 'object' && + environment != null && + 'unit' in environment && + 'environment_quantitative_id' in environment + ); +}; + +/** + * Check if an object is a `QualitativeEnvironmentTypeDefinition` + * + * Returns true if the object has the properties `options` and `environment_qualitative_id` + * + * @param {unknown} environment - The object to check + * @returns {boolean} True if the object is a QualitativeEnvironmentTypeDefinition + */ +export const isQualitativeEnvironmentTypeDefinition = ( + environment: unknown +): environment is QualitativeEnvironmentTypeDefinition => { + return ( + typeof environment === 'object' && + environment != null && + 'options' in environment && + 'environment_qualitative_id' in environment + ); +}; + +/** + * Check if an object is a qualitative environment stub + * + * Returns true if the object has the properties `environment_qualitative_option_id` and `environment_qualitative_id` + * + * Note: This function is NOT a typeguard, it is used to determine if an object + * contains the minimum required properties to create a qualitative environment. + * + * @param {unknown} environment - The object to check + * @returns {boolean} True if the object is a qualitative environment + */ +export const isQualitativeEnvironmentStub = (environment: unknown): boolean => { + return ( + typeof environment === 'object' && + environment != null && + 'environment_qualitative_option_id' in environment && + 'environment_qualitative_id' in environment + ); +}; + +/** + * Check if an object is a quantitative environment stub + * + * Returns true if the object has the properties `value` and `environment_quantitative_id` + * + * Note: This function is NOT a typeguard, it is used to determine if an object + * contains the minimum required properties to create a quantitative environment. + * + * @param {unknown} environment - The object to check + * @returns {boolean} True if the object is a quantitative environment + */ +export const isQuantitativeEnvironmentStub = (environment: unknown): boolean => { + return ( + typeof environment === 'object' && + environment != null && + 'value' in environment && + 'environment_quantitative_id' in environment + ); +}; + +/** + * Get the environment name type definition map for a survey - case insensitive + * + * @param {string[]} environmentNames The environment names + * @param {ObservationSubCountEnvironmentService} environmentService The environment service + * @return {*} {Promise} A mapping of environment names to their respective environment type definitions + */ +export const getEnvironmentNameTypeDefinitionMap = async ( + environmentNames: string[], + environmentService: ObservationSubCountEnvironmentService +): Promise => { + const environmentMap: EnvironmentNameTypeDefinitionMap = new CaseInsensitiveMap(); + + const [qualitativeEnvironments, quantitativeEnvironments] = await Promise.all([ + environmentService.findQualitativeEnvironmentTypeDefinitions(environmentNames), + environmentService.findQuantitativeEnvironmentTypeDefinitions(environmentNames) + ]); + + // Map environment names to their respective environment type definitions + for (const environment of qualitativeEnvironments) { + environmentMap.set(environment.name, environment); + } + + for (const environment of quantitativeEnvironments) { + environmentMap.set(environment.name, environment); + } + + return environmentMap; +}; diff --git a/api/src/services/import-services/utils/measurement.test.ts b/api/src/services/import-services/utils/measurement.test.ts index 81e319e120..674847bec0 100644 --- a/api/src/services/import-services/utils/measurement.test.ts +++ b/api/src/services/import-services/utils/measurement.test.ts @@ -1,9 +1,15 @@ import { expect } from 'chai'; import sinon from 'sinon'; -import { getTsnMeasurementDictionary } from './measurement'; +import { + getTsnMeasurementDictionary, + isCBQualitativeMeasurementStub, + isCBQualitativeMeasurementTypeDefinition, + isCBQuantitativeMeasurementStub, + isCBQuantitativeMeasurementTypeDefinition +} from './measurement'; describe('measurement', () => { - describe('_getTsnMeasurementDictionary', () => { + describe('getTsnMeasurementDictionary', () => { it('should get the tsn measurement dictionary', async () => { const measurements = { qualitative: [{ measurement_name: 'qualitative' }], @@ -25,4 +31,62 @@ describe('measurement', () => { expect(result.get(1, 'quantitative')).to.deep.equal({ measurement_name: 'quantitative' }); }); }); + + describe('isCBQualitativeMeasurementStub', () => { + it('should return true if the object is a qualitative measurement stub', () => { + expect(isCBQualitativeMeasurementStub({ qualitative_option_id: 1, taxon_measurement_id: 1 })).to.be.true; + }); + + it('should return false if the object is not a qualitative measurement stub', () => { + for (const value of [undefined, null, 1, 'string', [], { taxon_measurement_id: 1 }]) { + expect(isCBQualitativeMeasurementStub(value)).to.be.false; + } + }); + }); + + describe('isCBQuantitativeMeasurementStub', () => { + it('should return true if the object is a quantitative measurement stub', () => { + expect(isCBQuantitativeMeasurementStub({ value: 1, taxon_measurement_id: 1 })).to.be.true; + }); + + it('should return false if the object is not a quantitative measurement stub', () => { + for (const value of [undefined, null, 1, 'string', [], { value: 1 }, { taxon_measurement_id: 1 }]) { + expect(isCBQuantitativeMeasurementStub(value)).to.be.false; + } + }); + }); + + describe('isCBQualitativeMeasurementTypeDefinition', () => { + it('should return true if the object is a qualitative measurement type definition', () => { + expect( + isCBQualitativeMeasurementTypeDefinition({ + taxon_measurement_id: '1', + options: [{ qualitative_option_id: '1', option_label: 'option' }] + }) + ).to.be.true; + }); + + it('should return false if the object is not a qualitative measurement type definition', () => { + for (const value of [undefined, null, 1, 'string', [], { measurement_name: 1 }]) { + expect(isCBQualitativeMeasurementTypeDefinition(value)).to.be.false; + } + }); + }); + + describe('isCBQuantitativeMeausurementTypeDefinition', () => { + it('should return true if the object is a quantitative measurement type definition', () => { + expect( + isCBQuantitativeMeasurementTypeDefinition({ + taxon_measurement_id: '1', + unit: 'm' + }) + ).to.be.true; + }); + + it('should return false if the object is not a quantitative measurement type definition', () => { + for (const value of [undefined, null, 1, 'string', [], { measurement_name: 1 }]) { + expect(isCBQuantitativeMeasurementTypeDefinition(value)).to.be.false; + } + }); + }); }); diff --git a/api/src/services/import-services/utils/measurement.ts b/api/src/services/import-services/utils/measurement.ts index 0dc1769b60..39669f9f20 100644 --- a/api/src/services/import-services/utils/measurement.ts +++ b/api/src/services/import-services/utils/measurement.ts @@ -1,8 +1,6 @@ import { NestedRecord } from '../../../utils/nested-record'; import { - CBQualitativeMeasurement, CBQualitativeMeasurementTypeDefinition, - CBQuantitativeMeasurement, CBQuantitativeMeasurementTypeDefinition, CritterbaseService } from '../../critterbase-service'; @@ -50,14 +48,17 @@ export const isCBQualitativeMeasurementTypeDefinition = ( }; /** - * Check if an object is a `CBQualitativeMeasurement` - ie: the recorded mesasurement + * Check if an object is a qualitative measurement stub * * Returns true if the object has the properties `qualitative_option_id` and `taxon_measurement_id` * + * Note: This function is NOT a typeguard, it is used to determine if an object + * contains the minimum required properties to create a qualitative measurement. + * * @param {unknown} measurement - The object to check - * @returns {boolean} True if the object is a CBQualitativeMeasurement + * @returns {boolean} True if the object is a qualitative measurement */ -export const isCBQualitativeMeasurement = (measurement: unknown): measurement is CBQualitativeMeasurement => { +export const isCBQualitativeMeasurementStub = (measurement: unknown): boolean => { return ( typeof measurement === 'object' && measurement != null && @@ -67,14 +68,17 @@ export const isCBQualitativeMeasurement = (measurement: unknown): measurement is }; /** - * Check if an object is a `CBQuantitativeMeasurement` - ie: the recorded mesasurement + * Check if an object is a quantitative measurement stub * * Returns true if the object has the properties `value` and `taxon_measurement_id` * + * Note: This function is NOT a typeguard, it is used to determine if an object + * contains the minimum required properties to create a quantitative measurement. + * * @param {unknown} measurement - The object to check - * @returns {boolean} True if the object is a CBQuantitativeMeasurement + * @returns {boolean} True if the object is a quantitative measurement */ -export const isCBQuantitativeMeasurement = (measurement: unknown): measurement is CBQuantitativeMeasurement => { +export const isCBQuantitativeMeasurementStub = (measurement: unknown): boolean => { return ( typeof measurement === 'object' && measurement != null && diff --git a/api/src/services/import-services/utils/qualitative.test.ts b/api/src/services/import-services/utils/qualitative.test.ts new file mode 100644 index 0000000000..d4128bbe60 --- /dev/null +++ b/api/src/services/import-services/utils/qualitative.test.ts @@ -0,0 +1,65 @@ +import { expect } from 'chai'; +import { validateQualitativeValue } from './qualitative'; + +describe('validateQualitativeValue', () => { + it('should return the option id if the value is valid', () => { + const value = 'red'; + const qualitativeTypeDefinition = { + options: [ + { + option_id: '456', + option_name: 'red' + } + ] + }; + const result = validateQualitativeValue(value, qualitativeTypeDefinition); + + expect(result).to.be.equal('456'); + }); + + it('should return the option id if the value is valid and case insensitive', () => { + const value = 'Red'; + const qualitativeTypeDefinition = { + options: [ + { + option_id: '456', + option_name: 'red' + } + ] + }; + const result = validateQualitativeValue(value, qualitativeTypeDefinition); + + expect(result).to.be.equal('456'); + }); + + it('should return an error if the value is not a string', () => { + const value = 123; + const qualitativeTypeDefinition = { + options: [ + { + option_id: '456', + option_name: 'red' + } + ] + }; + const result = validateQualitativeValue(value, qualitativeTypeDefinition); + + expect(result[0]['error']).to.contain('a string'); + }); + + it('should return an error if the value is not a valid option', () => { + const value = 'blue'; + const qualitativeTypeDefinition = { + options: [ + { + option_id: '456', + option_name: 'red' + } + ] + }; + const result = validateQualitativeValue(value, qualitativeTypeDefinition); + + expect(result[0]['error']).to.contain('Invalid qualitative option'); + expect(result[0]['values']).to.be.an('array').that.has.a.lengthOf(1).and.includes('red'); + }); +}); diff --git a/api/src/services/import-services/utils/qualitative.ts b/api/src/services/import-services/utils/qualitative.ts new file mode 100644 index 0000000000..bf616cae88 --- /dev/null +++ b/api/src/services/import-services/utils/qualitative.ts @@ -0,0 +1,70 @@ +import { CSVError } from '../../../utils/csv-utils/csv-config-validation.interface'; + +interface IQualitativeTypeDefinitionStub { + /** + * The allowed options for the qualitative value + * + * @type {Array<{ option_id: string; option_name: string }>} + */ + options: { + /** + * The qualitative option ID + * + * @type {string} - UUID + */ + option_id: string; + /** + * The qualitative option name + * + * @type {string} + */ + option_name: string; + }[]; +} + +/** + * Validate a qualitative value against the qualitative type definition. + * + * Rules: + * 1. Qualitative values are strings ie: 'red' + * 2. Qualitative values must be one of the allowed options ie: ['red', 'blue', 'green'] + * 3. Qualitative values are case insensitive ie: 'Red' === 'red' + * + * @param {unknown} value - The value to validate + * @param {IQualitativeTypeDefinitionStub} qualitativeTypeDefinition - The qualitative type definition + * @param {string} [qualitativeTag] - The tag to inject into the error message ie: 'measurement' or 'environment' + * @returns {CSVError[] | string} - The list of errors or the qualitative value + */ +export const validateQualitativeValue = ( + value: unknown, + qualitativeTypeDefinition: IQualitativeTypeDefinitionStub, + qualitativeTag?: string +): CSVError[] | string => { + const qualitativeTagString = qualitativeTag ? ` ${qualitativeTag}` : ''; + + if (typeof value !== 'string') { + return [ + { + error: `Qualitative${qualitativeTagString} value must be a string`, + solution: 'Update the value to match the expected type' + } + ]; + } + + const matchingQualitativeOption = qualitativeTypeDefinition.options.find( + (option) => option.option_name.toLowerCase() === value.toLowerCase() + ); + + // Validate value is an alowed qualitative option + if (!matchingQualitativeOption) { + return [ + { + error: `Invalid qualitative${qualitativeTagString} option`, + solution: `Use a valid qualitative option`, + values: qualitativeTypeDefinition.options.map((option) => option.option_name) + } + ]; + } + + return matchingQualitativeOption.option_id; +}; diff --git a/api/src/services/import-services/utils/quantitative.test.ts b/api/src/services/import-services/utils/quantitative.test.ts new file mode 100644 index 0000000000..5a4d8387f6 --- /dev/null +++ b/api/src/services/import-services/utils/quantitative.test.ts @@ -0,0 +1,44 @@ +import { expect } from 'chai'; +import { validateQuantitativeValue } from './quantitative'; + +describe('validateQuantitativeValue', () => { + it('should return the value if it is valid', () => { + const quantitativeTypeDefinition = { + min: 0, + max: 10 + }; + + const result = validateQuantitativeValue(2, quantitativeTypeDefinition); + expect(result).to.equal(2); + }); + + it('should return an error if the value is not a number', () => { + const quantitativeTypeDefinition = { + min: 0, + max: 10 + }; + + const result = validateQuantitativeValue('2', quantitativeTypeDefinition); + expect(result[0]['error']).to.contain('a number'); + }); + + it('should return an error if the value is too large', () => { + const quantitativeTypeDefinition = { + min: 0, + max: 10 + }; + + const result = validateQuantitativeValue(11, quantitativeTypeDefinition); + expect(result[0]['error']).to.contain('too large'); + }); + + it('should return an error if the value is too small', () => { + const quantitativeTypeDefinition = { + min: 0, + max: 10 + }; + + const result = validateQuantitativeValue(-1, quantitativeTypeDefinition); + expect(result[0]['error']).to.contain('too small'); + }); +}); diff --git a/api/src/services/import-services/utils/quantitative.ts b/api/src/services/import-services/utils/quantitative.ts new file mode 100644 index 0000000000..7cfe4359e5 --- /dev/null +++ b/api/src/services/import-services/utils/quantitative.ts @@ -0,0 +1,69 @@ +import { CSVError } from '../../../utils/csv-utils/csv-config-validation.interface'; + +interface IQuantitativeTypeDefinitionStub { + /** + * The minimum value the quantitative value can be + * + * @type {number} + */ + min: number | null; + /** + * The maximum value the quantitative value can be + * + * @type {number} + */ + max: number | null; +} + +/** + * Validate a quantitative value against the quantitative type definition. + * + * Rules: + * 1. Quantitative values are numbers ie: 2 + * 2. Quantitative values must be within the min max bounds ie: 0 <= value <= 10 + * + * @param {unknown} value - The value to validate + * @param {IQuantitativeTypeDefinitionStub} quantitativeTypeDefinition - The quantitative type definition + * @param {string} [quantitativeTag] - The tag to inject into the error message ie: 'measurement' or 'environment' + * @returns {CSVError[] | number} - The list of errors or the quantitative value + */ +export const validateQuantitativeValue = ( + value: unknown, + quantitativeTypeDefinition: IQuantitativeTypeDefinitionStub, + quantitativeTag?: string +): CSVError[] | number => { + const errors: CSVError[] = []; + const quantitativeTagString = quantitativeTag ? ` ${quantitativeTag}` : ''; + + // Quantitative are numbers ie: antler count: 2 + if (typeof value !== 'number') { + return [ + { + error: `Quantitative${quantitativeTagString} value must be a number`, + solution: 'Update the value to match the expected type' + } + ]; + } + + // Validate value is within min max bounds + if (quantitativeTypeDefinition.max != null && value > quantitativeTypeDefinition.max) { + errors.push({ + error: `Quantitative${quantitativeTagString} value too large`, + solution: `Value must be less than or equal to ${quantitativeTypeDefinition.max}` + }); + } + + // Validate value is within the min max bounds + if (quantitativeTypeDefinition.min != null && value < quantitativeTypeDefinition.min) { + errors.push({ + error: `Quantitative${quantitativeTagString} value too small`, + solution: `Value must be greater than or equal to ${quantitativeTypeDefinition.min}` + }); + } + + if (errors.length) { + return errors; + } + + return value; +}; diff --git a/api/src/services/import-services/utils/row-state.test.ts b/api/src/services/import-services/utils/row-state.test.ts new file mode 100644 index 0000000000..6c80d3e0ca --- /dev/null +++ b/api/src/services/import-services/utils/row-state.test.ts @@ -0,0 +1,48 @@ +import { expect } from 'chai'; +import { z } from 'zod'; +import { CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { createRowStateGetter } from './row-state'; + +describe('row-state', () => { + describe('createRowStateGetter', () => { + it('should not throw an error when the row state is valid', () => { + const row = { + [CSVRowState]: { + id: '123' + } + }; + + const getRowState = createRowStateGetter(z.object({ id: z.string() })); + + expect(getRowState(row)).to.deep.equal({ id: '123' }); + }); + + it('should allow for the row state to be passed directly', () => { + const state = { + id: '123' + }; + + const getRowState = createRowStateGetter(z.object({ id: z.string() })); + + expect(getRowState(state)).to.deep.equal({ id: '123' }); + }); + + it('should throw an error when the row state is invalid', () => { + const row = { + [CSVRowState]: { + id: 123 + } + }; + + const getRowState = createRowStateGetter(z.object({ id: z.string() })); + + try { + getRowState(row); + expect.fail('Should have thrown an error'); + } catch (err: any) { + expect(err.message).to.equal('Invalid CSV row state'); + expect(err.errors[0].state).to.deep.equal({ id: 123 }); + } + }); + }); +}); diff --git a/api/src/services/import-services/utils/row-state.ts b/api/src/services/import-services/utils/row-state.ts new file mode 100644 index 0000000000..2f16db5af4 --- /dev/null +++ b/api/src/services/import-services/utils/row-state.ts @@ -0,0 +1,108 @@ +import { z } from 'zod'; +import { ApiGeneralError } from '../../../errors/api-error'; +import { CSVRow, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; + +/** + * Create a row state getter + * + * Note: This function allows both the row and the row state to be passed in. + * + * @example ` + * const getFromRowState = createRowStateGetter(z.object({ ... })); + * getFromRowState(row) + * getFromRowState(row[CSVRowState]) + * getFromRowState(row[CSVRowState].nestedState) + * ` + * + * @param {z.ZodSchema} schema - The Zod schema to validate the row state + * @returns {*} {function} - The row state getter + */ +export const createRowStateGetter = (schema: SchemaType) => { + return (rowOrState: CSVRow | CSVRow[typeof CSVRowState]): z.infer => { + let state = rowOrState; + + // Note: The row state is nested under the CSVRowState symbol + if (_isCSVRow(rowOrState)) { + // Get the state from the row + state = rowOrState?.[CSVRowState]; + } + + // Parse the row state using the schema + const parsedState = schema.safeParse(state); + + // Throw an error if unable to correctly parse the row state + if (!parsedState.success) { + throw new ApiGeneralError('Invalid CSV row state', [ + { + state: state, + errors: parsedState.error + } + ]); + } + + return parsedState.data; + }; +}; + +/** + * Check if the object is a CSV row + * + * @param {CSVRow | CSVRow[typeof CSVRowState]} rowOrState - The row or row state + * @returns {boolean} - Whether the row is a CSV row + */ +const _isCSVRow = (rowOrState: CSVRow | CSVRow[typeof CSVRowState]): rowOrState is CSVRow => { + return Object.getOwnPropertySymbols(rowOrState).includes(CSVRowState); +}; + +// Critter / Capture +export const getCritterCaptureFromRowState = createRowStateGetter( + z.object({ + critter_id: z.string().uuid(), + capture_id: z.string().uuid() + }) +); + +// Taxon +export const getTaxonFromRowState = createRowStateGetter( + z.object({ + itis_tsn: z.number(), + itis_scientific_name: z.string() + }) +); + +// Measurement +export const getQualitativeMeasurementFromRowState = createRowStateGetter( + z.object({ + taxon_measurement_id: z.string().uuid(), + qualitative_option_id: z.string().uuid() + }) +); + +export const getQuantitativeMeasurementFromRowState = createRowStateGetter( + z.object({ + taxon_measurement_id: z.string().uuid(), + value: z.number() + }) +); + +// Environment +export const getQualitativeEnvironmentFromRowState = createRowStateGetter( + z.object({ + environment_qualitative_id: z.string().uuid(), + environment_qualitative_option_id: z.string().uuid() + }) +); + +export const getQuantitativeEnvironmentFromRowState = createRowStateGetter( + z.object({ + environment_quantitative_id: z.string().uuid(), + value: z.number() + }) +); + +// Observation +export const getSamplePeriodIdFromRowState = createRowStateGetter( + z.object({ + sample_period_id: z.number().optional() + }) +); diff --git a/api/src/services/import-services/utils/taxon.test.ts b/api/src/services/import-services/utils/taxon.test.ts new file mode 100644 index 0000000000..07e83891ce --- /dev/null +++ b/api/src/services/import-services/utils/taxon.test.ts @@ -0,0 +1,38 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CaseInsensitiveMap } from '../../../utils/case-insensitive-map'; +import { getTaxonMap, getTsnsFromTaxonMap, TaxonMap } from './taxon'; + +describe('taxon', () => { + afterEach(() => { + sinon.restore(); + }); + + describe('getTaxonMap', () => { + it('should return a map of taxon identifiers to taxons', async () => { + const taxonIdentifiers = [1, 'taxon']; + const platformService: any = { + getTaxonomyByTsns: sinon.stub().resolves([{ tsn: 1, scientificName: 'Alces alces' }]), + getTaxonByScientificName: sinon.stub().resolves({ tsn: 2, scientificName: 'taxon' }) + }; + + const taxonMap = await getTaxonMap(taxonIdentifiers, platformService); + + expect(taxonMap.get(1)).to.deep.equal({ tsn: 1, scientificName: 'Alces alces' }); + expect(taxonMap.get('TAXON')).to.deep.equal({ tsn: 2, scientificName: 'taxon' }); + }); + }); + + describe('getTsnsFromTaxonMap', () => { + it('should return a list of tsns from a taxon map', () => { + const taxonMap: TaxonMap = new CaseInsensitiveMap([ + [1, { tsn: 1, scientificName: 'Alces alces' }], + [2, { tsn: 2, scientificName: 'taxon' }] + ]); + + const tsns = getTsnsFromTaxonMap(taxonMap); + + expect(tsns).to.deep.equal([1, 2]); + }); + }); +}); diff --git a/api/src/services/import-services/utils/taxon.ts b/api/src/services/import-services/utils/taxon.ts new file mode 100644 index 0000000000..50cd0ec745 --- /dev/null +++ b/api/src/services/import-services/utils/taxon.ts @@ -0,0 +1,54 @@ +import { isNumber, partition } from 'lodash'; +import { CaseInsensitiveMap } from '../../../utils/case-insensitive-map'; +import { IItisSearchResult, PlatformService } from '../../platform-service'; + +export type TaxonMap = CaseInsensitiveMap; + +/** + * Get a mapping of taxons from a list of taxon identifiers (TSN or scientific name) - case insensitive. + * + * @param {Array} taxonIdentifiers A list of taxon identifier (TSN or scientific name) + * @param {PlatformService} platformService The platform service + * @return {*} {Promise>} The taxon map - case insensitive + */ +export const getTaxonMap = async ( + taxonIdentifiers: Array, + platformService: PlatformService +): Promise => { + const taxonMap = new CaseInsensitiveMap(); + + // Partition the values into tsns (numbers) and scientific names (strings) + const [tsns, scientificNames] = partition(taxonIdentifiers, isNumber); + + const uniqueScientificNames = [...new Set(scientificNames.map((name) => name.toLowerCase()))]; + + // Fetch the taxons by scientific name in parallel + const scientificNameTaxons = await Promise.all( + uniqueScientificNames.map((name) => platformService.getTaxonByScientificName(name)) + ); + + // Bulk fetch the taxons by TSN + const tsnTaxons = await platformService.getTaxonomyByTsns(tsns); + + scientificNameTaxons.forEach((taxon) => { + if (taxon) { + taxonMap.set(taxon.scientificName, taxon); + } + }); + + tsnTaxons.forEach((taxon) => { + taxonMap.set(taxon.tsn, taxon); + }); + + return taxonMap; +}; + +/** + * Get the TSNs from a taxon map. + * + * @param {TaxonMap} taxonMap The taxon map + * @return {*} {number[]} The list of TSNs + */ +export const getTsnsFromTaxonMap = (taxonMap: TaxonMap) => { + return [...new Set(Array.from(taxonMap.values()).map((taxon) => taxon.tsn))]; +}; diff --git a/api/src/services/observation-services/observation-service.ts b/api/src/services/observation-services/observation-service.ts index e5ebea241d..d580d02432 100644 --- a/api/src/services/observation-services/observation-service.ts +++ b/api/src/services/observation-services/observation-service.ts @@ -1,8 +1,6 @@ import { SurveyObservationRecord } from '../../database-models/survey_observation'; import { IDBConnection } from '../../database/db'; -import { ApiGeneralError } from '../../errors/api-error'; import { IObservationAdvancedFilters } from '../../models/observation-view'; -import { CodeRepository } from '../../repositories/code-repository'; import { InsertObservation, ObservationGeometryRecord, @@ -23,33 +21,15 @@ import { InsertObservationSubCountQuantitativeMeasurementRecord } from '../../repositories/observation-subcount-measurement-repository'; import { SurveySamplePeriodDetails } from '../../repositories/sample-period-repository'; -import { generateS3FileKey, getFileFromS3 } from '../../utils/file-utils'; +import { generateS3FileKey } from '../../utils/file-utils'; import { getLogger } from '../../utils/logger'; -import { parseS3File } from '../../utils/media/media-utils'; -import { getCodeTypeDefinitions, validateCodes } from '../../utils/observation-xlsx-utils/code-column-utils'; import { isQuantitativeValueValid } from '../../utils/observation-xlsx-utils/common-utils'; +import { IEnvironmentDataToValidate } from '../../utils/observation-xlsx-utils/environment-column-utils'; import { - getEnvironmentColumnsTypeDefinitionMap, - getEnvironmentTypeDefinitionsFromColumnNames, - IEnvironmentDataToValidate, - validateEnvironments -} from '../../utils/observation-xlsx-utils/environment-column-utils'; -import { - getMeasurementColumnNames, getTsnMeasurementTypeDefinitionMap, IMeasurementDataToValidate, validateMeasurements } from '../../utils/observation-xlsx-utils/measurement-column-utils'; -import { CSV_COLUMN_ALIASES } from '../../utils/xlsx-utils/column-aliases'; -import { generateColumnCellGetterFromColumnValidator } from '../../utils/xlsx-utils/column-validator-utils'; -import { - constructXLSXWorkbook, - getDefaultWorksheet, - getNonStandardColumnNamesFromWorksheet, - getWorksheetRowObjects, - IXLSXCSVValidator, - validateCsvFile -} from '../../utils/xlsx-utils/worksheet-utils'; import { ApiPaginationOptions } from '../../zod-schema/pagination'; import { CBQualitativeMeasurementTypeDefinition, @@ -59,39 +39,10 @@ import { import { DBService } from '../db-service'; import { ObservationSubCountEnvironmentService } from '../observation-subcount-environment-service'; import { ObservationSubCountMeasurementService } from '../observation-subcount-measurement-service'; -import { PlatformService } from '../platform-service'; import { SamplePeriodService } from '../sample-period-service'; import { SubCountService } from '../subcount-service'; -import { - pullEnvironmentsFromWorkSheetRowObject, - pullMeasurementsFromWorkSheetRowObject, - pullSamplingDataFromWorksheetRowObject -} from './utils'; export const defaultLog = getLogger('services/observation-services/observation-service'); -const defaultSubcountSign = 'direct sighting'; - -/** - * An XLSX validation config for the standard columns of an Observation CSV. - * - * Note: `satisfies` allows `keyof` to correctly infer key types, while also - * enforcing uppercase object keys. - */ -export const observationStandardColumnValidator = { - ITIS_TSN: { type: 'number', aliases: CSV_COLUMN_ALIASES.ITIS_TSN }, - COUNT: { type: 'number' }, - OBSERVATION_SUBCOUNT_SIGN: { type: 'code', aliases: CSV_COLUMN_ALIASES.OBSERVATION_SUBCOUNT_SIGN, optional: true }, - DATE: { type: 'date', optional: true }, - TIME: { type: 'string', optional: true }, - LATITUDE: { type: 'number', aliases: CSV_COLUMN_ALIASES.LATITUDE, optional: true }, - LONGITUDE: { type: 'number', aliases: CSV_COLUMN_ALIASES.LONGITUDE, optional: true }, - SAMPLING_SITE: { type: 'string', aliases: CSV_COLUMN_ALIASES.SAMPLING_SITE, optional: true }, - METHOD_TECHNIQUE: { type: 'string', aliases: CSV_COLUMN_ALIASES.METHOD_TECHNIQUE, optional: true }, - SAMPLING_PERIOD: { type: 'string', aliases: CSV_COLUMN_ALIASES.SAMPLING_PERIOD, optional: true }, - COMMENT: { type: 'string', aliases: CSV_COLUMN_ALIASES.COMMENT, optional: true } -} satisfies IXLSXCSVValidator; - -export const getColumnCellValue = generateColumnCellGetterFromColumnValidator(observationStandardColumnValidator); export interface InsertSubCount { observation_subcount_id: number | null; @@ -186,10 +137,9 @@ export class ObservationService extends DBService { for (const observation of observations) { // Upsert observation standard columns - const upsertedObservationRecord = await this.observationRepository.insertUpdateSurveyObservations( - surveyId, - await this._attachItisScientificName([observation.standardColumns]) - ); + const upsertedObservationRecord = await this.observationRepository.insertUpdateSurveyObservations(surveyId, [ + observation.standardColumns + ]); const surveyObservationId = upsertedObservationRecord[0].survey_observation_id; @@ -471,285 +421,6 @@ export class ObservationService extends DBService { return this.observationRepository.getObservationsCountByTechniqueIds(surveyId, methodTechniqueIds); } - /** - * Processes an observation CSV file submission. - * - * This method: - * - Receives an id belonging to an observation submission, - * - Fetches the CSV file associated with the submission id - * - Validates the CSV file and its content, failing the entire process if any validation check fails - * - Appends all of the records in the CSV file to the observations for the survey. - * - * @param {number} surveyId - * @param {number} submissionId - * @param {{ surveySamplePeriodId?: number }} [options] - * @return {*} {Promise} - * @memberof ObservationService - */ - async processObservationCsvSubmission( - surveyId: number, - submissionId: number, - options?: { surveySamplePeriodId?: number } - ): Promise { - defaultLog.debug({ label: 'processObservationCsvSubmission', submissionId }); - - // Get the observation submission record - const observationSubmissionRecord = await this.getObservationSubmissionById(surveyId, submissionId); - - // Get the S3 object containing the uploaded CSV file - const s3Object = await getFileFromS3(observationSubmissionRecord.key); - - // Get the csv file from the S3 object - const mediaFile = await parseS3File(s3Object); - - // Validate the CSV file mime type - if (mediaFile.mimetype !== 'text/csv') { - throw new Error('Failed to process file for importing observations. Invalid CSV file.'); - } - - // Construct the XLSX workbook - const xlsxWorkBook = constructXLSXWorkbook(mediaFile); - - // Get the default XLSX worksheet - const xlsxWorksheet = getDefaultWorksheet(xlsxWorkBook); - - // Validate the standard columns in the CSV file - if (!validateCsvFile(xlsxWorksheet, observationStandardColumnValidator)) { - throw new Error('Failed to process file for importing observations. Column validator failed.'); - } - - // Filter out the standard columns from the worksheet - const nonStandardColumnNames = getNonStandardColumnNamesFromWorksheet( - xlsxWorksheet, - observationStandardColumnValidator - ); - - // Get the worksheet row objects - const worksheetRowObjects = getWorksheetRowObjects(xlsxWorksheet); - - // VALIDATE CODES ----------------------------------------------------------------------------------------- - - // TODO: This code column validation logic is specifically catered to the observation_subcount_signs code set, as - // it is the only code set currently being used in the observation CSVs, and is required. This logic will need to - // be updated to be more generic if other code sets are used in the future, or if they can be nullable. - - // Validate the Code columns in CSV file - const codeRepository = new CodeRepository(this.connection); - const codeTypeDefinitions = await getCodeTypeDefinitions(codeRepository); - - const codesToValidate = worksheetRowObjects.flatMap((row) => getColumnCellValue(row, 'OBSERVATION_SUBCOUNT_SIGN')); - - // Validate code column data - if (!validateCodes(codesToValidate, codeTypeDefinitions)) { - throw new Error('Failed to process file for importing observations. Code column validator failed.'); - } - - // VALIDATE MEASUREMENTS ----------------------------------------------------------------------------------------- - - // Validate the Measurement columns in CSV file - const critterBaseService = new CritterbaseService({ - keycloak_guid: this.connection.systemUserGUID(), - username: this.connection.systemUserIdentifier() - }); - - // Fetch all measurement type definitions from Critterbase for all unique TSNs - const tsns = worksheetRowObjects.map((row) => String(getColumnCellValue(row, 'ITIS_TSN').cell)); - - const tsnMeasurementTypeDefinitionMap = await getTsnMeasurementTypeDefinitionMap(tsns, critterBaseService); - - // Get all measurement columns names from the worksheet, that match a measurement in the TSN measurements - const measurementColumnNames = getMeasurementColumnNames(nonStandardColumnNames, tsnMeasurementTypeDefinitionMap); - - const measurementsToValidate: IMeasurementDataToValidate[] = worksheetRowObjects.flatMap((row) => { - return measurementColumnNames.map((columnName) => ({ - tsn: String(getColumnCellValue(row, 'ITIS_TSN').cell), - key: columnName, - value: row[columnName] - })); - }); - - // Validate measurement column data - if (!validateMeasurements(measurementsToValidate, tsnMeasurementTypeDefinitionMap)) { - throw new Error('Failed to process file for importing observations. Measurement column validator failed.'); - } - - // VALIDATE ENVIRONMENTS ----------------------------------------------------------------------------------------- - - // Filter out the measurement columns from the non-standard columns. - // Note: This assumes that after filtering out both standard and measurement columns, the remaining columns are the - // environment columns - const environmentColumnNames = nonStandardColumnNames.filter( - (nonStandardColumnHeader) => !measurementColumnNames.includes(nonStandardColumnHeader) - ); - - const observationSubCountEnvironmentService = new ObservationSubCountEnvironmentService(this.connection); - - // Fetch all environment type definitions from SIMS for all unique environment column names in the CSV file - const environmentTypeDefinitions = await getEnvironmentTypeDefinitionsFromColumnNames( - environmentColumnNames, - observationSubCountEnvironmentService - ); - - const environmentColumnsTypeDefinitionMap = getEnvironmentColumnsTypeDefinitionMap( - environmentColumnNames, - environmentTypeDefinitions - ); - - const environmentsToValidate: IEnvironmentDataToValidate[] = worksheetRowObjects.flatMap((row) => { - return environmentColumnNames.map((columnName) => ({ - key: columnName, - value: row[columnName] - })); - }); - - // Validate environment column data - if (!validateEnvironments(environmentsToValidate, environmentColumnsTypeDefinitionMap)) { - throw new Error('Failed to process file for importing observations. Environment column validator failed.'); - } - - // SAMPLING INFORMATION ----------------------------------------------------------------------------------------- - const samplePeriodService = new SamplePeriodService(this.connection); - - // Get sampling information for the survey to later validate - const samplingPeriods = await samplePeriodService.getSamplePeriodsForSurvey(surveyId); - - // -------------------------------------------------------------------------------------------------------------- - - // SamplePeriodHierarchyIds is only for when all records are being assigned to the same sampling period - let samplePeriodHierarchyIds: SurveySamplePeriodDetails; - - if (options?.surveySamplePeriodId) { - const samplePeriodService = new SamplePeriodService(this.connection); - samplePeriodHierarchyIds = await samplePeriodService.getSamplePeriodById(surveyId, options.surveySamplePeriodId); - } - - // Get subcount sign options and default option for when sign is null - const codeMap = new Map( - codeTypeDefinitions.OBSERVATION_SUBCOUNT_SIGN.map((option) => [option.name.toLowerCase(), option.id]) - ); - const defaultSubcountSignId = codeMap.get(defaultSubcountSign) || null; - - // Merge all the table rows into an array of InsertUpdateObservations[] - const newRowData: InsertUpdateObservations[] = worksheetRowObjects.map((row) => { - const observationSubcountSignId = this._getCodeIdFromCellValue( - getColumnCellValue(row, 'OBSERVATION_SUBCOUNT_SIGN').cell, - codeMap, - defaultSubcountSignId - ); - - const newSubcount: InsertSubCount = { - observation_subcount_id: null, - subcount: getColumnCellValue(row, 'COUNT').cell as number, - observation_subcount_sign_id: observationSubcountSignId ?? null, - comment: (getColumnCellValue(row, 'COMMENT').cell as string) ?? null, - qualitative_measurements: [], - quantitative_measurements: [], - qualitative_environments: [], - quantitative_environments: [] - }; - - const measurements = pullMeasurementsFromWorkSheetRowObject( - row, - measurementColumnNames, - tsnMeasurementTypeDefinitionMap - ); - newSubcount.qualitative_measurements = measurements.qualitative_measurements; - newSubcount.quantitative_measurements = measurements.quantitative_measurements; - - const environments = pullEnvironmentsFromWorkSheetRowObject( - row, - environmentColumnNames, - environmentColumnsTypeDefinitionMap - ); - newSubcount.qualitative_environments = environments.qualitative_environments; - newSubcount.quantitative_environments = environments.quantitative_environments; - - // If surveySamplePeriodId was included in the initial request, assign all rows to that sampling period - if (options?.surveySamplePeriodId) { - return { - standardColumns: { - survey_id: surveyId, - itis_tsn: getColumnCellValue(row, 'ITIS_TSN').cell as number, - itis_scientific_name: null, - survey_sample_site_id: samplePeriodHierarchyIds?.survey_sample_site_id ?? null, - method_technique_id: samplePeriodHierarchyIds?.method_technique_id ?? null, - survey_sample_period_id: samplePeriodHierarchyIds?.survey_sample_period_id ?? null, - latitude: getColumnCellValue(row, 'LATITUDE').cell as number, - longitude: getColumnCellValue(row, 'LONGITUDE').cell as number, - count: getColumnCellValue(row, 'COUNT').cell as number, - observation_time: getColumnCellValue(row, 'TIME').cell as string, - observation_date: getColumnCellValue(row, 'DATE').cell as string - }, - subcounts: [newSubcount] - }; - } - - // PROCESS AND VALIDATE SAMPLING INFORMATION ----------------------------------------------------------------------------------------- - const samplingData = pullSamplingDataFromWorksheetRowObject(row, samplingPeriods); - - if (!samplingData && getColumnCellValue(row, 'SAMPLING_SITE').cell) { - throw new Error('Failed to process file for importing observations. Sampling data validator failed.'); - } - - return { - standardColumns: { - survey_id: surveyId, - itis_tsn: getColumnCellValue(row, 'ITIS_TSN').cell as number, - itis_scientific_name: null, - survey_sample_site_id: samplingData?.sampleSiteId ?? null, - method_technique_id: samplingData?.methodTechniqueId ?? null, - survey_sample_period_id: samplingData?.samplePeriodId ?? null, - latitude: getColumnCellValue(row, 'LATITUDE').cell as number, - longitude: getColumnCellValue(row, 'LONGITUDE').cell as number, - count: getColumnCellValue(row, 'COUNT').cell as number, - observation_time: getColumnCellValue(row, 'TIME').cell as string, - observation_date: getColumnCellValue(row, 'DATE').cell as string - }, - subcounts: [newSubcount] - }; - }); - - // Insert the parsed observation rows - await this.insertUpdateManualSurveyObservations(surveyId, newRowData); - } - - /** - * Maps over an array of inserted/updated observation records in order to update its scientific - * name to match its ITIS TSN. - * - * @template RecordWithTaxonFields - * @param {RecordWithTaxonFields[]} recordsToPatch - * @return {*} {Promise} - * @memberof ObservationService - */ - async _attachItisScientificName< - RecordWithTaxonFields extends Pick - >(recordsToPatch: RecordWithTaxonFields[]): Promise { - defaultLog.debug({ label: '_attachItisScientificName' }); - - const platformService = new PlatformService(this.connection); - - const uniqueTsnSet: Set = recordsToPatch.reduce((acc: Set, record: RecordWithTaxonFields) => { - if (record.itis_tsn) { - acc.add(record.itis_tsn); - } - return acc; - }, new Set([])); - - const taxonomyResponse = await platformService.getTaxonomyByTsns(Array.from(uniqueTsnSet)).catch((error) => { - throw new ApiGeneralError( - `Failed to fetch scientific names for observation records. The request to BioHub failed: ${error}` - ); - }); - - return recordsToPatch.map((recordToPatch: RecordWithTaxonFields) => { - recordToPatch.itis_scientific_name = - taxonomyResponse.find((taxonItem) => taxonItem.tsn === recordToPatch.itis_tsn)?.scientificName ?? null; - - return recordToPatch; - }); - } - /** * Deletes all survey_observation records for the given survey observation ids. * @@ -933,29 +604,4 @@ export class ObservationService extends DBService { // Return true if both environments and measurements are valid return true; } - - /** - * Gets the code id value with a matching name from a pre-mapped set of options. If the function returns null, the - * request should probably throw an error. - * - * @param cellValue The name of a code to find the id for - * @param codeMap A Map where the key is the normalized code name and the value is the ID - * @param defaultCodeId A precomputed default code ID for cases where cellValue is null - * @returns The ID of the matching code, or the default ID, or null if no match is found - */ - _getCodeIdFromCellValue( - cellValue: string | null, - codeMap: Map, - defaultCodeId?: number | null - ): number | null { - const value = cellValue?.toLowerCase(); // Normalize the cell value - - // If no value exists, return the default code ID or null - if (!value) { - return defaultCodeId ?? null; - } - - // Return the ID from the map if it exists, otherwise return null - return codeMap.get(value) ?? null; - } } diff --git a/api/src/services/observation-services/utils.ts b/api/src/services/observation-services/utils.ts deleted file mode 100644 index 10e992ef19..0000000000 --- a/api/src/services/observation-services/utils.ts +++ /dev/null @@ -1,499 +0,0 @@ -import dayjs from 'dayjs'; -import isSameOrAfter from 'dayjs/plugin/isSameOrAfter'; -import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'; -import { DefaultDateFormat, DefaultTimeFormat, DefaultTimeFormatNoSeconds } from '../../constants/dates'; -import { SurveySamplePeriodDetails } from '../../repositories/sample-period-repository'; -import { isDateString, isDateTimeString, isTimeString } from '../../utils/date-time-utils'; -import { - EnvironmentNameTypeDefinitionMap, - isEnvironmentQualitativeTypeDefinition -} from '../../utils/observation-xlsx-utils/environment-column-utils'; -import { - getMeasurementFromTsnMeasurementTypeDefinitionMap, - isMeasurementCBQualitativeTypeDefinition, - TsnMeasurementTypeDefinitionMap -} from '../../utils/observation-xlsx-utils/measurement-column-utils'; -import { getColumnCellValue, InsertSubCount } from './observation-service'; - -dayjs.extend(isSameOrAfter); -dayjs.extend(isSameOrBefore); - -/** - * A helper function that will take a row object from a worksheet and attempt to find a matching sampling period. - * - * If the row contains a sampling site, technique, or period, then the function will attempt to find a unique period - * that matches all of the provided (non-null) values. - * - * If the row does not contain a sampling site, technique, and period, then the function will attempt to find a unique - * period that matches the observation date and time. - * - * @param {Record} row The current row of the worksheet being processed. - * @param {SurveySamplePeriodDetails[]} samplingPeriods All available sampling periods for the survey. - * @return {*} {({ samplePeriodId: number; sampleSiteId: number | null; methodTechniqueId: number | null } | null)} A - * matching sampling period object, or null if no periods match the row data. - */ -export function pullSamplingDataFromWorksheetRowObject( - row: Record, - samplingPeriods: SurveySamplePeriodDetails[] -): { samplePeriodId: number; sampleSiteId: number | null; methodTechniqueId: number | null } | null { - // Extract site, technique, and period data from the row - const worksheetSiteName = getColumnCellValue(row, 'SAMPLING_SITE').cell as string | null; - const worksheetTechniqueName = getColumnCellValue(row, 'METHOD_TECHNIQUE').cell as string | null; - const worksheetPeriod = getColumnCellValue(row, 'SAMPLING_PERIOD').cell as string | null; - - // If any of the site, technique, or period values are provided, then attempt to find a unique period - // that matches all of provided (non-null) values. - if (worksheetSiteName || worksheetTechniqueName || worksheetPeriod) { - // Find all periods that match the provided site, technique, and period - // Periods must match all non-null worksheet values to be considered a match - const matchingPeriodsBySamplingInformation = samplingPeriods.filter((period) => { - if (worksheetSiteName) { - const isMatch = matchSamplePeriodToWorksheetSiteName(worksheetSiteName, period); - - if (!isMatch) { - // If the worksheet site name is provided but does not match, then this period is not a match - return false; - } - } - - if (worksheetTechniqueName) { - const isMatch = matchSamplePeriodToWorksheetTechniqueName(worksheetTechniqueName, period); - - if (!isMatch) { - // If the worksheet technique name is provided but does not match, then this period is not a match - return false; - } - } - - if (worksheetPeriod) { - const isMatch = matchSamplePeriodToWorksheetPeriod(worksheetPeriod, period); - - if (!isMatch) { - // If the worksheet period is provided but does not match, then this period is not a match - return false; - } - } - - // If all provided (non-null) values match, then consider this period a match - return true; - }); - - if (matchingPeriodsBySamplingInformation.length === 1) { - // Found exactly one period record that uniquely matches some or all of the filters above - return formatMatchingPeriod(matchingPeriodsBySamplingInformation[0]); - } - - // Unable to match the observation sampling information to any existing period uniquely - return null; - } - - // If not site, technique, or period values are provided, then attempt to find a unique period that matches the - // observation date and time - const observationDate = getColumnCellValue(row, 'DATE').cell as string | null; - const observationTime = getColumnCellValue(row, 'TIME').cell as string | null; - - const matchingPeriodsByObservationDateTime = matchSamplePeriodsToObservationDateTime( - observationDate, - observationTime, - samplingPeriods - ); - - if (matchingPeriodsByObservationDateTime.length) { - // If at least one period record is found that satisfies the observation date and time, then return the first one - return formatMatchingPeriod(matchingPeriodsByObservationDateTime[0]); - } - - // Unable to match the observation date/time to any existing period uniquely - return null; -} - -/** - * This function is a helper method for the `pullSamplingDataFromWorksheetRowObject` function. - * - * It will compare a worksheet site name to a sampling site object and return true if the names match. - * - * @param {string} worksheetSiteName - * @param {SurveySamplePeriodDetails} samplingPeriod - * @return {*} {boolean} - */ -export function matchSamplePeriodToWorksheetSiteName( - worksheetSiteName: string, - samplingPeriod: SurveySamplePeriodDetails -): boolean { - return samplingPeriod.survey_sample_site?.name === worksheetSiteName; -} - -/** - * This function is a helper method for the `pullSamplingDataFromWorksheetRowObject` function. - * - * It will compare a worksheet technique name to a sampling technique object and return true if the names match. - * - * @param {string} worksheetTechniqueName - * @param {SurveySamplePeriodDetails} samplingPeriod - * @return {*} {boolean} - */ -export function matchSamplePeriodToWorksheetTechniqueName( - worksheetTechniqueName: string, - samplingPeriod: SurveySamplePeriodDetails -): boolean { - return samplingPeriod.method_technique?.name === worksheetTechniqueName; -} - -/** - * This function is a helper method for the `pullSamplingDataFromWorksheetRowObject` function. - * - * It will compare a worksheet period string to a sampling period object and return true if they match. - * - * They must match on start_date and end_date. And optionally match on start_time and end_time if they are present - * in the worksheet period string. - * - * Note: This function relies on the incoming period string to separate the start and end dates with a " - " delimiter. - * - * @param {string} period A string in the format "YYYY-MM-DDTHH:mm:ss - YYYY-MM-DDTHH:mm:ss", or a valid subset or - * superset. (Ex: "2024-07-28 - 2024-07-29", "2024-07-28T00:00:00 - 2024-07-29T23:59:59", etc) - * @param {SurveySamplePeriodDetails[]} samplingPeriods - * @return {*} {SurveySamplePeriodDetails} - */ -export function matchSamplePeriodToWorksheetPeriod( - worksheetPeriod: string, - samplingPeriod: SurveySamplePeriodDetails -): boolean { - const [worksheetStartDateTime, worksheetEndDateTime] = worksheetPeriod.split(' - '); - - if (!worksheetStartDateTime || !worksheetEndDateTime) { - // Failed to split the period string into expected start and end date strings - return false; - } - - if (!matchSamplePeriodDateToWorksheetPeriodDateTime(worksheetStartDateTime, samplingPeriod.start_date ?? '')) { - // Failed to match the start date - return false; - } - - if (!matchSamplePeriodDateToWorksheetPeriodDateTime(worksheetEndDateTime, samplingPeriod.end_date ?? '')) { - // Failed to match the end date - return false; - } - - if (!matchSamplePeriodTimeToWorksheetPeriodDateTime(worksheetStartDateTime, samplingPeriod.start_time ?? '')) { - // Failed to match the start time - return false; - } - - if (!matchSamplePeriodTimeToWorksheetPeriodDateTime(worksheetEndDateTime, samplingPeriod.end_time ?? '')) { - // Failed to match the end time - return false; - } - - // Successfully matched the period - return true; -} - -/** - * This function is a helper method for the `matchSamplePeriodToWorksheetPeriod` function. - * - * It will compare a worksheet date-time string to a sampling period date string and return true if they match. - * - * @example - * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01 11:00:00', '2021-01-01') // true - * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '2021-01-01') // true - * - * matchSamplePeriodDateToWorksheetPeriodDateTime('', '') // true - * - * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01 11:00:00', '2022-02-02') // false - * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '2022-02-02') // false - * - * matchSamplePeriodDateToWorksheetPeriodDateTime('2021-01-01', '') // false - * matchSamplePeriodDateToWorksheetPeriodDateTime('', '2021-01-01') // false - * - * @export - * @param {string} worksheetDateTimeString - * @param {string} samplingPeriodDateString - * @return {*} {boolean} - */ -export function matchSamplePeriodDateToWorksheetPeriodDateTime( - worksheetDateTimeString: string, - samplingPeriodDateString: string -): boolean { - const isWorksheetDateTimeString = isDateString(worksheetDateTimeString); - const isSamplingPeriodDateString = isDateString(samplingPeriodDateString); - - if (isWorksheetDateTimeString !== isSamplingPeriodDateString) { - // If either string contains date information, and the other does not, then they cannot possibly match - return false; - } - - if (!isWorksheetDateTimeString && !isSamplingPeriodDateString) { - // If neither string contains date information, then they are considered to match - return true; - } - - const formattedWorksheetDateString = dayjs(worksheetDateTimeString).format(DefaultDateFormat); - const formattedSamplingPeriodDateString = dayjs(samplingPeriodDateString).format(DefaultDateFormat); - - if (formattedWorksheetDateString !== formattedSamplingPeriodDateString) { - // Failed to match the date strings - return false; - } - - return true; -} - -/** - * This function is a helper method for the `matchSamplePeriodToWorksheetPeriod` function. - * - * It will compare a worksheet date-time string to a sampling period time string and return true if they match. - * - * @example - * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '11:00:00') // true - * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '11:00') // true - * - * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01', '') // true - * matchSamplePeriodTimeToWorksheetPeriodDateTime('', '') // true - * matchSamplePeriodTimeToWorksheetPeriodDateTime('not_a_time', 'invalid_time') // true - * - * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01 11:00:00', '12:00:00') // false - * matchSamplePeriodTimeToWorksheetPeriodDateTime('2021-01-01', '11:00:00') // false - * - * matchSamplePeriodTimeToWorksheetPeriodDateTime('11:00:00', '') // false - * matchSamplePeriodTimeToWorksheetPeriodDateTime('', '11:00:00') // false - * - * @export - * @param {string} worksheetDateTimeString - * @param {string} samplingPeriodTimeString - * @return {*} {boolean} - */ -export function matchSamplePeriodTimeToWorksheetPeriodDateTime( - worksheetDateTimeString: string, - samplingPeriodTimeString: string -): boolean { - const isWorksheetDateTimeString = isDateTimeString(worksheetDateTimeString); - const isSamplingPeriodTimeString = isTimeString(samplingPeriodTimeString); - - if (isWorksheetDateTimeString !== isSamplingPeriodTimeString) { - // If either string contains time information, and the other does not, then they cannot possibly match - return false; - } - - if (!isWorksheetDateTimeString && !isSamplingPeriodTimeString) { - // If neither string contains time information, then they are considered to match - return true; - } - - const formattedWorksheetTimeString = dayjs(worksheetDateTimeString).format(DefaultTimeFormat); - const formattedSamplingPeriodTimeString = dayjs(samplingPeriodTimeString, [ - DefaultTimeFormat, - DefaultTimeFormatNoSeconds - ]).format(DefaultTimeFormat); - - if (formattedWorksheetTimeString !== formattedSamplingPeriodTimeString) { - // Failed to match the time strings - return false; - } - - return true; -} - -/** - * This function is a helper method for the `pullSamplingDataFromWorksheetRowObject` function. It will take an - * observation date and time and find all matching sampling periods from the provided samplingPeriods. - * - * @param {(string | null)} observationDate - * @param {(string | null)} observationTime - * @param {SurveySamplePeriodDetails[]} samplingPeriods - * @return {*} - */ -export function matchSamplePeriodsToObservationDateTime( - observationDate: string | null, - observationTime: string | null, - samplingPeriods: SurveySamplePeriodDetails[] -) { - if (!observationDate) { - // If no observation date is provided, then no periods can be matched - return []; - } - - if (!samplingPeriods.length) { - // If no sampling periods are provided, then no periods can be matched - return []; - } - - const formattedObservationDateTime = observationTime - ? dayjs(`${observationDate} ${observationTime}`) - : dayjs(observationDate); - - const suitablePeriods = samplingPeriods.filter((samplingPeriod) => { - if (!samplingPeriod.start_date || !samplingPeriod.end_date) { - // If the sampling period does not have a start or end date, then it cannot be matched - return false; - } - - const formattedSamplingPeriodStartDateTime = samplingPeriod.start_time - ? dayjs(`${samplingPeriod.start_date} ${samplingPeriod.start_time}`) - : dayjs(samplingPeriod.start_date); - - const formattedSamplingPeriodEndDateTime = samplingPeriod.end_time - ? dayjs(`${samplingPeriod.end_date} ${samplingPeriod.end_time}`) - : dayjs(samplingPeriod.end_date); - - // The observation date time must be within the start and end date time of the sampling period - return ( - formattedObservationDateTime.isSameOrAfter(formattedSamplingPeriodStartDateTime) && - formattedObservationDateTime.isSameOrBefore(formattedSamplingPeriodEndDateTime) - ); - }); - - return suitablePeriods; -} - -/** - * This function is a helper method for the `pullSamplingDataFromWorksheetRowObject` function. It will take a matching - * period and format it to return the sample site, method technique, and sample period IDs. - * - * @param {SurveySamplePeriodDetails} period - * @return {*} - */ -export function formatMatchingPeriod(period: SurveySamplePeriodDetails) { - return { - samplePeriodId: period.survey_sample_period_id, - sampleSiteId: period.survey_sample_site_id, - methodTechniqueId: period.method_technique_id - }; -} - -/** - * This function is a helper method for the `processObservationCsvSubmission` function. It will take row data from an - * uploaded CSV and find and connect the CSV measurement data with proper measurement taxon ids (UUIDs) from the - * TsnMeasurementTypeDefinitionMap passed in. Any qualitative and quantitative measurements found are returned to be - * inserted into the database. This function assumes that the data in the CSV has already been validated. - * - * @param {Record} row A worksheet row object from a CSV that was uploaded for processing - * @param {string[]} measurementColumns A list of the measurement columns found in a CSV uploaded - * @param {TsnMeasurementTypeDefinitionMap} tsnMeasurements Map of TSNs and their valid measurements - * @return {*} {(Pick)} - * @memberof ObservationService - */ -export function pullMeasurementsFromWorkSheetRowObject( - row: Record, - measurementColumns: string[], - tsnMeasurements: TsnMeasurementTypeDefinitionMap -): Pick { - const foundMeasurements: Pick = { - qualitative_measurements: [], - quantitative_measurements: [] - }; - - measurementColumns.forEach((mColumn) => { - // Ignore blank columns - if (!mColumn) { - return; - } - - const rowData = row[mColumn]; - - // Ignore empty rows - if (rowData === undefined) { - return; - } - - const measurement = getMeasurementFromTsnMeasurementTypeDefinitionMap( - getColumnCellValue(row, 'ITIS_TSN').cell as string, - mColumn, - tsnMeasurements - ); - - // Ignore empty measurements - if (!measurement) { - return; - } - - // if measurement is qualitative, find the option uuid - if (isMeasurementCBQualitativeTypeDefinition(measurement)) { - const foundOption = measurement.options.find( - (option) => - option.option_label.toLowerCase() === String(rowData).toLowerCase() || - option.option_value === Number(rowData) || - option.qualitative_option_id === rowData - ); - - if (!foundOption) { - return; - } - - foundMeasurements.qualitative_measurements.push({ - measurement_id: measurement.taxon_measurement_id, - measurement_option_id: foundOption.qualitative_option_id - }); - } else { - foundMeasurements.quantitative_measurements.push({ - measurement_id: measurement.taxon_measurement_id, - measurement_value: Number(rowData) - }); - } - }); - - return foundMeasurements; -} - -/** - * This function is a helper method for the `processObservationCsvSubmission` function. It will take row data from an - * uploaded CSV. - * - * @export - * @param {Record} row - * @param {string[]} environmentColumns - * @param {EnvironmentNameTypeDefinitionMap} environmentNameTypeDefinitionMap - * @return {*} {(Pick)} - */ -export function pullEnvironmentsFromWorkSheetRowObject( - row: Record, - environmentColumns: string[], - environmentNameTypeDefinitionMap: EnvironmentNameTypeDefinitionMap -): Pick { - const foundEnvironments: Pick = { - qualitative_environments: [], - quantitative_environments: [] - }; - - environmentColumns.forEach((mColumn) => { - // Ignore blank columns - if (!mColumn) { - return; - } - - const rowData = row[mColumn]; - - // Ignore empty rows - if (rowData === undefined) { - return; - } - - const environment = environmentNameTypeDefinitionMap.get(mColumn); - - // Ignore empty environments - if (!environment) { - return; - } - - // if environment is qualitative, find the option id - if (isEnvironmentQualitativeTypeDefinition(environment)) { - const foundOption = environment.options.find((option) => option.name === String(rowData).toLowerCase()); - - if (!foundOption) { - return; - } - - foundEnvironments.qualitative_environments.push({ - environment_qualitative_id: foundOption.environment_qualitative_id, - environment_qualitative_option_id: foundOption.environment_qualitative_option_id - }); - } else { - foundEnvironments.quantitative_environments.push({ - environment_quantitative_id: environment.environment_quantitative_id, - value: Number(rowData) - }); - } - }); - - return foundEnvironments; -} diff --git a/api/src/services/platform-service.test.ts b/api/src/services/platform-service.test.ts index 10794525da..acea502eb4 100644 --- a/api/src/services/platform-service.test.ts +++ b/api/src/services/platform-service.test.ts @@ -4,6 +4,7 @@ import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; import { SurveyObservationRecord } from '../database-models/survey_observation'; +import * as envConfig from '../utils/env-config'; import * as featureFlagUtils from '../utils/feature-flag-utils'; import { getMockDBConnection } from '../__mocks__/db'; import { AttachmentService } from './attachment-service'; @@ -20,6 +21,90 @@ describe('PlatformService', () => { sinon.restore(); }); + describe('getTaxonByScientificName', () => { + it('should return a taxon by scientific name', async () => { + const mockDBConnection = getMockDBConnection(); + const platformService = new PlatformService(mockDBConnection); + + const mockAxiosResponse = { + data: { + searchResponse: [ + { + tsn: 1, + scientificName: 'alces alces' + } + ] + } + }; + + const getEnvironmentVariableStub = sinon.stub(envConfig, 'getEnvironmentVariable'); + sinon.stub(KeycloakService.prototype, 'getKeycloakServiceToken').resolves('token'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + getEnvironmentVariableStub.onCall(0).returns('/taxon'); + getEnvironmentVariableStub.onCall(1).returns('https://url.com'); + + const taxon = await platformService.getTaxonByScientificName('Alces alces'); + + expect(axiosStub.getCall(0).args[1]?.headers?.authorization).to.equal('Bearer token'); + expect(axiosStub.getCall(0).args[1]?.params.terms).to.deep.equal(['Alces', 'alces']); + + expect(taxon).to.deep.equal({ tsn: 1, scientificName: 'alces alces' }); + }); + + it('should return a null when unable to find match by scientific name', async () => { + const mockDBConnection = getMockDBConnection(); + const platformService = new PlatformService(mockDBConnection); + + const mockAxiosResponse = { + data: { + searchResponse: [ + { + tsn: 1, + scientificName: 'unknown taxon' + } + ] + } + }; + + const getEnvironmentVariableStub = sinon.stub(envConfig, 'getEnvironmentVariable'); + sinon.stub(KeycloakService.prototype, 'getKeycloakServiceToken').resolves('token'); + + const axiosStub = sinon.stub(axios, 'get').resolves(mockAxiosResponse); + + getEnvironmentVariableStub.onCall(0).returns('/taxon'); + getEnvironmentVariableStub.onCall(1).returns('https://url.com'); + + const taxon = await platformService.getTaxonByScientificName('Alces alces'); + + expect(axiosStub.getCall(0).args[1]?.headers?.authorization).to.equal('Bearer token'); + expect(axiosStub.getCall(0).args[1]?.params.terms).to.deep.equal(['Alces', 'alces']); + + expect(taxon).to.equal(null); + }); + + it('should return a null error thrown', async () => { + const mockDBConnection = getMockDBConnection(); + const platformService = new PlatformService(mockDBConnection); + + const getEnvironmentVariableStub = sinon.stub(envConfig, 'getEnvironmentVariable'); + sinon.stub(KeycloakService.prototype, 'getKeycloakServiceToken').resolves('token'); + + const axiosStub = sinon.stub(axios, 'get').rejects(new Error('error')); + + getEnvironmentVariableStub.onCall(0).returns('/taxon'); + getEnvironmentVariableStub.onCall(1).returns('https://url.com'); + + const taxon = await platformService.getTaxonByScientificName('Alces alces'); + + expect(axiosStub.getCall(0).args[1]?.headers?.authorization).to.equal('Bearer token'); + expect(axiosStub.getCall(0).args[1]?.params.terms).to.deep.equal(['Alces', 'alces']); + + expect(taxon).to.equal(null); + }); + }); + describe('submitSurveyToBioHub', () => { afterEach(() => { sinon.restore(); diff --git a/api/src/services/platform-service.ts b/api/src/services/platform-service.ts index fbb75cbecc..38db531692 100644 --- a/api/src/services/platform-service.ts +++ b/api/src/services/platform-service.ts @@ -6,8 +6,10 @@ import qs from 'qs'; import { URL } from 'url'; import { IDBConnection } from '../database/db'; import { ApiError, ApiErrorType, ApiGeneralError } from '../errors/api-error'; +import { formatAxiosError } from '../errors/axios-error'; import { PostSurveySubmissionToBioHubObject } from '../models/biohub-create'; import { ISurveyAttachment, ISurveyReportAttachment } from '../repositories/attachment-repository'; +import { getEnvironmentVariable } from '../utils/env-config'; import { isFeatureFlagPresent } from '../utils/feature-flag-utils'; import { getFileFromS3 } from '../utils/file-utils'; import { getLogger } from '../utils/logger'; @@ -62,10 +64,11 @@ export interface ITaxonomyWithEcologicalUnits extends ITaxonomy { ecological_units: IPostCollectionUnit[]; } -const getBackboneInternalApiHost = () => process.env.BACKBONE_INTERNAL_API_HOST || ''; -const getBackboneArtifactIntakePath = () => process.env.BACKBONE_ARTIFACT_INTAKE_PATH || ''; -const getBackboneSurveyIntakePath = () => process.env.BACKBONE_INTAKE_PATH || ''; -const getBackboneTaxonTsnPath = () => process.env.BIOHUB_TAXON_TSN_PATH || ''; +const getBackboneInternalApiHost = () => getEnvironmentVariable('BACKBONE_INTERNAL_API_HOST'); +const getBackboneArtifactIntakePath = () => getEnvironmentVariable('BACKBONE_ARTIFACT_INTAKE_PATH'); +const getBackboneSurveyIntakePath = () => getEnvironmentVariable('BACKBONE_INTAKE_PATH'); +const getBackboneTaxonTsnPath = () => getEnvironmentVariable('BIOHUB_TAXON_TSN_PATH'); +const getBackboneTaxonPath = () => getEnvironmentVariable('BIOHUB_TAXON_PATH'); export class PlatformService extends DBService { attachmentService: AttachmentService; @@ -117,6 +120,58 @@ export class PlatformService extends DBService { } } + /** + * Get taxon by scientific name from the BioHub. + * + * @param {string} scientificName - The scientific name of the taxon to search for + * @returns {*} {Promise} The first matching taxon by scientific name or null if not found + */ + async getTaxonByScientificName(scientificName: string): Promise { + defaultLog.debug({ label: 'getTaxonomyByScientificName', scientificName }); + + if (!scientificName) { + return null; + } + + try { + const keycloakService = new KeycloakService(); + + const token = await keycloakService.getKeycloakServiceToken(); + + const backboneTaxonSearchUrl = new URL(getBackboneTaxonPath(), getBackboneInternalApiHost()).href; + + const { data } = await axios.get<{ searchResponse: IItisSearchResult[] }>(backboneTaxonSearchUrl, { + headers: { + authorization: `Bearer ${token}` + }, + params: { + // Biohub searches ITIS by "terms" -> "alces alces" -> ["alces", "alces"] + terms: scientificName.split(' ') + }, + paramsSerializer: (params) => { + return qs.stringify(params); + } + }); + + // Find a matching taxon by scientific name (case-insensitive) + const matchingTaxon = data.searchResponse.find( + (taxon) => taxon.scientificName.toLowerCase() === scientificName.toLowerCase() + ); + + defaultLog.debug({ label: 'getTaxonByScientificName', matchingTaxon }); + + if (!matchingTaxon) { + return null; + } + + return matchingTaxon; + } catch (error) { + defaultLog.error({ label: 'getTaxonByScientificName', error: formatAxiosError(error) }); + + return null; + } + } + /** * Submit survey to BioHub. * diff --git a/api/src/utils/case-insensitive-map.test.ts b/api/src/utils/case-insensitive-map.test.ts new file mode 100644 index 0000000000..316f1174d5 --- /dev/null +++ b/api/src/utils/case-insensitive-map.test.ts @@ -0,0 +1,34 @@ +import { expect } from 'chai'; +import { CaseInsensitiveMap } from './case-insensitive-map'; + +describe('CaseInsensitiveMap', () => { + describe('set / get', () => { + it('should set a key-value pair in the map', () => { + const map = new CaseInsensitiveMap(); + + map.set('KEY', 1); + + expect(map.get('key')).to.be.equal(1); + expect(map.size).to.be.equal(1); + }); + + it('should set a key-value pair in the map with a number key', () => { + const map = new CaseInsensitiveMap(); + + map.set(1, 2); + expect(map.get(1)).to.be.equal(2); + expect(map.size).to.be.equal(1); + }); + }); + + describe('has', () => { + it('should return true if the map has the key', () => { + const map = new CaseInsensitiveMap(); + + map.set('KEY', 1); + + expect(map.has('kEy')).to.be.true; + expect(map.size).to.be.equal(1); + }); + }); +}); diff --git a/api/src/utils/case-insensitive-map.ts b/api/src/utils/case-insensitive-map.ts new file mode 100644 index 0000000000..a7c54d9d8c --- /dev/null +++ b/api/src/utils/case-insensitive-map.ts @@ -0,0 +1,56 @@ +/** + * A case-insensitive map - all string keys are normalized to lowercase. + * + * @example ` + * const map = new CaseInsensitiveMap(); + * map.set('KEY', 1); + * map.get('key'); // 1 + * map.has('kEy') // true + * ` + * @class CaseInsensitiveMap + * @template KeyType - The key type + * @template ValuesType - The value type + * @extends {Map} + */ +export class CaseInsensitiveMap extends Map { + /** + * Set a key-value pair in the map. + * + * @param {KeyType} key - The key + * @param {ValuesType} value - The value + * @returns {this} The map + */ + set(key: KeyType, value: ValuesType): this { + if (typeof key === 'string') { + key = key.toLowerCase() as KeyType; + } + + return super.set(key, value); + } + + /** + * Get a value from the map. + * + * @param {KeyType} key - The key + * @returns {ValuesType | undefined} The value + */ + get(key: KeyType): ValuesType | undefined { + if (typeof key === 'string') { + key = key.toLowerCase() as KeyType; + } + return super.get(key); + } + + /** + * Check if the map has a key. + * + * @param {KeyType} key - The key + * @returns {boolean} Whether the map has the key + */ + has(key: KeyType): boolean { + if (typeof key === 'string') { + key = key.toLowerCase() as KeyType; + } + return super.has(key); + } +} diff --git a/api/src/utils/csv-utils/csv-config-utils.test.ts b/api/src/utils/csv-utils/csv-config-utils.test.ts index 835ffe2b04..684db46b6f 100644 --- a/api/src/utils/csv-utils/csv-config-utils.test.ts +++ b/api/src/utils/csv-utils/csv-config-utils.test.ts @@ -21,7 +21,7 @@ describe('CSVConfigUtils', () => { const utils = new CSVConfigUtils(worksheet, mockConfig); expect(utils).to.be.instanceOf(CSVConfigUtils); - expect(utils._config).to.be.equal(mockConfig); + expect(utils.config).to.be.equal(mockConfig); expect(utils.worksheet).to.be.equal(worksheet); expect(utils.worksheetRows[0]).to.deep.equal({ @@ -224,7 +224,7 @@ describe('CSVConfigUtils', () => { TEST: { validateCell, setCellValue } }); - expect(utils._config).to.be.deep.equal({ + expect(utils.config).to.be.deep.equal({ staticHeadersConfig: { TEST: { aliases: [], validateCell, setCellValue } }, @@ -276,7 +276,7 @@ describe('CSVConfigUtils', () => { const header = utils.getWorksheetHeader('BAD', { TEST: 'cellValue' }); - expect(header).to.be.undefined; + expect(header).to.be.null; }); }); }); diff --git a/api/src/utils/csv-utils/csv-config-utils.ts b/api/src/utils/csv-utils/csv-config-utils.ts index 2398ffc98d..0716804d43 100644 --- a/api/src/utils/csv-utils/csv-config-utils.ts +++ b/api/src/utils/csv-utils/csv-config-utils.ts @@ -11,12 +11,12 @@ import { CSVCell, CSVConfig, CSVHeaderConfig, CSVRow } from './csv-config-valida * @class CSVConfigUtils */ export class CSVConfigUtils = Uppercase> { - _config: CSVConfig; + config: CSVConfig; worksheet: WorkSheet; worksheetRows: CSVRow[]; constructor(worksheet: WorkSheet, config: CSVConfig) { - this._config = config; + this.config = config; this.worksheet = worksheet; this.worksheetRows = getWorksheetRowObjects(worksheet); } @@ -27,7 +27,7 @@ export class CSVConfigUtils = Upperca * @returns {StaticHeaderType[]} - The config headers */ get configStaticHeaders(): StaticHeaderType[] { - return Object.keys(this._config.staticHeadersConfig) as StaticHeaderType[]; + return Object.keys(this.config.staticHeadersConfig) as StaticHeaderType[]; } /** @@ -61,7 +61,7 @@ export class CSVConfigUtils = Upperca staticHeaders.push(header); } - const aliases = this._config.staticHeadersConfig[header].aliases; + const aliases = this.config.staticHeadersConfig[header].aliases; for (const alias of aliases) { if (worksheetHeaders.has(alias)) { @@ -92,7 +92,7 @@ export class CSVConfigUtils = Upperca staticHeaders.push(header); } - const aliases = this._config.staticHeadersConfig[header].aliases; + const aliases = this.config.staticHeadersConfig[header].aliases; for (const alias of aliases) { if (worksheetHeaders.has(alias)) { @@ -126,20 +126,22 @@ export class CSVConfigUtils = Upperca * * @param {StaticHeaderType} header - The header name * @param {CSVRow} row - The CSV row - * @returns {Uppercase | undefined} - The header name + * @returns {Uppercase | null} - The header name or null if not found */ - getWorksheetHeader(header: StaticHeaderType, row: CSVRow): Uppercase | undefined { + getWorksheetHeader(header: StaticHeaderType, row: CSVRow): Uppercase | null { // Static header or dynamic header exact match if ((header as Uppercase) in row) { return header; } // Attempt to find the matching header from the header aliases - for (const alias of this._config.staticHeadersConfig[header]?.aliases ?? []) { + for (const alias of this.config.staticHeadersConfig[header]?.aliases ?? []) { if (alias in row) { return alias; } } + + return null; } /** @@ -150,7 +152,7 @@ export class CSVConfigUtils = Upperca * @returns {void} */ setStaticHeaderConfig(header: StaticHeaderType, headerConfig: CSVHeaderConfig): void { - this._config.staticHeadersConfig[header] = { ...this._config.staticHeadersConfig[header], ...headerConfig }; + this.config.staticHeadersConfig[header] = { ...this.config.staticHeadersConfig[header], ...headerConfig }; } /** @@ -173,12 +175,12 @@ export class CSVConfigUtils = Upperca */ getConfig(): CSVConfig { for (const header of this.configStaticHeaders) { - if (!this._config.staticHeadersConfig[header].validateCell) { + if (!this.config.staticHeadersConfig[header].validateCell) { throw new Error(`Invalid CSV config. Missing 'validateCell' for static header: ${header}`); } } - return this._config; + return this.config; } /** @@ -195,7 +197,7 @@ export class CSVConfigUtils = Upperca } // Attempt to find the cell value from the header aliases - for (const alias of this._config.staticHeadersConfig[header]?.aliases ?? []) { + for (const alias of this.config.staticHeadersConfig[header]?.aliases ?? []) { if (alias in row) { return row[alias]; } @@ -213,7 +215,7 @@ export class CSVConfigUtils = Upperca } /** - * Get all the unique cell values from a static header. + * Get all the unique cell values from a static header - case sensitive. * * @param {StaticHeaderType} header - The header name * @returns {any[]} - The unique cell values diff --git a/api/src/utils/csv-utils/csv-config-validation.interface.ts b/api/src/utils/csv-utils/csv-config-validation.interface.ts index c881b97fd9..21f5554e6e 100644 --- a/api/src/utils/csv-utils/csv-config-validation.interface.ts +++ b/api/src/utils/csv-utils/csv-config-validation.interface.ts @@ -255,7 +255,7 @@ export interface CSVError { * validators. This is to ensure the error object is consistent across the validators. * */ -type CSVRowError = Prettify | string | null }>; +export type CSVRowError = Prettify & { header: Uppercase | string | null }>; /** * The CSV row state symbol to store additional row metadata diff --git a/api/src/utils/csv-utils/csv-header-configs.test.ts b/api/src/utils/csv-utils/csv-header-configs.test.ts index c377eaa9f5..ffa52cdb11 100644 --- a/api/src/utils/csv-utils/csv-header-configs.test.ts +++ b/api/src/utils/csv-utils/csv-header-configs.test.ts @@ -2,9 +2,12 @@ import { expect } from 'chai'; import { z } from 'zod'; import { CSVParams, CSVRow, CSVRowState } from './csv-config-validation.interface'; import { + getDateRangeCellValidator, getDescriptionCellValidator, getLatitudeCellValidator, getLongitudeCellValidator, + getNonEmptyStringCellValidator, + getPositiveNumberCellValidator, getSurveyCritterAliasCellValidator, getTsnCellValidator, updateCSVRowState, @@ -109,19 +112,23 @@ describe('CSVHeaderConfigs', () => { it('should return an empty array if the cell is valid', () => { const descriptionValidator = getDescriptionCellValidator(); - const result = descriptionValidator({ - cell: 'description', - row: {}, - header: 'HEADER', - rowIndex: 0, - mutateCell: 'description' - }); + const validDescriptions = ['description', '1', 1, undefined, ' test']; - expect(result).to.be.deep.equal([]); + for (const validDescription of validDescriptions) { + const result = descriptionValidator({ + cell: validDescription, + row: {}, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'description' + }); + + expect(result).to.be.deep.equal([]); + } }); it('should return a single error when invalid', () => { - const badDescriptions = ['', 2, null, ' ']; + const badDescriptions = ['', null, ' ']; for (const badDescription of badDescriptions) { const descriptionValidator = getDescriptionCellValidator(); @@ -255,4 +262,106 @@ describe('CSVHeaderConfigs', () => { expect(params.row[CSVRowState]?.critterId).to.be.equal('uuid'); }); }); + + describe('getPositiveNumberCellValidator', () => { + it('should return an empty array if the cell is optional and undefined', () => { + const positiveNumberValidator = getPositiveNumberCellValidator({ optional: true }); + + const result = positiveNumberValidator({ cell: undefined } as CSVParams); + + expect(result).to.be.deep.equal([]); + }); + + it('should return an empty array if the cell is valid', () => { + const positiveNumberValidator = getPositiveNumberCellValidator({ optional: false }); + + const values = [1, 0.1, 100]; + + for (const value of values) { + const result = positiveNumberValidator({ cell: value } as CSVParams); + + expect(result).to.be.deep.equal([]); + } + }); + + it('should return a single error when invalid', () => { + const positiveNumberValidator = getPositiveNumberCellValidator({ optional: false }); + + const badValues = [0, -1, -0.1, 'string', null, undefined]; + + for (const badValue of badValues) { + const result = positiveNumberValidator({ cell: badValue } as CSVParams); + + expect(result.length).to.be.equal(1); + } + }); + }); + + describe('getNonEmptyStringCellValidator', () => { + it('should return an empty array if the cell is optional and undefined', () => { + const nonEmptyStringValidator = getNonEmptyStringCellValidator({ optional: true }); + + const result = nonEmptyStringValidator({ cell: undefined } as CSVParams); + + expect(result).to.be.deep.equal([]); + }); + + it('should return an empty array if the cell is optional and undefined', () => { + const nonEmptyStringValidator = getNonEmptyStringCellValidator({ optional: true }); + + const result = nonEmptyStringValidator({ cell: undefined } as CSVParams); + + expect(result).to.be.deep.equal([]); + }); + + it('should return an empty array if the cell is valid', () => { + const nonEmptyStringValidator = getNonEmptyStringCellValidator({ optional: false }); + + const values = ['string', '0', '0.1', ' test']; + + for (const value of values) { + const result = nonEmptyStringValidator({ cell: value } as CSVParams); + + expect(result).to.be.deep.equal([]); + } + }); + + it('should return a single error when invalid', () => { + const nonEmptyStringValidator = getNonEmptyStringCellValidator({ optional: false }); + + const badValues = ['', ' ', null, undefined]; + + for (const badValue of badValues) { + const result = nonEmptyStringValidator({ cell: badValue } as CSVParams); + + expect(result.length).to.be.equal(1); + } + }); + }); + + describe('getDateRangeCellValidator', () => { + it('should return an empty array when the cell is valid (timestamps)', () => { + const dateRangeValidator = getDateRangeCellValidator({ optional: false }); + const result = dateRangeValidator({ cell: '2021-01-01 10:10:10 - 2021-01-02 10:10:10' } as CSVParams); + expect(result).to.be.deep.equal([]); + }); + + it('should return an empty array if the cell is valid', () => { + const dateRangeValidator = getDateRangeCellValidator({ optional: false }); + const result = dateRangeValidator({ cell: '2021-01-01 - 2021-01-02' } as CSVParams); + expect(result).to.be.deep.equal([]); + }); + + it('should return a single error when invalid', () => { + const dateRangeValidator = getDateRangeCellValidator({ optional: false }); + const result = dateRangeValidator({ cell: '2021-01-01 - 2021-01-02 - 2021-01-03' } as CSVParams); + expect(result.length).to.be.equal(1); + }); + + it('shoud return an empty array if the cell is optional and undefined', () => { + const dateRangeValidator = getDateRangeCellValidator({ optional: true }); + const result = dateRangeValidator({ cell: undefined } as CSVParams); + expect(result).to.be.deep.equal([]); + }); + }); }); diff --git a/api/src/utils/csv-utils/csv-header-configs.ts b/api/src/utils/csv-utils/csv-header-configs.ts index d097e59c94..53b9805095 100644 --- a/api/src/utils/csv-utils/csv-header-configs.ts +++ b/api/src/utils/csv-utils/csv-header-configs.ts @@ -1,6 +1,7 @@ import { z } from 'zod'; import { ICritterDetailed } from '../../services/critterbase-service'; import { formatTimeString } from '../../services/import-services/utils/datetime'; +import { isDateString } from '../date-time-utils'; import { CSVCellSetter, CSVCellValidator, @@ -64,6 +65,46 @@ export const validateZodCell = (cell: unknown, schema: z.ZodSchema, solution?: s return errors; }; +/** + * Get the positive number header cell validator. + * + * Rules: + * 1. The cell must be a positive number + * 2. The cell is optional if the optional flag is set + * + * @param {CSVOptionalCell} [options] Optional cell config override + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getPositiveNumberCellValidator = (options?: CSVOptionalCell): CSVCellValidator => { + return (params: CSVParams) => { + if (options?.optional && params.cell === undefined) { + return []; + } + + return validateZodCell(params.cell, z.number().positive()); + }; +}; + +/** + * Get the non-empty string header cell validator. + * + * Rules: + * 1. The cell must be a non-empty string + * 2. The cell is optional if the optional flag is set + * + * @param {CSVOptionalCell} [options] Optional cell config override + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getNonEmptyStringCellValidator = (options?: CSVOptionalCell) => { + return (params: CSVParams) => { + if (options?.optional && params.cell === undefined) { + return []; + } + + return validateZodCell(params.cell, z.string().trim().min(1)); + }; +}; + /** * Get the TSN header cell validator. * @@ -92,20 +133,29 @@ export const getTsnCellValidator = (tsns: Set): CSVCellValidator => { /** * Get the description header cell validator. * + * TODO: Add optional flag to allow undefined values conditionally + * * Rules: - * 1. The cell must be a string or undefined with a maximum length of 250 + * 1. The cell must be a string with a maximum length of 250 or undefined * * @returns {*} {CSVCellValidator} The validate cell callback */ export const getDescriptionCellValidator = (): CSVCellValidator => { return (params: CSVParams) => { - return validateZodCell(params.cell, z.string().trim().min(1).max(250).optional()); + if (typeof params.cell === 'number') { + // Allow numbers to be converted to strings for descriptions + params.mutateCell = String(params.cell); + } + + return validateZodCell(params.mutateCell, z.string().trim().min(1).max(250).optional()); }; }; /** * Get the time header cell validator. * + * TODO: Add optional flag to allow undefined values conditionally + * * Rules: * 1. The cell must be a valid 24-hour time format 'HH:mm:ss' or 'HH:mm' or undefined * @@ -152,8 +202,8 @@ export const getTimeCellSetter = (): CSVCellSetter => { */ export const getLatitudeCellValidator = (options?: CSVOptionalCell): CSVCellValidator => { return (params) => { - if (options?.optional) { - return validateZodCell(params.cell, z.number().min(-90).max(90).optional()); + if (options?.optional && params.cell === undefined) { + return []; } return validateZodCell(params.cell, z.number().min(-90).max(90)); @@ -171,8 +221,8 @@ export const getLatitudeCellValidator = (options?: CSVOptionalCell): CSVCellVali */ export const getLongitudeCellValidator = (options?: CSVOptionalCell): CSVCellValidator => { return (params) => { - if (options?.optional) { - return validateZodCell(params.cell, z.number().min(-180).max(180).optional()); + if (options?.optional && params.cell === undefined) { + return []; } return validateZodCell(params.cell, z.number().min(-180).max(180)); @@ -190,8 +240,8 @@ export const getLongitudeCellValidator = (options?: CSVOptionalCell): CSVCellVal */ export const getDateCellValidator = (options?: CSVOptionalCell): CSVCellValidator => { return (params) => { - if (options?.optional) { - return validateZodCell(params.cell, z.string().date().optional()); + if (options?.optional && params.cell === undefined) { + return []; } return validateZodCell(params.cell, z.string().date()); @@ -228,3 +278,38 @@ export const getSurveyCritterAliasCellValidator = (surveyAliasMap: Map { + return (params) => { + if (options?.optional && params.cell === undefined) { + return []; + } + + const dateParts = String(params.cell).split(' - '); + + if (dateParts.length !== 2 || !dateParts.every(isDateString)) { + return [ + { + error: 'Invalid date range', + solution: + "Use a valid date range format: 'YYYY-MM-DD - YYYY-MM-DD' OR 'YYYY-MM-DD HH:mm:ss - YYYY-MM-DD HH:mm:ss'" + } + ]; + } + + return []; + }; +}; diff --git a/api/src/utils/csv-utils/row-validators/taxon-row-validator.test.ts b/api/src/utils/csv-utils/row-validators/taxon-row-validator.test.ts new file mode 100644 index 0000000000..11eed596a6 --- /dev/null +++ b/api/src/utils/csv-utils/row-validators/taxon-row-validator.test.ts @@ -0,0 +1,105 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getTaxonFromRowState } from '../../../services/import-services/utils/row-state'; +import { CaseInsensitiveMap } from '../../case-insensitive-map'; +import { getTaxonRowValidator } from './taxon-row-validator'; + +chai.use(sinonChai); + +describe('getTaxonRowValidator', () => { + afterEach(() => { + sinon.restore(); + }); + + it('should handle undefined cell values', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns(undefined), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + const errors = rowValidator({} as any); + + expect(errors[0].error).to.contain('is required'); + }); + + it('should return no errors', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns(1), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowMock = {}; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + const errors = rowValidator({ row: rowMock } as any); + + expect(errors).to.be.an('array').that.is.empty; + }); + + it('should update the row state and be retrievable with the getter', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns(1), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowMock = {}; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + rowValidator({ row: rowMock } as any); + + const state = getTaxonFromRowState(rowMock); + + expect(state.itis_tsn).to.equal(1); + expect(state.itis_scientific_name).to.equal('Alces alces'); + }); + + it('should return an error for an invalid TSN', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns(2), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + const errors = rowValidator({} as any); + + expect(errors[0].error).to.contain('Invalid ITIS TSN'); + }); + + it('should return an error for an invalid scientific name', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns('Invalid'), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + const errors = rowValidator({} as any); + + expect(errors[0].error).to.contain('Invalid scientific name'); + }); + + it('should return an error for an invalid cell type', () => { + const taxonMapMock = new CaseInsensitiveMap([[1, { tsn: 1, scientificName: 'Alces alces' }]]); + const utilsMock: any = { + getCellValue: sinon.stub().returns(true), + getWorksheetHeader: sinon.stub().returns('SPECIES') + }; + + const rowValidator = getTaxonRowValidator(taxonMapMock, utilsMock, 'SPECIES'); + + const errors = rowValidator({} as any); + + expect(errors[0].error).to.contain('Invalid species'); + }); +}); diff --git a/api/src/utils/csv-utils/row-validators/taxon-row-validator.ts b/api/src/utils/csv-utils/row-validators/taxon-row-validator.ts new file mode 100644 index 0000000000..18bd28a516 --- /dev/null +++ b/api/src/utils/csv-utils/row-validators/taxon-row-validator.ts @@ -0,0 +1,80 @@ +import { TaxonMap } from '../../../services/import-services/utils/taxon'; +import { CSVConfigUtils } from '../csv-config-utils'; +import { CSVRowParams, CSVRowValidator } from '../csv-config-validation.interface'; +import { updateCSVRowState } from '../csv-header-configs'; + +/** + * Get the taxon header cell validator. + * + * Rules: + * 1. The cell must be a valid ITIS TSN or scientific name + * 2. The cell must be a valid species from the provided taxons + * 3. The row state will be updated with the TSN and scientific name + * + * @param {TaxonMap} taxonMap The list of taxons + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getTaxonRowValidator = >( + taxonMap: TaxonMap, + utils: CSVConfigUtils, + taxonStaticHeader: StaticHeaderType +): CSVRowValidator => { + return (params: CSVRowParams) => { + const taxonIdentifierCell = utils.getCellValue(taxonStaticHeader, params.row); + const taxonHeader = utils.getWorksheetHeader(taxonStaticHeader, params.row); + + if (taxonIdentifierCell === undefined) { + return [ + { + error: 'Cell is required', + solution: 'Use a valid ITIS TSN or scientific name', + header: taxonHeader, + cell: taxonIdentifierCell + } + ]; + } + + const taxon = taxonMap.get(taxonIdentifierCell); + + // If a valid taxon + if (taxon) { + // Update the row state with the TSN and scientific name + updateCSVRowState(params.row, { itis_tsn: taxon.tsn, itis_scientific_name: taxon.scientificName }); + + return []; + } + + // If an invalid TSN + if (typeof taxonIdentifierCell === 'number') { + return [ + { + error: 'Invalid ITIS TSN', + solution: 'Use a valid ITIS TSN', + header: taxonHeader, + cell: taxonIdentifierCell + } + ]; + } + + // If an invalid scientific name + if (typeof taxonIdentifierCell === 'string') { + return [ + { + error: 'Invalid scientific name', + solution: 'Use a valid scientific name', + header: taxonHeader, + cell: taxonIdentifierCell + } + ]; + } + + return [ + { + error: 'Invalid species', + solution: 'Expecting a valid ITIS TSN or scientific name', + header: taxonHeader, + cell: taxonIdentifierCell + } + ]; + }; +}; diff --git a/api/src/utils/date-time-utils.test.ts b/api/src/utils/date-time-utils.test.ts index 66b9e4f641..9512ac790b 100644 --- a/api/src/utils/date-time-utils.test.ts +++ b/api/src/utils/date-time-utils.test.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { isDateString, isDateTimeString, isTimeString } from './date-time-utils'; +import { formatDateString, isDateString, isDateTimeString, isTimeString } from './date-time-utils'; describe('isDateString', () => { describe('returns true', () => { @@ -117,4 +117,80 @@ describe('isTimeString', () => { expect(isTimeString('invalid')).to.be.false; }); }); + + describe('formatStringDateCell', () => { + it('should return null when string is not shaped like a date', () => { + expect(formatDateString('TEST')).to.be.null; + }); + + it('should return null when string is not a 3 part delimited string', () => { + expect(formatDateString('01-01')).to.be.null; + expect(formatDateString('01-01-2024-01')).to.be.null; + expect(formatDateString('01/01')).to.be.null; + }); + + it('should return null when string is not a valid date', () => { + expect(formatDateString('99-99-9999')).to.be.null; + }); + + it('should format 2024-01-31', () => { + expect(formatDateString('2024-01-31')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 2024-01-02', () => { + expect(formatDateString('2024-01-02')).to.equal('2024-01-02'); + }); + + it('should format 2024/01/31', () => { + expect(formatDateString('2024/01/31')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 2024/01/02', () => { + expect(formatDateString('2024/01/02')).to.equal('2024-01-02'); + }); + + it('should format 31-01-2024', () => { + expect(formatDateString('31-01-2024')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 02-01-2024', () => { + expect(formatDateString('02-01-2024')).to.equal('2024-01-02'); + }); + + it('should format 31/01/2024', () => { + expect(formatDateString('31/01/2024')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 02/01/2024', () => { + expect(formatDateString('02/01/2024')).to.equal('2024-01-02'); + }); + + it('should format 01-31-2024', () => { + expect(formatDateString('01-31-2024')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 01-02-2024', () => { + expect(formatDateString('01-02-2024')).to.equal('2024-02-01'); + }); + + it('should format 01/31/2024', () => { + expect(formatDateString('01/31/2024')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 01/02/2024', () => { + expect(formatDateString('01/02/2024')).to.equal('2024-02-01'); + }); + + it('should format 2024-01-31', () => { + expect(formatDateString('2024-01-31')).to.equal('2024-01-31'); + }); + + it('should format 2024/01/31', () => { + expect(formatDateString('2024/01/31')).to.equal('2024-01-31'); + }); + + it('should format ambiguous 2024/01/02', () => { + expect(formatDateString('2024/01/02')).to.equal('2024-01-02'); + }); + }); }); diff --git a/api/src/utils/date-time-utils.ts b/api/src/utils/date-time-utils.ts index 50dc2a620a..3cbb2aa4fb 100644 --- a/api/src/utils/date-time-utils.ts +++ b/api/src/utils/date-time-utils.ts @@ -1,5 +1,12 @@ import dayjs from 'dayjs'; -import { DefaultTimeFormat, DefaultTimeFormatNoSeconds } from '../constants/dates'; +import { + DefaultDateFormat, + DefaultDateFormatReverse, + DefaultTimeFormat, + DefaultTimeFormatNoSeconds, + USDefaultDateFormat, + USDefaultDateFormatReverse +} from '../constants/dates'; /** * Check if a string is a date string. @@ -71,3 +78,48 @@ export function isTimeString(value: string): boolean { return dayjs(value, [DefaultTimeFormat, DefaultTimeFormatNoSeconds], true).isValid(); } + +/** + * Formats a date string to a date - prioritizes Canadian date formats over American date formats. + * + * @example formatDate('2025-01-01') // '2025-01-01' + * @example formatDate('01/31/2025') // '2025-01-31' + * @example formatDate('31-01-2025') // '2025-01-31' + * + * @export + * @param {string} value - + * @return {*} {string | null} - Date string or null if the cell value is not a date + */ +export function formatDateString(value: string): string | null { + const dateParts = String(value).replace(/\//g, '-').split('-'); + + // Check if the string is a 3 part delimited date + if (dateParts.length !== 3) { + return null; + } + + // Generate a dayjs date object for both Canadian and American date formats + // Why? There is a edge case where both the Canadian and American date formats are BOTH valid + // but the date is generated incorrectly (01/31/2024 -> 2026-07-01). + // We can determine the correct format by cross-referencing the year with the raw cell value. + const canadianDate = dayjs(String(value), [DefaultDateFormat, DefaultDateFormatReverse]); + const americanDate = dayjs(String(value), [USDefaultDateFormat, USDefaultDateFormatReverse]); + + if (!canadianDate.isValid() && !americanDate.isValid()) { + return null; + } + + // Grab the year from the date string + const dateYear = Number(dateParts[0].length === 4 ? dateParts[0] : dateParts[2]); + + // Always prioritize Canadian date formats over American date formats + if (canadianDate.year() === dateYear) { + return canadianDate.format(DefaultDateFormat); + } + + if (americanDate.year() === dateYear) { + return americanDate.format(DefaultDateFormat); + } + + return null; +} diff --git a/api/src/utils/xlsx-utils/cell-utils.test.ts b/api/src/utils/xlsx-utils/cell-utils.test.ts index cf05a27371..154d2c2409 100644 --- a/api/src/utils/xlsx-utils/cell-utils.test.ts +++ b/api/src/utils/xlsx-utils/cell-utils.test.ts @@ -1,12 +1,5 @@ import { expect } from 'chai'; -import { - formatDateCellValue, - isDateCell, - isStringCell, - isTimeCell, - replaceCellDates, - trimCellWhitespace -} from './cell-utils'; +import { isDateCell, isStringCell, isTimeCell, replaceCellDates, trimCellWhitespace } from './cell-utils'; import { CUSTOM_XLSX_DATE_FORMAT } from './worksheet-utils'; describe('cell-utils', () => { @@ -41,82 +34,6 @@ describe('cell-utils', () => { }); }); - describe('formatStringDateCell', () => { - it('should return null when string is not shaped like a date', () => { - expect(formatDateCellValue('TEST')).to.be.null; - }); - - it('should return null when string is not a 3 part delimited string', () => { - expect(formatDateCellValue('01-01')).to.be.null; - expect(formatDateCellValue('01-01-2024-01')).to.be.null; - expect(formatDateCellValue('01/01')).to.be.null; - }); - - it('should return null when string is not a valid date', () => { - expect(formatDateCellValue('99-99-9999')).to.be.null; - }); - - it('should format 2024-01-31', () => { - expect(formatDateCellValue('2024-01-31')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 2024-01-02', () => { - expect(formatDateCellValue('2024-01-02')).to.equal('2024-01-02'); - }); - - it('should format 2024/01/31', () => { - expect(formatDateCellValue('2024/01/31')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 2024/01/02', () => { - expect(formatDateCellValue('2024/01/02')).to.equal('2024-01-02'); - }); - - it('should format 31-01-2024', () => { - expect(formatDateCellValue('31-01-2024')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 02-01-2024', () => { - expect(formatDateCellValue('02-01-2024')).to.equal('2024-01-02'); - }); - - it('should format 31/01/2024', () => { - expect(formatDateCellValue('31/01/2024')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 02/01/2024', () => { - expect(formatDateCellValue('02/01/2024')).to.equal('2024-01-02'); - }); - - it('should format 01-31-2024', () => { - expect(formatDateCellValue('01-31-2024')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 01-02-2024', () => { - expect(formatDateCellValue('01-02-2024')).to.equal('2024-02-01'); - }); - - it('should format 01/31/2024', () => { - expect(formatDateCellValue('01/31/2024')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 01/02/2024', () => { - expect(formatDateCellValue('01/02/2024')).to.equal('2024-02-01'); - }); - - it('should format 2024-01-31', () => { - expect(formatDateCellValue('2024-01-31')).to.equal('2024-01-31'); - }); - - it('should format 2024/01/31', () => { - expect(formatDateCellValue('2024/01/31')).to.equal('2024-01-31'); - }); - - it('should format ambiguous 2024/01/02', () => { - expect(formatDateCellValue('2024/01/02')).to.equal('2024-01-02'); - }); - }); - describe('trimCellWhitespace', () => { it('should trim cell value', () => { const cell = trimCellWhitespace({ t: 's', v: ' TEST ', w: ' OTHER ' }); diff --git a/api/src/utils/xlsx-utils/cell-utils.ts b/api/src/utils/xlsx-utils/cell-utils.ts index b7524cc417..869495a551 100644 --- a/api/src/utils/xlsx-utils/cell-utils.ts +++ b/api/src/utils/xlsx-utils/cell-utils.ts @@ -1,20 +1,13 @@ import dayjs from 'dayjs'; import duration from 'dayjs/plugin/duration'; import { CellObject } from 'xlsx'; -import { - DefaultDateFormat, - DefaultDateFormatReverse, - DefaultTimeFormat, - USDefaultDateFormat, - USDefaultDateFormatReverse -} from '../../constants/dates'; +import { DefaultDateFormat, DefaultTimeFormat } from '../../constants/dates'; +import { formatDateString } from '../date-time-utils'; import { safeTrim } from '../string-utils'; import { CUSTOM_XLSX_DATE_FORMAT } from './worksheet-utils'; dayjs.extend(duration); -type CellValue = CellObject['v']; - /** * Trims whitespace from the value of a string type cell. * Trims whitespace from the formatted text value of a cell, if present. @@ -56,19 +49,22 @@ export function replaceCellDates(cell: CellObject): CellObject { // Use the formatted value ('2024-01-01') instead of the epoch number (434565) // Why? The epoch number is inconsistent and is affected by the dateNF option. // Same dates with different incomming formats will have different epoch values. - const date = formatDateCellValue(cell.w); + const date = formatDateString(String(cell.w)); return { ...cell, z: DefaultDateFormat, v: date ?? 'Invalid Date Format' }; } // If time cell - convert the epoch value (ie: 0.5) to a time string (ie: '12:00:00') else if (isTimeCell(cell)) { - const time = dayjs.duration(Number(cell.v), 'days'); - + // Round the time fraction to the nearest millisecond to avoid floating point errors + // Excel stores time as a fraction of a day, where each day has 86400000 milliseconds. + // Ex: 0.25 is 6:00:00 AM, 0.5 is 12:00:00 PM, 0.75 is 6:00:00 PM + const roundedTimeFraction = Math.round(Number(cell.v) * 86400000) / 86400000; + const time = dayjs.duration(roundedTimeFraction, 'days'); return { ...cell, z: DefaultTimeFormat, v: time.format(DefaultTimeFormat) }; } // If a string cell - check if the string is a date and convert it to a date string else if (cell.z !== CUSTOM_XLSX_DATE_FORMAT && isStringCell(cell)) { - const date = formatDateCellValue(cell.v); + const date = formatDateString(String(cell.v)); // If the string is a date, update the cell value to the formatted date string if (date) { @@ -79,47 +75,6 @@ export function replaceCellDates(cell: CellObject): CellObject { return cell; } -/** - * Converts a cell value to a date string - prioritizes Canadian date formats over American date formats. - * - * @export - * @param {CellValue} cellValue - Cell value - * @return {*} {string | null} - Date string or null if the cell value is not a date - */ -export function formatDateCellValue(cellValue: CellValue): string | null { - const dateParts = String(cellValue).replace(/\//g, '-').split('-'); - - // Check if the string is a 3 part delimited date - if (dateParts.length !== 3) { - return null; - } - - // Generate a dayjs date object for both Canadian and American date formats - // Why? There is a edge case where both the Canadian and American date formats are BOTH valid - // but the date is generated incorrectly (01/31/2024 -> 2026-07-01). - // We can determine the correct format by cross-referencing the year with the raw cell value. - const canadianDate = dayjs(String(cellValue), [DefaultDateFormat, DefaultDateFormatReverse]); - const americanDate = dayjs(String(cellValue), [USDefaultDateFormat, USDefaultDateFormatReverse]); - - if (!canadianDate.isValid() && !americanDate.isValid()) { - return null; - } - - // Grab the year from the date string - const dateYear = Number(dateParts[0].length === 4 ? dateParts[0] : dateParts[2]); - - // Always prioritize Canadian date formats over American date formats - if (canadianDate.year() === dateYear) { - return canadianDate.format(DefaultDateFormat); - } - - if (americanDate.year() === dateYear) { - return americanDate.format(DefaultDateFormat); - } - - return null; -} - /** * Checks if the cell has type string. * diff --git a/api/src/utils/xlsx-utils/worksheet-utils.test.ts b/api/src/utils/xlsx-utils/worksheet-utils.test.ts index a13dbf668d..e9254ea565 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.test.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.test.ts @@ -16,6 +16,7 @@ const xlsxWorksheet: xlsx.WorkSheet = { F1: { t: 's', v: 'Longitude' }, G1: { t: 's', v: 'Antler Configuration' }, H1: { t: 's', v: 'Wind Direction' }, + I1: { t: 's', v: ' ' }, A2: { t: 'n', w: '180703', v: 180703 }, B2: { t: 'n', w: '1', v: 1 }, C2: { t: 's', v: '1970-01-01T08:00:00.000Z' }, @@ -24,6 +25,7 @@ const xlsxWorksheet: xlsx.WorkSheet = { F2: { t: 'n', w: '-123', v: -123 }, G2: { t: 's', v: 'more than 3 points' }, H2: { t: 's', v: 'North' }, + I2: { t: 's', v: undefined }, A3: { t: 'n', w: '180596', v: 180596 }, B3: { t: 'n', w: '2', v: 2 }, C3: { t: 's', v: '1970-01-01T08:00:00.000Z' }, @@ -31,6 +33,7 @@ const xlsxWorksheet: xlsx.WorkSheet = { E3: { t: 'n', w: '-57', v: -57 }, F3: { t: 'n', w: '-122', v: -122 }, H3: { t: 's', v: 'North' }, + I3: { t: 's', v: undefined }, A4: { t: 'n', w: '180713', v: 180713 }, B4: { t: 'n', w: '3', v: 3 }, C4: { t: 's', v: '1970-01-01T08:00:00.000Z' }, @@ -38,6 +41,7 @@ const xlsxWorksheet: xlsx.WorkSheet = { E4: { t: 'n', w: '-56', v: -56 }, F4: { t: 'n', w: '-121', v: -121 }, H4: { t: 's', v: 'North' }, + I4: { t: 's', v: undefined }, '!ref': 'A1:H9' }; diff --git a/api/src/utils/xlsx-utils/worksheet-utils.ts b/api/src/utils/xlsx-utils/worksheet-utils.ts index 0682b094fb..bdd51b9a4a 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.ts @@ -97,8 +97,8 @@ export const getHeadersUpperCase = (worksheet: xlsx.WorkSheet): string[] => { // Parse the headers array from the array of arrays produced by calling `xlsx.utils.sheet_to_json` headers = aoaHeaders[0] .map(String) - .filter(Boolean) - .map((header) => header.trim().toUpperCase()); + .map((header) => header.trim().toUpperCase()) + .filter(Boolean); } return headers; @@ -149,7 +149,12 @@ export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record { dialogContext.setSnackbar({ open: true, snackbarAutoCloseMs: 2000, - snackbarMessage: 'Successfully imported telemetry' + snackbarMessage: 'Successfully imported CSV file' }); handleClose(); @@ -107,7 +107,7 @@ export const CSVSingleImportDialog = (props: CSVSingleImportDialogProps) => { dialogContext.setSnackbar({ open: true, snackbarAutoCloseMs: 2000, - snackbarMessage: 'Failed to import telemetry' + snackbarMessage: 'Failed to import CSV file' }); setUploadStatus(UploadFileStatus.FAILED); @@ -119,7 +119,7 @@ export const CSVSingleImportDialog = (props: CSVSingleImportDialogProps) => { } return ( - + { export const getTelemetryCSVTemplate = (): CSVEncodedTemplate => { return getCSVTemplate(['VENDOR', 'SERIAL', 'LATITUDE', 'LONGITUDE', 'DATE', 'TIME']); }; + +/** + * Get CSV template for observations. + * + * @returns {CSVEncodedTemplate} Encoded CSV template + */ +export const getObservationCSVTemplate = (): CSVEncodedTemplate => { + return getCSVTemplate(['SPECIES', 'SITE', 'TECHNIQUE', 'PERIOD', 'SIGN', 'COUNT', 'DATE', 'TIME', 'COMMENT']); +}; diff --git a/app/src/features/surveys/observations/SurveyObservationPage.tsx b/app/src/features/surveys/observations/SurveyObservationPage.tsx index ffce32625f..914759c492 100644 --- a/app/src/features/surveys/observations/SurveyObservationPage.tsx +++ b/app/src/features/surveys/observations/SurveyObservationPage.tsx @@ -8,9 +8,9 @@ import { ProjectContext } from 'contexts/projectContext'; import { SurveyContext } from 'contexts/surveyContext'; import { TaxonomyContextProvider } from 'contexts/taxonomyContext'; import { useContext } from 'react'; +import SurveyObservationHeader from './components/SurveyObservationHeader'; import ObservationsTableContainer from './observations-table/ObservationsTableContainer'; import { SamplingSiteListContainer } from './sampling-sites/SamplingSiteListContainer'; -import SurveyObservationHeader from './SurveyObservationHeader'; export const SurveyObservationPage = () => { const surveyContext = useContext(SurveyContext); diff --git a/app/src/features/surveys/observations/components/ImportObservationsButton.tsx b/app/src/features/surveys/observations/components/ImportObservationsButton.tsx new file mode 100644 index 0000000000..1746d6b935 --- /dev/null +++ b/app/src/features/surveys/observations/components/ImportObservationsButton.tsx @@ -0,0 +1,124 @@ +import Button, { ButtonProps } from '@mui/material/Button'; +import axios, { AxiosProgressEvent } from 'axios'; +import { CSVSingleImportDialog } from 'components/csv/CSVSingleImportDialog'; +import { SurveyContext } from 'contexts/surveyContext'; +import { getObservationCSVTemplate } from 'features/surveys/animals/profile/captures/import-captures/utils/templates'; +import { useBiohubApi } from 'hooks/useBioHubApi'; +import { useContext, useState } from 'react'; +import { downloadFile } from 'utils/file-utils'; + +export interface IImportObservationsButtonProps { + /** + * If true, the button will be disabled. + * + * @type {boolean} + * @memberof IImportObservationsButtonProps + */ + disabled?: boolean; + /** + * Callback fired when the import process is started. + * + * @memberof IImportObservationsButtonProps + */ + onStart?: () => void; + /** + * Callback fired when the import process is successful. + * + * @memberof IImportObservationsButtonProps + */ + onSuccess?: () => void; + /** + * Callback fired when the import process encounters an error. + * + * @memberof IImportObservationsButtonProps + */ + onError?: () => void; + /** + * Callback fired when the import process is complete (success or error). + * + * @memberof IImportObservationsButtonProps + */ + onFinish?: () => void; + /** + * An optional survey sample period id. All imported observation records will be associated to this sample period. + * + * @type {number} + */ + surveySamplePeriodId?: number; + /** + * Optional button props to pass to the button component. + * + * @type {ButtonProps} + */ + buttonProps?: ButtonProps; +} + +/** + * Renders a button that allows the user to import observation records from a CSV file. + * + * @param {IImportObservationsButtonProps} props + * @return {*} + */ +export const ImportObservationsButton = (props: IImportObservationsButtonProps) => { + const { disabled, surveySamplePeriodId, onStart, onSuccess, onError, onFinish } = props; + + const biohubApi = useBiohubApi(); + + const surveyContext = useContext(SurveyContext); + const { projectId, surveyId } = surveyContext; + + const [open, setOpen] = useState(false); + + const cancelTokenSource = axios.CancelToken.source(); + + /** + * Callback fired when the user attempts to import observations. + * + * @param {File} file + * @return {*} + */ + const handleImportObservations = async (file: File, onProgress: (progressEvent: AxiosProgressEvent) => void) => { + try { + onStart?.(); + + await biohubApi.observation.importObservationCSV({ + projectId, + surveyId, + surveySamplePeriodId, + file, + onProgress, + cancelTokenSource + }); + + onSuccess?.(); + } catch (error) { + onError?.(); + // re-throw the error so the dialog can display the CSVErrors + throw error; + } finally { + onFinish?.(); + } + }; + + return ( + <> + + setOpen(false)} + onImport={handleImportObservations} + onDownloadTemplate={() => downloadFile(getObservationCSVTemplate(), 'SIMS-observations-template.csv')} + /> + + ); +}; diff --git a/app/src/features/surveys/observations/SurveyObservationHeader.tsx b/app/src/features/surveys/observations/components/SurveyObservationHeader.tsx similarity index 100% rename from app/src/features/surveys/observations/SurveyObservationHeader.tsx rename to app/src/features/surveys/observations/components/SurveyObservationHeader.tsx diff --git a/app/src/features/surveys/observations/observations-table/ObservationsTableContainer.tsx b/app/src/features/surveys/observations/observations-table/ObservationsTableContainer.tsx index 454f318f38..9200a511bc 100644 --- a/app/src/features/surveys/observations/observations-table/ObservationsTableContainer.tsx +++ b/app/src/features/surveys/observations/observations-table/ObservationsTableContainer.tsx @@ -31,7 +31,6 @@ import { TaxonomyColDef } from 'features/surveys/observations/observations-table/grid-column-definitions/GridColumnDefinitions'; import { useSamplingInformationCache } from 'features/surveys/observations/observations-table/grid-column-definitions/sampling-information/useSamplingInformationCache'; -import { ImportObservationsButton } from 'features/surveys/observations/observations-table/import-obsevations/ImportObservationsButton'; import ObservationsTable from 'features/surveys/observations/observations-table/ObservationsTable'; import { useCodesContext, @@ -41,6 +40,7 @@ import { } from 'hooks/useContext'; import { MarkdownTypeNameEnum } from 'interfaces/useMarkdownApi.interface'; import { useEffect, useMemo } from 'react'; +import { ImportObservationsButton } from '../components/ImportObservationsButton'; import { ConfigureColumnsButton } from './configure-columns/ConfigureColumnsButton'; import ExportHeadersButton from './export-button/ExportHeadersButton'; import { ObservationSubcountCommentDialog } from './grid-column-definitions/comment/ObservationSubcountCommentDialog'; diff --git a/app/src/features/surveys/observations/observations-table/import-obsevations/ImportObservationsButton.tsx b/app/src/features/surveys/observations/observations-table/import-obsevations/ImportObservationsButton.tsx deleted file mode 100644 index b3e5fd4e9e..0000000000 --- a/app/src/features/surveys/observations/observations-table/import-obsevations/ImportObservationsButton.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { mdiImport } from '@mdi/js'; -import Icon from '@mdi/react'; -import Button from '@mui/material/Button'; -import Typography from '@mui/material/Typography'; -import { FileUploadSingleItemDialog } from 'components/dialog/attachments/FileUploadSingleItemDialog'; -import { ObservationsTableI18N } from 'constants/i18n'; -import { DialogContext } from 'contexts/dialogContext'; -import { SurveyContext } from 'contexts/surveyContext'; -import { APIError } from 'hooks/api/useAxios'; -import { useBiohubApi } from 'hooks/useBioHubApi'; -import { useContext, useState } from 'react'; - -export interface IImportObservationsButtonProps { - /** - * If true, the button will be disabled. - * - * @type {boolean} - * @memberof IImportObservationsButtonProps - */ - disabled?: boolean; - /** - * Callback fired when the import process is started. - * - * @memberof IImportObservationsButtonProps - */ - onStart?: () => void; - /** - * Callback fired when the import process is successful. - * - * @memberof IImportObservationsButtonProps - */ - onSuccess?: () => void; - /** - * Callback fired when the import process encounters an error. - * - * @memberof IImportObservationsButtonProps - */ - onError?: () => void; - /** - * Callback fired when the import process is complete (success or error). - * - * @memberof IImportObservationsButtonProps - */ - onFinish?: () => void; - /** - * Options to pass to the process csv submission endpoint. - * - * @type {{ - * surveySamplePeriodId?: number; - * }} - * @memberof IImportObservationsButtonProps - */ -} - -export const ImportObservationsButton = (props: IImportObservationsButtonProps) => { - const { disabled, onStart, onSuccess, onError, onFinish } = props; - - const biohubApi = useBiohubApi(); - - const surveyContext = useContext(SurveyContext); - const { projectId, surveyId } = surveyContext; - - const dialogContext = useContext(DialogContext); - - const [open, setOpen] = useState(false); - - /** - * Callback fired when the user attempts to import observations. - * - * @param {File} file - * @return {*} - */ - const handleImportObservations = async (file: File) => { - try { - onStart?.(); - - const uploadResponse = await biohubApi.observation.uploadCsvForImport(projectId, surveyId, file); - - await biohubApi.observation.processCsvSubmission(projectId, surveyId, uploadResponse.submissionId); - - setOpen(false); - - dialogContext.setSnackbar({ - snackbarMessage: ( - - {ObservationsTableI18N.importRecordsSuccessSnackbarMessage} - - ), - open: true - }); - - onSuccess?.(); - } catch (error) { - const apiError = error as APIError; - - dialogContext.setErrorDialog({ - dialogTitle: ObservationsTableI18N.importRecordsErrorDialogTitle, - dialogText: ObservationsTableI18N.importRecordsErrorDialogText, - dialogError: apiError.message, - dialogErrorDetails: apiError.errors, - open: true, - onClose: () => { - dialogContext.setErrorDialog({ open: false }); - }, - onOk: () => { - dialogContext.setErrorDialog({ open: false }); - } - }); - - onError?.(); - } finally { - onFinish?.(); - } - }; - - return ( - <> - - setOpen(false)} - onUpload={handleImportObservations} - uploadButtonLabel="Import" - dropZoneProps={{ acceptedFileExtensions: '.csv' }} - /> - - ); -}; diff --git a/app/src/features/surveys/observations/sampling-sites/components/ImportObservationsButton.tsx b/app/src/features/surveys/observations/sampling-sites/components/ImportObservationsButton.tsx deleted file mode 100644 index 9efab72aae..0000000000 --- a/app/src/features/surveys/observations/sampling-sites/components/ImportObservationsButton.tsx +++ /dev/null @@ -1,154 +0,0 @@ -import Button from '@mui/material/Button'; -import Typography from '@mui/material/Typography'; -import { FileUploadSingleItemDialog } from 'components/dialog/attachments/FileUploadSingleItemDialog'; -import { ObservationsTableI18N } from 'constants/i18n'; -import { DialogContext } from 'contexts/dialogContext'; -import { SurveyContext } from 'contexts/surveyContext'; -import { APIError } from 'hooks/api/useAxios'; -import { useBiohubApi } from 'hooks/useBioHubApi'; -import { useContext, useState } from 'react'; - -export interface IImportObservationsButtonProps { - /** - * If true, the button will be disabled. - * - * @type {boolean} - * @memberof IImportObservationsButtonProps - */ - disabled?: boolean; - /** - * Callback fired when the import process is started. - * - * @memberof IImportObservationsButtonProps - */ - onStart?: () => void; - /** - * Callback fired when the import process is successful. - * - * @memberof IImportObservationsButtonProps - */ - onSuccess?: () => void; - /** - * Callback fired when the import process encounters an error. - * - * @memberof IImportObservationsButtonProps - */ - onError?: () => void; - /** - * Callback fired when the import process is complete (success or error). - * - * @memberof IImportObservationsButtonProps - */ - onFinish?: () => void; - /** - * Options to pass to the process csv submission endpoint. - * - * @type {{ - * surveySamplePeriodId?: number; - * }} - * @memberof IImportObservationsButtonProps - */ - processOptions?: { - /** - * An optional survey sample period id. All imported observation records will be associated to this sample period. - * - * @type {number} - */ - surveySamplePeriodId?: number; - }; -} - -/** - * Renders a button that allows the user to import observation records from a CSV file. - * - * @param {IImportObservationsButtonProps} props - * @return {*} - */ -export const ImportObservationsButton = (props: IImportObservationsButtonProps) => { - const { disabled, onStart, onSuccess, onError, onFinish, processOptions } = props; - - const biohubApi = useBiohubApi(); - - const surveyContext = useContext(SurveyContext); - const { projectId, surveyId } = surveyContext; - - const dialogContext = useContext(DialogContext); - - const [open, setOpen] = useState(false); - - /** - * Callback fired when the user attempts to import observations. - * - * @param {File} file - * @return {*} - */ - const handleImportObservations = async (file: File) => { - try { - onStart?.(); - - const uploadResponse = await biohubApi.observation.uploadCsvForImport(projectId, surveyId, file); - - await biohubApi.observation.processCsvSubmission( - projectId, - surveyId, - uploadResponse.submissionId, - processOptions - ); - - setOpen(false); - - dialogContext.setSnackbar({ - snackbarMessage: ( - - {ObservationsTableI18N.importRecordsSuccessSnackbarMessage} - - ), - open: true - }); - - onSuccess?.(); - } catch (error) { - const apiError = error as APIError; - - dialogContext.setErrorDialog({ - dialogTitle: ObservationsTableI18N.importRecordsErrorDialogTitle, - dialogText: ObservationsTableI18N.importRecordsErrorDialogText, - dialogError: apiError.message, - dialogErrorDetails: apiError.errors, - open: true, - onClose: () => { - dialogContext.setErrorDialog({ open: false }); - }, - onOk: () => { - dialogContext.setErrorDialog({ open: false }); - } - }); - - onError?.(); - } finally { - onFinish?.(); - } - }; - - return ( - <> - - setOpen(false)} - onUpload={handleImportObservations} - uploadButtonLabel="Import" - dropZoneProps={{ acceptedFileExtensions: '.csv' }} - /> - - ); -}; diff --git a/app/src/features/surveys/observations/sampling-sites/site/accordion-details/period/SamplingSiteListPeriod.tsx b/app/src/features/surveys/observations/sampling-sites/site/accordion-details/period/SamplingSiteListPeriod.tsx index 7f0b52454a..ce6c4afc66 100644 --- a/app/src/features/surveys/observations/sampling-sites/site/accordion-details/period/SamplingSiteListPeriod.tsx +++ b/app/src/features/surveys/observations/sampling-sites/site/accordion-details/period/SamplingSiteListPeriod.tsx @@ -7,7 +7,7 @@ import Typography from '@mui/material/Typography'; import { IObservationsContext } from 'contexts/observationsContext'; import { IObservationsPageContext } from 'contexts/observationsPageContext'; import dayjs from 'dayjs'; -import { ImportObservationsButton } from 'features/surveys/observations/sampling-sites/components/ImportObservationsButton'; +import { ImportObservationsButton } from 'features/surveys/observations/components/ImportObservationsButton'; import { GetSamplingPeriod } from 'interfaces/useSamplingPeriodApi.interface'; interface ISamplingSiteListPeriodProps { @@ -145,7 +145,14 @@ export const SamplingSiteListPeriod = (props: ISamplingSiteListPeriodProps) => { observationsPageContext.setIsDisabled(false); observationsPageContext.setIsLoading(false); }} - processOptions={{ surveySamplePeriodId: samplePeriod.survey_sample_period_id }} + surveySamplePeriodId={samplePeriod.survey_sample_period_id} + buttonProps={{ + size: 'small', + sx: { + borderRadius: '3px', + fontSize: '0.6rem' + } + }} /> )} diff --git a/app/src/hooks/api/useObservationApi.test.ts b/app/src/hooks/api/useObservationApi.test.ts index 35f8a6226f..b1a8add453 100644 --- a/app/src/hooks/api/useObservationApi.test.ts +++ b/app/src/hooks/api/useObservationApi.test.ts @@ -56,54 +56,21 @@ describe('useObservationApi', () => { expect(result).toEqual(mockResponse); }); - describe('uploadCsvForImport', () => { + describe('importObservationCSV', () => { it('works as expected', async () => { const projectId = 1; const surveyId = 2; const file = new File([''], 'file.txt', { type: 'application/plain' }); - const res = { - submissionId: 1 - }; + mock.onPost(`/api/project/${projectId}/survey/${surveyId}/observations/import`).reply(200, undefined); - mock.onPost(`/api/project/${projectId}/survey/${surveyId}/observations/upload`).reply(200, res); + const result = await useObservationApi(axios).importObservationCSV({ + projectId, + surveyId, + file + }); - const result = await useObservationApi(axios).uploadCsvForImport(projectId, surveyId, file); - - expect(result).toEqual(res); - }); - }); - - describe('processCsvSubmission', () => { - it('works as expected', async () => { - const projectId = 1; - const surveyId = 2; - const submissionId = 3; - - const res = undefined; - - mock.onPost(`/api/project/${projectId}/survey/${surveyId}/observations/process`).reply(200, res); - - const result = await useObservationApi(axios).processCsvSubmission(projectId, surveyId, submissionId); - - expect(result).toEqual(res); - }); - - it('works as expected with options', async () => { - const projectId = 1; - const surveyId = 2; - const submissionId = 3; - const options = { - surveySamplePeriodId: 4 - }; - - const res = undefined; - - mock.onPost(`/api/project/${projectId}/survey/${surveyId}/observations/process`).reply(200, res); - - const result = await useObservationApi(axios).processCsvSubmission(projectId, surveyId, submissionId, options); - - expect(result).toEqual(res); + expect(result).toEqual(undefined); }); }); }); diff --git a/app/src/hooks/api/useObservationApi.ts b/app/src/hooks/api/useObservationApi.ts index 2592daaf35..d88ad2153d 100644 --- a/app/src/hooks/api/useObservationApi.ts +++ b/app/src/hooks/api/useObservationApi.ts @@ -195,66 +195,38 @@ const useObservationApi = (axios: AxiosInstance) => { }; /** - * Uploads an observation CSV for import. + * Imports observation records from a CSV file. * - * @param {number} projectId - * @param {number} surveyId - * @param {File} file * @param {{ - * samplingPeriodId: number; - * }} [options] - * @param {CancelTokenSource} [cancelTokenSource] - * @param {(progressEvent: AxiosProgressEvent) => void} [onProgress] + * projectId: number; + * surveyId: number; + * file: File; // The CSV file to import. + * surveySamplePeriodId?: number; // Optional sample period id to associate all imported records with. + * cancelTokenSource?: CancelTokenSource; + * onProgress?: (progressEvent: AxiosProgressEvent) => void; + * }} params * @return {*} {Promise<{ submissionId: number }>} */ - const uploadCsvForImport = async ( - projectId: number, - surveyId: number, - file: File, - cancelTokenSource?: CancelTokenSource, - onProgress?: (progressEvent: AxiosProgressEvent) => void - ): Promise<{ submissionId: number }> => { + const importObservationCSV = async (params: { + projectId: number; + surveyId: number; + file: File; + surveySamplePeriodId?: number; + cancelTokenSource?: CancelTokenSource; + onProgress?: (progressEvent: AxiosProgressEvent) => void; + }): Promise => { const formData = new FormData(); - formData.append('media', file); - - const { data } = await axios.post<{ submissionId: number }>( - `/api/project/${projectId}/survey/${surveyId}/observations/upload`, - formData, - { - cancelToken: cancelTokenSource?.token, - onUploadProgress: onProgress - } - ); - - return data; - }; + formData.append('media', params.file); - /** - * Begins processing an uploaded observation CSV for import - * - * @param {number} projectId - * @param {number} surveyId - * @param {number} submissionId - * @param {{ - * surveySamplePeriodId?: number; - * }} [options] - * @return {*} {Promise} - */ - const processCsvSubmission = async ( - projectId: number, - surveyId: number, - submissionId: number, - options?: { - surveySamplePeriodId?: number; + if (params.surveySamplePeriodId) { + formData.append('surveySamplePeriodId', params.surveySamplePeriodId.toString()); } - ): Promise => { - const { data } = await axios.post(`/api/project/${projectId}/survey/${surveyId}/observations/process`, { - observation_submission_id: submissionId, - options - }); - return data; + await axios.post(`/api/project/${params.projectId}/survey/${params.surveyId}/observations/import`, formData, { + cancelToken: params.cancelTokenSource?.token, + onUploadProgress: params.onProgress + }); }; /** @@ -331,8 +303,7 @@ const useObservationApi = (axios: AxiosInstance) => { deleteObservationRecords, deleteObservationMeasurements, deleteObservationEnvironments, - uploadCsvForImport, - processCsvSubmission + importObservationCSV }; };