From 225c891c1b8de3ae1c2a30e22722edcf5f2e4513 Mon Sep 17 00:00:00 2001 From: Mac Deluca Date: Thu, 18 Jul 2024 12:04:56 -0700 Subject: [PATCH] feat: updated tests for import services and util files --- .../survey/{surveyId}/critters/import.ts | 2 +- .../capture/import-captures-service.ts | 2 +- .../critter/import-critters-service.test.ts | 272 ++++++++---------- .../critter/import-critters-service.ts | 6 +- ...es.ts => csv-import-strategy.interface.ts} | 3 + .../import-services/csv-import-strategy.ts | 2 +- .../xlsx-utils/column-validator-utils.test.ts | 44 +++ .../xlsx-utils/column-validator-utils.ts | 24 +- .../utils/xlsx-utils/worksheet-utils.test.ts | 25 +- api/src/utils/xlsx-utils/worksheet-utils.ts | 20 +- 10 files changed, 216 insertions(+), 184 deletions(-) rename api/src/services/import-services/{import-types.ts => csv-import-strategy.interface.ts} (99%) create mode 100644 api/src/utils/xlsx-utils/column-validator-utils.test.ts diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts index 8993219ee4..9c0b1333f4 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts @@ -152,7 +152,7 @@ export function importCsv(): RequestHandler { // Critter CSV import service - child of CSVImportStrategy const importCsvCritters = new ImportCrittersService(connection, surveyId); - // CSV import strategy with injected import service - parent + // Inject the import service into the CSV import strategy const csvImportStrategry = new CSVImportStrategy(importCsvCritters); // Import CSV data with strategy class diff --git a/api/src/services/import-services/capture/import-captures-service.ts b/api/src/services/import-services/capture/import-captures-service.ts index d931db9a9e..9d306ced94 100644 --- a/api/src/services/import-services/capture/import-captures-service.ts +++ b/api/src/services/import-services/capture/import-captures-service.ts @@ -6,7 +6,7 @@ import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { CritterbaseService, IBulkCreate } from '../../critterbase-service'; import { DBService } from '../../db-service'; -import { CSVImportService, Row } from '../import-types'; +import { CSVImportService, Row } from '../csv-import-strategy.interface'; import { CsvCapture, CsvCaptureSchema, PartialCsvCapture } from './import-captures-service.interface'; /** diff --git a/api/src/services/import-services/critter/import-critters-service.test.ts b/api/src/services/import-services/critter/import-critters-service.test.ts index 137f0f9333..e74ec85b5b 100644 --- a/api/src/services/import-services/critter/import-critters-service.test.ts +++ b/api/src/services/import-services/critter/import-critters-service.test.ts @@ -2,8 +2,6 @@ import chai, { expect } from 'chai'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; import { WorkSheet } from 'xlsx'; -import { MediaFile } from '../../../utils/media/media-file'; -import * as xlsxUtils from '../../../utils/xlsx-utils/worksheet-utils'; import { getMockDBConnection } from '../../../__mocks__/db'; import { IBulkCreateResponse } from '../../critterbase-service'; import { ImportCrittersService } from './import-critters-service'; @@ -14,7 +12,7 @@ chai.use(sinonChai); const mockConnection = getMockDBConnection(); describe('ImportCrittersService', () => { - describe('_getCritterRowsToValidate', () => { + describe('getRowsToValidate', () => { it('it should correctly format rows', () => { const rows = [ { @@ -29,7 +27,9 @@ describe('ImportCrittersService', () => { ]; const service = new ImportCrittersService(mockConnection, 1); - const parsedRow = service.getRowsToValidate(rows, ['COLLECTION', 'TEST'])[0]; + sinon.stub(service, '_getNonStandardColumns').returns(['COLLECTION']); + + const parsedRow = service.getRowsToValidate(rows, {})[0]; expect(parsedRow.sex).to.be.eq('Male'); expect(parsedRow.itis_tsn).to.be.eq(1); @@ -99,18 +99,16 @@ describe('ImportCrittersService', () => { it('should return unique list of tsns', async () => { const service = new ImportCrittersService(mockConnection, 1); - const mockWorksheet = {} as unknown as WorkSheet; - - const getRowsStub = sinon - .stub(service, 'getRows') - .returns([{ itis_tsn: 1 }, { itis_tsn: 2 }, { itis_tsn: 2 }] as any); - const getTaxonomyStub = sinon.stub(service.platformService, 'getTaxonomyByTsns').resolves([ { tsn: '1', scientificName: 'a' }, { tsn: '2', scientificName: 'b' } ]); - const tsns = await service._getValidTsns(mockWorksheet); - expect(getRowsStub).to.have.been.calledWith(mockWorksheet); + + const tsns = await service._getValidTsns([ + { critter_id: 'a', itis_tsn: 1 }, + { critter_id: 'b', itis_tsn: 2 } + ]); + expect(getTaxonomyStub).to.have.been.calledWith(['1', '2']); expect(tsns).to.deep.equal(['1', '2']); }); @@ -156,7 +154,7 @@ describe('ImportCrittersService', () => { ]; it('should return collection unit mapping', async () => { - const service = new ImportCrittersService(mockConnection); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const mockWorksheet = {} as unknown as WorkSheet; @@ -177,7 +175,7 @@ describe('ImportCrittersService', () => { }); it('should return empty map when no collection unit columns', async () => { - const service = new ImportCrittersService(mockConnection); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const mockWorksheet = {} as unknown as WorkSheet; @@ -193,7 +191,7 @@ describe('ImportCrittersService', () => { }); }); - describe('_insertCsvCrittersIntoSimsAndCritterbase', () => { + describe('insert', () => { afterEach(() => { sinon.restore(); }); @@ -220,7 +218,7 @@ describe('ImportCrittersService', () => { ]; it('should correctly parse collection units and critters and insert into sims / critterbase', async () => { - const service = new ImportCrittersService(mockConnection); + const service = new ImportCrittersService(mockConnection, 1); const critterbaseBulkCreateStub = sinon.stub(service.critterbaseService, 'bulkCreate'); const simsAddSurveyCrittersStub = sinon.stub(service.surveyCritterService, 'addCrittersToSurvey'); @@ -228,7 +226,7 @@ describe('ImportCrittersService', () => { critterbaseBulkCreateStub.resolves({ created: { critters: 2, collections: 1 } } as IBulkCreateResponse); simsAddSurveyCrittersStub.resolves([1]); - const ids = await service._insertCsvCrittersIntoSimsAndCritterbase(1, critters); + const ids = await service.insert(critters); expect(critterbaseBulkCreateStub).to.have.been.calledWithExactly({ critters: [ @@ -259,7 +257,7 @@ describe('ImportCrittersService', () => { }); it('should throw error if response from critterbase is less than provided critters', async () => { - const service = new ImportCrittersService(mockConnection); + const service = new ImportCrittersService(mockConnection, 1); const critterbaseBulkCreateStub = sinon.stub(service.critterbaseService, 'bulkCreate'); const simsAddSurveyCrittersStub = sinon.stub(service.surveyCritterService, 'addCrittersToSurvey'); @@ -268,7 +266,7 @@ describe('ImportCrittersService', () => { simsAddSurveyCrittersStub.resolves([1]); try { - await service._insertCsvCrittersIntoSimsAndCritterbase(1, critters); + await service.insert(critters); expect.fail(); } catch (err: any) { expect(err.message).to.be.equal('Unable to fully import critters from CSV'); @@ -278,7 +276,7 @@ describe('ImportCrittersService', () => { }); }); - describe('_validateRows', () => { + describe('validateRows', () => { afterEach(() => { sinon.restore(); }); @@ -318,8 +316,7 @@ describe('ImportCrittersService', () => { ]; it('should return successful', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -356,9 +353,7 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(validRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(validRows, {} as WorkSheet); expect(validation.success).to.be.true; @@ -387,8 +382,7 @@ describe('ImportCrittersService', () => { }); it('should push error when sex is undefined or invalid', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -425,14 +419,12 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(invalidRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(invalidRows, {} as WorkSheet); expect(validation.success).to.be.false; if (!validation.success) { - expect(validation.errors.length).to.be.eq(2); - expect(validation.errors).to.be.deep.equal([ + expect(validation.error.issues.length).to.be.eq(2); + expect(validation.error.issues).to.be.deep.equal([ { message: 'Invalid SEX. Expecting: UNKNOWN, MALE, FEMALE.', row: 0 @@ -446,8 +438,7 @@ describe('ImportCrittersService', () => { }); it('should push error when wlh_id is invalid regex / shape', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -484,14 +475,12 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(invalidRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(invalidRows, {} as WorkSheet); expect(validation.success).to.be.false; if (!validation.success) { - expect(validation.errors.length).to.be.eq(2); - expect(validation.errors).to.be.deep.equal([ + expect(validation.error.issues.length).to.be.eq(2); + expect(validation.error.issues).to.be.deep.equal([ { message: `Invalid WLH_ID. Example format '10-1000R'.`, row: 0 @@ -505,8 +494,7 @@ describe('ImportCrittersService', () => { }); it('should push error when itis_tsn undefined or invalid option', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -543,14 +531,12 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(invalidRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(invalidRows, {} as WorkSheet); expect(validation.success).to.be.false; if (!validation.success) { - expect(validation.errors.length).to.be.eq(4); - expect(validation.errors).to.be.deep.equal([ + expect(validation.error.issues.length).to.be.eq(4); + expect(validation.error.issues).to.be.deep.equal([ { message: `Invalid ITIS_TSN.`, row: 0 @@ -572,8 +558,7 @@ describe('ImportCrittersService', () => { }); it('should push error when itis_tsn undefined or invalid option', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -610,14 +595,12 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(invalidRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(invalidRows, {} as WorkSheet); expect(validation.success).to.be.false; if (!validation.success) { - expect(validation.errors.length).to.be.eq(4); - expect(validation.errors).to.be.deep.equal([ + expect(validation.error.issues.length).to.be.eq(4); + expect(validation.error.issues).to.be.deep.equal([ { message: `Invalid ITIS_TSN.`, row: 0 @@ -639,8 +622,7 @@ describe('ImportCrittersService', () => { }); it('should push error if alias undefined, duplicate or exists in survey', async () => { - const service = new ImportCrittersService(mockConnection); - const getRowsStub = sinon.stub(service, '_getRows'); + const service = new ImportCrittersService(mockConnection, 1); const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); @@ -677,14 +659,12 @@ describe('ImportCrittersService', () => { } ]; - getRowsStub.returns(invalidRows); - - const validation = await service._validateRows(1, {} as WorkSheet); + const validation = await service.validateRows(invalidRows, {} as WorkSheet); expect(validation.success).to.be.false; if (!validation.success) { - expect(validation.errors.length).to.be.eq(1); - expect(validation.errors).to.be.deep.equal([ + expect(validation.error.issues.length).to.be.eq(1); + expect(validation.error.issues).to.be.deep.equal([ { message: `Invalid ALIAS. Must be unique in Survey and CSV.`, row: 1 @@ -694,89 +674,91 @@ describe('ImportCrittersService', () => { }); }); - describe('_validate', () => { - afterEach(() => { - sinon.restore(); - }); - - it('should throw error when csv validation fails', async () => { - const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile').returns(false); - - const service = new ImportCrittersService(mockConnection); - - sinon.stub(service, '_validateRows'); - - try { - await service._validate(1, {} as WorkSheet); - expect.fail(); - } catch (err: any) { - expect(err.message).to.contain('Column validator failed.'); - } - expect(validateCsvStub).to.have.been.calledOnceWithExactly({}, critterStandardColumnValidator); - }); - - it('should call _validateRows if csv validation succeeds', async () => { - const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); - - const service = new ImportCrittersService(mockConnection); - - const validateRowsStub = sinon.stub(service, '_validateRows'); - - validateCsvStub.returns(true); - validateRowsStub.resolves({ success: true, data: [] }); - - const data = await service._validate(1, {} as WorkSheet); - expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); - expect(data).to.be.deep.equal([]); - }); - - it('should throw error if row validation fails', async () => { - const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); - - const service = new ImportCrittersService(mockConnection); - - const validateRowsStub = sinon.stub(service, '_validateRows'); - - validateCsvStub.returns(true); - validateRowsStub.resolves({ success: false, errors: [] }); - - try { - await service._validate(1, {} as WorkSheet); - - expect.fail(); - } catch (err: any) { - expect(err.message).to.contain('Failed to import Critter CSV.'); - } - expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); - }); - }); - - describe('import', () => { - it('should pass values to correct methods', async () => { - const service = new ImportCrittersService(mockConnection); - const csv = new MediaFile('file', 'mime', Buffer.alloc(1)); - - const critter: CsvCritter = { - critter_id: 'id', - sex: 'Male', - itis_tsn: 1, - animal_id: 'Carl', - wlh_id: '10-1000', - critter_comment: 'comment', - COLLECTION: 'Unit' - }; - - const getWorksheetStub = sinon.stub(service, '_getWorksheet').returns({} as unknown as WorkSheet); - const validateStub = sinon.stub(service, '_validate').resolves([critter]); - const insertStub = sinon.stub(service, '_insertCsvCrittersIntoSimsAndCritterbase').resolves([1]); - - const data = await service.import(1, csv); - - expect(getWorksheetStub).to.have.been.calledWithExactly(csv); - expect(validateStub).to.have.been.calledWithExactly(1, {}); - expect(insertStub).to.have.been.calledWithExactly(1, [critter]); - - expect(data).to.be.deep.equal([1]); - }); - }); + //TODO: Move these to the strategy test suite + // + //describe('_validate', () => { + // afterEach(() => { + // sinon.restore(); + // }); + // + // it('should throw error when csv validation fails', async () => { + // const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile').returns(false); + // + // const service = new ImportCrittersService(mockConnection, 1); + // + // sinon.stub(service, 'validateRows'); + // + // try { + // await service._validate(1, {} as WorkSheet); + // expect.fail(); + // } catch (err: any) { + // expect(err.message).to.contain('Column validator failed.'); + // } + // expect(validateCsvStub).to.have.been.calledOnceWithExactly({}, critterStandardColumnValidator); + // }); + // + // it('should call _validateRows if csv validation succeeds', async () => { + // const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); + // + // const service = new ImportCrittersService(mockConnection); + // + // const validateRowsStub = sinon.stub(service, '_validateRows'); + // + // validateCsvStub.returns(true); + // validateRowsStub.resolves({ success: true, data: [] }); + // + // const data = await service._validate(1, {} as WorkSheet); + // expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); + // expect(data).to.be.deep.equal([]); + // }); + // + // it('should throw error if row validation fails', async () => { + // const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); + // + // const service = new ImportCrittersService(mockConnection); + // + // const validateRowsStub = sinon.stub(service, '_validateRows'); + // + // validateCsvStub.returns(true); + // validateRowsStub.resolves({ success: false, errors: [] }); + // + // try { + // await service._validate(1, {} as WorkSheet); + // + // expect.fail(); + // } catch (err: any) { + // expect(err.message).to.contain('Failed to import Critter CSV.'); + // } + // expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); + // }); + //}); + // + //describe('import', () => { + // it('should pass values to correct methods', async () => { + // const service = new ImportCrittersService(mockConnection); + // const csv = new MediaFile('file', 'mime', Buffer.alloc(1)); + // + // const critter: CsvCritter = { + // critter_id: 'id', + // sex: 'Male', + // itis_tsn: 1, + // animal_id: 'Carl', + // wlh_id: '10-1000', + // critter_comment: 'comment', + // COLLECTION: 'Unit' + // }; + // + // const getWorksheetStub = sinon.stub(service, '_getWorksheet').returns({} as unknown as WorkSheet); + // const validateStub = sinon.stub(service, '_validate').resolves([critter]); + // const insertStub = sinon.stub(service, '_insertCsvCrittersIntoSimsAndCritterbase').resolves([1]); + // + // const data = await service.import(1, csv); + // + // expect(getWorksheetStub).to.have.been.calledWithExactly(csv); + // expect(validateStub).to.have.been.calledWithExactly(1, {}); + // expect(insertStub).to.have.been.calledWithExactly(1, [critter]); + // + // expect(data).to.be.deep.equal([1]); + // }); + //}); }); diff --git a/api/src/services/import-services/critter/import-critters-service.ts b/api/src/services/import-services/critter/import-critters-service.ts index e2c743a121..747752dfcb 100644 --- a/api/src/services/import-services/critter/import-critters-service.ts +++ b/api/src/services/import-services/critter/import-critters-service.ts @@ -17,7 +17,7 @@ import { import { DBService } from '../../db-service'; import { PlatformService } from '../../platform-service'; import { SurveyCritterService } from '../../survey-critter-service'; -import { CSVImportService, Row, Validation, ValidationError } from '../import-types'; +import { CSVImportService, Row, Validation, ValidationError } from '../csv-import-strategy.interface'; import { CsvCritter, PartialCsvCritter } from './import-critters-service.interface'; const defaultLog = getLogger('services/import/import-critters-service'); @@ -123,14 +123,12 @@ export class ImportCrittersService extends DBService implements CSVImportService } /** - * Get a Set of valid ITIS TSNS from xlsx worksheet. + * Get a Set of valid ITIS TSNS from xlsx worksheet rows. * * @async * @returns {Promise} Unique Set of valid TSNS from worksheet. */ async _getValidTsns(rows: PartialCsvCritter[]): Promise { - //const rows = this._getRows(worksheet); - // Get a unique list of tsns from worksheet const critterTsns = uniq(rows.map((row) => String(row.itis_tsn))); diff --git a/api/src/services/import-services/import-types.ts b/api/src/services/import-services/csv-import-strategy.interface.ts similarity index 99% rename from api/src/services/import-services/import-types.ts rename to api/src/services/import-services/csv-import-strategy.interface.ts index 5df80f2e3f..1027f75149 100644 --- a/api/src/services/import-services/import-types.ts +++ b/api/src/services/import-services/csv-import-strategy.interface.ts @@ -54,14 +54,17 @@ export interface CSVImportService { /** * CSV validation error + * */ export type ValidationError = { /** * CSV row index + * */ row: number; /** * CSV row error message + * */ message: string; }; diff --git a/api/src/services/import-services/csv-import-strategy.ts b/api/src/services/import-services/csv-import-strategy.ts index cf0f7a3792..8c3de66d8b 100644 --- a/api/src/services/import-services/csv-import-strategy.ts +++ b/api/src/services/import-services/csv-import-strategy.ts @@ -8,7 +8,7 @@ import { getWorksheetRowObjects, validateCsvFile } from '../../utils/xlsx-utils/worksheet-utils'; -import { CSVImportService } from './import-types'; +import { CSVImportService } from './csv-import-strategy.interface'; /** * CSV Import Strategy - Used with `CSVImportService` classes. diff --git a/api/src/utils/xlsx-utils/column-validator-utils.test.ts b/api/src/utils/xlsx-utils/column-validator-utils.test.ts new file mode 100644 index 0000000000..5a38253798 --- /dev/null +++ b/api/src/utils/xlsx-utils/column-validator-utils.test.ts @@ -0,0 +1,44 @@ +import { expect } from 'chai'; +import { + generateCellGetterFromColumnValidator, + getColumnAliasesFromValidator, + getColumnNamesFromValidator +} from './column-validator-utils'; +import { IXLSXCSVValidator } from './worksheet-utils'; + +const columnValidator = { + NAME: { type: 'string' }, + ID: { type: 'number', aliases: ['IDENTIFIER'] }, + AGE: { type: 'number' }, + BIRTH_DATE: { type: 'date' } +} satisfies IXLSXCSVValidator; + +describe.only('column-validator-utils', () => { + describe('getColumnNamesFromValidator', () => { + it('should return all column names from validator', () => { + expect(getColumnNamesFromValidator(columnValidator)).to.be.eql(['NAME', 'ID', 'AGE', 'BIRTH_DATE']); + }); + }); + + describe('getColumnAliasesFromValidator', () => { + it('should return all column aliases from validator', () => { + expect(getColumnAliasesFromValidator(columnValidator)).to.be.eql(['IDENTIFIER']); + }); + }); + + describe('generateCellGetterFromColumnValidator', () => { + const getCellValue = generateCellGetterFromColumnValidator(columnValidator); + + it('should return the cell value for a known column name', () => { + expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME')).to.be.eql('Dr. Steve Brule'); + }); + + it('should return the cell value for a known alias name', () => { + expect(getCellValue({ IDENTIFIER: 1 }, 'ID')).to.be.eql(1); + }); + + it('should return undefined if row cannot find cell value', () => { + expect(getCellValue({ BAD_NAME: 1 }, 'NAME')).to.be.eql(undefined); + }); + }); +}); diff --git a/api/src/utils/xlsx-utils/column-validator-utils.ts b/api/src/utils/xlsx-utils/column-validator-utils.ts index 086120d859..dc0d51bf3a 100644 --- a/api/src/utils/xlsx-utils/column-validator-utils.ts +++ b/api/src/utils/xlsx-utils/column-validator-utils.ts @@ -1,9 +1,13 @@ -import { Row } from '../../services/import-services/import-types'; +import { Row } from '../../services/import-services/csv-import-strategy.interface'; import { IXLSXCSVColumn, IXLSXCSVValidator } from './worksheet-utils'; +// TODO: Move the IXLSXCSVValidator type to this file + /** * Get column names / headers from column validator. * + * Note: This actually returns Uppercase[] but for convenience we define the return as string[] + * * @param {IXLSXCSVValidator} columnValidator * @returns {*} {string[]} Column names / headers */ @@ -14,22 +18,16 @@ export const getColumnNamesFromValidator = (columnValidator: IXLSXCSVValidator): /** * Get flattened list of ALL column aliases from column validator. * + * Note: This actually returns Uppercase[] but for convenience we define the return as string[] + * * @param {IXLSXCSVValidator} columnValidator * @returns {*} {string[]} Column aliases */ export const getColumnAliasesFromValidator = (columnValidator: IXLSXCSVValidator): string[] => { const columnNames = getColumnNamesFromValidator(columnValidator); - let columnAliases: string[] = []; - - for (const columnName of columnNames) { - const columnSpec: IXLSXCSVColumn = columnValidator[columnName]; - if (columnSpec.aliases?.length) { - columnAliases = columnAliases.concat(columnSpec.aliases); - } - } - - return columnAliases; + // Return flattened list of column validator aliases + return columnNames.flatMap((columnName) => (columnValidator[columnName] as IXLSXCSVColumn).aliases ?? []); }; /** @@ -50,12 +48,12 @@ export const getColumnValidatorSpecification = (columnValidator: IXLSXCSVValidat }; /** - * Generate a cell value getter from a column validator. + * Generate a cell getter from a column validator. * * Note: This will attempt to retrive the cell value from the row by the known header first. * If not found, it will then attempt to retrieve the value by the column header aliases. * - * TODO: Can the internal typing for this be improved? + * TODO: Can the internal typing for this be improved (without the `as` cast)? * * @example * const getCellValue = generateCellGetterFromColumnValidator(columnValidator) diff --git a/api/src/utils/xlsx-utils/worksheet-utils.test.ts b/api/src/utils/xlsx-utils/worksheet-utils.test.ts index f358b99c3e..a05483ec26 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.test.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.test.ts @@ -64,13 +64,12 @@ describe('worksheet-utils', () => { it('should validate aliases', () => { const observationCSVColumnValidator: IXLSXCSVValidator = { - columnNames: ['SPECIES', 'COUNT', 'DATE', 'TIME', 'LATITUDE', 'LONGITUDE'], - columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'], - columnAliases: { - LATITUDE: ['LAT'], - LONGITUDE: ['LON', 'LONG', 'LNG'], - SPECIES: ['TAXON'] - } + SPECIES: { type: 'string', aliases: ['TAXON'] }, + COUNT: { type: 'number' }, + DATE: { type: 'string' }, + TIME: { type: 'string' }, + LATITUDE: { type: 'number', aliases: ['LAT'] }, + LONGITUDE: { type: 'number', aliases: ['LON', 'LONG', 'LNG'] } }; const mockWorksheet = {} as unknown as xlsx.WorkSheet; @@ -87,12 +86,12 @@ describe('worksheet-utils', () => { it('should fail for unknown aliases', () => { const observationCSVColumnValidator: IXLSXCSVValidator = { - columnNames: ['SPECIES', 'COUNT', 'DATE', 'TIME', 'LATITUDE', 'LONGITUDE'], - columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'], - columnAliases: { - LATITUDE: ['LAT'], - LONGITUDE: ['LON', 'LONG', 'LNG'] - } + SPECIES: { type: 'string', aliases: ['TAXON'] }, + COUNT: { type: 'number' }, + DATE: { type: 'string' }, + TIME: { type: 'string' }, + LATITUDE: { type: 'number', aliases: ['LAT'] }, + LONGITUDE: { type: 'number', aliases: ['LON', 'LONG', 'LNG'] } }; const mockWorksheet = {} as unknown as xlsx.WorkSheet; diff --git a/api/src/utils/xlsx-utils/worksheet-utils.ts b/api/src/utils/xlsx-utils/worksheet-utils.ts index f9eb4f79b3..f7627f7e2c 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.ts @@ -1,4 +1,5 @@ import { default as dayjs } from 'dayjs'; +import { intersection } from 'lodash'; import xlsx, { CellObject } from 'xlsx'; import { getLogger } from '../logger'; import { MediaFile } from '../media/media-file'; @@ -195,13 +196,18 @@ export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record { // Get column names and aliases from validator - const columnNames = getColumnNamesFromValidator(columnValidator); - const columnAliases = getColumnAliasesFromValidator(columnValidator); + const validatorHeaders = getColumnNamesFromValidator(columnValidator); + // Get column names from actual worksheet const worksheetHeaders = getHeadersUpperCase(worksheet); - return columnNames.every((expectedHeader) => { - return columnAliases.some((alias) => worksheetHeaders.includes(alias)) || worksheetHeaders.includes(expectedHeader); + // Check that every validator header has matching header or alias in worksheet + return validatorHeaders.every((header) => { + const aliases: string[] = columnValidator[header]?.aliases ?? []; + const columnHeaderAndAliases = [header, ...aliases]; + + // Intersect the worksheet headers against the column header and aliases + return intersection(columnHeaderAndAliases, worksheetHeaders).length; }); }; @@ -354,8 +360,10 @@ export function getNonStandardColumnNamesFromWorksheet( const columnValidatorAliases = getColumnAliasesFromValidator(columnValidator); // Combine the column validator headers and all aliases - const standardColumNames = [...columnValidatorHeaders, ...columnValidatorAliases]; + const standardColumnNames = new Set([...columnValidatorHeaders, ...columnValidatorAliases]); + + console.log({ columns }); // Only return column names not in the validation CSV Column validator (ie: only return the non-standard columns) - return columns.filter((column) => !standardColumNames.includes(column)); + return columns.filter((column) => !standardColumnNames.has(column)); }