feat: updated tests for import services and util files

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts

@@ -152,7 +152,7 @@ export function importCsv(): RequestHandler {
       // Critter CSV import service - child of CSVImportStrategy
       const importCsvCritters = new ImportCrittersService(connection, surveyId);
 
-      // CSV import strategy with injected import service - parent
+      // Inject the import service into the CSV import strategy
       const csvImportStrategry = new CSVImportStrategy(importCsvCritters);
 
       // Import CSV data with strategy class

api/src/services/import-services/capture/import-captures-service.ts

@@ -6,7 +6,7 @@ import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils
 import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils';
 import { CritterbaseService, IBulkCreate } from '../../critterbase-service';
 import { DBService } from '../../db-service';
-import { CSVImportService, Row } from '../import-types';
+import { CSVImportService, Row } from '../csv-import-strategy.interface';
 import { CsvCapture, CsvCaptureSchema, PartialCsvCapture } from './import-captures-service.interface';
 
 /**

api/src/services/import-services/critter/import-critters-service.ts

@@ -17,7 +17,7 @@ import {
 import { DBService } from '../../db-service';
 import { PlatformService } from '../../platform-service';
 import { SurveyCritterService } from '../../survey-critter-service';
-import { CSVImportService, Row, Validation, ValidationError } from '../import-types';
+import { CSVImportService, Row, Validation, ValidationError } from '../csv-import-strategy.interface';
 import { CsvCritter, PartialCsvCritter } from './import-critters-service.interface';
 
 const defaultLog = getLogger('services/import/import-critters-service');
@@ -123,14 +123,12 @@ export class ImportCrittersService extends DBService implements CSVImportService
   }
 
   /**
-   * Get a Set of valid ITIS TSNS from xlsx worksheet.
+   * Get a Set of valid ITIS TSNS from xlsx worksheet rows.
    *
    * @async
    * @returns {Promise<string[]>} Unique Set of valid TSNS from worksheet.
    */
   async _getValidTsns(rows: PartialCsvCritter[]): Promise<string[]> {
-    //const rows = this._getRows(worksheet);
-
     // Get a unique list of tsns from worksheet
     const critterTsns = uniq(rows.map((row) => String(row.itis_tsn)));
 

api/src/services/import-services/import-types.ts → api/src/services/import-services/csv-import-strategy.interface.ts

@@ -54,14 +54,17 @@ export interface CSVImportService<ValidatedRow, PartialRow> {
 
 /**
  * CSV validation error
+ *
  */
 export type ValidationError = {
   /**
    * CSV row index
+   *
    */
   row: number;
   /**
    * CSV row error message
+   *
    */
   message: string;
 };

api/src/services/import-services/csv-import-strategy.ts

@@ -8,7 +8,7 @@ import {
   getWorksheetRowObjects,
   validateCsvFile
 } from '../../utils/xlsx-utils/worksheet-utils';
-import { CSVImportService } from './import-types';
+import { CSVImportService } from './csv-import-strategy.interface';
 
 /**
  * CSV Import Strategy - Used with `CSVImportService` classes.

api/src/utils/xlsx-utils/column-validator-utils.test.ts

@@ -0,0 +1,44 @@
+import { expect } from 'chai';
+import {
+  generateCellGetterFromColumnValidator,
+  getColumnAliasesFromValidator,
+  getColumnNamesFromValidator
+} from './column-validator-utils';
+import { IXLSXCSVValidator } from './worksheet-utils';
+
+const columnValidator = {
+  NAME: { type: 'string' },
+  ID: { type: 'number', aliases: ['IDENTIFIER'] },
+  AGE: { type: 'number' },
+  BIRTH_DATE: { type: 'date' }
+} satisfies IXLSXCSVValidator;
+
+describe.only('column-validator-utils', () => {
+  describe('getColumnNamesFromValidator', () => {
+    it('should return all column names from validator', () => {
+      expect(getColumnNamesFromValidator(columnValidator)).to.be.eql(['NAME', 'ID', 'AGE', 'BIRTH_DATE']);
+    });
+  });
+
+  describe('getColumnAliasesFromValidator', () => {
+    it('should return all column aliases from validator', () => {
+      expect(getColumnAliasesFromValidator(columnValidator)).to.be.eql(['IDENTIFIER']);
+    });
+  });
+
+  describe('generateCellGetterFromColumnValidator', () => {
+    const getCellValue = generateCellGetterFromColumnValidator(columnValidator);
+
+    it('should return the cell value for a known column name', () => {
+      expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME')).to.be.eql('Dr. Steve Brule');
+    });
+
+    it('should return the cell value for a known alias name', () => {
+      expect(getCellValue({ IDENTIFIER: 1 }, 'ID')).to.be.eql(1);
+    });
+
+    it('should return undefined if row cannot find cell value', () => {
+      expect(getCellValue({ BAD_NAME: 1 }, 'NAME')).to.be.eql(undefined);
+    });
+  });
+});

api/src/utils/xlsx-utils/column-validator-utils.ts

@@ -1,9 +1,13 @@
-import { Row } from '../../services/import-services/import-types';
+import { Row } from '../../services/import-services/csv-import-strategy.interface';
 import { IXLSXCSVColumn, IXLSXCSVValidator } from './worksheet-utils';
 
+// TODO: Move the IXLSXCSVValidator type to this file
+
 /**
  * Get column names / headers from column validator.
  *
+ * Note: This actually returns Uppercase<string>[] but for convenience we define the return as string[]
+ *
  * @param {IXLSXCSVValidator} columnValidator
  * @returns {*} {string[]} Column names / headers
  */
@@ -14,22 +18,16 @@ export const getColumnNamesFromValidator = (columnValidator: IXLSXCSVValidator):
 /**
  * Get flattened list of ALL column aliases from column validator.
  *
+ * Note: This actually returns Uppercase<string>[] but for convenience we define the return as string[]
+ *
  * @param {IXLSXCSVValidator} columnValidator
  * @returns {*} {string[]} Column aliases
  */
 export const getColumnAliasesFromValidator = (columnValidator: IXLSXCSVValidator): string[] => {
   const columnNames = getColumnNamesFromValidator(columnValidator);
 
-  let columnAliases: string[] = [];
-
-  for (const columnName of columnNames) {
-    const columnSpec: IXLSXCSVColumn = columnValidator[columnName];
-    if (columnSpec.aliases?.length) {
-      columnAliases = columnAliases.concat(columnSpec.aliases);
-    }
-  }
-
-  return columnAliases;
+  // Return flattened list of column validator aliases
+  return columnNames.flatMap((columnName) => (columnValidator[columnName] as IXLSXCSVColumn).aliases ?? []);
 };
 
 /**
@@ -50,12 +48,12 @@ export const getColumnValidatorSpecification = (columnValidator: IXLSXCSVValidat
 };
 
 /**
- * Generate a cell value getter from a column validator.
+ * Generate a cell getter from a column validator.
  *
  * Note: This will attempt to retrive the cell value from the row by the known header first.
  * If not found, it will then attempt to retrieve the value by the column header aliases.
  *
- * TODO: Can the internal typing for this be improved?
+ * TODO: Can the internal typing for this be improved (without the `as` cast)?
  *
  * @example
  * const getCellValue = generateCellGetterFromColumnValidator(columnValidator)

api/src/utils/xlsx-utils/worksheet-utils.test.ts

@@ -64,13 +64,12 @@ describe('worksheet-utils', () => {
 
     it('should validate aliases', () => {
       const observationCSVColumnValidator: IXLSXCSVValidator = {
-        columnNames: ['SPECIES', 'COUNT', 'DATE', 'TIME', 'LATITUDE', 'LONGITUDE'],
-        columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'],
-        columnAliases: {
-          LATITUDE: ['LAT'],
-          LONGITUDE: ['LON', 'LONG', 'LNG'],
-          SPECIES: ['TAXON']
-        }
+        SPECIES: { type: 'string', aliases: ['TAXON'] },
+        COUNT: { type: 'number' },
+        DATE: { type: 'string' },
+        TIME: { type: 'string' },
+        LATITUDE: { type: 'number', aliases: ['LAT'] },
+        LONGITUDE: { type: 'number', aliases: ['LON', 'LONG', 'LNG'] }
       };
 
       const mockWorksheet = {} as unknown as xlsx.WorkSheet;
@@ -87,12 +86,12 @@ describe('worksheet-utils', () => {
 
     it('should fail for unknown aliases', () => {
       const observationCSVColumnValidator: IXLSXCSVValidator = {
-        columnNames: ['SPECIES', 'COUNT', 'DATE', 'TIME', 'LATITUDE', 'LONGITUDE'],
-        columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'],
-        columnAliases: {
-          LATITUDE: ['LAT'],
-          LONGITUDE: ['LON', 'LONG', 'LNG']
-        }
+        SPECIES: { type: 'string', aliases: ['TAXON'] },
+        COUNT: { type: 'number' },
+        DATE: { type: 'string' },
+        TIME: { type: 'string' },
+        LATITUDE: { type: 'number', aliases: ['LAT'] },
+        LONGITUDE: { type: 'number', aliases: ['LON', 'LONG', 'LNG'] }
       };
 
       const mockWorksheet = {} as unknown as xlsx.WorkSheet;

api/src/utils/xlsx-utils/worksheet-utils.ts

@@ -1,4 +1,5 @@
 import { default as dayjs } from 'dayjs';
+import { intersection } from 'lodash';
 import xlsx, { CellObject } from 'xlsx';
 import { getLogger } from '../logger';
 import { MediaFile } from '../media/media-file';
@@ -195,13 +196,18 @@ export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record<string
  */
 export const validateWorksheetHeaders = (worksheet: xlsx.WorkSheet, columnValidator: IXLSXCSVValidator): boolean => {
   // Get column names and aliases from validator
-  const columnNames = getColumnNamesFromValidator(columnValidator);
-  const columnAliases = getColumnAliasesFromValidator(columnValidator);
+  const validatorHeaders = getColumnNamesFromValidator(columnValidator);
 
+  // Get column names from actual worksheet
   const worksheetHeaders = getHeadersUpperCase(worksheet);
 
-  return columnNames.every((expectedHeader) => {
-    return columnAliases.some((alias) => worksheetHeaders.includes(alias)) || worksheetHeaders.includes(expectedHeader);
+  // Check that every validator header has matching header or alias in worksheet
+  return validatorHeaders.every((header) => {
+    const aliases: string[] = columnValidator[header]?.aliases ?? [];
+    const columnHeaderAndAliases = [header, ...aliases];
+
+    // Intersect the worksheet headers against the column header and aliases
+    return intersection(columnHeaderAndAliases, worksheetHeaders).length;
   });
 };
 
@@ -354,8 +360,10 @@ export function getNonStandardColumnNamesFromWorksheet(
   const columnValidatorAliases = getColumnAliasesFromValidator(columnValidator);
 
   // Combine the column validator headers and all aliases
-  const standardColumNames = [...columnValidatorHeaders, ...columnValidatorAliases];
+  const standardColumnNames = new Set([...columnValidatorHeaders, ...columnValidatorAliases]);
+
+  console.log({ columns });
 
   // Only return column names not in the validation CSV Column validator (ie: only return the non-standard columns)
-  return columns.filter((column) => !standardColumNames.includes(column));
+  return columns.filter((column) => !standardColumnNames.has(column));
 }