From d7f84b96ae023065d67348611f681e70111c0e76 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 17 Sep 2024 16:04:45 -0700 Subject: [PATCH] Added separation to validate and submission based on what the user clicks on the frontend --- backend/src/cron-job/cron-job.service.ts | 15 ++- .../file_parse_and_validation.service.ts | 104 ++++++++++-------- 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index 9c78cf28..1b935e5f 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -442,21 +442,20 @@ export class CronJobService { if (filesToValidate.length < 1) { console.log("************** NO FILES TO VALIDATE **************"); + return } else { for (const file of filesToValidate) { const fileBinary = await this.objectStore.getFileData(file.file_name); - await this.fileSubmissionsService.updateFileStatus( + + this.fileParser.parseFile( + fileBinary, + file.file_name, file.submission_id, - "INPROGRESS", + file.file_operation_code, ); - - // this.fileParser.parseFile( - // fileBinary, - // file.file_name, - // file.submission_id, - // ); } this.dataPullDownComplete = false; + return } } } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index acca6542..7da2a0a2 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -852,7 +852,12 @@ export class FileParseValidateService { return finalErrorLog; } - async parseFile(file: string, fileName: string, file_submission_id: string) { + async parseFile( + file: string, + fileName: string, + file_submission_id: string, + file_operation_code: string, + ) { const path = require("path"); const extention = path.extname(fileName); if (extention == ".xlsx") { @@ -921,6 +926,11 @@ export class FileParseValidateService { * Do the local validation for each section here - if passed then go to the API calls - else create the message/file/email for the errors */ + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "INPROGRESS", + ); + const localValidationResults = this.localValidation( allRecords, ObsFilePath, @@ -937,58 +947,66 @@ export class FileParseValidateService { "REJECTED", ); console.log(await localValidationResults); + + return; } else if (!(await localValidationResults).includes("ERROR")) { await this.fileSubmissionsService.updateFileStatus( file_submission_id, "VALIDATED", ); - /* - * If the local validation passed then split the file into 4 and process with the AQI API calls - * Get unique records to prevent redundant API calls - * Post the unique records to the API - * Expand the returned list of object - this will be used for finding unique activities - */ - const uniqueVisitsWithCounts = this.getUniqueWithCounts(allFieldVisits); - let visitInfo = await this.postFieldVisits(uniqueVisitsWithCounts); - let expandedVisitInfo = visitInfo.flatMap((visit) => - Array(visit.count).fill(visit.rec), - ); - /* - * Merge the expanded visitInfo with allFieldActivities - * Collapse allFieldActivities with a dupe count - * Post the unique records to the API - * Expand the returned list of object - this will be used for finding unique specimens - */ + if (file_operation_code === "VALIDATE") { + return; + } else { + /* + * If the local validation passed then split the file into 4 and process with the AQI API calls + * Get unique records to prevent redundant API calls + * Post the unique records to the API + * Expand the returned list of object - this will be used for finding unique activities + */ + const uniqueVisitsWithCounts = + this.getUniqueWithCounts(allFieldVisits); + let visitInfo = await this.postFieldVisits(uniqueVisitsWithCounts); + let expandedVisitInfo = visitInfo.flatMap((visit) => + Array(visit.count).fill(visit.rec), + ); - allFieldActivities = allFieldActivities.map((obj2, index) => { - const obj1 = expandedVisitInfo[index]; - return { ...obj2, ...obj1 }; - }); + /* + * Merge the expanded visitInfo with allFieldActivities + * Collapse allFieldActivities with a dupe count + * Post the unique records to the API + * Expand the returned list of object - this will be used for finding unique specimens + */ - const uniqueActivitiesWithCounts = - this.getUniqueWithCounts(allFieldActivities); - let activityInfo = await this.postFieldActivities( - uniqueActivitiesWithCounts, - ); - let expandedActivityInfo = activityInfo.flatMap((activity) => - Array(activity.count).fill(activity.rec), - ); + allFieldActivities = allFieldActivities.map((obj2, index) => { + const obj1 = expandedVisitInfo[index]; + return { ...obj2, ...obj1 }; + }); - /* - * Merge the expanded activityInfo with allSpecimens - * Collapse allSpecimens with a dupe count - * Post the unique records to the API - */ - allSpecimens = allSpecimens.map((obj2, index) => { - const obj1 = expandedActivityInfo[index]; - return { ...obj2, ...obj1 }; - }); - const uniqueSpecimensWithCounts = - this.getUniqueWithCounts(allSpecimens); - await this.postFieldSpecimens(uniqueSpecimensWithCounts); + const uniqueActivitiesWithCounts = + this.getUniqueWithCounts(allFieldActivities); + let activityInfo = await this.postFieldActivities( + uniqueActivitiesWithCounts, + ); + let expandedActivityInfo = activityInfo.flatMap((activity) => + Array(activity.count).fill(activity.rec), + ); - await this.aqiService.importObservations(ObsFilePath, ""); + /* + * Merge the expanded activityInfo with allSpecimens + * Collapse allSpecimens with a dupe count + * Post the unique records to the API + */ + allSpecimens = allSpecimens.map((obj2, index) => { + const obj1 = expandedActivityInfo[index]; + return { ...obj2, ...obj1 }; + }); + const uniqueSpecimensWithCounts = + this.getUniqueWithCounts(allSpecimens); + await this.postFieldSpecimens(uniqueSpecimensWithCounts); + + await this.aqiService.importObservations(ObsFilePath, ""); + } } } }