From 4d294509b97acf83c2ce1cfb4a017a1175b3c0ac Mon Sep 17 00:00:00 2001 From: jhflorey Date: Wed, 26 Jun 2024 10:16:02 -0400 Subject: [PATCH 01/34] Fix: Notifications getting destroyed too soon --- utils/firestore.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index a909da3f..4af2b43b 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -450,7 +450,6 @@ const removeParticipantsDataDestruction = async () => { // Check each participant if they are already registered or more than 60 days from the date of their request // then the system will delete their data except the stub records and update the dataHasBeenDestroyed flag to yes. for (const doc of currSnapshot.docs) { - const batch = db.batch(); const participant = doc.data(); const timeDiff = isIsoDate(participant[dateRequestedDataDestroyCId]) ? new Date().getTime() - @@ -462,6 +461,7 @@ const removeParticipantsDataDestruction = async () => { requestedAndSignCId || timeDiff > millisecondsWait ) { + const batch = db.batch(); let hasRemovedField = false; const fieldKeys = Object.keys(participant); const participantRef = doc.ref; @@ -492,12 +492,12 @@ const removeParticipantsDataDestruction = async () => { }); count++; } + await batch.commit(); + await removeDocumentFromCollection( + participant["Connect_ID"], + participant["token"] + ); } - await batch.commit(); - await removeDocumentFromCollection( - participant["Connect_ID"], - participant["token"] - ); } console.log( From 260317d86e1c9df0bd13b2e76ebae440bdb79139 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Wed, 26 Jun 2024 14:41:21 -0400 Subject: [PATCH 02/34] Fix: addressing comment --- utils/firestore.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/firestore.js b/utils/firestore.js index 4af2b43b..b92e7fc6 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -447,6 +447,8 @@ const removeParticipantsDataDestruction = async () => { .where(dataHasBeenDestroyed, "!=", fieldMapping.yes) .get(); + const batch = db.batch(); + // Check each participant if they are already registered or more than 60 days from the date of their request // then the system will delete their data except the stub records and update the dataHasBeenDestroyed flag to yes. for (const doc of currSnapshot.docs) { @@ -461,7 +463,6 @@ const removeParticipantsDataDestruction = async () => { requestedAndSignCId || timeDiff > millisecondsWait ) { - const batch = db.batch(); let hasRemovedField = false; const fieldKeys = Object.keys(participant); const participantRef = doc.ref; From c97b5f71c012cd1972a2e07b0f1dd0ff8e5d8538 Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:06:47 -0400 Subject: [PATCH 03/34] Revert "Revert "Revert "Revert "RCA Variables -> add new variables"""" --- updateParticipantData.json | 17 +++++++++++++++++ utils/fieldToConceptIdMapping.js | 9 ++++++++- utils/shared.js | 24 +++++++++++++++++++++++- utils/sites.js | 2 +- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/updateParticipantData.json b/updateParticipantData.json index 0e5abfe0..ca1d8adb 100644 --- a/updateParticipantData.json +++ b/updateParticipantData.json @@ -540,6 +540,23 @@ "dataType": "object", "mustExist": false }, + "637153953[].844209241": { + "values": [104430631, 353358909, 178420302], + "dataType": "number", + "mustExist": false, + "required": true + }, + "637153953[].114227122": { + "values": [337516613, 646675764, 178420302], + "dataType": "number", + "mustExist": false, + "required": true + }, + "637153953[].421730068": { + "dataType": "string", + "mustExist": false, + "maxLength": 800 + }, "637153953[].740819233.149205077": { "values": [ 939782495, 135725957, 518416174, 847945207, 283025574, 942970912, 596122041, diff --git a/utils/fieldToConceptIdMapping.js b/utils/fieldToConceptIdMapping.js index 87402ca9..d28d5a7d 100644 --- a/utils/fieldToConceptIdMapping.js +++ b/utils/fieldToConceptIdMapping.js @@ -228,7 +228,14 @@ module.exports = { isCancerDiagnosis: 525972260, primaryCancerSiteObject: 740819233, primaryCancerSiteCategorical: 149205077, - preliminaryStageInformation: 457270069, + participantDiagnosisAwareness: 844209241, + pathologyAccessionNumber: 421730068, + vitalStatusCategorical: 114227122, + vitalStatus: { + alive: 337516613, + dead: 646675764, + unknown: 178420302, + }, anotherTypeOfCancerText: 868006655, // Text response for 'other' cancer site cancerSites: { anal: 939782495, diff --git a/utils/shared.js b/utils/shared.js index 453c64c6..61684b9f 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1224,6 +1224,11 @@ const handleCancerOccurrences = async (incomingCancerOccurrenceArray, requiredOc if (cancerSiteValidationObj.error === true) { return cancerSiteValidationObj; } + + const diagnosisAwarenessValidationObj = validateDiagnosisAwareness(occurrence[fieldMapping.vitalStatusCategorical], occurrence[fieldMapping.participantDiagnosisAwareness]); + if (diagnosisAwarenessValidationObj.error === true) { + return diagnosisAwarenessValidationObj; + } } // Query existing occurrences for the participant @@ -1254,7 +1259,7 @@ const handleCancerOccurrences = async (incomingCancerOccurrenceArray, requiredOc * If the 'fieldMapping.cancerSites.other' cancer site is selected, the 'fieldMapping.anotherTypeOfCancerText' field is required. * Else, the 'anotherTypeOfCancerText' field should not be present. * @param {object} cancerSitesObject - property (740819233) in the cancer occurrence object (637153953). - * @returns {boolean} - Returns true the above requirements are met, false otherwise. + * @returns {object} - Returns an object with error (boolean), message (string), and data (array). */ const validateCancerOccurrence = (cancerSitesObject) => { if (!cancerSitesObject || Object.keys(cancerSitesObject).length === 0 || !cancerSitesObject[fieldMapping.primaryCancerSiteCategorical]) { @@ -1275,6 +1280,23 @@ const validateCancerOccurrence = (cancerSitesObject) => { return { error: hasError, message: hasError ? otherCancerSiteErrorMessage : '', data: [] }; } +/** + * Rules: if vitalStatusCategorical is 'alive' at chart review (114227122: 337516613), participant must be aware of diagnosis (844209241: 353358909). Else, block API request. + * If vitalStatusCategorical is 'dead' or 'unknown' (114227122: 646675764 or 178420302), participant awareness can be yes, no, or unknown (844209241: 353358909 or 104430631 or 178420302). + * @param {number} vitalStatusCategorical - the participant's vital status (conceptID). + * @param {number} participantDiagnosisAwareness - the participant's awareness of diagnosis (conceptID). + * @returns {object} - Returns an object with error (boolean), message (string), and data (array). + */ +const validateDiagnosisAwareness = (vitalStatusCategorical, participantDiagnosisAwareness) => { + const isAliveAtChartReview = vitalStatusCategorical === fieldMapping.vitalStatus.alive; + const isParticipantAwareOfDiagnosis = participantDiagnosisAwareness === fieldMapping.yes; + + const isAwarenessValid = isAliveAtChartReview ? isParticipantAwareOfDiagnosis : true; + const awarenessErrorMessage = "Participant must be aware of diagnosis if alive at chart review. Otherwise, awareness can be 'yes (353358909)', 'no (104430631)', or 'unknown (178420302)'."; + + return { error: !isAwarenessValid, message: !isAwarenessValid ? awarenessErrorMessage : '', data: [] }; +} + /** * Check for duplicate cancer occurrences. Occurrences are considered duplicates if the timestamp and primary cancer sites match. * @param {array} newOccurrenceArray - the new occurrence array to check. diff --git a/utils/sites.js b/utils/sites.js index 1fb6f68f..8a9754ea 100644 --- a/utils/sites.js +++ b/utils/sites.js @@ -418,7 +418,7 @@ const flatValidationHandler = (newData, existingData, rules, validationFunction) * Validate data submitted to the updateParticipantData endpoint. * @param {string|number|array|object} value - The value to validate. From a key:value pair submitted in the POST request. * @param {string|number|array|object} existingValue - The existing value to validate against. From the existing participant data in the database. - * @param {string} path - The flattened path to the value in the data object. Example: 'state.123456789' or '637153953[].457270069' <- where [] is an array with any index value. + * @param {string} path - The flattened path to the value in the data object. Example: 'state.123456789' or '637153953[].149205077' <- where [] is an array with any index value. * @param {object} rule - The validation rule to use from updateParticipantData.json. Example: { "dataType": "string", "maxLength": 100 } * @returns null for success, or an error message for failure. */ From 699ff19a29953732aaa823792fee09b6c12e9cc3 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Thu, 27 Jun 2024 10:42:49 -0400 Subject: [PATCH 04/34] Revert code --- utils/firestore.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index b92e7fc6..4af2b43b 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -447,8 +447,6 @@ const removeParticipantsDataDestruction = async () => { .where(dataHasBeenDestroyed, "!=", fieldMapping.yes) .get(); - const batch = db.batch(); - // Check each participant if they are already registered or more than 60 days from the date of their request // then the system will delete their data except the stub records and update the dataHasBeenDestroyed flag to yes. for (const doc of currSnapshot.docs) { @@ -463,6 +461,7 @@ const removeParticipantsDataDestruction = async () => { requestedAndSignCId || timeDiff > millisecondsWait ) { + const batch = db.batch(); let hasRemovedField = false; const fieldKeys = Object.keys(participant); const participantRef = doc.ref; From a596d183707c11fd09926d3d03a67291f350b0ee Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:05:14 -0400 Subject: [PATCH 05/34] add getAppSettings endpoint --- utils/connectApp.js | 20 ++++++++++++++++++++ utils/firestore.js | 31 ++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/utils/connectApp.js b/utils/connectApp.js index 70cba8fc..a2b54527 100644 --- a/utils/connectApp.js +++ b/utils/connectApp.js @@ -73,6 +73,7 @@ const connectApp = async (req, res) => { return res.status(200).json({data: shaResult, code: 200}); } + else if (api === 'getSHAFromGitHubCommitData') { if (req.method !== 'GET') { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); @@ -91,6 +92,25 @@ const connectApp = async (req, res) => { return res.status(200).json({data: shaResult, code: 200}); } + else if (api === 'getAppSettings') { + if (req.method !== 'GET') { + return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); + } + + const selectedParamsArray = req.query.selectedParamsArray + ? req.query.selectedParamsArray.split(',') + : []; + + if (selectedParamsArray && !Array.isArray(selectedParamsArray)) { + return res.status(400).json(getResponseJSON("Error: selectedParamsArray is optional. If present, it must be an array.", 400)); + } + + const { getAppSettings } = require('./firestore'); + const appSettings = await getAppSettings('connectApp', selectedParamsArray); + + return res.status(200).json({data: appSettings, code: 200}); + } + else return res.status(400).json(getResponseJSON('Bad request!', 400)); } catch (error) { diff --git a/utils/firestore.js b/utils/firestore.js index b92e7fc6..da4f32f2 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -3296,6 +3296,34 @@ const generateSignInWithEmailLink = async (email, continueUrl) => { }); }; +/** + * Get the app settings from Firestore. + * @param {String} appName - Name of the app (e.g. 'connectApp', 'biospecimen', 'smdb') + * @param {Array} selectedParamsArray - (Optional) array of parameters to retrieve from the document. If not provided, all parameters will be retrieved. + * @returns {Object} - App settings object. + */ +const getAppSettings = async (appName, selectedParamsArray) => { + try { + let query = db.collection('appSettings').where('appName', '==', appName); + + if (selectedParamsArray && selectedParamsArray.length > 0) { + query = query.select(...selectedParamsArray); + } + + const snapshot = await query.get(); + + if (!snapshot.empty) { + return snapshot.docs[0].data(); + } else { + console.error(`No settings found for ${appName}`); + return {}; + } + } catch (error) { + console.error(`Error fetching app settings for ${appName}.`, error); + throw new Error("Error fetching app settings.", { cause: error }); + } +} + module.exports = { updateResponse, retrieveParticipants, @@ -3422,5 +3450,6 @@ module.exports = { writeCancerOccurrences, updateParticipantCorrection, updateSurveyEligibility, - generateSignInWithEmailLink + generateSignInWithEmailLink, + getAppSettings } From ba9ee19d766f41ac9e3866a75084f2c510988fc5 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Fri, 28 Jun 2024 10:02:28 -0400 Subject: [PATCH 06/34] Use db collection instead of batch --- utils/firestore.js | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index 4af2b43b..5ece393d 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -451,6 +451,7 @@ const removeParticipantsDataDestruction = async () => { // then the system will delete their data except the stub records and update the dataHasBeenDestroyed flag to yes. for (const doc of currSnapshot.docs) { const participant = doc.data(); + const participantId = doc.id; const timeDiff = isIsoDate(participant[dateRequestedDataDestroyCId]) ? new Date().getTime() - new Date(participant[dateRequestedDataDestroyCId]).getTime() @@ -461,38 +462,30 @@ const removeParticipantsDataDestruction = async () => { requestedAndSignCId || timeDiff > millisecondsWait ) { - const batch = db.batch(); + const updatedData = {}; let hasRemovedField = false; const fieldKeys = Object.keys(participant); - const participantRef = doc.ref; fieldKeys.forEach((key) => { if (!stubFieldArray.includes(key)) { - batch.update(participantRef, { - [key]: admin.firestore.FieldValue.delete(), - }); + updatedData[key] = admin.firestore.FieldValue.delete(); hasRemovedField = true; } else { if (key === "query" || key === "state") { const subFieldKeys = Object.keys(participant[key]); subFieldKeys.forEach((subKey) => { if (!subStubFieldArray.includes(subKey)) { - batch.update(participantRef, { - [`${key}.${subKey}`]: - admin.firestore.FieldValue.delete(), - }); + updatedData[`${key}.${subKey}`] = admin.firestore.FieldValue.delete(); } }); } } }); if (hasRemovedField) { - batch.update(participantRef, { - [dataHasBeenDestroyed]: fieldMapping.yes, - [fieldMapping.participationStatus]: fieldMapping.participantMap.dataDestroyedStatus, - }); + updatedData[dataHasBeenDestroyed] = fieldMapping.yes; + updatedData[fieldMapping.participationStatus] = fieldMapping.participantMap.dataDestroyedStatus; count++; } - await batch.commit(); + await db.collection('participants').doc(participantId).update(updatedData); await removeDocumentFromCollection( participant["Connect_ID"], participant["token"] From 5d5344818bc205ad193d8f1dfab4ed87baa86cb3 Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:35:42 -0400 Subject: [PATCH 07/34] make selectedParamsArray required (not optional) --- utils/connectApp.js | 9 +++------ utils/firestore.js | 15 ++++++--------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/utils/connectApp.js b/utils/connectApp.js index a2b54527..92f62b1c 100644 --- a/utils/connectApp.js +++ b/utils/connectApp.js @@ -97,12 +97,9 @@ const connectApp = async (req, res) => { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - const selectedParamsArray = req.query.selectedParamsArray - ? req.query.selectedParamsArray.split(',') - : []; - - if (selectedParamsArray && !Array.isArray(selectedParamsArray)) { - return res.status(400).json(getResponseJSON("Error: selectedParamsArray is optional. If present, it must be an array.", 400)); + const selectedParamsArray = req.query.selectedParamsArray?.split(','); + if (!selectedParamsArray || !Array.isArray(selectedParamsArray) || selectedParamsArray.length === 0) { + return res.status(400).json(getResponseJSON("Error: selectedParamsArray is required. Please specify parameters to return.", 400)); } const { getAppSettings } = require('./firestore'); diff --git a/utils/firestore.js b/utils/firestore.js index 88ae6d88..2db75795 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -3291,23 +3291,20 @@ const generateSignInWithEmailLink = async (email, continueUrl) => { /** * Get the app settings from Firestore. * @param {String} appName - Name of the app (e.g. 'connectApp', 'biospecimen', 'smdb') - * @param {Array} selectedParamsArray - (Optional) array of parameters to retrieve from the document. If not provided, all parameters will be retrieved. + * @param {Array} selectedParamsArray - Array of parameters to retrieve from the document. * @returns {Object} - App settings object. */ const getAppSettings = async (appName, selectedParamsArray) => { try { - let query = db.collection('appSettings').where('appName', '==', appName); - - if (selectedParamsArray && selectedParamsArray.length > 0) { - query = query.select(...selectedParamsArray); - } - - const snapshot = await query.get(); + const snapshot = await db.collection('appSettings') + .where('appName', '==', appName) + .select(...selectedParamsArray) + .get(); if (!snapshot.empty) { return snapshot.docs[0].data(); } else { - console.error(`No settings found for ${appName}`); + console.error(`No app settings found for ${appName}. Parameters requested: ${selectedParamsArray.join(', ')}`); return {}; } } catch (error) { From 1ad60598a14a2675531893832f981bdf6606479d Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Tue, 2 Jul 2024 10:04:09 -0500 Subject: [PATCH 08/34] updating version --- config/dev/.env.yaml | 2 +- config/prod/.env.yaml | 2 +- config/stage/.env.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/dev/.env.yaml b/config/dev/.env.yaml index 999f22bb..34099ba4 100644 --- a/config/dev/.env.yaml +++ b/config/dev/.env.yaml @@ -9,5 +9,5 @@ TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-dev/secrets/twilio-sms-auth-tok TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-dev/secrets/twilio-messaging-service-sid/versions/1 APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-dev/secrets/app-registration-tenant-id/versions/1 APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-dev/secrets/app-registration-client-id/versions/1 -APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-dev/secrets/app-registration-client-secret/versions/1 +APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-dev/secrets/app-registration-client-secret/versions/2 GITHUB_TOKEN: projects/nih-nci-dceg-connect-dev/secrets/questionnaire-sha-tracking/versions/1 \ No newline at end of file diff --git a/config/prod/.env.yaml b/config/prod/.env.yaml index 95118549..0823ad2c 100644 --- a/config/prod/.env.yaml +++ b/config/prod/.env.yaml @@ -9,5 +9,5 @@ TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-sms-au TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-messaging-service-sid/versions/1 APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-tenant-id/versions/1 APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-client-id/versions/1 -APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-client-secret/versions/1 +APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-client-secret/versions/2 GITHUB_TOKEN: projects/nih-nci-dceg-connect-prod-6d04/secrets/questionnaire-sha-tracking/versions/1 \ No newline at end of file diff --git a/config/stage/.env.yaml b/config/stage/.env.yaml index e30c6c01..eba982cc 100644 --- a/config/stage/.env.yaml +++ b/config/stage/.env.yaml @@ -9,7 +9,7 @@ TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-sms-aut TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-messaging-service-sid/versions/1 APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-tenant-id/versions/1 APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-client-id/versions/1 -APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-client-secret/versions/1 +APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-client-secret/versions/2 GITHUB_TOKEN: projects/nih-nci-dceg-connect-stg-5519/secrets/questionnaire-sha-tracking/versions/1 PROMIS_UOID: projects/nih-nci-dceg-connect-stg-5519/secrets/promis-uoid/versions/1 PROMIS_TOKEN: projects/nih-nci-dceg-connect-stg-5519/secrets/promis-token/versions/1 \ No newline at end of file From 7de4a1d78f7e3d1f2bc93d1098cf83f5a7d3d7d4 Mon Sep 17 00:00:00 2001 From: amber-emmes Date: Mon, 8 Jul 2024 12:21:42 -0400 Subject: [PATCH 09/34] Duplicate tracking numbers now prohibited (1038) --- utils/biospecimen.js | 3 ++- utils/firestore.js | 48 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/utils/biospecimen.js b/utils/biospecimen.js index fa4b612c..9ddf2ae3 100644 --- a/utils/biospecimen.js +++ b/utils/biospecimen.js @@ -673,11 +673,12 @@ const biospecimenAPIs = async (req, res) => { } const supplyQuery = req.query.supplyKitId; const collectionQuery = (req.query.collectionId?.slice(0, -4) || "") + " " + (req.query.collectionId?.slice(-4) || ""); // add space to collection + const returnKitTrackingNumberQuery = req.query.returnKitTrackingNumber; if (Object.keys(query).length === 0) return res.status(404).json(getResponseJSON('Please include id to check uniqueness.', 400)); if (collectionQuery.length < 14) return res.status(200).json({data: 'Check Collection ID', code:200}); try { const { checkCollectionUniqueness } = require('./firestore'); - const response = await checkCollectionUniqueness(supplyQuery, collectionQuery); + const response = await checkCollectionUniqueness(supplyQuery, collectionQuery, returnKitTrackingNumberQuery); return res.status(200).json({data: response, code:200}); } catch (error) { diff --git a/utils/firestore.js b/utils/firestore.js index 2db75795..ec6f90d8 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -2152,16 +2152,29 @@ const updateKitAssemblyData = async (data) => { } } -const checkCollectionUniqueness = async (supplyId, collectionId) => { +const checkCollectionUniqueness = async (supplyId, collectionId, returnKitTrackingNumber) => { try { const supplySnapShot = await db.collection('kitAssembly').where('690210658', '==', supplyId).get(); const collectionSnapShot = await db.collection('kitAssembly').where('259846815', '==', collectionId).get(); - if (supplySnapShot.docs.length === 0 && collectionSnapShot.docs.length === 0) { + let returnKitTrackingNumberSnapshot = {docs: []}; + let supplyKitTrackingNumberSnapshot = {docs: []}; + if(returnKitTrackingNumber) { + [returnKitTrackingNumberSnapshot, supplyKitTrackingNumberSnapshot] = await Promise.all([ + db.collection('kitAssembly').where(fieldMapping.returnKitTrackingNum.toString(), '==', returnKitTrackingNumber).get(), + db.collection('kitAssembly').where(fieldMapping.supplyKitTrackingNum.toString(), '==', returnKitTrackingNumber).get() + ]); + } + if (supplySnapShot.docs.length === 0 && collectionSnapShot.docs.length === 0 + && returnKitTrackingNumberSnapshot.docs.length === 0 && supplyKitTrackingNumberSnapshot.docs.length === 0) { return true; } else if (supplySnapShot.docs.length !== 0) { return 'duplicate supplykit id'; } else if (collectionSnapShot.docs.length !== 0) { return 'duplicate collection id'; + } else if (returnKitTrackingNumberSnapshot.docs.length !== 0) { + return 'duplicate return kit tracking number'; + } else if (supplyKitTrackingNumberSnapshot.docs.length !== 0) { + return 'return kit tracking number is for supply kit'; } } catch (error) { return new Error(error); @@ -2295,10 +2308,39 @@ const processParticipantHomeMouthwashKitData = (record, printLabel) => { const assignKitToParticipant = async (data) => { try { - const { supplyKitId, kitStatus, pending, uniqueKitID, supplyKitTrackingNum, + const { supplyKitId, kitStatus, pending, uniqueKitID, supplyKitTrackingNum, returnKitTrackingNum, assigned, collectionRound, collectionDetails, baseline, bioKitMouthwash, kitType, mouthwashKit } = fieldMapping; + + // Check the supply kit tracking number and see if it matches the return kit tracking number + // of any kits including this one or the supply kit tracking number of any other kits + + const kitsWithDuplicateReturnTrackingNumbers = await db.collection("kitAssembly") + .where(`${returnKitTrackingNum}`, '==', data[supplyKitTrackingNum]) + .get(); + + if(kitsWithDuplicateReturnTrackingNumbers.size > 0) { + return false; + } + + const otherKitsUsingSupplyKitTrackingNumber = await db.collection("kitAssembly") + .where(`${supplyKitTrackingNum}`, '==', data[supplyKitTrackingNum]) + .get(); + // .where(`${supplyKitId}`, '!=', data[supplyKitId]).get(); + + if(otherKitsUsingSupplyKitTrackingNumber.size > 1) { + return false; + } else if (otherKitsUsingSupplyKitTrackingNumber.size === 1) { + // check if the kit found is the current kit + // Doing this instead of including it in the query to avoid creating an unnecessary composite index + const possibleDuplicate = otherKitsUsingSupplyKitTrackingNumber.docs[0]; + const possibleDuplicateKitId = possibleDuplicate.data()[supplyKitId]; + if(possibleDuplicateKitId !== data[supplyKitId]) { + return false; + } + } + const kitSnapshot = await db.collection("kitAssembly") .where(`${supplyKitId}`, '==', data[supplyKitId]) .where(`${kitStatus}`, '==', pending).get(); From 8a80e7f656cf9058e6c05106a743992039af395a Mon Sep 17 00:00:00 2001 From: Warren Lu Date: Tue, 9 Jul 2024 15:50:35 -0400 Subject: [PATCH 10/34] Print docs count and clean up --- utils/firestore.js | 347 ++++++++++++++++++++++++++------------------- utils/shared.js | 26 +++- 2 files changed, 228 insertions(+), 145 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index a909da3f..919bca93 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -1,10 +1,9 @@ const functions = require('firebase-functions'); const admin = require('firebase-admin'); +const { Transaction } = require('firebase-admin/firestore'); admin.initializeApp(functions.config().firebase); const db = admin.firestore(); -const increment = admin.firestore.FieldValue.increment(1); -const decrement = admin.firestore.FieldValue.increment(-1); -const { tubeConceptIds, collectionIdConversion, swapObjKeysAndValues, batchLimit, listOfCollectionsRelatedToDataDestruction, createChunkArray, twilioErrorMessages, cidToLangMapper } = require('./shared'); +const { tubeConceptIds, collectionIdConversion, swapObjKeysAndValues, batchLimit, listOfCollectionsRelatedToDataDestruction, createChunkArray, twilioErrorMessages, cidToLangMapper, printDocsCount } = require('./shared'); const fieldMapping = require('./fieldToConceptIdMapping'); const { isIsoDate } = require('./validation'); @@ -24,6 +23,7 @@ const verifyTokenOrPin = async ({ token = null, pin = null }) => { } const snapshot = await query.get(); + printDocsCount(snapshot, "verifyTokenOrPin"); if (snapshot.size === 1) { const participantData = snapshot.docs[0].data(); if ( @@ -81,8 +81,9 @@ const linkParticipanttoFirebaseUID = async (docID, uID) => { const participantExists = async (uid) => { try{ - const response = await db.collection('participants').where('state.uid', '==', uid).get(); - if(response.size === 0){ + const snapshot = await db.collection('participants').where('state.uid', '==', uid).get(); + printDocsCount(snapshot, "participantExists"); + if(snapshot.size === 0){ return false; } else{ @@ -97,9 +98,10 @@ const participantExists = async (uid) => { const updateResponse = async (data, uid) => { try{ - const response = await db.collection('participants').where('state.uid', '==', uid).get(); - if(response.size === 1) { - for(let doc of response.docs){ + const snapshot = await db.collection('participants').where('state.uid', '==', uid).get(); + printDocsCount(snapshot, "updateResponse"); + if(snapshot.size === 1) { + for(let doc of snapshot.docs){ await db.collection('participants').doc(doc.id).update(data); return true; } @@ -111,20 +113,6 @@ const updateResponse = async (data, uid) => { } } -const incrementCounter = async (field, siteCode) => { - const snapShot = await db.collection('stats').where('siteCode', '==', siteCode).get(); - let obj = {} - obj[field] = increment; - await db.collection('stats').doc(snapShot.docs[0].id).update(obj); -} - -const decrementCounter = async (field, siteCode) => { - const snapShot = await db.collection('stats').where('siteCode', '==', siteCode).get(); - let obj = {} - obj[field] = decrement; - await db.collection('stats').doc(snapShot.docs[0].id).update(obj); -} - const createRecord = async (data) => { try{ await db.collection('participants').add(data); @@ -138,13 +126,13 @@ const createRecord = async (data) => { const recordExists = async (studyId, siteCode) => { try{ - const snapShot = await db.collection('participants').where('state.studyId', '==', studyId).where('827220437', '==', siteCode).get(); - if(snapShot.size === 1){ - return snapShot.docs[0].data(); - } - else { - return false; + const snapshot = await db.collection('participants').where('state.studyId', '==', studyId).where('827220437', '==', siteCode).get(); + printDocsCount(snapshot, "recordExists"); + if (snapshot.size === 1) { + return snapshot.docs[0].data(); } + + return false; } catch(error){ console.error(error); @@ -154,15 +142,15 @@ const recordExists = async (studyId, siteCode) => { const validateSiteSAEmail = async (saEmail) => { try{ - const snapShot = await db.collection('siteDetails') + const snapshot = await db.collection('siteDetails') .where('saEmail', '==', saEmail) .get(); - if(snapShot.size === 1) { - return snapShot.docs[0].data(); + printDocsCount(snapshot, "validateSiteSAEmail"); + if(snapshot.size === 1) { + return snapshot.docs[0].data(); } - else{ - return false; - }; + + return false; } catch(error){ console.error(error); @@ -173,16 +161,16 @@ const validateSiteSAEmail = async (saEmail) => { const getParticipantData = async (token, siteCode, isParent) => { try{ const operator = isParent ? 'in' : '=='; - const snapShot = await db.collection('participants') + const snapshot = await db.collection('participants') .where('token', '==', token) .where('827220437', operator, siteCode) .get(); - if(snapShot.size === 1) { - return {id: snapShot.docs[0].id, data: snapShot.docs[0].data()}; + printDocsCount(snapshot, "getParticipantData"); + if (snapshot.size === 1) { + return {id: snapshot.docs[0].id, data: snapshot.docs[0].data()}; } - else{ - return false; - }; + + return false; } catch(error){ console.error(error); @@ -197,10 +185,11 @@ const updateParticipantData = async (id, data) => { .update(data); } +// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. const retrieveParticipants = async (siteCode, decider, isParent, limit, page, site, from, to) => { try{ const operator = isParent ? 'in' : '=='; - let participants = {}; + let snapshot; const offset = (page-1)*limit; if(decider === 'verified') { let query = db.collection('participants') @@ -211,7 +200,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .offset(offset) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'notyetverified') { let query = db.collection('participants') @@ -222,7 +211,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .limit(limit) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'cannotbeverified') { let query = db.collection('participants') @@ -233,7 +222,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .limit(limit) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'profileNotSubmitted') { let query = db.collection('participants') @@ -244,7 +233,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .limit(limit) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'consentNotSubmitted') { let query = db.collection('participants') @@ -256,7 +245,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .limit(limit) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'notSignedIn') { let query = db.collection('participants') @@ -268,7 +257,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si .limit(limit) if(site) query = query.where('827220437', '==', site) // Get for a specific site else query = query.where('827220437', operator, siteCode) // Get for all site if parent - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'all') { let query = db.collection('participants') @@ -281,7 +270,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si else query = query.where('827220437', operator, siteCode) // Get for all site if parent if(from) query = query.where('914594314', '>=', from) if(to) query = query.where('914594314', '<=', to) - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'active') { let query = db.collection('participants') @@ -295,7 +284,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si else query = query.where('827220437', operator, siteCode) // Get for all site if parent if(from) query = query.where('914594314', '>=', from) if(to) query = query.where('914594314', '<=', to) - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'notactive') { let query = db.collection('participants') @@ -309,7 +298,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si else query = query.where('827220437', operator, siteCode) // Get for all site if parent if(from) query = query.where('914594314', '>=', from) if(to) query = query.where('914594314', '<=', to) - participants = await query.get(); + snapshot = await query.get(); } if(decider === 'passive') { let query = db.collection('participants') @@ -323,12 +312,11 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si else query = query.where('827220437', operator, siteCode) // Get for all site if parent if(from) query = query.where('914594314', '>=', from) if(to) query = query.where('914594314', '<=', to) - participants = await query.get(); + snapshot = await query.get(); } - return participants.docs.map(document => { - let data = document.data(); - return data; - }); + + printDocsCount(snapshot, "retrieveParticipants"); + return snapshot.docs.map(doc => doc.data()); } catch(error){ console.error(error); @@ -336,28 +324,29 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si } } +// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. const retrieveRefusalWithdrawalParticipants = async (siteCode, isParent, concept, limit, page) => { try { const operator = isParent ? 'in' : '=='; const offset = (page - 1) * limit; - let participants = await db.collection('participants') + const snpashot = await db.collection('participants') .where('827220437', operator, siteCode) .where(concept, '==', 353358909) .orderBy('Connect_ID', 'asc') .offset(offset) .limit(limit) .get(); + printDocsCount(snpashot, "retrieveRefusalWithdrawalParticipants"); - return participants.docs.map(document => { - return document.data(); - }); + return snpashot.docs.map(doc => doc.data()); } catch (error) { console.error(error); return new Error(error) } } +// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, isParent, limit, page) => { try { @@ -367,7 +356,7 @@ const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, is const { incentiveConcepts } = require('./shared'); const object = incentiveConcepts[roundType] - let participants = await db.collection('participants') + const snpashot = await db.collection('participants') .where('827220437', operator, siteCode) .where('821247024', '==', 197316935) .where(`${object}.222373868`, "==", 353358909) @@ -377,8 +366,9 @@ const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, is .offset(offset) .limit(limit) .get(); - - return participants.docs.map(document => { + printDocsCount(snpashot, "retrieveParticipantsEligibleForIncentives"); + + return snpashot.docs.map(document => { let data = document.data(); return {firstName: data['399159511'], email: data['869588347'], token: data['token'], site: data['827220437']} }); @@ -392,13 +382,14 @@ const removeDocumentFromCollection = async (connectID, token) => { try { for (const collection of listOfCollectionsRelatedToDataDestruction) { const query = db.collection(collection) - const data = + const snapshot = collection === "notifications" || collection === "ssn" ? await query.where("token", "==", token).get() : await query.where("Connect_ID", "==", connectID).get(); - - if (data.size !== 0) { - for (const dt of data.docs) { + printDocsCount(snapshot, "removeDocumentFromCollection"); + + if (snapshot.size !== 0) { + for (const dt of snapshot.docs) { await db.collection(collection).doc(dt.id).delete(); } } @@ -446,6 +437,7 @@ const removeParticipantsDataDestruction = async () => { .where(destroyDataCId, "==", fieldMapping.yes) .where(dataHasBeenDestroyed, "!=", fieldMapping.yes) .get(); + printDocsCount(currSnapshot, "removeParticipantsDataDestruction"); // Check each participant if they are already registered or more than 60 days from the date of their request // then the system will delete their data except the stub records and update the dataHasBeenDestroyed flag to yes. @@ -516,10 +508,11 @@ const removeUninvitedParticipants = async () => { while (willContinue) { const currSnapshot = await db - .collection('participants') - .where(uninvitedRecruitsCId, '==', fieldMapping.yes) - .limit(batchLimit) - .get(); + .collection("participants") + .where(uninvitedRecruitsCId, "==", fieldMapping.yes) + .limit(batchLimit) + .get(); + printDocsCount(currSnapshot, "removeUninvitedParticipants"); willContinue = currSnapshot.docs.length === batchLimit; const batch = db.batch(); @@ -544,13 +537,15 @@ const removeUninvitedParticipants = async () => { */ const getChildren = async (id) => { try{ - const snapShot = await db.collection('siteDetails') + const snapshot = await db.collection('siteDetails') .where('state.parentID', 'array-contains', id) .get(); - if(snapShot.size > 0) { + printDocsCount(snapshot, "getChildren"); + + if(snapshot.size > 0) { /** @type {number[]} */ const siteCodes = []; - snapShot.docs.forEach(document => { + snapshot.docs.forEach(document => { if(document.data().siteCode){ siteCodes.push(document.data().siteCode); } @@ -567,13 +562,15 @@ const getChildren = async (id) => { const verifyIdentity = async (type, token, siteCode) => { try{ - const snapShot = await db.collection('participants') + const snapshot = await db.collection('participants') .where('token', '==', token) .where('827220437', '==', siteCode) .get(); - if(snapShot.size > 0){ - const docId = snapShot.docs[0].id; - const docData = snapShot.docs[0].data(); + printDocsCount(snapshot, "verifyIdentity"); + + if(snapshot.size > 0){ + const docId = snapshot.docs[0].id; + const docData = snapshot.docs[0].data(); const existingVerificationStatus = docData[821247024]; const { conceptMappings } = require('./shared'); const concept = conceptMappings[type]; @@ -602,12 +599,13 @@ const verifyIdentity = async (type, token, siteCode) => { const retrieveUserProfile = async (uid) => { try{ - const snapShot = await db.collection('participants') + const snapshot = await db.collection('participants') .where('state.uid', '==', uid) .get(); + printDocsCount(snapshot, "retrieveUserProfile"); - if(snapShot.size > 0) { - let data = snapShot.docs[0].data(); + if(snapshot.size > 0) { + let data = snapshot.docs[0].data(); delete data.state; return data; @@ -627,6 +625,8 @@ const retrieveConnectID = async (uid) => { const snapshot = await db.collection('participants') .where('state.uid', '==', uid) .get(); + printDocsCount(snapshot, "retrieveConnectID"); + if(snapshot.size === 1){ if(snapshot.docs[0].data()['Connect_ID']) { return snapshot.docs[0].data()['Connect_ID']; @@ -665,6 +665,7 @@ const retrieveUserSurveys = async (uid, concepts) => { const snapshot = await db.collection(moduleConceptsToCollections[concept]) .where('uid', '==', uid) .get(); + printDocsCount(snapshot, "retrieveUserSurveys"); if (snapshot.size > 0) { return { concept, data: snapshot.docs[0].data() }; @@ -693,6 +694,8 @@ const retrieveUserSurveys = async (uid, concepts) => { const surveyExists = async (collection, uid) => { const snapshot = await db.collection(collection).where('uid', '==', uid).get(); + printDocsCount(snapshot, "surveyExists"); + if (snapshot.size > 0) { return snapshot.docs[0]; } @@ -727,8 +730,9 @@ const updateSurvey = async (data, collection, doc) => { const sanityCheckConnectID = async (ID) => { try{ const snapshot = await db.collection('participants').where('Connect_ID', '==', ID).get(); - if(snapshot.size === 0) return true; - else return false; + printDocsCount(snapshot, "sanityCheckConnectID"); + + return snapshot.size === 0; } catch(error){ console.error(error); @@ -739,8 +743,9 @@ const sanityCheckConnectID = async (ID) => { const sanityCheckPIN = async (pin) => { try{ const snapshot = await db.collection('participants').where('pin', '==', pin).get(); - if(snapshot.size === 0) return true; - else return false; + printDocsCount(snapshot, "sanityCheckPIN"); + + return snapshot.size === 0; } catch(error){ console.error(error); @@ -752,6 +757,7 @@ const individualParticipant = async (key, value, siteCode, isParent) => { try { const operator = isParent ? 'in' : '=='; const snapshot = await db.collection('participants').where(key, '==', value).where('827220437', operator, siteCode).get(); + printDocsCount(snapshot, "individualParticipant"); if(snapshot.size > 0) { return snapshot.docs.map(document => { let data = document.data(); @@ -770,6 +776,7 @@ const updateParticipantRecord = async (key, value, siteCode, isParent, obj) => { try { const operator = isParent ? 'in' : '=='; const snapshot = await db.collection('participants').where(key, '==', value).where('827220437', operator, siteCode).get(); + printDocsCount(snapshot, "updateParticipantRecord"); const docId = snapshot.docs[0].id; await db.collection('participants').doc(docId).update(obj); } @@ -781,9 +788,11 @@ const updateParticipantRecord = async (key, value, siteCode, isParent, obj) => { const deleteFirestoreDocuments = async (siteCode) => { try{ - const data = await db.collection('participants').where('827220437', '==', siteCode).get(); - if(data.size !== 0){ - data.docs.forEach(async dt =>{ + const snapshot = await db.collection('participants').where('827220437', '==', siteCode).get(); + printDocsCount(snapshot, "deleteFirestoreDocuments"); + + if(snapshot.size !== 0){ + snapshot.docs.forEach(async dt =>{ await db.collection('participants').doc(dt.id).delete() }) } @@ -800,11 +809,13 @@ const storeNotificationTokens = (data) => { } const notificationTokenExists = async (token) => { - const snapShot = await db.collection('notificationRegistration') + const snapshot = await db.collection('notificationRegistration') .where('notificationToken', '==', token) .get(); - if(snapShot.size === 1){ - return snapShot.docs[0].data().uid; + printDocsCount(snapshot, "notificationTokenExists"); + + if(snapshot.size === 1){ + return snapshot.docs[0].data().uid; } else { return false; @@ -821,6 +832,7 @@ const retrieveUserNotifications = async (uid) => { .where("uid", "==", uid) .orderBy("notification.time", "desc") .get(); + printDocsCount(snapshot, "retrieveUserNotifications"); return snapshot.docs.map((doc) => doc.data()); }; @@ -829,11 +841,12 @@ const retrieveSiteNotifications = async (siteId, isParent) => { try { let query = db.collection('siteNotifications'); if(!isParent) query = query.where('siteId', '==', siteId); - const snapShot = await query.orderBy('notification.time', 'desc') - .get(); + const snapshot = await query.orderBy('notification.time', 'desc') + .get(); + printDocsCount(snapshot, "retrieveSiteNotifications"); - if(snapShot.size > 0){ - return snapShot.docs.map(document => { + if(snapshot.size > 0){ + return snapshot.docs.map(document => { let data = document.data(); return data; }); @@ -925,6 +938,7 @@ const filterDB = async (queries, siteCode, isParent) => { const executeQuery = async (query) => { const operator = isParent ? 'in' : '=='; const snapshot = await (queries['allSiteSearch'] === 'true' ? query.get() : query.where('827220437', operator, siteCode).get()); + printDocsCount(snapshot, "executeQuery"); if (snapshot.size !== 0) { snapshot.docs.forEach(doc => { fetchedResults.push(doc.data()); @@ -1013,6 +1027,7 @@ const filterDB = async (queries, siteCode, isParent) => { const validateBiospecimenUser = async (email) => { try { const snapshot = await db.collection('biospecimenUsers').where('email', '==', email).get(); + printDocsCount(snapshot, "validateBiospecimenUser"); if(snapshot.size === 1) { const role = snapshot.docs[0].data().role; const siteCode = snapshot.docs[0].data().siteCode; @@ -1031,9 +1046,10 @@ const biospecimenUserList = async (siteCode, email) => { try { let query = db.collection('biospecimenUsers').where('siteCode', '==', siteCode) if(email) query = query.where('addedBy', '==', email) - const snapShot = await query.orderBy('role').orderBy('email').get(); - if(snapShot.size !== 0){ - return snapShot.docs.map(document => document.data()); + const snapshot = await query.orderBy('role').orderBy('email').get(); + printDocsCount(snapshot, "biospecimenUserList"); + if(snapshot.size !== 0){ + return snapshot.docs.map(document => document.data()); } else{ return []; @@ -1047,6 +1063,7 @@ const biospecimenUserList = async (siteCode, email) => { const biospecimenUserExists = async (email) => { try { const snapshot = await db.collection('biospecimenUsers').where('email', '==', email).get(); + printDocsCount(snapshot, "biospecimenUserExists"); if(snapshot.size === 0) return false; else return true; } catch (error) { @@ -1069,6 +1086,7 @@ const removeUser = async (userEmail, siteCode, email, manager) => { let query = db.collection('biospecimenUsers').where('email', '==', userEmail).where('siteCode', '==', siteCode); if(manager) query = query.where('addedBy', '==', email); const snapshot = await query.get(); + printDocsCount(snapshot, "removeUser"); if(snapshot.size === 1) { console.log('Removing', userEmail); const docId = snapshot.docs[0].id; @@ -1088,6 +1106,7 @@ const storeSpecimen = async (data) => { const updateSpecimen = async (id, data) => { const snapshot = await db.collection('biospecimen').where('820476880', '==', id).get(); + printDocsCount(snapshot, "updateSpecimen"); const docId = snapshot.docs[0].id; if (!data[fieldMapping.tubesBagsCids.streckTube]) { // Check for streck data due to intermittent null streck values in Firestore (11/2023). @@ -1135,6 +1154,7 @@ const getUnshippedBoxes = async (siteCode, isBPTL = false) => { let query = db.collection('boxes').where(fieldMapping.submitShipmentFlag.toString(), '==', fieldMapping.no); if (!isBPTL) query = query.where(fieldMapping.loginSite.toString(), '==', siteCode); const snapshot = await query.get(); + printDocsCount(snapshot, "getUnshippedBoxes"); return snapshot.docs.map(document => document.data()); } catch (error) { @@ -1156,6 +1176,7 @@ const getSpecimensByBoxedStatus = async (siteCode, boxedStatusConceptId, isBPTL if (!isBPTL) query = query.where(fieldMapping.healthCareProvider.toString(), '==', siteCode); const snapshot = await query.get(); + printDocsCount(snapshot, "getSpecimensByBoxedStatus"); return snapshot.docs.map(document => document.data()); } catch (error) { @@ -1190,6 +1211,7 @@ const updateBox = async (id, boxAndTubesData, addedTubes, loginSite) => { const boxSnapshot = snapshotResponse[0].docs.map(doc => ({ ref: doc.ref, data: doc.data() })); const specimenSnapshot = snapshotResponse[1].docs.map(doc => ({ ref: doc.ref, data: doc.data() })); + printDocsCount(snapshotResponse, "updateBox"); if (boxSnapshot.length !== 1 || specimenSnapshot.length !== 1) { throw new Error('Couldn\'t find Matching documents.'); @@ -1251,6 +1273,7 @@ const removeBag = async (siteCode, requestData) => { } const [boxSnapshot, ...specimenSnapshots] = await Promise.all([boxQuery, ...chunkedSpecimenQueries]); + printDocsCount([boxSnapshot, ...specimenSnapshots], "removeBag"); if (boxSnapshot.size !== 1 || specimenSnapshots.some(snapshot => snapshot.empty)) { throw new Error('Couldn\'t find Matching documents.'); @@ -1324,6 +1347,7 @@ const reportMissingSpecimen = async (siteAcronym, requestData) => { let conceptTube = conversion[tubeId]; const snapshot = await db.collection('biospecimen').where('820476880', '==', masterSpecimenId).where('siteAcronym', '==', siteAcronym).get(); + printDocsCount(snapshot, "reportMissingSpecimen"); if(snapshot.size === 1 && conceptTube != undefined){ const docId = snapshot.docs[0].id; let currDoc = snapshot.docs[0].data(); @@ -1355,6 +1379,7 @@ const getSpecimenAndParticipant = async (collectionId, siteCode, isBPTL) => { let query = db.collection('biospecimen').where(fieldMapping.collectionId.toString(), '==', collectionId); if (!isBPTL) query = query.where(fieldMapping.healthCareProvider.toString(), '==', siteCode); const specimenSnapshot = await query.get(); + printDocsCount(specimenSnapshot, "getSpecimenAndParticipant; collection: biospecimen"); if (specimenSnapshot.size !== 1) { throw new Error('Couldn\'t find matching specimen document.'); @@ -1363,6 +1388,7 @@ const getSpecimenAndParticipant = async (collectionId, siteCode, isBPTL) => { // Use the Connect_ID in the specimen doc to fetch the participant const participantSnapshot = await db.collection('participants').where('Connect_ID', '==', specimenData['Connect_ID']).get(); + printDocsCount(participantSnapshot, "getSpecimenAndParticipant; collection: participants"); if (participantSnapshot.size !== 1) { throw new Error('Couldn\'t find matching participant document.'); @@ -1377,6 +1403,7 @@ const getSpecimenAndParticipant = async (collectionId, siteCode, isBPTL) => { const searchSpecimen = async (masterSpecimenId, siteCode, allSitesFlag) => { const snapshot = await db.collection('biospecimen').where('820476880', '==', masterSpecimenId).get(); + printDocsCount(snapshot, "searchSpecimen"); if (snapshot.size === 1) { if (allSitesFlag) return snapshot.docs[0].data(); const token = snapshot.docs[0].data().token; @@ -1398,6 +1425,7 @@ const getSiteLocationBox = async (requestedSite, boxId) => { const snapshot = await db.collection('boxes') .where(fieldMapping.loginSite.toString(), "==", requestedSite) .where(fieldMapping.shippingBoxId.toString(), "==", boxId).get(); + printDocsCount(snapshot, "getSiteLocationBox"); const boxMatch = []; for(let document of snapshot.docs) { boxMatch.push(document.data()); @@ -1476,6 +1504,7 @@ const searchSpecimenBySiteAndBoxId = async (requestedSite, boxId) => { }); const promiseResults = await Promise.all(chunkedPromises); + printDocsCount(promiseResults, "searchSpecimenBySiteAndBoxId"); const biospecimenDocs = []; promiseResults.forEach(snapshot => { @@ -1496,6 +1525,7 @@ const searchShipments = async (siteCode) => { const tubesBagsCidKeys = swapObjKeysAndValues(tubesBagsCids); const snapshot = await db.collection('biospecimen').where(healthCareProvider, '==', siteCode).get(); let shipmentData = []; + printDocsCount(snapshot, "searchShipments"); if (snapshot.size !== 0) { for (let document of snapshot.docs) { @@ -1526,18 +1556,21 @@ const searchShipments = async (siteCode) => { const specimenExists = async (id) => { const snapshot = await db.collection('biospecimen').where('820476880', '==', id).get(); + printDocsCount(snapshot, "specimenExists"); if(snapshot.size === 1) return true; else return false; } const boxExists = async (boxId, loginSite) => { const snapshot = await db.collection('boxes').where('132929440', '==', boxId).where('789843387', '==', loginSite).get(); + printDocsCount(snapshot, "boxExists"); if(snapshot.size === 1) return true; else return false; } const accessionIdExists = async (accessionId, accessionIdType, siteCode) => { const snapshot = await db.collection('biospecimen').where(accessionIdType, '==', accessionId).get(); + printDocsCount(snapshot, "accessionIdExists"); if(snapshot.size === 1) { const token = snapshot.docs[0].data().token; const response = await db.collection('participants').where('token', '==', token).get(); @@ -1553,18 +1586,18 @@ const updateTempCheckDate = async (institute) => { let randomStart = Math.floor(Math.random()*5)+15 - currDate.getDay(); currDate.setDate(currDate.getDate() + randomStart); const snapshot = await db.collection('SiteLocations').where('Site', '==',institute).get(); + printDocsCount(snapshot, "updateTempCheckDate"); if(snapshot.size === 1) { const docId = snapshot.docs[0].id; await db.collection('SiteLocations').doc(docId).update({'nextTempMonitor':currDate.toString()}); - //console.log(currDate.toString()); } } /** * - * @param {Array} boxIdArray - array of box ids to fetch + * @param {Array} boxIdArray - array of box ids to fetch * @param {string} siteCode - site code of the user (number) - * @param {transaction} transaction - firestore transation object + * @param {Transaction} transaction - firestore transation object * @returns boxes object with data and docRef * If boxIdArray.length > 15, chunk the array into multiple queries to support the use of 'in' operator. */ @@ -1597,6 +1630,7 @@ const getBoxesByBoxId = async (boxIdArray, siteCode, transaction = null) => { const chunkPromises = chunksToSend.map(chunk => getSnapshot(chunk)); const snapshots = await Promise.all(chunkPromises); + printDocsCount(snapshots, "getBoxesByBoxId"); snapshots.forEach(snapshot => { const docsArray = snapshot.docs.map(document => ({ data: document.data(), docRef: document.ref })); @@ -1606,6 +1640,7 @@ const getBoxesByBoxId = async (boxIdArray, siteCode, transaction = null) => { } else { const snapshot = await getSnapshot(boxIdArray); resultsArray = snapshot.docs.map(document => ({ data: document.data(), docRef: document.ref })); + printDocsCount(snapshot, "getBoxesByBoxId"); } return resultsArray; @@ -1654,6 +1689,7 @@ const shipBatchBoxes = async (boxIdAndShipmentDataArray, siteCode) => { const shipBox = async (boxId, siteCode, shippingData, trackingNumbers) => { const snapshot = await db.collection('boxes').where('132929440', '==', boxId).where('789843387', '==',siteCode).get(); + printDocsCount(snapshot, "shipBox"); if(snapshot.size === 1) { let currDate = new Date().toISOString(); shippingData['656548982'] = currDate; @@ -1670,6 +1706,7 @@ const shipBox = async (boxId, siteCode, shippingData, trackingNumbers) => { const getLocations = async (institute) => { const snapshot = await db.collection('SiteLocations').where('siteAcronym', '==', institute).get(); + printDocsCount(snapshot, "getLocations"); console.log(institute) if(snapshot.size !== 0) { return snapshot.docs.map(document => document.data()); @@ -1698,6 +1735,7 @@ const searchBoxes = async (institute, flag) => { } if (snapshot.size !== 0){ + printDocsCount(snapshot, "searchBoxes"); return snapshot.docs.map(document => document.data()); } else { return []; @@ -1708,7 +1746,7 @@ const searchBoxesByLocation = async (institute, location) => { console.log("institute" + institute); console.log("location" + location); const snapshot = await db.collection('boxes').where('789843387', '==', institute).where('560975149','==',location).get(); - console.log(snapshot); + printDocsCount(snapshot, "searchBoxesByLocation"); if(snapshot.size !== 0){ let result = snapshot.docs.map(document => document.data()); // console.log(JSON.stringify(result)); @@ -1724,6 +1762,7 @@ const searchBoxesByLocation = async (institute, location) => { const getSpecimenCollections = async (token, siteCode) => { const snapshot = await db.collection('biospecimen').where('token', '==', token).where('827220437', '==', siteCode).get(); + printDocsCount(snapshot, "getSpecimenCollections"); if(snapshot.size !== 0){ return snapshot.docs.map(document => document.data()); } @@ -1750,6 +1789,7 @@ const buildQueryWithFilters = (query, trackingId, endDate, startDate, source, si return query } +// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. const getBoxesPagination = async (siteCode, body) => { const currPage = body.pageNumber; const orderByField = body.orderBy; @@ -1764,6 +1804,7 @@ const getBoxesPagination = async (siteCode, body) => { query = preQueryBuilder(filters, query, trackingId, endDate, startDate, source, siteCode); query = query.orderBy(orderByField, 'desc').limit(elementsPerPage).offset(currPage * elementsPerPage); const snapshot = await query.get(); + printDocsCount(snapshot, "getBoxesPagination"); const result = snapshot.docs.map(document => document.data()); return result; } @@ -1797,6 +1838,7 @@ const getNotificationSpecifications = async (notificationType, notificationCateg if(notificationCategory) snapshot = snapshot.where('category', '==', notificationCategory); snapshot = snapshot.where('scheduleAt', '==', scheduleAt); snapshot = await snapshot.get(); + printDocsCount(snapshot, "getNotificationSpecifications"); return snapshot.docs.map(document => { return document.data(); }); @@ -1806,6 +1848,7 @@ const getNotificationSpecifications = async (notificationType, notificationCateg } } +// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. const retrieveParticipantsByStatus = async (conditions, limit, offset) => { try { let query = db.collection('participants') @@ -1871,23 +1914,15 @@ const retrieveParticipantsByStatus = async (conditions, limit, offset) => { } query = query.where(obj, operator, values); } - const participants = await query.get(); - return participants.docs.map(doc => doc.data()); + const snapshot = await query.get(); + printDocsCount(snapshot, "retrieveParticipantsByStatus"); + return snapshot.docs.map(doc => doc.data()); } catch (error) { console.error(error); return new Error(error); } } -const notificationAlreadySent = async (token, notificationSpecificationsID) => { - try { - const snapshot = await db.collection('notifications').where('token', '==', token).where('notificationSpecificationsID', '==', notificationSpecificationsID).get(); - return snapshot.size !== 0; - } catch (error) { - return new Error(error); - } -} - const sendClientEmail = async (data) => { const { sendEmail } = require('./notifications'); @@ -1934,9 +1969,10 @@ const checkIsNotificationSent = async (userToken, specId) => { .collection("notifications") .where("token", "==", userToken) .where("notificationSpecificationsID", "==", specId) + .count() .get(); - return !snapshot.empty; + return snapshot.data().count > 0; }; /** @@ -1963,6 +1999,7 @@ const saveNotificationBatch = async (notificationRecordArray) => { const markNotificationAsRead = async (id, collection) => { const snapshot = await db.collection(collection).where('id', '==', id).get(); + printDocsCount(snapshot, "markNotificationAsRead"); const docId = snapshot.docs[0].id; await db.collection(collection).doc(docId).update({read: true}); } @@ -1970,6 +2007,7 @@ const markNotificationAsRead = async (id, collection) => { const storeSSN = async (data) => { try{ const response = await db.collection('ssn').where('uid', '==', data.uid).get(); + printDocsCount(response, "storeSSN"); if(response.size === 1) { for(let doc of response.docs){ await db.collection('ssn').doc(doc.id).update(data); @@ -1989,16 +2027,19 @@ const storeSSN = async (data) => { const getTokenForParticipant = async (uid) => { const snapshot = await db.collection('participants').where('state.uid', '==', uid).get(); + printDocsCount(snapshot, "getTokenForParticipant"); return snapshot.docs[0].data()['token']; } const getSiteDetailsWithSignInProvider = async (acronym) => { const snapshot = await db.collection('siteDetails').where('acronym', '==', acronym).get(); + printDocsCount(snapshot, "getSiteDetailsWithSignInProvider"); return snapshot.docs[0].data(); } const retrieveNotificationSchemaByID = async (id) => { const snapshot = await db.collection("notificationSpecifications").where("id", "==", id).get(); + printDocsCount(snapshot, "retrieveNotificationSchemaByID"); if (snapshot.size === 1) { return snapshot.docs[0].id; } @@ -2015,6 +2056,7 @@ const retrieveNotificationSchemaByCategory = async (category, getDrafts = false, } const snapshot = await query.orderBy("attempt").get(); + printDocsCount(snapshot, "retrieveNotificationSchemaByCategory"); return snapshot.docs.map((doc) => doc.data()); }; @@ -2034,11 +2076,15 @@ const getNotificationHistoryByParticipant = async (token, siteCode, isParent) => .where('token', '==', token) .where('827220437', operator, siteCode) .get(); + printDocsCount(participantRecord, "getNotificationHistoryByParticipant; collection: participants"); + if(participantRecord.size === 1) { const snapshot = await db.collection('notifications') .where('token', '==', token) .orderBy('notification.time', 'desc') .get(); + printDocsCount(snapshot, "getNotificationHistoryByParticipant; collection: notifications"); + return snapshot.docs.map(dt => dt.data()); } else return false; @@ -2046,6 +2092,7 @@ const getNotificationHistoryByParticipant = async (token, siteCode, isParent) => const getNotificationsCategories = async (scheduleAt) => { const snapshot = await db.collection('notificationSpecifications').where('scheduleAt', '==', scheduleAt).get(); + printDocsCount(snapshot, "getNotificationsCategories"); const categories = []; snapshot.forEach(dt => { const category = dt.data().category; @@ -2056,6 +2103,7 @@ const getNotificationsCategories = async (scheduleAt) => { const getEmailNotifications = async (scheduleAt) => { const snapshot = await db.collection('notificationSpecifications').where('scheduleAt', '==', scheduleAt).get(); + printDocsCount(snapshot, "getEmailNotifications"); const notifications = []; snapshot.forEach(dt => { const notification = dt.data(); @@ -2071,6 +2119,7 @@ const getEmailNotifications = async (scheduleAt) => { */ const getNotificationSpecsBySchedule = async (scheduleAt) => { const snapshot = await db.collection("notificationSpecifications").where("scheduleAt", "==", scheduleAt).get(); + printDocsCount(snapshot, "getNotificationSpecsBySchedule"); let notificationSpecArray = []; for (const doc of snapshot.docs) { const docData = doc.data(); @@ -2092,6 +2141,7 @@ const getNotificationSpecsByScheduleOncePerDay = async (scheduleAt) => { const currTimeIsoStr = currTime.toISOString(); const batch = db.batch(); const snapshot = await db.collection("notificationSpecifications").where("scheduleAt", "==", scheduleAt).get(); + printDocsCount(snapshot, "getNotificationSpecsByScheduleOncePerDay"); let notificationSpecArray = []; for (const doc of snapshot.docs) { const docData = doc.data(); @@ -2109,6 +2159,7 @@ const getNotificationSpecsByScheduleOncePerDay = async (scheduleAt) => { const getNotificationSpecById = async (id) => { const snapshot = await db.collection('notificationSpecifications').where('id', '==', id).get(); + printDocsCount(snapshot, "getNotificationSpecById"); return snapshot.empty ? null : snapshot.docs[0].data(); } @@ -2121,6 +2172,7 @@ const getNotificationSpecByCategoryAndAttempt = async (category = "", attempt = .where("category", "==", category) .where("attempt", "==", attempt) .get(); + printDocsCount(snapshot, "getNotificationSpecByCategoryAndAttempt"); return snapshot.empty ? null : snapshot.docs[0].data(); }; @@ -2140,6 +2192,7 @@ const addKitAssemblyData = async (data) => { const updateKitAssemblyData = async (data) => { try { const snapShot = await db.collection('kitAssembly').where('687158491', '==', data['687158491']).get(); + printDocsCount(snapShot, "updateKitAssemblyData"); if (snapShot.empty) return false const docId = snapShot.docs[0].id; @@ -2163,6 +2216,7 @@ const checkCollectionUniqueness = async (supplyId, collectionId) => { try { const supplySnapShot = await db.collection('kitAssembly').where('690210658', '==', supplyId).get(); const collectionSnapShot = await db.collection('kitAssembly').where('259846815', '==', collectionId).get(); + printDocsCount([supplySnapShot, collectionSnapShot], "checkCollectionUniqueness"); if (supplySnapShot.docs.length === 0 && collectionSnapShot.docs.length === 0) { return true; } else if (supplySnapShot.docs.length !== 0) { @@ -2235,6 +2289,7 @@ const eligibleParticipantsForKitAssignment = async () => { .select(...participantHomeCollectionKitFields) .orderBy(`${collectionDetails}.${baseline}.${bloodOrUrineCollectedTimestamp}`) .get(); + printDocsCount(snapshot, "eligibleParticipantsForKitAssignment"); return snapshot.size === 0 ? [] @@ -2251,6 +2306,7 @@ const addKitStatusToParticipant = async (participantsCID) => { // Create an array of promises to update participants in parallel const updatePromises = participantsCID.map(async (participantCID) => { const snapshot = await db.collection("participants").where('Connect_ID', '==', parseInt(participantCID)).get(); + printDocsCount(snapshot, "addKitStatusToParticipant"); if (snapshot.size === 0) { // No matching document found, stop the update return false; @@ -2309,6 +2365,7 @@ const assignKitToParticipant = async (data) => { const kitSnapshot = await db.collection("kitAssembly") .where(`${supplyKitId}`, '==', data[supplyKitId]) .where(`${kitStatus}`, '==', pending).get(); + printDocsCount(kitSnapshot, "assignKitToParticipant; collection: kitAssembly"); if (kitSnapshot.size !== 1) { return false; @@ -2326,6 +2383,7 @@ const assignKitToParticipant = async (data) => { await kitDoc.ref.update(kitData); const participantSnapshot = await db.collection("participants").where('Connect_ID', '==', parseInt(data['Connect_ID'])).get(); + printDocsCount(participantSnapshot, "assignKitToParticipant; collection: participants"); if (participantSnapshot.size !== 1) { return false; @@ -2359,6 +2417,7 @@ const assignKitToParticipant = async (data) => { const processVerifyScannedCode = async (id) => { try { const snapShot = await db.collection('kitAssembly').where('531858099', '==', id).where('221592017', '==', 241974920).get(); + printDocsCount(snapShot, "processVerifyScannedCode"); if (snapShot.docs.length === 1) { return { valid: true, uniqueKitID: snapShot.docs[0].data()[687158491] } } @@ -2374,6 +2433,7 @@ const confirmShipmentKit = async (shipmentData) => { const { collectionDetails, baseline, bioKitMouthwash, uniqueKitID } = fieldMapping; const kitSnapshot = await db.collection("kitAssembly").where('687158491', '==', shipmentData['687158491']).get(); + printDocsCount(kitSnapshot, "confirmShipmentKit; collection: kitAssembly"); if (kitSnapshot.size === 0) { return false; @@ -2388,6 +2448,7 @@ const confirmShipmentKit = async (shipmentData) => { await kitDoc.ref.update(kitData); const participantSnapshot = await db.collection("participants") .where(`${collectionDetails}.${baseline}.${bioKitMouthwash}.${uniqueKitID}`, '==', shipmentData[uniqueKitID]).get(); + printDocsCount(participantSnapshot, "confirmShipmentKit; collection: participants"); if (participantSnapshot.size === 0) { return false; @@ -2431,6 +2492,7 @@ const storeKitReceipt = async (package) => { let toReturn; await db.runTransaction(async (transaction) => { const kitSnapshot = await transaction.get(db.collection("kitAssembly").where('972453354', '==', package['972453354']).where('221592017', '==', 277438316)); + printDocsCount(kitSnapshot, "storeKitReceipt"); if (kitSnapshot.size === 0) { toReturn = false; return; @@ -2545,6 +2607,7 @@ const processPackageConditions = (pkgConditions) => { const getKitAssemblyData = async () => { try { const snapshot = await db.collection("kitAssembly").get(); + printDocsCount(snapshot, "getKitAssemblyData"); if(snapshot.size !== 0) return snapshot.docs.map(doc => doc.data()) else return false; } @@ -2566,6 +2629,7 @@ const storeSiteNotifications = async (reminder) => { const getCoordinatingCenterEmail = async () => { try { const snapshot = await db.collection('siteDetails').where('coordinatingCenter', '==', true).get(); + printDocsCount(snapshot, "getCoordinatingCenterEmail"); if(snapshot.size > 0) return snapshot.docs[0].data().email; } catch (error) { console.error(error); @@ -2576,6 +2640,7 @@ const getCoordinatingCenterEmail = async () => { const getSiteEmail = async (siteCode) => { try { const snapshot = await db.collection('siteDetails').where('siteCode', '==', siteCode).get(); + printDocsCount(snapshot, "getSiteEmail"); if(snapshot.size > 0) return snapshot.docs[0].data().email; } catch (error) { console.error(error); @@ -2586,6 +2651,7 @@ const getSiteEmail = async (siteCode) => { const getSiteAcronym = async (siteCode) => { try { const snapshot = await db.collection('siteDetails').where('siteCode', '==', siteCode).get(); + printDocsCount(snapshot, "getSiteAcronym"); if(snapshot.size > 0) return snapshot.docs[0].data().acronym; } catch (error) { console.error(error); @@ -2596,6 +2662,7 @@ const getSiteAcronym = async (siteCode) => { const getSiteMostRecentBoxId = async (siteCode) => { try { const snapshot = await db.collection('siteDetails').where('siteCode', '==', siteCode).get(); + printDocsCount(snapshot, "getSiteMostRecentBoxId"); if (snapshot.size > 0) { const doc = snapshot.docs[0]; return { @@ -2687,11 +2754,12 @@ const shippedKitStatusParticipants = async () => { ); } - const kitAssemblySnapshot = await Promise.all(kitAssemblyPromises); + const kitAssemblySnapshots = await Promise.all(kitAssemblyPromises); + printDocsCount(kitAssemblySnapshots, "shippedKitStatusParticipants"); - kitAssemblySnapshot.forEach((result, index) => { - if(!result.empty) { - const kitData = result.docs[0].data(); + kitAssemblySnapshots.forEach((snapshot, index) => { + if(!snapshot.empty) { + const kitData = snapshot.docs[0].data(); Object.assign(participants[index], kitData); } }); @@ -2733,6 +2801,7 @@ const storePackageReceipt = async (data) => { const setPackageReceiptUSPS = async (data) => { try { const snapshot = await db.collection("participantSelection").where('usps_trackingNum', '==', data.scannedBarcode).get(); + printDocsCount(snapshot, "setPackageReceiptUSPS"); const docId = snapshot.docs[0].id; await db.collection("participantSelection").doc(docId).update( { @@ -2759,6 +2828,7 @@ const setPackageReceiptFedex = async (boxUpdateData) => { } const boxSnapshot = await query.get(); + printDocsCount(boxSnapshot, "setPackageReceiptFedex"); if (boxSnapshot.empty) { console.error('Box not found'); @@ -2822,6 +2892,7 @@ const processReceiptData = async (collectionIdHolder, boxUpdateData, boxDocRef) }); const specimenQuerySnapshots = await Promise.all(specimenDocsToFetch); + printDocsCount(specimenQuerySnapshots, "processReceiptData"); let specimenDocs = []; specimenQuerySnapshots.forEach(snapshot => { snapshot.docs.forEach(specimenDoc => { @@ -2878,10 +2949,10 @@ const processReceiptData = async (collectionIdHolder, boxUpdateData, boxDocRef) const kitStatusCounterVariation = async (currentkitStatus, prevKitStatus) => { try { await db.collection("bptlMetrics").doc('--metrics--').update({ - [currentkitStatus]: increment + [currentkitStatus]: admin.firestore.FieldValue.increment(1) }) await db.collection("bptlMetrics").doc('--metrics--').update({ - [prevKitStatus]: decrement + [prevKitStatus]: admin.firestore.FieldValue.increment(-1) }) return true; } @@ -2893,6 +2964,7 @@ const kitStatusCounterVariation = async (currentkitStatus, prevKitStatus) => { const getBptlMetrics = async () => { const snapshot = await db.collection("bptlMetrics").get(); + printDocsCount(snapshot, "getBptlMetrics"); return snapshot.docs.map(doc => doc.data()) } @@ -2900,6 +2972,7 @@ const getBptlMetricsForShipped = async () => { try { let response = [] const snapshot = await db.collection("participantSelection").where('kit_status', '==', 'shipped').get(); + printDocsCount(snapshot, "getBptlMetricsForShipped"); let shipedParticipants = snapshot.docs.map(doc => doc.data()) const keys = ['first_name', 'last_name', 'pickup_date', 'participation_status'] shipedParticipants.forEach( i => { response.push(pick(i, keys) )}); @@ -2915,27 +2988,6 @@ const pick = (obj, arr) => { return arr.reduce((acc, record) => (record in obj && (acc[record] = obj[record]), acc), {}) } -const processBsiData = async (tubeConceptIds, query) => { - return await Promise.all(tubeConceptIds.map( async tubeConceptId => { // using await promise.all waits until all the ele in a map are processed - const snapshot = await db.collection("biospecimen").where(`${tubeConceptId}.926457119`, '==', query).get();// perform query on tube level - return snapshot.docs.map(doc => { - let collectionIdInfo = {} - collectionIdInfo['825582494'] = doc.data()[tubeConceptId]['825582494'] - collectionIdInfo['926457119'] = doc.data()[tubeConceptId]['926457119'] - collectionIdInfo['678166505'] = doc.data()['678166505'] - collectionIdInfo['Connect_ID'] = doc.data()['Connect_ID'] - // collectionIdInfo['789843387'] = i['789843387'] - collectionIdInfo['827220437'] = doc.data()['827220437'] - collectionIdInfo['951355211'] = doc.data()['951355211'] - collectionIdInfo['915838974'] = doc.data()['915838974'] - collectionIdInfo['650516960'] = doc.data()['650516960'] - collectionIdInfo['762124027'] = doc.data()[tubeConceptId]['762124027'] === undefined ? `` : doc.data()[tubeConceptId]['762124027'] - collectionIdInfo['982885431'] = doc.data()[tubeConceptId]['248868659'] === undefined ? `` : doc.data()[tubeConceptId]['248868659']['982885431'] - return collectionIdInfo - }) // push query results to holdBiospecimenMatches array - })); -} - const verifyUsersEmailOrPhone = async (req) => { const queries = req.query if(queries.email) { @@ -3045,6 +3097,7 @@ const queryDailyReportParticipants = async (sitecode) => { let query = db.collection('participants'); try { const snapshot = await query.where('331584571.266600170.840048338', '>=', twoDaysAgo).where('827220437', '==', sitecode).get(); + printDocsCount(snapshot, "queryDailyReportParticipants"); if (snapshot.size !== 0) { const promises = snapshot.docs.map(async (document) => { return processQueryDailyReportParticipants(document); @@ -3064,6 +3117,7 @@ const queryDailyReportParticipants = async (sitecode) => { const processQueryDailyReportParticipants = async (document) => { try { const secondSnapshot = await db.collection('biospecimen').where('Connect_ID', '==', document.data()['Connect_ID']).get(); + printDocsCount(secondSnapshot, "processQueryDailyReportParticipants"); if (secondSnapshot.size !== 0) { const dailyReport = {}; if (secondSnapshot.docs[0].data()['951355211'] !== undefined) { @@ -3088,6 +3142,7 @@ const processQueryDailyReportParticipants = async (document) => { const getRestrictedFields = async () => { const snapshot = await db.collection('siteDetails').where('coordinatingCenter', '==', true).get(); + printDocsCount(snapshot, "getRestrictedFields"); return snapshot.docs[0].data().restrictedFields; } @@ -3121,6 +3176,7 @@ const getSpecimensByReceivedDate = async (receivedTimestamp) => { */ const getBoxesByReceivedDate = async (receivedTimestamp) => { const snapshot = await db.collection('boxes').where('926457119', '==', receivedTimestamp).get(); + printDocsCount(snapshot, "getBoxesByReceivedDate"); return snapshot.docs.map(doc => doc.data()); } @@ -3152,6 +3208,7 @@ const getSpecimensByCollectionIds = async (collectionIdsArray, siteCode, isBPTL const chunkQueries = chunksToSend.map(chunk => getSnapshot(chunk)); const snapshots = await Promise.all(chunkQueries); + printDocsCount(snapshots, "getSpecimensByCollectionIds"); snapshots.forEach(snapshot => { const docsArray = snapshot.docs.map(document => ({ data: document.data(), docRef: document.ref })); @@ -3173,6 +3230,7 @@ const processSendGridEvent = async (event) => { .collection("notifications") .where("id", "==", event.notification_id) .get(); + printDocsCount(snapshot, "processSendGridEvent"); if (snapshot.size > 0) { const doc = snapshot.docs[0]; @@ -3197,6 +3255,7 @@ const processTwilioEvent = async (event) => { .collection("notifications") .where("messageSid", "==", event.MessageSid) .get(); + printDocsCount(snapshot, "processTwilioEvent"); if (snapshot.size > 0) { const doc = snapshot.docs[0]; @@ -3217,6 +3276,7 @@ const processTwilioEvent = async (event) => { const getParticipantCancerOccurrences = async (participantToken) => { try { const snapshot = await db.collection('cancerOccurrence').where('token', '==', participantToken).get(); + printDocsCount(snapshot, "getParticipantCancerOccurrences"); return snapshot.docs.map(doc => doc.data()); } catch (error) { throw new Error("Error fetching cancer occurrences.", { cause: error }); @@ -3245,6 +3305,7 @@ const writeCancerOccurrences = async (cancerOccurrenceArray) => { const updateParticipantCorrection = async (participantData) => { try { const snapshot = await db.collection('participants').where('token', '==', participantData['token']).get(); + printDocsCount(snapshot, "updateParticipantCorrection"); if (snapshot.empty) return false const docId = snapshot.docs[0].id; delete participantData['token'] @@ -3271,6 +3332,7 @@ const updateSurveyEligibility = async (token, survey) => { try { const snapshot = await db.collection('participants').where('token', '==', token).get(); + printDocsCount(snapshot, "updateSurveyEligibility"); if (snapshot.empty) return; @@ -3347,15 +3409,12 @@ module.exports = { getSpecimenCollections, getBoxesPagination, getNumBoxesShipped, - incrementCounter, - decrementCounter, updateParticipantRecord, retrieveParticipantsEligibleForIncentives, removeParticipantsDataDestruction, removeUninvitedParticipants, getNotificationSpecifications, retrieveParticipantsByStatus, - notificationAlreadySent, storeNotification, checkIsNotificationSent, validateSiteSAEmail, @@ -3421,5 +3480,5 @@ module.exports = { writeCancerOccurrences, updateParticipantCorrection, updateSurveyEligibility, - generateSignInWithEmailLink + generateSignInWithEmailLink,db } diff --git a/utils/shared.js b/utils/shared.js index 61684b9f..d690abc3 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1,5 +1,6 @@ -const fieldMapping = require('./fieldToConceptIdMapping'); const {SecretManagerServiceClient} = require('@google-cloud/secret-manager'); +const { QuerySnapshot } = require('firebase-admin/firestore'); +const fieldMapping = require('./fieldToConceptIdMapping'); const getResponseJSON = (message, code) => { return { message, code }; @@ -1572,6 +1573,28 @@ const cidToLangMapper = { [fieldMapping.spanish]: "spanish", }; +/** + * @param {QuerySnapshot | QuerySnapshot[]} snapshot A query snapshot or an array of snapshots + * @param {string} infoStr Name of the function and other info to be printed + * @returns {void} + */ +const printDocsCount = (snapshot, infoStr = "") => { + let count = 0; + if (Array.isArray(snapshot)) { + for (const snap of snapshot) { + if (snap.constructor.name !== "QuerySnapshot" || snap.empty) continue; + count += snap.size; + } + } else { + if (snapshot.constructor.name !== "QuerySnapshot" || snapshot.empty) return; + count = snapshot.size; + } + + if (count > 0) { + console.log(`Docs read from Firestore: ${count}; function: ${infoStr}`); + } +}; + module.exports = { getResponseJSON, setHeaders, @@ -1635,4 +1658,5 @@ module.exports = { twilioErrorMessages, getSecret, cidToLangMapper, + printDocsCount, }; From 33b82989d5976228fd863db6baddd2655ca87cd4 Mon Sep 17 00:00:00 2001 From: Warren Lu Date: Tue, 9 Jul 2024 17:12:51 -0400 Subject: [PATCH 11/34] use consistent `snapshot` per suggestions --- utils/firestore.js | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index 2ebe2e00..768b7391 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -330,16 +330,16 @@ const retrieveRefusalWithdrawalParticipants = async (siteCode, isParent, concept const operator = isParent ? 'in' : '=='; const offset = (page - 1) * limit; - const snpashot = await db.collection('participants') + const snapshot = await db.collection('participants') .where('827220437', operator, siteCode) .where(concept, '==', 353358909) .orderBy('Connect_ID', 'asc') .offset(offset) .limit(limit) .get(); - printDocsCount(snpashot, "retrieveRefusalWithdrawalParticipants"); + printDocsCount(snapshot, "retrieveRefusalWithdrawalParticipants"); - return snpashot.docs.map(doc => doc.data()); + return snapshot.docs.map(doc => doc.data()); } catch (error) { console.error(error); return new Error(error) @@ -356,7 +356,7 @@ const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, is const { incentiveConcepts } = require('./shared'); const object = incentiveConcepts[roundType] - const snpashot = await db.collection('participants') + const snapshot = await db.collection('participants') .where('827220437', operator, siteCode) .where('821247024', '==', 197316935) .where(`${object}.222373868`, "==", 353358909) @@ -366,9 +366,9 @@ const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, is .offset(offset) .limit(limit) .get(); - printDocsCount(snpashot, "retrieveParticipantsEligibleForIncentives"); + printDocsCount(snapshot, "retrieveParticipantsEligibleForIncentives"); - return snpashot.docs.map(document => { + return snapshot.docs.map(document => { let data = document.data(); return {firstName: data['399159511'], email: data['869588347'], token: data['token'], site: data['827220437']} }); @@ -500,16 +500,16 @@ const removeUninvitedParticipants = async () => { const uninvitedRecruitsCId = fieldMapping.participantMap.uninvitedRecruits.toString(); while (willContinue) { - const currSnapshot = await db + const snapshot = await db .collection("participants") .where(uninvitedRecruitsCId, "==", fieldMapping.yes) .limit(batchLimit) .get(); - printDocsCount(currSnapshot, "removeUninvitedParticipants"); + printDocsCount(snapshot, "removeUninvitedParticipants"); - willContinue = currSnapshot.docs.length === batchLimit; + willContinue = snapshot.docs.length === batchLimit; const batch = db.batch(); - for (const doc of currSnapshot.docs) { + for (const doc of snapshot.docs) { batch.delete(doc.ref); count++ } @@ -1371,13 +1371,13 @@ const getSpecimenAndParticipant = async (collectionId, siteCode, isBPTL) => { // Fetch the specimen let query = db.collection('biospecimen').where(fieldMapping.collectionId.toString(), '==', collectionId); if (!isBPTL) query = query.where(fieldMapping.healthCareProvider.toString(), '==', siteCode); - const specimenSnapshot = await query.get(); - printDocsCount(specimenSnapshot, "getSpecimenAndParticipant; collection: biospecimen"); + const snapshot = await query.get(); + printDocsCount(snapshot, "getSpecimenAndParticipant; collection: biospecimen"); - if (specimenSnapshot.size !== 1) { + if (snapshot.size !== 1) { throw new Error('Couldn\'t find matching specimen document.'); } - const specimenData = specimenSnapshot.docs[0].data(); + const specimenData = snapshot.docs[0].data(); // Use the Connect_ID in the specimen doc to fetch the participant const participantSnapshot = await db.collection('participants').where('Connect_ID', '==', specimenData['Connect_ID']).get(); @@ -2184,11 +2184,11 @@ const addKitAssemblyData = async (data) => { const updateKitAssemblyData = async (data) => { try { - const snapShot = await db.collection('kitAssembly').where('687158491', '==', data['687158491']).get(); - printDocsCount(snapShot, "updateKitAssemblyData"); + const snapshot = await db.collection('kitAssembly').where('687158491', '==', data['687158491']).get(); + printDocsCount(snapshot, "updateKitAssemblyData"); - if (snapShot.empty) return false - const docId = snapShot.docs[0].id; + if (snapshot.empty) return false + const docId = snapshot.docs[0].id; await db.collection('kitAssembly').doc(docId).update({ '194252513': data[fieldMapping.returnKitId], @@ -2820,15 +2820,15 @@ const setPackageReceiptFedex = async (boxUpdateData) => { delete boxUpdateData['shipmentTimestamp']; } - const boxSnapshot = await query.get(); - printDocsCount(boxSnapshot, "setPackageReceiptFedex"); + const snapshot = await query.get(); + printDocsCount(snapshot, "setPackageReceiptFedex"); - if (boxSnapshot.empty) { + if (snapshot.empty) { console.error('Box not found'); return { message: 'Box Not Found', data: null }; } - const boxListData = boxSnapshot.docs.map(doc => ({ + const boxListData = snapshot.docs.map(doc => ({ boxDocRef: doc.ref, boxData: doc.data(), })); From 77965e383f75b100706712b189b4dff1884b7170 Mon Sep 17 00:00:00 2001 From: Warren Lu Date: Tue, 9 Jul 2024 17:15:10 -0400 Subject: [PATCH 12/34] one more fix --- utils/firestore.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index 768b7391..fc80d856 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -2409,10 +2409,10 @@ const assignKitToParticipant = async (data) => { const processVerifyScannedCode = async (id) => { try { - const snapShot = await db.collection('kitAssembly').where('531858099', '==', id).where('221592017', '==', 241974920).get(); - printDocsCount(snapShot, "processVerifyScannedCode"); - if (snapShot.docs.length === 1) { - return { valid: true, uniqueKitID: snapShot.docs[0].data()[687158491] } + const snapshot = await db.collection('kitAssembly').where('531858099', '==', id).where('221592017', '==', 241974920).get(); + printDocsCount(snapshot, "processVerifyScannedCode"); + if (snapshot.docs.length === 1) { + return { valid: true, uniqueKitID: snapshot.docs[0].data()[687158491] } } else { return false } } catch (error) { From 538576a849a28921c9c9ca8a39c67353cf4cf76f Mon Sep 17 00:00:00 2001 From: Warren Lu Date: Wed, 10 Jul 2024 10:19:18 -0400 Subject: [PATCH 13/34] include offset in printed logs --- utils/firestore.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index fc80d856..93cbe764 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -315,7 +315,7 @@ const retrieveParticipants = async (siteCode, decider, isParent, limit, page, si snapshot = await query.get(); } - printDocsCount(snapshot, "retrieveParticipants"); + printDocsCount(snapshot, `retrieveParticipants; offset: ${offset}`); return snapshot.docs.map(doc => doc.data()); } catch(error){ @@ -337,7 +337,7 @@ const retrieveRefusalWithdrawalParticipants = async (siteCode, isParent, concept .offset(offset) .limit(limit) .get(); - printDocsCount(snapshot, "retrieveRefusalWithdrawalParticipants"); + printDocsCount(snapshot, `retrieveRefusalWithdrawalParticipants; offset: ${offset}`); return snapshot.docs.map(doc => doc.data()); } catch (error) { @@ -366,7 +366,7 @@ const retrieveParticipantsEligibleForIncentives = async (siteCode, roundType, is .offset(offset) .limit(limit) .get(); - printDocsCount(snapshot, "retrieveParticipantsEligibleForIncentives"); + printDocsCount(snapshot, `retrieveParticipantsEligibleForIncentives; offset: ${offset}`); return snapshot.docs.map(document => { let data = document.data(); @@ -1797,7 +1797,7 @@ const getBoxesPagination = async (siteCode, body) => { query = preQueryBuilder(filters, query, trackingId, endDate, startDate, source, siteCode); query = query.orderBy(orderByField, 'desc').limit(elementsPerPage).offset(currPage * elementsPerPage); const snapshot = await query.get(); - printDocsCount(snapshot, "getBoxesPagination"); + printDocsCount(snapshot, `getBoxesPagination; offset: ${currPage * elementsPerPage}`); const result = snapshot.docs.map(document => document.data()); return result; } @@ -1908,7 +1908,7 @@ const retrieveParticipantsByStatus = async (conditions, limit, offset) => { query = query.where(obj, operator, values); } const snapshot = await query.get(); - printDocsCount(snapshot, "retrieveParticipantsByStatus"); + printDocsCount(snapshot, `retrieveParticipantsByStatus; offset: ${offset}`); return snapshot.docs.map(doc => doc.data()); } catch (error) { console.error(error); From ccc2bcff9b32643f816442f854292c45cd3f6371 Mon Sep 17 00:00:00 2001 From: Warren Lu Date: Wed, 10 Jul 2024 11:53:47 -0400 Subject: [PATCH 14/34] remove function retrieveParticipantsByStatus --- utils/firestore.js | 76 ---------------------------------------------- 1 file changed, 76 deletions(-) diff --git a/utils/firestore.js b/utils/firestore.js index 93cbe764..27b6ab77 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -1841,81 +1841,6 @@ const getNotificationSpecifications = async (notificationType, notificationCateg } } -// TODO: Avoid using `offset` for pagination, because offset documents are still read and charged. -const retrieveParticipantsByStatus = async (conditions, limit, offset) => { - try { - let query = db.collection('participants') - .limit(limit) - .offset(offset); - - for(let obj in conditions) { - let operator = ''; - let values = ''; - if(conditions[obj]['equals']) { - if(typeof conditions[obj]['equals'] == 'string') { - values = conditions[obj]['equals']; - } - else if(typeof conditions[obj]['equals'] == 'number') { - values = parseInt(conditions[obj]['equals']); - } - operator = '=='; - } - if(conditions[obj]['notequals']) { - if(typeof conditions[obj]['notequals'] == 'string') { - values = conditions[obj]['notequals']; - } - else if(typeof conditions[obj]['notequals'] == 'number') { - values = parseInt(conditions[obj]['notequals']); - } - operator = '!='; - } - if(conditions[obj]['greater']) { - if(typeof conditions[obj]['greater'] == 'string') { - values = conditions[obj]['greater']; - } - else if(typeof conditions[obj]['greater'] == 'number') { - values = parseInt(conditions[obj]['greater']); - } - operator = '>'; - } - if(conditions[obj]['greaterequals']) { - if(typeof conditions[obj]['greaterequals'] == 'string') { - values = conditions[obj]['greaterequals']; - } - else if(typeof conditions[obj]['greaterequals'] == 'number') { - values = parseInt(conditions[obj]['greaterequals']); - } - operator = '>='; - } - if(conditions[obj]['less']) { - if(typeof conditions[obj]['less'] == 'string') { - values = conditions[obj]['less']; - } - else if(typeof conditions[obj]['less'] == 'number') { - values = parseInt(conditions[obj]['less']); - } - operator = '<'; - } - if(conditions[obj]['lessequals']) { - if(typeof conditions[obj]['lessequals'] == 'string') { - values = conditions[obj]['lessequals']; - } - else if(typeof conditions[obj]['lessequals'] == 'number') { - values = parseInt(conditions[obj]['lessequals']); - } - operator = '<='; - } - query = query.where(obj, operator, values); - } - const snapshot = await query.get(); - printDocsCount(snapshot, `retrieveParticipantsByStatus; offset: ${offset}`); - return snapshot.docs.map(doc => doc.data()); - } catch (error) { - console.error(error); - return new Error(error); - } -} - const sendClientEmail = async (data) => { const { sendEmail } = require('./notifications'); @@ -3432,7 +3357,6 @@ module.exports = { removeParticipantsDataDestruction, removeUninvitedParticipants, getNotificationSpecifications, - retrieveParticipantsByStatus, storeNotification, checkIsNotificationSent, validateSiteSAEmail, From 71a4f73ab838cad36c7586281b78a66986e77118 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Mon, 15 Jul 2024 11:20:47 -0400 Subject: [PATCH 15/34] Spanish Translation: magic link emails for login --- utils/notifications.js | 15 +++++++++++---- utils/shared.js | 24 ++++++++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/utils/notifications.js b/utils/notifications.js index 5e2012a5..f08cde78 100644 --- a/utils/notifications.js +++ b/utils/notifications.js @@ -663,7 +663,7 @@ const sendEmailLink = async (req, res) => { .json(getResponseJSON("Only POST requests are accepted!", 405)); } try { - const { email, continueUrl } = req.body; + const { email, continueUrl, preferredLanguage } = req.body; const [clientId, clientSecret, tenantId, magicLink] = await Promise.all( [ getSecret(process.env.APP_REGISTRATION_CLIENT_ID), @@ -692,13 +692,20 @@ const sendEmailLink = async (req, res) => { ); const { access_token } = await resAuthorize.json(); - const body = { message: { - subject: `Sign in to Connect for Cancer Prevention Study`, + subject: + preferredLanguage === + conceptIds.spanish.toString() + ? "Inicie sesión para Estudio Connect para la Prevención del Cáncer" + : "Sign in to Connect for Cancer Prevention Study", body: { contentType: "html", - content: getTemplateForEmailLink(email, magicLink), + content: getTemplateForEmailLink( + email, + magicLink, + preferredLanguage + ), }, toRecipients: [ { diff --git a/utils/shared.js b/utils/shared.js index 61684b9f..36a1a116 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1540,10 +1540,27 @@ const filterSelectedFields = (dataObjArray, selectedFieldsArray) => { }); } -const getTemplateForEmailLink = (email, continueUrl) => { - return ` +const getTemplateForEmailLink = ( + email, + continueUrl, + preferredLanguage = fieldMapping.english.toString() +) => { + return preferredLanguage === fieldMapping.spanish.toString() + ? ` + +

Hola,

+

Recibimos una solicitud para iniciar sesión en el Estudio Connect para la Prevención del Cáncer usando esta dirección de correo electrónico. Si desea iniciar sesión con su cuenta ${email}, haga clic en este enlace:

+

Iniciar sesión para Estudio Connect para la Prevención del Cáncer:

+

Si no solicitó este enlace, puede ignorar este correo electrónico de forma segura.

+

Gracias,

+

Su equipo del Estudio Connect para la Prevención del Cáncer

+ + + ` + : ` +

Hello,

We received a request to sign in to Connect for Cancer Prevention Study using this email address. If you want to sign in with your ${email} account, click this link:

@@ -1552,8 +1569,7 @@ const getTemplateForEmailLink = (email, continueUrl) => {

Thanks,

Your Connect for Cancer Prevention Study team

- - `; + `; }; const nihMailbox = 'NCIConnectStudy@mail.nih.gov' From 38db5618bf2cb2e5776da0538b4d60d358fb3c3f Mon Sep 17 00:00:00 2001 From: jhflorey Date: Mon, 15 Jul 2024 12:13:27 -0400 Subject: [PATCH 16/34] Spanish Translation of Email Signature for Notifications --- utils/notifications.js | 32 ++++++++++++++++++++++++-------- utils/shared.js | 8 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/utils/notifications.js b/utils/notifications.js index 5e2012a5..8ac7d20e 100644 --- a/utils/notifications.js +++ b/utils/notifications.js @@ -2,7 +2,7 @@ const { v4: uuid } = require("uuid"); const sgMail = require("@sendgrid/mail"); const showdown = require("showdown"); const {SecretManagerServiceClient} = require('@google-cloud/secret-manager'); -const {getResponseJSON, setHeadersDomainRestricted, setHeaders, logIPAdddress, redactEmailLoginInfo, redactPhoneLoginInfo, createChunkArray, validEmailFormat, getTemplateForEmailLink, nihMailbox, getSecret, cidToLangMapper} = require("./shared"); +const {getResponseJSON, setHeadersDomainRestricted, setHeaders, logIPAdddress, redactEmailLoginInfo, redactPhoneLoginInfo, createChunkArray, validEmailFormat, getTemplateForEmailLink, nihMailbox, getSecret, cidToLangMapper, unsubscribeTextObj} = require("./shared"); const {getNotificationSpecById, getNotificationSpecByCategoryAndAttempt, getNotificationSpecsByScheduleOncePerDay, saveNotificationBatch, updateSurveyEligibility, generateSignInWithEmailLink, storeNotification, checkIsNotificationSent, getNotificationSpecsBySchedule} = require("./firestore"); const {getParticipantsForNotificationsBQ} = require("./bigquery"); const conceptIds = require("./fieldToConceptIdMapping"); @@ -427,13 +427,23 @@ async function getParticipantsAndSendNotifications({ notificationSpec, cutoffTim let { emailRecordArray, emailPersonalizationArray, smsRecordArray } = notificationData[lang]; if (emailPersonalizationArray.length > 0) { const emailBatch = { - from: { - name: process.env.SG_FROM_NAME || "Connect for Cancer Prevention Study", - email: process.env.SG_FROM_EMAIL || "donotreply@myconnect.cancer.gov", - }, - subject: emailInSpec[lang].subject, - html: emailInSpec[lang].body, - personalizations: emailPersonalizationArray, + from: { + name: + process.env.SG_FROM_NAME || + "Connect for Cancer Prevention Study", + email: + process.env.SG_FROM_EMAIL || + "donotreply@myconnect.cancer.gov", + }, + subject: emailInSpec[lang].subject, + html: emailInSpec[lang].body, + personalizations: emailPersonalizationArray, + tracking_settings: { + subscription_tracking: { + enable: true, + html: unsubscribeTextObj[lang] || unsubscribeTextObj.english + }, + }, }; try { @@ -875,6 +885,12 @@ const sendInstantNotification = async (requestData) => { }, }, ], + tracking_settings: { + subscription_tracking: { + enable: true, + html: unsubscribeTextObj[requestData.preferredLanguage] || unsubscribeTextObj.english + }, + }, }; await sgMail.send(emailDataToSg); diff --git a/utils/shared.js b/utils/shared.js index d690abc3..3f4a1b4e 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1595,6 +1595,13 @@ const printDocsCount = (snapshot, infoStr = "") => { } }; +const unsubscribeTextObj = { + english: + "

To unsubscribe from emails about Connect from the National Cancer Institute (NCI), <% click here %> .

", + spanish: + "

Si desea darse de baja de Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %> .

", +}; + module.exports = { getResponseJSON, setHeaders, @@ -1659,4 +1666,5 @@ module.exports = { getSecret, cidToLangMapper, printDocsCount, + unsubscribeTextObj }; From ec5cae7334fe3f4261f02e0325eb3522fa8ba148 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Thu, 18 Jul 2024 10:00:56 -0400 Subject: [PATCH 17/34] Add new stub variables --- utils/fieldToConceptIdMapping.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/fieldToConceptIdMapping.js b/utils/fieldToConceptIdMapping.js index d28d5a7d..214573eb 100644 --- a/utils/fieldToConceptIdMapping.js +++ b/utils/fieldToConceptIdMapping.js @@ -394,6 +394,8 @@ module.exports = { baselineBloodAndUrineIsRefused: 526455436, baselineMouthwashCollected: 684635302, baselineBloodSampleCollected: 878865966, - bioSpmVisitV1r0: 331584571 + bioSpmVisitV1r0: 331584571, + allBaselineSurveysCompleted: 100767870, + firebaseAuthenticationEmail: 421823980, } }; From f8dedca31ce395db4041b6e47211fd1d1ad9aa3a Mon Sep 17 00:00:00 2001 From: amber-emmes Date: Thu, 18 Jul 2024 10:13:37 -0400 Subject: [PATCH 18/34] Removed redundant line --- utils/firestore.js | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/firestore.js b/utils/firestore.js index ec6f90d8..fc7f4c7c 100644 --- a/utils/firestore.js +++ b/utils/firestore.js @@ -2327,7 +2327,6 @@ const assignKitToParticipant = async (data) => { const otherKitsUsingSupplyKitTrackingNumber = await db.collection("kitAssembly") .where(`${supplyKitTrackingNum}`, '==', data[supplyKitTrackingNum]) .get(); - // .where(`${supplyKitId}`, '!=', data[supplyKitId]).get(); if(otherKitsUsingSupplyKitTrackingNumber.size > 1) { return false; From d98694a6ea0ba68da5d2e57509c53a85295bc40d Mon Sep 17 00:00:00 2001 From: amber-emmes Date: Thu, 18 Jul 2024 10:17:27 -0400 Subject: [PATCH 19/34] Variable name updates per feedback --- utils/biospecimen.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/biospecimen.js b/utils/biospecimen.js index 9ddf2ae3..9abef476 100644 --- a/utils/biospecimen.js +++ b/utils/biospecimen.js @@ -671,14 +671,14 @@ const biospecimenAPIs = async (req, res) => { if( req.method !== 'GET') { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - const supplyQuery = req.query.supplyKitId; - const collectionQuery = (req.query.collectionId?.slice(0, -4) || "") + " " + (req.query.collectionId?.slice(-4) || ""); // add space to collection - const returnKitTrackingNumberQuery = req.query.returnKitTrackingNumber; + const supplyKitId = req.query.supplyKitId; + const collectionIdSuffix = (req.query.collectionId?.slice(0, -4) || "") + " " + (req.query.collectionId?.slice(-4) || ""); // add space to collection + const returnKitTrackingNumberNumber = req.query.returnKitTrackingNumber; if (Object.keys(query).length === 0) return res.status(404).json(getResponseJSON('Please include id to check uniqueness.', 400)); - if (collectionQuery.length < 14) return res.status(200).json({data: 'Check Collection ID', code:200}); + if (collectionIdSuffix.length < 14) return res.status(200).json({data: 'Check Collection ID', code:200}); try { const { checkCollectionUniqueness } = require('./firestore'); - const response = await checkCollectionUniqueness(supplyQuery, collectionQuery, returnKitTrackingNumberQuery); + const response = await checkCollectionUniqueness(supplyKitId, collectionIdSuffix, returnKitTrackingNumberNumber); return res.status(200).json({data: response, code:200}); } catch (error) { From 6c44a5214dda93b4380eaa444db58c89cd5b1f21 Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Thu, 18 Jul 2024 13:39:42 -0500 Subject: [PATCH 20/34] adding heartbeat API --- config/dev/cloudbuild1.yaml | 4 ++++ config/prod/cloudbuild1.yaml | 4 ++++ config/stage/cloudbuild1.yaml | 4 ++++ index.js | 10 +++++++--- utils/heartbeat.js | 28 ++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 utils/heartbeat.js diff --git a/config/dev/cloudbuild1.yaml b/config/dev/cloudbuild1.yaml index 6f29d027..3bdf28e0 100644 --- a/config/dev/cloudbuild1.yaml +++ b/config/dev/cloudbuild1.yaml @@ -7,6 +7,10 @@ steps: args: ['functions', 'deploy', 'biospecimen', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/dev/.env.yaml'] - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'biospecimen', '--member=allUsers', '--role=${_ROLE}'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'add-iam-policy-binding', 'heartbeat', '--member=allUsers', '--role=${_ROLE}'] substitutions: _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-dev/repos/github_episphere_connectfaas/moveable-aliases/dev _RUNTIME: nodejs20 diff --git a/config/prod/cloudbuild1.yaml b/config/prod/cloudbuild1.yaml index 4ff649d0..712ccecf 100644 --- a/config/prod/cloudbuild1.yaml +++ b/config/prod/cloudbuild1.yaml @@ -7,6 +7,10 @@ steps: args: ['functions', 'deploy', 'biospecimen', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'biospecimen', '--member=allUsers', '--role=${_ROLE}'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'add-iam-policy-binding', 'heartbeat', '--member=allUsers', '--role=${_ROLE}'] substitutions: _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-prod-6d04/repos/github_episphere_connectfaas _RUNTIME: nodejs20 diff --git a/config/stage/cloudbuild1.yaml b/config/stage/cloudbuild1.yaml index d77337f0..5294702c 100644 --- a/config/stage/cloudbuild1.yaml +++ b/config/stage/cloudbuild1.yaml @@ -7,6 +7,10 @@ steps: args: ['functions', 'deploy', 'biospecimen', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/stage/.env.yaml'] - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'biospecimen', '--member=allUsers', '--role=${_ROLE}'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['functions', 'add-iam-policy-binding', 'heartbeat', '--member=allUsers', '--role=${_ROLE}'] substitutions: _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-stg-5519/repos/github_episphere_connectfaas/moveable-aliases/stage _RUNTIME: nodejs20 diff --git a/index.js b/index.js index 9c33981a..ba2dc857 100644 --- a/index.js +++ b/index.js @@ -1,15 +1,15 @@ const { getToken } = require('./utils/validation'); const { getFilteredParticipants, getParticipants, identifyParticipant } = require('./utils/submission'); const { submitParticipantsData, updateParticipantData } = require('./utils/sites'); -const { sendScheduledNotifications } = require('./utils/notifications'); +const { getParticipantNotification, sendScheduledNotifications } = require('./utils/notifications'); const { connectApp } = require('./utils/connectApp'); const { biospecimenAPIs } = require('./utils/biospecimen'); const { incentiveCompleted, eligibleForIncentive } = require('./utils/incentive'); const { dashboard } = require('./utils/dashboard'); -const { getParticipantNotification } = require('./utils/notifications'); const { importToBigQuery, firestoreExport, exportNotificationsToBucket, importNotificationsToBigquery } = require('./utils/events'); const { participantDataCleanup } = require('./utils/participantDataCleanup'); const { webhook } = require('./utils/webhook'); +const { heartbeat } = require('./utils/heartbeat'); // API End-Points for Sites @@ -68,4 +68,8 @@ exports.participantDataCleanup = participantDataCleanup; // End-Points for Event Webhook -exports.webhook = webhook; \ No newline at end of file +exports.webhook = webhook; + +// End-Points for Public Heartbeat + +exports.heartbeat = heartbeat; \ No newline at end of file diff --git a/utils/heartbeat.js b/utils/heartbeat.js new file mode 100644 index 00000000..cbf4ac5b --- /dev/null +++ b/utils/heartbeat.js @@ -0,0 +1,28 @@ +const { logIPAdddress, setHeaders } = require('./shared'); + +const heartbeat = async (req, res) => { + logIPAdddress(req); + setHeaders(res); + + if(req.method === 'OPTIONS') { + return res.status(200).json({code: 200}); + } + + if(req.method !== 'GET') { + return res.status(405).json({ code: 405, data: 'Only GET requests are accepted!'}); + } + + const currentTime = new Date(); + + const hours = currentTime.getUTCHours().toString().padStart(2, '0'); + const minutes = currentTime.getUTCMinutes().toString().padStart(2, '0'); + const seconds = currentTime.getUTCSeconds().toString().padStart(2, '0'); + + const message = `Current time (UTC) is: ${hours}:${minutes}:${seconds}`; + + return res.status(200).json({ code: 200, data: message}); +} + +module.exports = { + heartbeat +} \ No newline at end of file From d9194fb4028e07f407c8505694868a85bfc90098 Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Thu, 18 Jul 2024 13:58:05 -0500 Subject: [PATCH 21/34] fixes --- config/dev/cloudbuild1.yaml | 2 +- config/stage/cloudbuild1.yaml | 2 +- utils/biospecimen.js | 4 ++-- utils/dashboard.js | 4 ++-- utils/heartbeat.js | 4 ++-- utils/incentive.js | 6 +++--- utils/notifications.js | 10 +++++----- utils/shared.js | 4 ++-- utils/sites.js | 8 ++++---- utils/stats.js | 4 ++-- utils/submission.js | 8 ++++---- utils/validation.js | 6 +++--- 12 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config/dev/cloudbuild1.yaml b/config/dev/cloudbuild1.yaml index 3bdf28e0..c9ecf93a 100644 --- a/config/dev/cloudbuild1.yaml +++ b/config/dev/cloudbuild1.yaml @@ -8,7 +8,7 @@ steps: - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'biospecimen', '--member=allUsers', '--role=${_ROLE}'] - name: 'gcr.io/cloud-builders/gcloud' - args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] + args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/dev/.env.yaml'] - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'heartbeat', '--member=allUsers', '--role=${_ROLE}'] substitutions: diff --git a/config/stage/cloudbuild1.yaml b/config/stage/cloudbuild1.yaml index 5294702c..d0eae02e 100644 --- a/config/stage/cloudbuild1.yaml +++ b/config/stage/cloudbuild1.yaml @@ -8,7 +8,7 @@ steps: - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'biospecimen', '--member=allUsers', '--role=${_ROLE}'] - name: 'gcr.io/cloud-builders/gcloud' - args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml'] + args: ['functions', 'deploy', 'heartbeat', '--trigger-http', '--runtime=${_RUNTIME}', '--source=${_SOURCE}', '--env-vars-file=config/stage/.env.yaml'] - name: 'gcr.io/cloud-builders/gcloud' args: ['functions', 'add-iam-policy-binding', 'heartbeat', '--member=allUsers', '--role=${_ROLE}'] substitutions: diff --git a/utils/biospecimen.js b/utils/biospecimen.js index 9abef476..b0a0f6a8 100644 --- a/utils/biospecimen.js +++ b/utils/biospecimen.js @@ -1,9 +1,9 @@ -const { getResponseJSON, setHeaders, logIPAdddress, SSOValidation, convertSiteLoginToNumber } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress, SSOValidation, convertSiteLoginToNumber } = require('./shared'); const fieldMapping = require('./fieldToConceptIdMapping'); const { sendInstantNotification } = require("./notifications"); const biospecimenAPIs = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); diff --git a/utils/dashboard.js b/utils/dashboard.js index 1476e184..fe24d1be 100644 --- a/utils/dashboard.js +++ b/utils/dashboard.js @@ -1,7 +1,7 @@ -const { getResponseJSON, setHeaders, logIPAdddress } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress } = require('./shared'); const dashboard = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === 'OPTIONS') return res.status(200).json({code: 200}); if (!req.headers.authorization || req.headers.authorization.trim() === "") { diff --git a/utils/heartbeat.js b/utils/heartbeat.js index cbf4ac5b..19e009fe 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -1,7 +1,7 @@ -const { logIPAdddress, setHeaders } = require('./shared'); +const { logIPAddress, setHeaders } = require('./shared'); const heartbeat = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') { diff --git a/utils/incentive.js b/utils/incentive.js index 9da71cc9..80547f33 100644 --- a/utils/incentive.js +++ b/utils/incentive.js @@ -1,7 +1,7 @@ -const { setHeaders, getResponseJSON, logIPAdddress, APIAuthorization, isParentEntity, isDateTimeFormat } = require('./shared'); +const { setHeaders, getResponseJSON, logIPAddress, APIAuthorization, isParentEntity, isDateTimeFormat } = require('./shared'); const incentiveCompleted = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -124,7 +124,7 @@ const incentiveCompleted = async (req, res) => { } const eligibleForIncentive = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); diff --git a/utils/notifications.js b/utils/notifications.js index 1b36170c..f1bfe356 100644 --- a/utils/notifications.js +++ b/utils/notifications.js @@ -2,7 +2,7 @@ const { v4: uuid } = require("uuid"); const sgMail = require("@sendgrid/mail"); const showdown = require("showdown"); const {SecretManagerServiceClient} = require('@google-cloud/secret-manager'); -const {getResponseJSON, setHeadersDomainRestricted, setHeaders, logIPAdddress, redactEmailLoginInfo, redactPhoneLoginInfo, createChunkArray, validEmailFormat, getTemplateForEmailLink, nihMailbox, getSecret, cidToLangMapper, unsubscribeTextObj} = require("./shared"); +const {getResponseJSON, setHeadersDomainRestricted, setHeaders, logIPAddress, redactEmailLoginInfo, redactPhoneLoginInfo, createChunkArray, validEmailFormat, getTemplateForEmailLink, nihMailbox, getSecret, cidToLangMapper, unsubscribeTextObj} = require("./shared"); const {getNotificationSpecById, getNotificationSpecByCategoryAndAttempt, getNotificationSpecsByScheduleOncePerDay, saveNotificationBatch, updateSurveyEligibility, generateSignInWithEmailLink, storeNotification, checkIsNotificationSent, getNotificationSpecsBySchedule} = require("./firestore"); const {getParticipantsForNotificationsBQ} = require("./bigquery"); const conceptIds = require("./fieldToConceptIdMapping"); @@ -527,7 +527,7 @@ async function getParticipantsAndSendNotifications({ notificationSpec, cutoffTim } const storeNotificationSchema = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === "OPTIONS") return res.status(200).json({ code: 200 }); @@ -567,7 +567,7 @@ const storeNotificationSchema = async (req, res, authObj) => { }; const retrieveNotificationSchema = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === "OPTIONS") return res.status(200).json({ code: 200 }); @@ -596,7 +596,7 @@ const retrieveNotificationSchema = async (req, res, authObj) => { }; const getParticipantNotification = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -632,7 +632,7 @@ const getParticipantNotification = async (req, res, authObj) => { } const getSiteNotification = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === 'OPTIONS') return res.status(200).json({code: 200}); diff --git a/utils/shared.js b/utils/shared.js index ca7c79a5..a49b6b34 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -624,7 +624,7 @@ const isParentEntity = async (siteDetails) => { return {...siteDetails, isParent, siteCodes}; }; -const logIPAdddress = (req) => { +const logIPAddress = (req) => { const ipAddress = req.headers['x-forwarded-for'] || req.connection.remoteAddress; console.log(ipAddress) } @@ -1638,7 +1638,7 @@ module.exports = { defaultStateFlags, SSOValidation, conceptMappings, - logIPAdddress, + logIPAddress, decodingJWT, initializeTimestamps, tubeKeytoNum, diff --git a/utils/sites.js b/utils/sites.js index 8a9754ea..93e93a5e 100644 --- a/utils/sites.js +++ b/utils/sites.js @@ -1,10 +1,10 @@ const rules = require("../updateParticipantData.json"); const submitRules = require("../submitParticipantData.json"); -const { getResponseJSON, setHeaders, logIPAdddress, validIso8601Format, validPhoneFormat, validEmailFormat, refusalWithdrawalConcepts } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress, validIso8601Format, validPhoneFormat, validEmailFormat, refusalWithdrawalConcepts } = require('./shared'); const fieldMapping = require('./fieldToConceptIdMapping'); const submitParticipantsData = async (req, res, site) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -176,7 +176,7 @@ const updateParticipantData = async (req, res, authObj) => { const { checkForQueryFields, flattenObject, initializeTimestamps, userProfileHistoryKeys } = require('./shared'); const { checkDerivedVariables } = require('./validation'); - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -517,7 +517,7 @@ const updateUserAuthentication = async (req, res, authObj) => { } const participantDataCorrection = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if (req.method === 'OPTIONS') return res.status(200).json({ code: 200 }); diff --git a/utils/stats.js b/utils/stats.js index dbeb8b0b..0b276c8e 100644 --- a/utils/stats.js +++ b/utils/stats.js @@ -1,8 +1,8 @@ -const { getResponseJSON, setHeaders, logIPAdddress } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress } = require('./shared'); const { getStatsFromBQ } = require('./bigquery'); const stats = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); diff --git a/utils/submission.js b/utils/submission.js index 14871dc1..6086c68f 100644 --- a/utils/submission.js +++ b/utils/submission.js @@ -1,4 +1,4 @@ -const { getResponseJSON, setHeaders, logIPAdddress } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress } = require('./shared'); const fieldMapping = require('./fieldToConceptIdMapping'); const submit = async (res, data, uid) => { @@ -212,7 +212,7 @@ const submitSocial = async (req, res, uid) => { } const getParticipants = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -348,7 +348,7 @@ const removeRestrictedFields = (data, restriectedFields, isParent) => { * @returns {array} - A filtered array of participant data objects. */ const getFilteredParticipants = async (req, res, authObj) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -430,7 +430,7 @@ const getFilteredParticipants = async (req, res, authObj) => { } const identifyParticipant = async (req, res, site) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); diff --git a/utils/validation.js b/utils/validation.js index bf8352d7..49974950 100644 --- a/utils/validation.js +++ b/utils/validation.js @@ -1,4 +1,4 @@ -const { getResponseJSON, setHeaders, logIPAdddress } = require('./shared'); +const { getResponseJSON, setHeaders, logIPAddress } = require('./shared'); const conceptIds = require('./fieldToConceptIdMapping') const generateToken = async (req, res, uid) => { @@ -104,7 +104,7 @@ const validateToken = async (req, res, uid) => { }; const getToken = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method === 'OPTIONS') return res.status(200).json({code: 200}); @@ -457,7 +457,7 @@ const checkRefusalWithdrawals = (data) => { } const validateUsersEmailPhone = async (req, res) => { - logIPAdddress(req); + logIPAddress(req); setHeaders(res); if(req.method !== 'GET') { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); From 11e3e9d8efe9faa36d61e5c1a7b960e5d58cb041 Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:20:51 -0400 Subject: [PATCH 22/34] fetch survey modules through GitHub API (authenticated) --- utils/connectApp.js | 22 +++++++++++++++++ utils/submission.js | 59 +++++++++++++++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/utils/connectApp.js b/utils/connectApp.js index 92f62b1c..580b16f1 100644 --- a/utils/connectApp.js +++ b/utils/connectApp.js @@ -74,6 +74,28 @@ const connectApp = async (req, res) => { return res.status(200).json({data: shaResult, code: 200}); } + else if (api === 'getQuestSurveyFromGitHub') { + if (req.method !== 'GET') { + return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); + } + + if (!req.query.sha || req.query.sha === '') { + return res.status(400).json(getResponseJSON('Sha parameter is required!', 400)); + } + + if (!req.query.path || req.query.path === '') { + return res.status(400).json(getResponseJSON('Path parameter is required!', 400)); + } + + const sha = req.query.sha; + const path = req.query.path; + + const { getQuestSurveyFromGitHub } = require('./submission'); + const moduleTextAndVersionResult = await getQuestSurveyFromGitHub(sha, path); + + return res.status(200).json({data: moduleTextAndVersionResult, code: 200}); + } + else if (api === 'getSHAFromGitHubCommitData') { if (req.method !== 'GET') { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); diff --git a/utils/submission.js b/utils/submission.js index 14871dc1..5a191b3a 100644 --- a/utils/submission.js +++ b/utils/submission.js @@ -582,17 +582,29 @@ const getUserCollections = async (req, res, uid) => { } /** - * Get the sha of a module in the questionnaire repository. - * @param {string} path - the path to the module in the questionnaire repository. - * @returns {string} - the sha of the module. + * Fetch the GitHub API token from Secret Manager. + * @returns {String} - The GitHub API token. */ -const getModuleSHA = async (path) => { +const getGitHubToken = async () => { try { const { SecretManagerServiceClient } = require('@google-cloud/secret-manager'); const client = new SecretManagerServiceClient(); const [version] = await client.accessSecretVersion({ name: process.env.GITHUB_TOKEN }); - const token = version.payload.data.toString(); + return version.payload.data.toString(); + } catch (error) { + console.error('Error fetching GitHub token:'); + throw new Error('Error fetching GitHub token.', { cause: error }); + } +} +/** + * Get the sha of a module in the questionnaire repository. + * @param {string} path - the path to the module in the questionnaire repository. + * @returns {string} - the sha of the module. + */ +const getModuleSHA = async (path) => { + try { + const token = await getGitHubToken(); const gitHubApiResponse = await fetch(`https://api.github.com/repos/episphere/questionnaire/commits?path=${path}&sha=main&per_page=1`, { headers: { 'Authorization': `token ${token}`, @@ -632,11 +644,7 @@ const getModuleSHA = async (path) => { */ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { try { - const { SecretManagerServiceClient } = require('@google-cloud/secret-manager'); - const client = new SecretManagerServiceClient(); - const [version] = await client.accessSecretVersion({ name: process.env.GITHUB_TOKEN }); - const token = version.payload.data.toString(); - + const token = await getGitHubToken(); const gitHubApiResponse = await fetch(`https://api.github.com/repos/episphere/questionnaire/commits?path=${path}&sha=main`, { headers: { 'Authorization': `token ${token}`, @@ -672,7 +680,8 @@ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { throw new Error(`Module SHA not found for path ${path}. Most recent SHA also failed.`); } - const surveyVersion = await getVersionNumberFromGitHubCommit(sha, path, token); + const textAndVersionResponse = await getTextAndVersionNumberFromGitHubCommit(sha, path, token); + const surveyVersion = textAndVersionResponse.surveyVersion; return { sha, surveyVersion }; @@ -682,6 +691,24 @@ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { } } +/** + * Fetch a raw file from the questionnaire repository using the GitHub API. + * @param {String} sha - The SHA of the raw file commit to fetch. + * @param {String} path - The path to the file in the questionnaire repository. + * @param {String} token - The GitHub API token.} surveyStartTimestamp + * @returns {Object} - The survey's module text and version number. + */ +const getQuestSurveyFromGitHub = async (sha, path) => { + try { + const token = await getGitHubToken(); + return await getTextAndVersionNumberFromGitHubCommit(sha, path, token); + + } catch (error) { + console.error('Error fetching Quest survey from GitHub.', error); + throw new Error(`Error fetching Quest survey from GitHub. ${error.message}`, { cause: error }); + } +} + /** * Search the GitHub API (Raw file) by commit SHA for the version number of a module at a specific commit. * Early versions didn't have a versioning convention. Return '1.0' for this case. @@ -690,7 +717,7 @@ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { * @param {String} token - The GitHub API token. * @returns {String} - The version number of the module or '1.0' if not versioned. */ -const getVersionNumberFromGitHubCommit = async (sha, path, token) => { +const getTextAndVersionNumberFromGitHubCommit = async (sha, path, token) => { try { const response = await fetch(`https://api.github.com/repos/episphere/questionnaire/contents/${path}?ref=${sha}`, { headers: { @@ -703,10 +730,11 @@ const getVersionNumberFromGitHubCommit = async (sha, path, token) => { throw new Error(`Github RAW File API response error. ${response.statusText}`); } - const rawTextData = await response.text(); - const versionMatch = rawTextData.match("{\"version\":\\s*\"([0-9]{1,2}\\.[0-9]{1,3})\"}"); + const moduleText = await response.text(); + const versionMatch = moduleText.match("{\"version\":\\s*\"([0-9]{1,2}\\.[0-9]{1,3})\"}"); + const surveyVersion = versionMatch ? versionMatch[1] : '1.0'; - return versionMatch ? versionMatch[1] : '1.0'; + return { moduleText, surveyVersion }; } catch (error) { console.error('Error fetching raw GitHub file from commit sha.', error); @@ -724,5 +752,6 @@ module.exports = { getUserSurveys, getUserCollections, getModuleSHA, + getQuestSurveyFromGitHub, getSHAFromGitHubCommitData, } \ No newline at end of file From b9b3c5ab406178780d70075ae3d14a910790166b Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:44:21 -0400 Subject: [PATCH 23/34] clean up redundant checks --- utils/connectApp.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/connectApp.js b/utils/connectApp.js index 580b16f1..0ba6d58c 100644 --- a/utils/connectApp.js +++ b/utils/connectApp.js @@ -62,7 +62,7 @@ const connectApp = async (req, res) => { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - if (!req.query.path || req.query.path === '') { + if (!req.query.path) { return res.status(400).json(getResponseJSON('Path parameter is required!', 400)); } @@ -79,11 +79,11 @@ const connectApp = async (req, res) => { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - if (!req.query.sha || req.query.sha === '') { + if (!req.query.sha) { return res.status(400).json(getResponseJSON('Sha parameter is required!', 400)); } - if (!req.query.path || req.query.path === '') { + if (!req.query.path) { return res.status(400).json(getResponseJSON('Path parameter is required!', 400)); } @@ -101,7 +101,7 @@ const connectApp = async (req, res) => { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - if (!req.query.path || req.query.path === '') { + if (!req.query.path) { return res.status(400).json(getResponseJSON('Path parameter is required!', 400)); } From 013dc8dc1e89dce5b3e85ef33ff26a68a09c3902 Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:33:06 -0400 Subject: [PATCH 24/34] updates per Warren's review --- utils/connectApp.js | 9 ++++----- utils/submission.js | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/utils/connectApp.js b/utils/connectApp.js index 0ba6d58c..492aee54 100644 --- a/utils/connectApp.js +++ b/utils/connectApp.js @@ -79,17 +79,16 @@ const connectApp = async (req, res) => { return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405)); } - if (!req.query.sha) { + const { sha, path } = req.query; + + if (!sha) { return res.status(400).json(getResponseJSON('Sha parameter is required!', 400)); } - if (!req.query.path) { + if (!path) { return res.status(400).json(getResponseJSON('Path parameter is required!', 400)); } - const sha = req.query.sha; - const path = req.query.path; - const { getQuestSurveyFromGitHub } = require('./submission'); const moduleTextAndVersionResult = await getQuestSurveyFromGitHub(sha, path); diff --git a/utils/submission.js b/utils/submission.js index 5a191b3a..82db9710 100644 --- a/utils/submission.js +++ b/utils/submission.js @@ -583,7 +583,7 @@ const getUserCollections = async (req, res, uid) => { /** * Fetch the GitHub API token from Secret Manager. - * @returns {String} - The GitHub API token. + * @returns {string} - The GitHub API token. */ const getGitHubToken = async () => { try { @@ -638,9 +638,9 @@ const getModuleSHA = async (path) => { * Repair: missing SHA value in the survey data. This property should exist for all survey data in all modules. * Ex: Firestore -> Module3_v1 -> any document > sha: "". * This function compares the survey start date with the commit history of the module. - * @param {String} surveyStartTimestamp - The timestamp of the survey start date (ISO 8601 String). - * @param {String} path - the file name of the module in the questionnaire repository. - * @returns {String} - The SHA of the commit that was active when the survey was started. + * @param {string} surveyStartTimestamp - The timestamp of the survey start date (ISO 8601 String). + * @param {string} path - the file name of the module in the questionnaire repository. + * @returns {string} - The SHA of the commit that was active when the survey was started. */ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { try { @@ -693,9 +693,9 @@ const getSHAFromGitHubCommitData = async (surveyStartTimestamp, path) => { /** * Fetch a raw file from the questionnaire repository using the GitHub API. - * @param {String} sha - The SHA of the raw file commit to fetch. - * @param {String} path - The path to the file in the questionnaire repository. - * @param {String} token - The GitHub API token.} surveyStartTimestamp + * @param {string} sha - The SHA of the raw file commit to fetch. + * @param {string} path - The path to the file in the questionnaire repository. + * @param {string} token - The GitHub API token. * @returns {Object} - The survey's module text and version number. */ const getQuestSurveyFromGitHub = async (sha, path) => { @@ -712,10 +712,10 @@ const getQuestSurveyFromGitHub = async (sha, path) => { /** * Search the GitHub API (Raw file) by commit SHA for the version number of a module at a specific commit. * Early versions didn't have a versioning convention. Return '1.0' for this case. - * @param {String} sha - The SHA of the raw file commit to fetch. - * @param {String} path - The path to the file in the questionnaire repository. - * @param {String} token - The GitHub API token. - * @returns {String} - The version number of the module or '1.0' if not versioned. + * @param {string} sha - The SHA of the raw file commit to fetch. + * @param {string} path - The path to the file in the questionnaire repository. + * @param {string} token - The GitHub API token. + * @returns {string} - The version number of the module or '1.0' if not versioned. */ const getTextAndVersionNumberFromGitHubCommit = async (sha, path, token) => { try { From 92893d9e9969e4a32f1ac53eec5ccd345fba6851 Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Fri, 19 Jul 2024 14:21:04 -0500 Subject: [PATCH 25/34] test --- utils/heartbeat.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils/heartbeat.js b/utils/heartbeat.js index 19e009fe..7387cf2e 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -1,4 +1,6 @@ const { logIPAddress, setHeaders } = require('./shared'); +const {BigQuery} = require('@google-cloud/bigquery'); +const bigquery = new BigQuery(); const heartbeat = async (req, res) => { logIPAddress(req); @@ -18,9 +20,17 @@ const heartbeat = async (req, res) => { const minutes = currentTime.getUTCMinutes().toString().padStart(2, '0'); const seconds = currentTime.getUTCSeconds().toString().padStart(2, '0'); - const message = `Current time (UTC) is: ${hours}:${minutes}:${seconds}`; + const queryStr = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; + const [rows] = await bigquery.query(queryStr); - return res.status(200).json({ code: 200, data: message}); + console.log(rows); + + const payload = { + utc: `${hours}:${minutes}:${seconds}`, + + } + + return res.status(200).json({ code: 200, data: payload}); } module.exports = { From 9f2073e63de9a59273301ea2c9c030bd802cf1c0 Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Fri, 19 Jul 2024 14:34:34 -0500 Subject: [PATCH 26/34] update --- utils/heartbeat.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/heartbeat.js b/utils/heartbeat.js index 7387cf2e..cb0f0b6b 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -20,14 +20,14 @@ const heartbeat = async (req, res) => { const minutes = currentTime.getUTCMinutes().toString().padStart(2, '0'); const seconds = currentTime.getUTCSeconds().toString().padStart(2, '0'); - const queryStr = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; - const [rows] = await bigquery.query(queryStr); - - console.log(rows); + const query = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; + const rows = await bigquery.query(query); const payload = { utc: `${hours}:${minutes}:${seconds}`, - + activeParticipants: rows.num_active_participants, + maleParticipants: rows.num_male_participants, + femaleParticipants: rows.num_female_participants } return res.status(200).json({ code: 200, data: payload}); From f0560e66f59c7b41a9b685a9a09527b2485956de Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Fri, 19 Jul 2024 14:58:02 -0500 Subject: [PATCH 27/34] test --- utils/heartbeat.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/heartbeat.js b/utils/heartbeat.js index cb0f0b6b..e15ac0a3 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -23,6 +23,8 @@ const heartbeat = async (req, res) => { const query = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; const rows = await bigquery.query(query); + console.log(rows); + const payload = { utc: `${hours}:${minutes}:${seconds}`, activeParticipants: rows.num_active_participants, From 35c51d6ca9effc1c8a33da9d97b9202dc52cfdbb Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Fri, 19 Jul 2024 15:04:36 -0500 Subject: [PATCH 28/34] syntax --- utils/heartbeat.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/heartbeat.js b/utils/heartbeat.js index e15ac0a3..7a2293ea 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -21,15 +21,13 @@ const heartbeat = async (req, res) => { const seconds = currentTime.getUTCSeconds().toString().padStart(2, '0'); const query = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; - const rows = await bigquery.query(query); - - console.log(rows); + const [rows] = await bigquery.query(query); const payload = { utc: `${hours}:${minutes}:${seconds}`, - activeParticipants: rows.num_active_participants, - maleParticipants: rows.num_male_participants, - femaleParticipants: rows.num_female_participants + activeParticipants: rows[0].num_active_participants, + maleParticipants: rows[0].num_male_participants, + femaleParticipants: rows[0].num_female_participants } return res.status(200).json({ code: 200, data: payload}); From b85d3b8f1380a056f44a19eeb0fb3ea2d7ac4934 Mon Sep 17 00:00:00 2001 From: Joe Armani <93854858+JoeArmani@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:05:51 -0400 Subject: [PATCH 29/34] Revert "Revert "Revert "Revert "Revert "RCA Variables -> add new variables""""" --- updateParticipantData.json | 17 ----------------- utils/fieldToConceptIdMapping.js | 9 +-------- utils/shared.js | 24 +----------------------- utils/sites.js | 2 +- 4 files changed, 3 insertions(+), 49 deletions(-) diff --git a/updateParticipantData.json b/updateParticipantData.json index ca1d8adb..0e5abfe0 100644 --- a/updateParticipantData.json +++ b/updateParticipantData.json @@ -540,23 +540,6 @@ "dataType": "object", "mustExist": false }, - "637153953[].844209241": { - "values": [104430631, 353358909, 178420302], - "dataType": "number", - "mustExist": false, - "required": true - }, - "637153953[].114227122": { - "values": [337516613, 646675764, 178420302], - "dataType": "number", - "mustExist": false, - "required": true - }, - "637153953[].421730068": { - "dataType": "string", - "mustExist": false, - "maxLength": 800 - }, "637153953[].740819233.149205077": { "values": [ 939782495, 135725957, 518416174, 847945207, 283025574, 942970912, 596122041, diff --git a/utils/fieldToConceptIdMapping.js b/utils/fieldToConceptIdMapping.js index 214573eb..4ecaa25f 100644 --- a/utils/fieldToConceptIdMapping.js +++ b/utils/fieldToConceptIdMapping.js @@ -228,14 +228,7 @@ module.exports = { isCancerDiagnosis: 525972260, primaryCancerSiteObject: 740819233, primaryCancerSiteCategorical: 149205077, - participantDiagnosisAwareness: 844209241, - pathologyAccessionNumber: 421730068, - vitalStatusCategorical: 114227122, - vitalStatus: { - alive: 337516613, - dead: 646675764, - unknown: 178420302, - }, + preliminaryStageInformation: 457270069, anotherTypeOfCancerText: 868006655, // Text response for 'other' cancer site cancerSites: { anal: 939782495, diff --git a/utils/shared.js b/utils/shared.js index a49b6b34..445e0336 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1225,11 +1225,6 @@ const handleCancerOccurrences = async (incomingCancerOccurrenceArray, requiredOc if (cancerSiteValidationObj.error === true) { return cancerSiteValidationObj; } - - const diagnosisAwarenessValidationObj = validateDiagnosisAwareness(occurrence[fieldMapping.vitalStatusCategorical], occurrence[fieldMapping.participantDiagnosisAwareness]); - if (diagnosisAwarenessValidationObj.error === true) { - return diagnosisAwarenessValidationObj; - } } // Query existing occurrences for the participant @@ -1260,7 +1255,7 @@ const handleCancerOccurrences = async (incomingCancerOccurrenceArray, requiredOc * If the 'fieldMapping.cancerSites.other' cancer site is selected, the 'fieldMapping.anotherTypeOfCancerText' field is required. * Else, the 'anotherTypeOfCancerText' field should not be present. * @param {object} cancerSitesObject - property (740819233) in the cancer occurrence object (637153953). - * @returns {object} - Returns an object with error (boolean), message (string), and data (array). + * @returns {boolean} - Returns true the above requirements are met, false otherwise. */ const validateCancerOccurrence = (cancerSitesObject) => { if (!cancerSitesObject || Object.keys(cancerSitesObject).length === 0 || !cancerSitesObject[fieldMapping.primaryCancerSiteCategorical]) { @@ -1281,23 +1276,6 @@ const validateCancerOccurrence = (cancerSitesObject) => { return { error: hasError, message: hasError ? otherCancerSiteErrorMessage : '', data: [] }; } -/** - * Rules: if vitalStatusCategorical is 'alive' at chart review (114227122: 337516613), participant must be aware of diagnosis (844209241: 353358909). Else, block API request. - * If vitalStatusCategorical is 'dead' or 'unknown' (114227122: 646675764 or 178420302), participant awareness can be yes, no, or unknown (844209241: 353358909 or 104430631 or 178420302). - * @param {number} vitalStatusCategorical - the participant's vital status (conceptID). - * @param {number} participantDiagnosisAwareness - the participant's awareness of diagnosis (conceptID). - * @returns {object} - Returns an object with error (boolean), message (string), and data (array). - */ -const validateDiagnosisAwareness = (vitalStatusCategorical, participantDiagnosisAwareness) => { - const isAliveAtChartReview = vitalStatusCategorical === fieldMapping.vitalStatus.alive; - const isParticipantAwareOfDiagnosis = participantDiagnosisAwareness === fieldMapping.yes; - - const isAwarenessValid = isAliveAtChartReview ? isParticipantAwareOfDiagnosis : true; - const awarenessErrorMessage = "Participant must be aware of diagnosis if alive at chart review. Otherwise, awareness can be 'yes (353358909)', 'no (104430631)', or 'unknown (178420302)'."; - - return { error: !isAwarenessValid, message: !isAwarenessValid ? awarenessErrorMessage : '', data: [] }; -} - /** * Check for duplicate cancer occurrences. Occurrences are considered duplicates if the timestamp and primary cancer sites match. * @param {array} newOccurrenceArray - the new occurrence array to check. diff --git a/utils/sites.js b/utils/sites.js index 93e93a5e..569e9e4b 100644 --- a/utils/sites.js +++ b/utils/sites.js @@ -418,7 +418,7 @@ const flatValidationHandler = (newData, existingData, rules, validationFunction) * Validate data submitted to the updateParticipantData endpoint. * @param {string|number|array|object} value - The value to validate. From a key:value pair submitted in the POST request. * @param {string|number|array|object} existingValue - The existing value to validate against. From the existing participant data in the database. - * @param {string} path - The flattened path to the value in the data object. Example: 'state.123456789' or '637153953[].149205077' <- where [] is an array with any index value. + * @param {string} path - The flattened path to the value in the data object. Example: 'state.123456789' or '637153953[].457270069' <- where [] is an array with any index value. * @param {object} rule - The validation rule to use from updateParticipantData.json. Example: { "dataType": "string", "maxLength": 100 } * @returns null for success, or an error message for failure. */ From 1145b53daca0c5fe8f71777dfc1e9e459c55c6f8 Mon Sep 17 00:00:00 2001 From: jhflorey Date: Sat, 20 Jul 2024 10:37:59 -0400 Subject: [PATCH 30/34] Type conversation --- utils/notifications.js | 3 +-- utils/shared.js | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/notifications.js b/utils/notifications.js index f1bfe356..1e1b77a9 100644 --- a/utils/notifications.js +++ b/utils/notifications.js @@ -705,8 +705,7 @@ const sendEmailLink = async (req, res) => { const body = { message: { subject: - preferredLanguage === - conceptIds.spanish.toString() + preferredLanguage === conceptIds.spanish ? "Inicie sesión para Estudio Connect para la Prevención del Cáncer" : "Sign in to Connect for Cancer Prevention Study", body: { diff --git a/utils/shared.js b/utils/shared.js index a49b6b34..8e906187 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1544,9 +1544,9 @@ const filterSelectedFields = (dataObjArray, selectedFieldsArray) => { const getTemplateForEmailLink = ( email, continueUrl, - preferredLanguage = fieldMapping.english.toString() + preferredLanguage = fieldMapping.english ) => { - return preferredLanguage === fieldMapping.spanish.toString() + return preferredLanguage === fieldMapping.spanish ? ` From 82f18e44e58bd622d5403f27eaa7f00208e5e3bc Mon Sep 17 00:00:00 2001 From: jhflorey Date: Tue, 23 Jul 2024 10:39:33 -0400 Subject: [PATCH 31/34] Change text --- utils/shared.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/shared.js b/utils/shared.js index 066dc15d..b78700c6 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1593,7 +1593,7 @@ const unsubscribeTextObj = { english: "

To unsubscribe from emails about Connect from the National Cancer Institute (NCI), <% click here %> .

", spanish: - "

Si desea darse de baja de Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %> .

", + "

Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %> .

", }; module.exports = { From b2cd64ab976f95552564b18f4306b60c9847c10f Mon Sep 17 00:00:00 2001 From: Jessica Florey Date: Tue, 23 Jul 2024 10:54:39 -0400 Subject: [PATCH 32/34] Update utils/shared.js Co-authored-by: Warren Lu --- utils/shared.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/shared.js b/utils/shared.js index b78700c6..c32633c4 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1593,7 +1593,7 @@ const unsubscribeTextObj = { english: "

To unsubscribe from emails about Connect from the National Cancer Institute (NCI), <% click here %> .

", spanish: - "

Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %> .

", + "

Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %>.

", }; module.exports = { From 8c6d9458c08436ab5fa712295a70995265c255ec Mon Sep 17 00:00:00 2001 From: Jessica Florey Date: Tue, 23 Jul 2024 11:03:47 -0400 Subject: [PATCH 33/34] Update utils/shared.js Co-authored-by: Warren Lu --- utils/shared.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/shared.js b/utils/shared.js index c32633c4..c7586e9d 100644 --- a/utils/shared.js +++ b/utils/shared.js @@ -1591,7 +1591,7 @@ const printDocsCount = (snapshot, infoStr = "") => { const unsubscribeTextObj = { english: - "

To unsubscribe from emails about Connect from the National Cancer Institute (NCI), <% click here %> .

", + "

To unsubscribe from emails about Connect from the National Cancer Institute (NCI), <% click here %>.

", spanish: "

Para cancelar la suscripción a los correos electrónicos sobre Connect del Instituto Nacional del Cáncer (NCI), <% haga clic aquí %>.

", }; From 346245dab32c3bb3e4bdaf363405092df8dba8d6 Mon Sep 17 00:00:00 2001 From: anthonypetersen Date: Mon, 29 Jul 2024 10:05:50 -0500 Subject: [PATCH 34/34] heartbeat update --- utils/heartbeat.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/heartbeat.js b/utils/heartbeat.js index 7a2293ea..7f82a544 100644 --- a/utils/heartbeat.js +++ b/utils/heartbeat.js @@ -20,7 +20,7 @@ const heartbeat = async (req, res) => { const minutes = currentTime.getUTCMinutes().toString().padStart(2, '0'); const seconds = currentTime.getUTCSeconds().toString().padStart(2, '0'); - const query = `SELECT * FROM \`nih-nci-dceg-connect-dev.heartbeat.recruitment_summary\``; + const query = `SELECT * FROM \`${process.env.GCLOUD_PROJECT}.heartbeat.recruitment_summary\``; const [rows] = await bigquery.query(query); const payload = {