diff --git a/Dockerfile b/Dockerfile index 66ef7b4..35cb565 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,10 @@ LABEL uk.gov.defra.ffc.parent-image=defradigital/node-development:${PARENT_VERSI ARG PORT_DEBUG EXPOSE ${PORT_DEBUG} +USER root +RUN apk add --update --no-cache openjdk17-jre + +USER node COPY --chown=node:node package*.json ./ RUN npm install diff --git a/app/config/index.js b/app/config/index.js index 4d788d1..f3c8310 100644 --- a/app/config/index.js +++ b/app/config/index.js @@ -2,14 +2,16 @@ const Joi = require('joi') const mqConfig = require('./message') const dbConfig = require('./database') const storageConfig = require('./storage') +const reportConfig = require('./report') const schema = Joi.object({ env: Joi.string().valid('development', 'test', 'production').default('development'), - deliveryCheckInterval: Joi.number().default(30000), + deliveryCheckInterval: Joi.number().required(), + reportingCheckInterval: Joi.number().required(), notifyApiKey: Joi.string().required(), notifyApiKeyLetter: Joi.string().required(), notifyEmailTemplateKey: Joi.string().required(), - retentionPeriodInWeeks: Joi.number().default(78), + retentionPeriodInWeeks: Joi.number().required(), statementReceiverApiVersion: Joi.string().required(), statementReceiverEndpoint: Joi.string().required() }) @@ -17,6 +19,7 @@ const schema = Joi.object({ const config = { env: process.env.NODE_ENV, deliveryCheckInterval: process.env.DELIVERY_CHECK_INTERVAL, + reportingCheckInterval: process.env.REPORTING_CHECK_INTERVAL, notifyApiKey: process.env.NOTIFY_API_KEY, notifyApiKeyLetter: process.env.NOTIFY_API_KEY_LETTER, notifyEmailTemplateKey: process.env.NOTIFY_EMAIL_TEMPLATE_KEY, @@ -42,5 +45,6 @@ value.publishSubscription = mqConfig.publishSubscription value.crmTopic = mqConfig.crmTopic value.dbConfig = dbConfig value.storageConfig = storageConfig +value.reportConfig = reportConfig module.exports = value diff --git a/app/config/report.js b/app/config/report.js new file mode 100644 index 0000000..7a4e83c --- /dev/null +++ b/app/config/report.js @@ -0,0 +1,67 @@ +const Joi = require('joi') +const { DELINKED: DELINKED_SCHEME_NAME, SFI } = require('../constants/scheme-names').LONG_NAMES + +// Define config schema +const schema = Joi.object({ + schemes: Joi.array().items( + Joi.object({ + schemeName: Joi.string().required(), + schedule: Joi.object({ + dayOfMonth: Joi.number().integer(), + dayOfYear: Joi.number().integer(), + monthOfYear: Joi.number().integer(), + hour: Joi.number().integer(), + minute: Joi.number().integer(), + second: Joi.number().integer(), + intervalNumber: Joi.number().integer().min(1).required(), + intervalType: Joi.string().valid('days', 'weeks', 'months', 'years').required() + }).required(), + dateRange: Joi.object({ + durationNumber: Joi.number().integer().min(1).required(), + durationType: Joi.string().valid('days', 'weeks', 'months', 'years').required() + }).required() + }) + ).required() +}) + +// Build config +const config = { + schemes: [ + { + schemeName: process.env.DELINKED_SCHEME_NAME || DELINKED_SCHEME_NAME, + schedule: { + intervalNumber: process.env.DELINKED_INTERVAL_NUMBER, + intervalType: process.env.DELINKED_INTERVAL_TYPE, + dayOfMonth: process.env.DELINKED_DAY_OF_MONTH + }, + dateRange: { + durationNumber: process.env.DELINKED_DURATION_NUMBER, + durationType: process.env.DELINKED_DURATION_TYPE + } + }, + { + schemeName: process.env.SFI_SCHEME_NAME || SFI, + schedule: { + intervalNumber: process.env.SFI_INTERVAL_NUMBER, + intervalType: process.env.SFI_INTERVAL_TYPE, + dayOfMonth: process.env.SFI_DAY_OF_MONTH + }, + dateRange: { + durationNumber: process.env.SFI_DURATION_NUMBER, + durationType: process.env.SFI_DURATION_TYPE + } + } + ] +} + +// Validate config +const result = schema.validate(config, { + abortEarly: false +}) + +// Throw if config is invalid +if (result.error) { + throw new Error(`The report config is invalid. ${result.error.message}`) +} + +module.exports = result.value diff --git a/app/config/storage.js b/app/config/storage.js index 794974c..a1a71e3 100644 --- a/app/config/storage.js +++ b/app/config/storage.js @@ -6,6 +6,7 @@ const schema = Joi.object({ storageAccount: Joi.string().required(), container: Joi.string().default('statements'), folder: Joi.string().default('outbound'), + reportFolder: Joi.string().default('reports'), useConnectionStr: Joi.boolean().default(false), createContainers: Joi.boolean().default(true) }) @@ -16,6 +17,7 @@ const config = { storageAccount: process.env.AZURE_STORAGE_ACCOUNT_NAME, container: process.env.AZURE_STORAGE_CONTAINER, folder: process.env.AZURE_STORAGE_FOLDER, + reportFolder: process.AZURE_STORAGE_REPORT_FOLDER, useConnectionStr: process.env.AZURE_STORAGE_USE_CONNECTION_STRING, createContainers: process.env.AZURE_STORAGE_CREATE_CONTAINERS } diff --git a/app/constants/publish.js b/app/constants/publish.js new file mode 100644 index 0000000..258b35c --- /dev/null +++ b/app/constants/publish.js @@ -0,0 +1,4 @@ +module.exports = { + RETRIES: 3, + RETRY_INTERVAL: 100 +} diff --git a/app/constants/report-headers.js b/app/constants/report-headers.js new file mode 100644 index 0000000..7c1381f --- /dev/null +++ b/app/constants/report-headers.js @@ -0,0 +1,20 @@ +module.exports = [ + 'Status', + 'Error(s)', + 'FRN', + 'SBI', + 'Payment Reference', + 'Scheme Name', + 'Scheme Short Name', + 'Scheme Year', + 'Delivery Method', + 'Business Name', + 'Address', + 'Email', + 'Filename', + 'Document DB ID', + 'Statement Data Received', + 'Notify Email Requested', + 'Statement Failure Notification', + 'Statement Delivery Notification' +] diff --git a/app/constants/report.js b/app/constants/report.js new file mode 100644 index 0000000..2496001 --- /dev/null +++ b/app/constants/report.js @@ -0,0 +1,5 @@ +module.exports = { + FAILED: 'Failed', + PENDING: 'Pending', + SUCCESS: 'Success' +} diff --git a/app/data/index.js b/app/data/index.js index 21fae61..3fcc179 100644 --- a/app/data/index.js +++ b/app/data/index.js @@ -1,6 +1,6 @@ const fs = require('fs') const path = require('path') -const { Sequelize, DataTypes } = require('sequelize') +const { Sequelize, DataTypes, Op } = require('sequelize') const config = require('../config') const dbConfig = config.dbConfig[config.env] const modelPath = path.join(__dirname, 'models') @@ -25,5 +25,6 @@ Object.keys(db).forEach(modelName => { db.sequelize = sequelize db.Sequelize = Sequelize +db.Op = Op module.exports = db diff --git a/app/data/models/report.js b/app/data/models/report.js new file mode 100644 index 0000000..9570678 --- /dev/null +++ b/app/data/models/report.js @@ -0,0 +1,17 @@ +module.exports = (sequelize, DataTypes) => { + const report = sequelize.define('report', { + reportId: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true }, + lastDeliveryId: DataTypes.INTEGER, + schemeName: DataTypes.STRING, + reportStartDate: DataTypes.DATE, + reportEndDate: DataTypes.DATE, + requested: DataTypes.DATE, + sent: DataTypes.DATE + }, + { + tableName: 'reports', + freezeTableName: true, + timestamps: false + }) + return report +} diff --git a/app/data/models/statement.js b/app/data/models/statement.js index b5f9e44..3cb154d 100644 --- a/app/data/models/statement.js +++ b/app/data/models/statement.js @@ -18,7 +18,8 @@ module.exports = (sequelize, DataTypes) => { schemeFrequency: DataTypes.STRING, received: DataTypes.DATE, documentReference: DataTypes.INTEGER, - emailTemplate: DataTypes.STRING + emailTemplate: DataTypes.STRING, + paymentReference: DataTypes.STRING }, { tableName: 'statements', diff --git a/app/index.js b/app/index.js index 0f1570e..0298f83 100644 --- a/app/index.js +++ b/app/index.js @@ -2,6 +2,7 @@ require('./insights').setup() require('log-timestamp') const messaging = require('./messaging') const monitoring = require('./monitoring') +const reporting = require('./reporting') const { initialiseContainers } = require('./storage') process.on('SIGTERM', async () => { @@ -18,4 +19,5 @@ module.exports = (async () => { await initialiseContainers() await messaging.start() await monitoring.start() + await reporting.start() })() diff --git a/app/publishing/publish-by-email.js b/app/publishing/publish-by-email.js index 55c8c2e..5eea641 100644 --- a/app/publishing/publish-by-email.js +++ b/app/publishing/publish-by-email.js @@ -2,6 +2,7 @@ const moment = require('moment') const config = require('../config') const { NotifyClient } = require('notifications-node-client') const { retry } = require('../retry') +const { RETRIES, RETRY_INTERVAL } = require('../constants/publish') function setupEmailRequest (template, email, linkToFile, personalisation, notifyClient) { const latestDownloadDate = moment(new Date()).add(config.retentionPeriodInWeeks, 'weeks').format('LL') @@ -16,18 +17,22 @@ function setupEmailRequest (template, email, linkToFile, personalisation, notify } } -const publishByEmail = async (template, email, file, personalisation) => { +const publishByEmail = async (template, email, file, personalisation, filename = null) => { moment.locale('en-gb') const notifyClient = new NotifyClient(config.notifyApiKey) + const fileOptions = { + confirmEmailBeforeDownload: true, + retentionPeriod: `${config.retentionPeriodInWeeks} weeks` + } + if (filename) { + fileOptions.filename = filename + } const linkToFile = await notifyClient.prepareUpload( file, - { - confirmEmailBeforeDownload: true, - retentionPeriod: `${config.retentionPeriodInWeeks} weeks` - } + fileOptions ) const emailRequest = setupEmailRequest(template, email, linkToFile, personalisation, notifyClient) - return retry(emailRequest, 3, 100, true) + return retry(emailRequest, RETRIES, RETRY_INTERVAL, true) .then(result => { return result }) diff --git a/app/reporting/complete-report.js b/app/reporting/complete-report.js new file mode 100644 index 0000000..8b89559 --- /dev/null +++ b/app/reporting/complete-report.js @@ -0,0 +1,10 @@ +const db = require('../data') + +const completeReport = async (reportId, transaction) => { + await db.report.update( + { sent: new Date() }, + { where: { reportId } }, + { transaction }) +} + +module.exports = completeReport diff --git a/app/reporting/create-report.js b/app/reporting/create-report.js new file mode 100644 index 0000000..ac092c7 --- /dev/null +++ b/app/reporting/create-report.js @@ -0,0 +1,13 @@ +const db = require('../data') + +const createReport = async (schemeName, lastDeliveryId, reportStartDate, reportEndDate, requested, transaction) => { + return db.report.create({ + lastDeliveryId, + schemeName, + reportStartDate, + reportEndDate, + requested + }, { transaction }) +} + +module.exports = createReport diff --git a/app/reporting/get-deliveries-for-report.js b/app/reporting/get-deliveries-for-report.js new file mode 100644 index 0000000..5f96486 --- /dev/null +++ b/app/reporting/get-deliveries-for-report.js @@ -0,0 +1,27 @@ +const db = require('../data') +const QueryStream = require('pg-query-stream') + +const getDeliveriesForReport = async (schemeName, start, end) => { + console.log('get deliveries', { + schemeName, + start, + end + }) + + const query = ` + SELECT d.*, s.*, f.* + FROM deliveries d + INNER JOIN statements s ON d."statementId" = s."statementId" + LEFT JOIN failures f ON d."deliveryId" = f."deliveryId" + WHERE s."schemeName" = $1 AND d.requested BETWEEN $2 AND $3 + ORDER BY d."deliveryId" ASC, d."method" + ` + + const client = await db.sequelize.connectionManager.getConnection() + const stream = new QueryStream(query, [schemeName, start, end]) + const queryStream = client.query(stream) + + return queryStream +} + +module.exports = getDeliveriesForReport diff --git a/app/reporting/get-todays-report.js b/app/reporting/get-todays-report.js new file mode 100644 index 0000000..3e5b629 --- /dev/null +++ b/app/reporting/get-todays-report.js @@ -0,0 +1,23 @@ +const db = require('../data') +const hour = 23 +const minute = 59 +const second = 59 +const millisecond = 999 + +const getTodaysReport = async (schemeName) => { + const today = new Date() + const startOfDay = new Date(today.setHours(0, 0, 0, 0)) + const endOfDay = new Date(today.setHours(hour, minute, second, millisecond)) + + return db.report.findAll({ + where: { + schemeName, + sent: { + [db.Op.between]: [startOfDay, endOfDay], + [db.Op.ne]: null + } + } + }) +} + +module.exports = getTodaysReport diff --git a/app/reporting/index.js b/app/reporting/index.js new file mode 100644 index 0000000..3852c0c --- /dev/null +++ b/app/reporting/index.js @@ -0,0 +1,67 @@ +const config = require('../config') +const getTodaysReport = require('./get-todays-report') +const { sendReport } = require('./send-report') +const moment = require('moment') + +const startSchemeReport = async (schemeName, startDate, endDate) => { + console.log('[REPORTING] Starting report for scheme: ', schemeName) + const existingReport = await getTodaysReport(schemeName) + if (!existingReport?.length) { + await sendReport(schemeName, startDate, endDate) + } else { + console.log('[REPORTING] A report has already run today for scheme: ', schemeName) + } +} + +const isToday = (date) => { + return moment(date).isSame(moment(), 'day') +} + +const getRunDate = (schedule) => { + const { intervalType, dayOfMonth, dayOfYear, monthOfYear, hour, minute, second } = schedule + const baseDate = moment().hour(hour || 0).minute(minute || 0).second(second || 0).startOf('day') + + if (intervalType === 'months') { + return baseDate.date(dayOfMonth || 1) + } else if (intervalType === 'years') { + return baseDate.month((monthOfYear || 1) - 1).date(dayOfYear || 1) + } else { + return baseDate + } +} + +const processScheme = async (scheme) => { + const { schemeName, schedule, dateRange } = scheme + const runDate = getRunDate(schedule) + + if (isToday(runDate)) { + console.log('[REPORTING] A report is due to run today for scheme: ', schemeName) + const startDate = moment().subtract(dateRange.durationNumber, dateRange.durationType).startOf('day').toDate() + const endDate = moment().endOf('day').toDate() + await startSchemeReport(schemeName, startDate, endDate) + } else { + console.log('[REPORTING] No report is due to run today for scheme: ', schemeName) + } +} + +const start = async () => { + try { + console.log('[REPORTING] Starting reporting') + const schemes = config.reportConfig.schemes + for (const scheme of schemes) { + try { + await processScheme(scheme) + } catch (error) { + console.error('Error processing scheme:', scheme.schemeName, error) + } + } + } catch (err) { + console.error(err) + } finally { + setTimeout(start, config.reportingCheckInterval) + } +} + +module.exports = { + start +} diff --git a/app/reporting/send-report.js b/app/reporting/send-report.js new file mode 100644 index 0000000..be0257b --- /dev/null +++ b/app/reporting/send-report.js @@ -0,0 +1,138 @@ +const db = require('../data') +const getDeliveriesForReport = require('./get-deliveries-for-report') +const createReport = require('./create-report') +const { saveReportFile } = require('../storage') +const completeReport = require('./complete-report') +const { format } = require('@fast-csv/format') +const { FAILED, PENDING, SUCCESS } = require('../constants/report') +const headers = require('../constants/report-headers') + +const getReportFilename = (schemeName, date) => { + const formattedDateTime = date.toISOString() + const formattedName = schemeName.toLowerCase().replace(/ /g, '-') + return `${formattedName}-${formattedDateTime}.csv` +} + +const getErrors = (data) => { + return [ + data.statusCode ? `Status Code: ${data.statusCode}` : '', + data.reason ? `Reason: ${data.reason}` : '', + data.error ? `Error: ${data.error}` : '', + data.message ? `Message: ${data.message}` : '' + ].filter(Boolean).join(', ') +} + +const getAddress = (data) => { + return [ + data.addressLine1, + data.addressLine2, + data.addressLine3, + data.addressLine4, + data.addressLine5, + data.postcode + ].filter(Boolean).join(', ') +} + +const formatDate = (date) => date ? new Date(date).toISOString().replace('T', ' ').split('.')[0] : '' + +const getFieldValue = (field) => field ? field.toString() : '' + +const getFormattedData = (data) => ({ + FRN: getFieldValue(data.frn), + SBI: getFieldValue(data.sbi), + 'Payment Reference': getFieldValue(data.PaymentReference), + 'Scheme Name': getFieldValue(data.schemeName), + 'Scheme Short Name': getFieldValue(data.schemeShortName), + 'Scheme Year': getFieldValue(data.schemeYear), + 'Delivery Method': getFieldValue(data.method), + 'Business Name': getFieldValue(data.businessName), + Email: getFieldValue(data.email), + Filename: getFieldValue(data.filename), + 'Document DB ID': getFieldValue(data.deliveryId), + 'Statement Data Received': formatDate(data.received), + 'Notify Email Requested': formatDate(data.requested), + 'Statement Failure Notification': formatDate(data.failed), + 'Statement Delivery Notification': formatDate(data.completed) +}) + +const getDataRow = (data, status, address, errors) => { + const formattedData = getFormattedData(data) + return { + Status: status, + 'Error(s)': errors, + 'Business Address': address, + ...formattedData + } +} + +const determineStatus = (data) => { + if (data.failureId) { + return FAILED + } else if (data.completed) { + return SUCCESS + } else { + return PENDING + } +} + +const sendReport = async (schemeName, startDate, endDate) => { + const transaction = await db.sequelize.transaction() + console.log('[REPORTING] start send report for scheme: ', schemeName) + + try { + const deliveriesStream = await getDeliveriesForReport(schemeName, startDate, endDate, transaction) + let hasData = false + let lastDeliveryId = null + const reportDate = new Date() + + const filename = getReportFilename(schemeName, reportDate) + const csvStream = format({ + headers + }) + + await new Promise((resolve, reject) => { + deliveriesStream.on('error', (error) => { + csvStream.end() + reject(error) + }) + + deliveriesStream.on('data', (data) => { + hasData = true + lastDeliveryId = data.deliveryId + + const status = determineStatus(data) + const errors = getErrors(data) + const address = getAddress(data) + + csvStream.write(getDataRow(data, status, address, errors)) + }) + + deliveriesStream.on('end', async () => { + try { + csvStream.end() + if (hasData) { + const report = await createReport(schemeName, lastDeliveryId, startDate, endDate, reportDate, transaction) + await saveReportFile(filename, csvStream) + await completeReport(report.reportId, transaction) + await transaction.commit() + resolve() + } else { + await transaction.rollback() + resolve() + } + } catch (error) { + await transaction.rollback() + reject(error) + } + }) + }) + } catch (error) { + await transaction.rollback() + throw error + } +} + +module.exports = { + getDataRow, + sendReport +} diff --git a/app/storage.js b/app/storage.js index 8b48847..084428c 100644 --- a/app/storage.js +++ b/app/storage.js @@ -1,6 +1,10 @@ const { DefaultAzureCredential } = require('@azure/identity') const { BlobServiceClient } = require('@azure/storage-blob') const config = require('./config').storageConfig + +const BUFFER_SIZE = 4 * 1024 * 1024 // 4 MB +const MAX_CONCURRENCY = 5 + let blobServiceClient let containersInitialised @@ -27,7 +31,9 @@ const initialiseContainers = async () => { const initialiseFolders = async () => { const placeHolderText = 'Placeholder' const client = container.getBlockBlobClient(`${config.folder}/default.txt`) + const reportClient = container.getBlockBlobClient(`${config.reportFolder}/default.txt`) await client.upload(placeHolderText, placeHolderText.length) + await reportClient.upload(placeHolderText, placeHolderText.length) } const getBlob = async (filename) => { @@ -40,9 +46,65 @@ const getFile = async (filename) => { return blob.downloadToBuffer() } +const saveReportFile = async (filename, readableStream) => { + try { + console.log('[STORAGE] Starting report file save:', filename) + containersInitialised ?? await initialiseContainers() + + const client = container.getBlockBlobClient(`${config.reportFolder}/${filename}`) + const options = { + blobHTTPHeaders: { + blobContentType: 'text/csv' + } + } + + return new Promise((resolve, reject) => { + let hasData = false + + readableStream.on('data', (chunk) => { + hasData = true + console.debug('[STORAGE] Received chunk:', chunk) + }) + + readableStream.on('end', () => { + console.debug('[STORAGE] Stream ended, had data:', hasData) + }) + + readableStream.on('error', (err) => { + console.error('[STORAGE] Stream error:', err) + reject(err) + }) + + client.uploadStream( + readableStream, + BUFFER_SIZE, + MAX_CONCURRENCY, + options + ) + .then(() => { + console.log('[STORAGE] Upload completed') + resolve() + }) + .catch(reject) + }) + } catch (error) { + console.error('[STORAGE] Error saving report file:', error) + throw error + } +} + +module.exports = { saveReportFile } + +const getReportFile = async (filename) => { + containersInitialised ?? await initialiseContainers() + const client = container.getBlockBlobClient(`${config.reportFolder}/${filename}`) + return client.downloadToBuffer() +} + module.exports = { initialiseContainers, - blobServiceClient, getBlob, - getFile + getFile, + saveReportFile, + getReportFile } diff --git a/changelog/db.changelog-1.10.xml b/changelog/db.changelog-1.10.xml new file mode 100644 index 0000000..7747654 --- /dev/null +++ b/changelog/db.changelog-1.10.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/changelog/db.changelog-1.8.xml b/changelog/db.changelog-1.8.xml new file mode 100644 index 0000000..bd256ee --- /dev/null +++ b/changelog/db.changelog-1.8.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/changelog/db.changelog-1.9.xml b/changelog/db.changelog-1.9.xml new file mode 100644 index 0000000..fb560ce --- /dev/null +++ b/changelog/db.changelog-1.9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/changelog/db.changelog.xml b/changelog/db.changelog.xml index 0c10b1d..331c175 100644 --- a/changelog/db.changelog.xml +++ b/changelog/db.changelog.xml @@ -13,4 +13,7 @@ + + + diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml index 0514645..2b408ff 100644 --- a/docker-compose.test.yaml +++ b/docker-compose.test.yaml @@ -22,7 +22,19 @@ services: DOC_NOTIFY_API_KEY: A_TEST_KEY DOC_NOTIFY_API_KEY_LETTER: A_TEST_KEY DOC_NOTIFY_EMAIL_TEMPLATE_KEY: A_TEST_KEY - + DELINKED_INTERVAL_NUMBER: 1 + DELINKED_INTERVAL_TYPE: months + DELINKED_DAY_OF_MONTH: 15 + DELINKED_DURATION_NUMBER: 1 + DELINKED_DURATION_TYPE: months + SFI_INTERVAL_NUMBER: 1 + SFI_INTERVAL_TYPE: months + SFI_DAY_OF_MONTH: 15 + SFI_DURATION_NUMBER: 1 + SFI_DURATION_TYPE: months + DELIVERY_CHECK_INTERVAL: 30000 + REPORTING_CHECK_INTERVAL: 30000 + RETENTION_PERIOD_IN_WEEKS: 78 ffc-doc-statement-publisher-postgres: volumes: - postgres_data:/var/lib/postgresql/data diff --git a/docker-compose.yaml b/docker-compose.yaml index aac0a6c..abebaa1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,6 +29,16 @@ services: NOTIFY_EMAIL_TEMPLATE_KEY: ${NOTIFY_EMAIL_TEMPLATE_KEY} STATEMENT_RECEIVER_API_VERSION: v1 STATEMENT_RECEIVER_ENDPOINT: http://host.docker.internal:3022 + DELINKED_INTERVAL_NUMBER: ${DELINKED_INTERVAL_NUMBER} + DELINKED_INTERVAL_TYPE: ${DELINKED_INTERVAL_TYPE} + DELINKED_DAY_OF_MONTH: ${DELINKED_DAY_OF_MONTH} + DELINKED_DURATION_NUMBER: ${DELINKED_DURATION_NUMBER} + DELINKED_DURATION_TYPE: ${DELINKED_DURATION_TYPE} + SFI_INTERVAL_NUMBER: ${SFI_INTERVAL_NUMBER} + SFI_INTERVAL_TYPE: ${SFI_INTERVAL_TYPE} + SFI_DAY_OF_MONTH: ${SFI_DAY_OF_MONTH} + SFI_DURATION_NUMBER: ${SFI_DURATION_NUMBER} + SFI_DURATION_TYPE: ${SFI_DURATION_TYPE} ffc-doc-statement-publisher-postgres: image: postgres:11.4-alpine diff --git a/helm/ffc-doc-statement-publisher/templates/config-map.yaml b/helm/ffc-doc-statement-publisher/templates/config-map.yaml index 6dc60fa..e5df628 100644 --- a/helm/ffc-doc-statement-publisher/templates/config-map.yaml +++ b/helm/ffc-doc-statement-publisher/templates/config-map.yaml @@ -23,13 +23,27 @@ data: AZURE_STORAGE_CREATE_CONTAINERS: {{ quote .Values.container.azureStorageCreateContainers }} AZURE_STORAGE_CONTAINER: {{ quote .Values.container.storageContainer }} AZURE_STORAGE_FOLDER: {{ quote .Values.container.storageFolder }} + AZURE_STORAGE_REPORT_FOLDER: {{ quote .Values.container.storageReportFolder }} NOTIFY_API_KEY: {{ quote .Values.container.notifyApiKey }} NOTIFY_API_KEY_LETTER: {{ quote .Values.container.notifyApiKeyLetter }} NOTIFY_EMAIL_TEMPLATE_KEY: {{ quote .Values.container.notifyEmailTemplateKey }} STATEMENT_RECEIVER_API_VERSION: {{ quote .Values.container.statementReceiverApiVersion }} + DELINKED_INTERVAL_NUMBER: {{ quote .Values.container.DelinkedIntervalNumber }} + DELINKED_INTERVAL_TYPE: {{ quote .Values.container.DelinkedIntervalType }} + DELINKED_DAY_OF_MONTH: {{ quote .Values.container.DelinkedDayOfMonth }} + DELINKED_DURATION_NUMBER: {{ quote .Values.container.DelinkedDurationNumber }} + DELINKED_DURATION_TYPE: {{ quote .Values.container.DelinkedDurationType }} + SFI_INTERVAL_NUMBER: {{ quote .Values.container.SfiIntervalNumber }} + SFI_INTERVAL_TYPE: {{ quote .Values.container.SfiIntervalType }} + SFI_DAY_OF_MONTH: {{ quote .Values.container.SfiDayOfMonth }} + SFI_DURATION_NUMBER: {{ quote .Values.container.SfiDurationNumber }} + SFI_DURATION_TYPE: {{ quote .Values.container.SfiDurationType }} {{- if and (.Values.environmentCode) (ne (.Values.environmentCode | toString ) "snd") }} STATEMENT_RECEIVER_ENDPOINT: {{ .Values.container.statementReceiverEndpoint }}-{{ .Values.environmentCode }}.{{ .Values.ingress.server }} {{ else }} STATEMENT_RECEIVER_ENDPOINT: {{ .Values.container.statementReceiverEndpoint }}.{{ .Values.ingress.server }} {{- end }} + DELIVERY_CHECK_INTERVAL: {{ quote .Values.container.deliveryCheckInterval }} + REPORTING_CHECK_INTERVAL: {{ quote .Values.container.reportingCheckInterval }} + RETENTION_PERIOD_IN_WEEKS: {{ quote .Values.container.retentionPeriodInWeeks }} {{- end -}} diff --git a/helm/ffc-doc-statement-publisher/values.yaml b/helm/ffc-doc-statement-publisher/values.yaml index 4a52029..2f413a1 100644 --- a/helm/ffc-doc-statement-publisher/values.yaml +++ b/helm/ffc-doc-statement-publisher/values.yaml @@ -39,11 +39,25 @@ container: azureStorageCreateContainers: false storageContainer: statements storageFolder: outbound + storageReportFolder: reports notifyApiKey: dummy notifyApiKeyLetter: dummy notifyEmailTemplateKey: dummy statementReceiverApiVersion: v1 - statementReceiverEndpoint: https://ffc-doc-statement-receiver + statementReceiverEndpoint: https://ffc-doc-statement-receiver + DelinkedIntervalNumber: 1 + DelinkedIntervalType: months + DelinkedDayOfMonth: 15 + DelinkedDurationNumber: 1 + DelinkedDurationType: months + SfiIntervalNumber: 1 + SfiIntervalType: months + SfiDayOfMonth: 15 + SfiDurationNumber: 1 + SfiDurationType: months + deliveryCheckInterval: 30000 + reportingCheckInterval: 30000 + retentionPeriodInWeeks: 78 ingress: server: example.com diff --git a/package-lock.json b/package-lock.json index 5d0caf8..1f13c86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "ffc-doc-statement-publisher", - "version": "2.2.3", + "version": "2.2.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ffc-doc-statement-publisher", - "version": "2.2.3", + "version": "2.2.5", "license": "OGL-UK-3.0", "dependencies": { "@azure/identity": "4.2.1", "@azure/storage-blob": "12.10.0", + "@fast-csv/format": "^5.0.2", "applicationinsights": "2.9.6", "axios": "1.7.7", "ffc-messaging": "2.9.1", @@ -19,6 +20,7 @@ "notifications-node-client": "7.0.4", "pg": "8.7.3", "pg-hstore": "2.3.4", + "pg-query-stream": "^4.7.1", "sequelize": "6.29.3" }, "devDependencies": { @@ -1143,6 +1145,19 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fast-csv/format": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.2.tgz", + "integrity": "sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -6030,6 +6045,12 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -6040,11 +6061,29 @@ "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -6713,6 +6752,15 @@ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" }, + "node_modules/pg-cursor": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.12.1.tgz", + "integrity": "sha512-V13tEaA9Oq1w+V6Q3UBIB/blxJrwbbr35/dY54r/86soBJ7xkP236bXaORUTVXUPt9B6Ql2BQu+uwQiuMfRVgg==", + "license": "MIT", + "peerDependencies": { + "pg": "^8" + } + }, "node_modules/pg-hstore": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", @@ -6745,6 +6793,18 @@ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, + "node_modules/pg-query-stream": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/pg-query-stream/-/pg-query-stream-4.7.1.tgz", + "integrity": "sha512-UMgsgn/pOIYsIifRySp59vwlpTpLADMK9HWJtq5ff0Z3MxBnPMGnCQeaQl5VuL+7ov4F96mSzIRIcz+Duo6OiQ==", + "license": "MIT", + "dependencies": { + "pg-cursor": "^2.12.1" + }, + "peerDependencies": { + "pg": "^8" + } + }, "node_modules/pg-types": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", diff --git a/package.json b/package.json index 24edcb5..15fdfe4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ffc-doc-statement-publisher", - "version": "2.2.3", + "version": "2.2.5", "description": "Publish statements", "homepage": "https://github.com/DEFRA/ffc-doc-statement-publisher", "main": "app/index.js", @@ -26,6 +26,7 @@ "dependencies": { "@azure/identity": "4.2.1", "@azure/storage-blob": "12.10.0", + "@fast-csv/format": "^5.0.2", "applicationinsights": "2.9.6", "axios": "1.7.7", "ffc-messaging": "2.9.1", @@ -34,6 +35,7 @@ "notifications-node-client": "7.0.4", "pg": "8.7.3", "pg-hstore": "2.3.4", + "pg-query-stream": "^4.7.1", "sequelize": "6.29.3" }, "devDependencies": { diff --git a/test/integration/monitoring/update-deliveries.test.js b/test/integration/monitoring/update-deliveries.test.js index 42f724f..2c3b6e0 100644 --- a/test/integration/monitoring/update-deliveries.test.js +++ b/test/integration/monitoring/update-deliveries.test.js @@ -21,7 +21,6 @@ jest.mock('notifications-node-client', () => { jest.mock('ffc-messaging') jest.mock('../../../app/publishing/get-statement-file-url', () => mockGetStatementFileUrl) jest.mock('../../../app/publishing/fetch-statement-file', () => mockFetchStatementFile) - const { BlobServiceClient } = require('@azure/storage-blob') const config = require('../../../app/config/storage') const db = require('../../../app/data') diff --git a/test/integration/reporting/complete-report.test.js b/test/integration/reporting/complete-report.test.js new file mode 100644 index 0000000..4875fd4 --- /dev/null +++ b/test/integration/reporting/complete-report.test.js @@ -0,0 +1,24 @@ +const db = require('../../../app/data') +const completeReport = require('../../../app/reporting/complete-report') +const { mockReport1, mockReport2 } = require('../../mocks/report') + +describe('completeReport', () => { + beforeEach(async () => { + jest.clearAllMocks() + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + + await db.sequelize.truncate({ cascade: true }) + await db.report.bulkCreate([mockReport1, mockReport2]) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + }) + + test('marks the report as sent', async () => { + const transaction = await db.sequelize.transaction() + await completeReport(mockReport1.reportId, transaction) + const updatedReport = await db.report.findByPk(mockReport1.reportId) + expect(updatedReport.sent).toEqual(new Date(2022, 7, 5, 15, 30, 10, 120)) + }) +}) diff --git a/test/integration/reporting/create-report.test.js b/test/integration/reporting/create-report.test.js new file mode 100644 index 0000000..b9075af --- /dev/null +++ b/test/integration/reporting/create-report.test.js @@ -0,0 +1,56 @@ +const db = require('../../../app/data') +const createReport = require('../../../app/reporting/create-report') +const { mockReport1 } = require('../../mocks/report') + +describe('createReport', () => { + beforeEach(async () => { + jest.clearAllMocks() + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + + await db.sequelize.truncate({ cascade: true }) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + await db.sequelize.close() + }) + + test('creates a new report', async () => { + const transaction = await db.sequelize.transaction() + const result = await createReport( + mockReport1.schemeName, + mockReport1.lastDeliveryId, + mockReport1.reportStartDate, + mockReport1.reportEndDate, + new Date(), + transaction + ) + + await transaction.commit() + + expect(result).toMatchObject({ + schemeName: mockReport1.schemeName, + lastDeliveryId: mockReport1.lastDeliveryId, + reportStartDate: mockReport1.reportStartDate, + reportEndDate: mockReport1.reportEndDate, + requested: new Date(2022, 7, 5, 15, 30, 10, 120), + sent: null + }) + }) + + test('does not create a report if transaction is rolled back', async () => { + const transaction = await db.sequelize.transaction() + await createReport( + mockReport1.schemeName, + mockReport1.lastDeliveryId, + mockReport1.reportStartDate, + mockReport1.reportEndDate, + mockReport1.requested, + transaction + ) + await transaction.rollback() + + const reports = await db.report.findAll() + expect(reports.length).toBe(0) + }) +}) diff --git a/test/integration/reporting/get-deliveries-for-report.test.js b/test/integration/reporting/get-deliveries-for-report.test.js new file mode 100644 index 0000000..5e48c62 --- /dev/null +++ b/test/integration/reporting/get-deliveries-for-report.test.js @@ -0,0 +1,61 @@ +const db = require('../../../app/data') +const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') +const QueryStream = require('pg-query-stream') + +jest.mock('../../../app/data') +jest.mock('pg-query-stream') + +describe('getDeliveriesForReport', () => { + const mockClient = { + query: jest.fn() + } + + beforeEach(() => { + jest.clearAllMocks() + db.sequelize.connectionManager.getConnection.mockResolvedValue(mockClient) + }) + + test('returns stream of deliveries for date range and scheme', async () => { + const schemeName = 'TEST' + const start = new Date('2024-12-01T00:00:00Z') + const end = new Date('2024-12-31T23:59:59Z') + const transaction = {} + + const mockDeliveries = [ + { deliveryId: 1, statementId: 101, method: 'email', reference: '123e4567-e89b-12d3-a456-426614174000', requested: new Date('2024-12-01T10:00:00Z'), completed: new Date('2024-12-02T10:00:00Z') }, + { deliveryId: 2, statementId: 102, method: 'sms', reference: '123e4567-e89b-12d3-a456-426614174001', requested: new Date('2024-12-03T10:00:00Z'), completed: new Date('2024-12-04T10:00:00Z') } + ] + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'data') { + mockDeliveries.forEach(delivery => callback(delivery)) + } + if (event === 'end') { + callback() + } + return mockStream + }) + } + + mockClient.query.mockReturnValue(mockStream) + + const result = await getDeliveriesForReport(schemeName, start, end, transaction) + + expect(QueryStream).toHaveBeenCalledWith( + expect.stringContaining('SELECT d.*, s.*'), + [schemeName, start, end] + ) + expect(db.sequelize.connectionManager.getConnection).toHaveBeenCalled() + expect(mockClient.query).toHaveBeenCalled() + expect(result).toBe(mockStream) + + const collected = [] + await new Promise(resolve => { + result.on('data', data => collected.push(data)) + result.on('end', resolve) + }) + + expect(collected).toEqual(mockDeliveries) + }) +}) diff --git a/test/integration/reporting/get-todays-report.test.js b/test/integration/reporting/get-todays-report.test.js new file mode 100644 index 0000000..6fec903 --- /dev/null +++ b/test/integration/reporting/get-todays-report.test.js @@ -0,0 +1,39 @@ +const db = require('../../../app/data') +const getTodaysReport = require('../../../app/reporting/get-todays-report') +const { mockReport1 } = require('../../mocks/report') + +describe('getTodaysReport', () => { + beforeEach(async () => { + jest.clearAllMocks() + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + + await db.sequelize.truncate({ cascade: true }) + await db.report.bulkCreate([{ ...mockReport1, ...{ sent: new Date() } }]) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + await db.sequelize.close() + }) + + test('returns reports sent today for the specified scheme name', async () => { + const result = await getTodaysReport(mockReport1.schemeName) + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ reportId: mockReport1.reportId }) + ]) + ) + }) + + test('does not return reports sent on a different day', async () => { + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 6, 15, 30, 10, 120)) + const result = await getTodaysReport(mockReport1.schemeName) + expect(result.length).toBe(0) + }) + + test('does not return reports for a different scheme name', async () => { + const result = await getTodaysReport('DifferentSchemeName') + console.log('whahahah', result) + expect(result.length).toBe(0) + }) +}) diff --git a/test/integration/reporting/index.load.test.js b/test/integration/reporting/index.load.test.js new file mode 100644 index 0000000..7d2d92e --- /dev/null +++ b/test/integration/reporting/index.load.test.js @@ -0,0 +1,123 @@ +const { v4: uuidv4 } = require('uuid') +const db = require('../../../app/data') +const getTodaysReport = require('../../../app/reporting/get-todays-report') +const { sendReport } = require('../../../app/reporting/send-report') +const config = require('../../../app/config') + +jest.mock('../../../app/reporting/get-todays-report') +jest.mock('../../../app/reporting/send-report') + +const { start } = require('../../../app/reporting/index') +const currentTimestamp = Math.floor(Date.now() / 1000) +const numberOfRecords = 10000 + +const generateMockStatements = (count) => { + const mockScheme = { + schemeName: 'TEST', + schemeShortName: 'TT', + schemeYear: '2022', + schemeFrequency: 'Quarterly' + } + + return Array.from({ length: count }, (_, i) => ({ + statementId: currentTimestamp + i + 1, + businessName: `Business ${i + 1}`, + addressLine1: 'Line1', + addressLine2: 'Line2', + addressLine3: 'Line 3', + addressLine4: 'Line4', + addressLine5: 'Line5', + postcode: `SW${i + 1} ${i + 1}A`.substring(0, 8), + filename: `FFC_PaymentStatement_SFI_2022_${i + 1}_2022080515301012.pdf`, + sbi: 123456789 + i, + frn: 1234567890 + i, + email: `farmer${i + 1}@farm.com`, + emailTemplate: 'template', + received: new Date(2022, 7, 5, 15, 30, 10, 120), + ...mockScheme + })) +} + +const generateMockDeliveries = (statements) => { + return statements.map((statement, i) => ({ + deliveryId: currentTimestamp + i + 1, + statementId: statement.statementId, + reference: uuidv4(), + method: 'EMAIL', + requested: new Date(2022, 7, 5, 15, 30, 10, 120), + completed: i % 2 === 0 ? null : new Date(2022, 7, 5, 15, 30, 10, 120) + })) +} + +const generateMockFailures = (deliveries, failureCount) => { + return deliveries.slice(0, failureCount).map((delivery, i) => ({ + failureId: currentTimestamp + i + 1, + deliveryId: delivery.deliveryId, + statusCode: 500, + reason: 'Server Error', + error: 'Internal Server Error', + message: 'Failed to deliver', + failed: new Date(2022, 7, 5, 15, 30, 10, 120) + })) +} + +describe('load test for reporting', () => { + const mockScheme = { + schemeName: 'TEST', + template: 'test-template', + email: 'test@test.com', + schedule: { + intervalType: 'months', + dayOfMonth: 5, + hour: 15, + minute: 30 + }, + dateRange: { + durationNumber: 1, + durationType: 'months' + } + } + + beforeEach(async () => { + jest.clearAllMocks() + config.reportConfig.schemes = [mockScheme] + }) + + beforeAll(async () => { + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + await db.sequelize.truncate({ cascade: true }) + const mockStatements = generateMockStatements(numberOfRecords) + await db.statement.bulkCreate(mockStatements) + const mockDeliveries = generateMockDeliveries(mockStatements) + await db.delivery.bulkCreate(mockDeliveries) + const mockFailures = generateMockFailures(mockDeliveries, Math.floor(numberOfRecords / 4)) + await db.failure.bulkCreate(mockFailures) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + await db.sequelize.close() + }) + + test('should process large number of records efficiently', async () => { + getTodaysReport.mockResolvedValue([]) + sendReport.mockResolvedValue() + console.log = jest.fn() + + await start() + + expect(console.log).toHaveBeenCalledWith('[REPORTING] Starting reporting') + expect(console.log).toHaveBeenCalledWith('[REPORTING] A report is due to run today for scheme: ', 'TEST') + expect(getTodaysReport).toHaveBeenCalledWith('TEST') + expect(sendReport).toHaveBeenCalled() + }) + + test('should handle errors and continue execution', async () => { + getTodaysReport.mockRejectedValue(new Error('Test error')) + console.error = jest.fn() + + await start() + + expect(console.error).toHaveBeenCalledWith('Error processing scheme:', 'TEST', expect.any(Error)) + }) +}) diff --git a/test/integration/reporting/index.test.js b/test/integration/reporting/index.test.js new file mode 100644 index 0000000..5d60e21 --- /dev/null +++ b/test/integration/reporting/index.test.js @@ -0,0 +1,78 @@ +const db = require('../../../app/data') +const getTodaysReport = require('../../../app/reporting/get-todays-report') +const { sendReport } = require('../../../app/reporting/send-report') +const config = require('../../../app/config') +const { start } = require('../../../app/reporting/index') +const { mockDelivery1, mockDelivery2 } = require('../../mocks/delivery') +const { mockStatement1, mockStatement2 } = require('../../mocks/statement') + +jest.mock('../../../app/reporting/get-todays-report') +jest.mock('../../../app/reporting/send-report') + +describe('start', () => { + const mockScheme = { + schemeName: 'TEST', + schedule: { + intervalType: 'months', + dayOfMonth: 5, + hour: 15, + minute: 30 + }, + dateRange: { + durationNumber: 1, + durationType: 'months' + } + } + + beforeEach(async () => { + jest.clearAllMocks() + config.reportConfig.schemes = [mockScheme] + }) + + beforeAll(async () => { + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + await db.sequelize.truncate({ cascade: true }) + await db.statement.bulkCreate([mockStatement1, mockStatement2]) + await db.delivery.bulkCreate([mockDelivery1, mockDelivery2]) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + await db.sequelize.close() + }) + + test('should process scheme when report is due', async () => { + getTodaysReport.mockResolvedValue([]) + console.log = jest.fn() + + await start() + + expect(console.log).toHaveBeenCalledWith('[REPORTING] Starting reporting') + expect(console.log).toHaveBeenCalledWith('[REPORTING] A report is due to run today for scheme: ', 'TEST') + expect(getTodaysReport).toHaveBeenCalledWith('TEST') + expect(sendReport).toHaveBeenCalledWith( + 'TEST', + new Date('2022-07-05T00:00:00.000Z'), + new Date('2022-08-05T23:59:59.999Z') + ) + }) + + test('should skip report generation when report already exists', async () => { + getTodaysReport.mockResolvedValue([{ id: 1 }]) + console.log = jest.fn() + + await start() + + expect(console.log).toHaveBeenCalledWith('[REPORTING] A report has already run today for scheme: ', 'TEST') + expect(sendReport).not.toHaveBeenCalled() + }) + + test('should handle errors and continue execution', async () => { + getTodaysReport.mockRejectedValue(new Error('Test error')) + console.error = jest.fn() + + await start() + + expect(console.error).toHaveBeenCalledWith('Error processing scheme:', 'TEST', expect.any(Error)) + }) +}) diff --git a/test/integration/reporting/send-report.test.js b/test/integration/reporting/send-report.test.js new file mode 100644 index 0000000..17fe7b4 --- /dev/null +++ b/test/integration/reporting/send-report.test.js @@ -0,0 +1,104 @@ +const db = require('../../../app/data') +const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') +const createReport = require('../../../app/reporting/create-report') +const { saveReportFile } = require('../../../app/storage') +const completeReport = require('../../../app/reporting/complete-report') +const { sendReport } = require('../../../app/reporting/send-report') +const { mockStatement1, mockStatement2 } = require('../../mocks/statement') +const { mockDelivery1, mockDelivery2 } = require('../../mocks/delivery') + +jest.mock('../../../app/publishing/publish-by-email') +jest.mock('../../../app/reporting/get-deliveries-for-report') +jest.mock('../../../app/reporting/create-report') +jest.mock('../../../app/storage') +jest.mock('../../../app/reporting/complete-report') + +describe('sendReport', () => { + beforeEach(async () => { + jest.clearAllMocks() + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + + await db.sequelize.truncate({ cascade: true }) + await db.statement.bulkCreate([mockStatement1, mockStatement2]) + await db.delivery.bulkCreate([mockDelivery1, mockDelivery2]) + }) + + afterAll(async () => { + await db.sequelize.truncate({ cascade: true }) + await db.sequelize.close() + }) + + test('should create and send report when deliveries are found', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + const mockDeliveries = [ + { deliveryId: 1, statementId: 101, method: 'email', reference: '123e4567-e89b-12d3-a456-426614174000', requested: new Date('2022-07-01T10:00:00Z'), completed: new Date('2022-07-02T10:00:00Z') }, + { deliveryId: 2, statementId: 102, method: 'sms', reference: '123e4567-e89b-12d3-a456-426614174001', requested: new Date('2022-07-03T10:00:00Z'), completed: new Date('2022-07-04T10:00:00Z') } + ] + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'data') { + mockDeliveries.forEach(delivery => callback(delivery)) + } + if (event === 'end') { + callback() + } + return mockStream + }) + } + + getDeliveriesForReport.mockResolvedValue(mockStream) + createReport.mockResolvedValue({ reportId: 1 }) + saveReportFile.mockResolvedValue() + completeReport.mockResolvedValue() + + await sendReport(schemeName, startDate, endDate) + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).toHaveBeenCalledWith(schemeName, 2, startDate, endDate, expect.any(Date), expect.any(Object)) + expect(saveReportFile).toHaveBeenCalledWith(expect.stringContaining('test-'), expect.any(Object)) + expect(completeReport).toHaveBeenCalledWith(1, expect.any(Object)) + }) + + test('should handle no deliveries found', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'end') { + callback() + } + return mockStream + }) + } + + getDeliveriesForReport.mockResolvedValue(mockStream) + + await sendReport(schemeName, startDate, endDate) + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).not.toHaveBeenCalled() + expect(saveReportFile).not.toHaveBeenCalled() + expect(completeReport).not.toHaveBeenCalled() + }) + + test('should handle errors and rollback transaction', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + getDeliveriesForReport.mockRejectedValue(new Error('Test error')) + + await expect(sendReport(schemeName, startDate, endDate)).rejects.toThrow('Test error') + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).not.toHaveBeenCalled() + expect(saveReportFile).not.toHaveBeenCalled() + expect(completeReport).not.toHaveBeenCalled() + }) +}) diff --git a/test/mocks/report.js b/test/mocks/report.js new file mode 100644 index 0000000..93aa5f2 --- /dev/null +++ b/test/mocks/report.js @@ -0,0 +1,20 @@ +const mockReport1 = { + reportId: 1, + lastDeliveryId: 1, + schemeName: 'testScheme', + reportStartDate: new Date(), + reportEndDate: new Date() +} + +const mockReport2 = { + reportId: 2, + lastDeliveryId: 2, + schemeName: 'testScheme', + reportStartDate: new Date(), + reportEndDate: new Date() +} + +module.exports = { + mockReport1, + mockReport2 +} diff --git a/test/unit/index.test.js b/test/unit/index.test.js index 6d4e24e..8f93fa8 100644 --- a/test/unit/index.test.js +++ b/test/unit/index.test.js @@ -4,6 +4,8 @@ jest.mock('../../app/monitoring') const mockMonitoring = require('../../app/monitoring') jest.mock('../../app/storage') const mockStorage = require('../../app/storage') +jest.mock('../../app/reporting') +const mockReporting = require('../../app/reporting') describe('app', () => { beforeEach(() => { @@ -18,6 +20,10 @@ describe('app', () => { expect(mockMonitoring.start).toHaveBeenCalled() }) + test('initialises reporting', async () => { + expect(mockReporting.start).toHaveBeenCalled() + }) + test('initialises containers', async () => { expect(mockStorage.initialiseContainers).toHaveBeenCalled() }) diff --git a/test/unit/publishing/publish-by-email.test.js b/test/unit/publishing/publish-by-email.test.js index 6c0a023..b704f49 100644 --- a/test/unit/publishing/publish-by-email.test.js +++ b/test/unit/publishing/publish-by-email.test.js @@ -101,4 +101,10 @@ describe('Publish by email', () => { const result = await publishByEmail(EMAIL_TEMPLATE, EMAIL, FILE_BUFFER, PERSONALISATION) expect(result).toBe(await mockNotifyClient().sendEmail()) }) + + test('should call mockNotifyClient.prepareUpload with FILE_BUFFER and { confirmEmailBeforeDownload: true, retentionPeriod: config.retentionPeriodInWeeks weeks, filename: "testfile.txt" } when filename is provided', async () => { + const filename = 'testfile.txt' + await publishByEmail(EMAIL_TEMPLATE, EMAIL, FILE_BUFFER, PERSONALISATION, filename) + expect(mockNotifyClient().prepareUpload).toHaveBeenCalledWith(FILE_BUFFER, { confirmEmailBeforeDownload: true, retentionPeriod: `${config.retentionPeriodInWeeks} weeks`, filename }) + }) }) diff --git a/test/unit/reporting/__snapshots__/send-report.test.js.snap b/test/unit/reporting/__snapshots__/send-report.test.js.snap new file mode 100644 index 0000000..cbb30f1 --- /dev/null +++ b/test/unit/reporting/__snapshots__/send-report.test.js.snap @@ -0,0 +1,8 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`sendReport should rollback transaction and throw error on stream error 1`] = ` +"Status,Error(s),FRN,SBI,Payment Reference,Scheme Name,Scheme Short Name,Scheme Year,Delivery Method,Business Name,Address,Email,Filename,Document DB ID,Statement Data Received,Notify Email Requested,Statement Failure Notification,Statement Delivery Notification +Success,,,,,,,,email,,,,,1,,2022-07-01 10:00:00,,2022-07-02 10:00:00 +Success,,,,,,,,sms,,,,,2,,2022-07-03 10:00:00,,2022-07-04 10:00:00 +Success,"Status Code: 500, Reason: Server Error, Error: Internal Server Error, Message: Failed to deliver",,,,,,,letter,,,,,3,,2022-07-03 10:00:00,2022-07-04 12:00:00,2022-07-04 10:00:00" +`; diff --git a/test/unit/reporting/complete-report.test.js b/test/unit/reporting/complete-report.test.js new file mode 100644 index 0000000..a534ee2 --- /dev/null +++ b/test/unit/reporting/complete-report.test.js @@ -0,0 +1,26 @@ +const completeReport = require('../../../app/reporting/complete-report') +const db = require('../../../app/data') + +jest.mock('../../../app/data') + +describe('completeReport', () => { + test('should update the report with the current date', async () => { + const reportId = 1 + const transaction = {} + const mockDate = new Date('2024-12-12T00:00:00Z') + + // Mock the Date object + global.Date = jest.fn(() => mockDate) + + // Mock the update method + db.report.update.mockResolvedValue([1]) + + await completeReport(reportId, transaction) + + expect(db.report.update).toHaveBeenCalledWith( + { sent: mockDate }, + { where: { reportId } }, + { transaction } + ) + }) +}) diff --git a/test/unit/reporting/create-report.test.js b/test/unit/reporting/create-report.test.js new file mode 100644 index 0000000..16a5123 --- /dev/null +++ b/test/unit/reporting/create-report.test.js @@ -0,0 +1,41 @@ +const createReport = require('../../../app/reporting/create-report') +const db = require('../../../app/data') + +jest.mock('../../../app/data') + +describe('createReport', () => { + test('should create a report with the given parameters', async () => { + const schemeName = 'Test Scheme' + const lastDeliveryId = 123 + const reportStartDate = new Date('2024-12-01') + const reportEndDate = new Date('2024-12-31') + const requested = new Date('2024-12-12') + const transaction = {} + + const mockReport = { + lastDeliveryId, + schemeName, + reportStartDate, + reportEndDate, + requested + } + + // Mock the create method + db.report.create.mockResolvedValue(mockReport) + + const result = await createReport(schemeName, lastDeliveryId, reportStartDate, reportEndDate, requested, transaction) + + expect(db.report.create).toHaveBeenCalledWith( + { + lastDeliveryId, + schemeName, + reportStartDate, + reportEndDate, + requested + }, + { transaction } + ) + + expect(result).toEqual(mockReport) + }) +}) diff --git a/test/unit/reporting/get-deliveries-for-report.test.js b/test/unit/reporting/get-deliveries-for-report.test.js new file mode 100644 index 0000000..5e48c62 --- /dev/null +++ b/test/unit/reporting/get-deliveries-for-report.test.js @@ -0,0 +1,61 @@ +const db = require('../../../app/data') +const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') +const QueryStream = require('pg-query-stream') + +jest.mock('../../../app/data') +jest.mock('pg-query-stream') + +describe('getDeliveriesForReport', () => { + const mockClient = { + query: jest.fn() + } + + beforeEach(() => { + jest.clearAllMocks() + db.sequelize.connectionManager.getConnection.mockResolvedValue(mockClient) + }) + + test('returns stream of deliveries for date range and scheme', async () => { + const schemeName = 'TEST' + const start = new Date('2024-12-01T00:00:00Z') + const end = new Date('2024-12-31T23:59:59Z') + const transaction = {} + + const mockDeliveries = [ + { deliveryId: 1, statementId: 101, method: 'email', reference: '123e4567-e89b-12d3-a456-426614174000', requested: new Date('2024-12-01T10:00:00Z'), completed: new Date('2024-12-02T10:00:00Z') }, + { deliveryId: 2, statementId: 102, method: 'sms', reference: '123e4567-e89b-12d3-a456-426614174001', requested: new Date('2024-12-03T10:00:00Z'), completed: new Date('2024-12-04T10:00:00Z') } + ] + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'data') { + mockDeliveries.forEach(delivery => callback(delivery)) + } + if (event === 'end') { + callback() + } + return mockStream + }) + } + + mockClient.query.mockReturnValue(mockStream) + + const result = await getDeliveriesForReport(schemeName, start, end, transaction) + + expect(QueryStream).toHaveBeenCalledWith( + expect.stringContaining('SELECT d.*, s.*'), + [schemeName, start, end] + ) + expect(db.sequelize.connectionManager.getConnection).toHaveBeenCalled() + expect(mockClient.query).toHaveBeenCalled() + expect(result).toBe(mockStream) + + const collected = [] + await new Promise(resolve => { + result.on('data', data => collected.push(data)) + result.on('end', resolve) + }) + + expect(collected).toEqual(mockDeliveries) + }) +}) diff --git a/test/unit/reporting/get-todays-report.test.js b/test/unit/reporting/get-todays-report.test.js new file mode 100644 index 0000000..f66d1a3 --- /dev/null +++ b/test/unit/reporting/get-todays-report.test.js @@ -0,0 +1,51 @@ +jest.mock('../../../app/data', () => ({ + report: { + findAll: jest.fn() + }, + Op: { + gte: Symbol('gte'), + lt: Symbol('lt'), + between: Symbol('between'), + ne: Symbol('ne') + } +})) + +const db = require('../../../app/data') +const getTodaysReport = require('../../../app/reporting/get-todays-report') + +describe('getTodaysReport', () => { + test('should fetch reports for the given scheme name and today\'s date', async () => { + const schemeName = 'Test Scheme' + const today = new Date() + const startOfDay = new Date(today.getTime()) + startOfDay.setHours(0, 0, 0, 0) + const endOfDay = new Date(today.getTime()) + endOfDay.setHours(23, 59, 59, 999) + + const report1SentDate = new Date() + report1SentDate.setHours(10, 0, 0, 0) + const report2SentDate = new Date() + report2SentDate.setHours(15, 0, 0, 0) + + const mockReports = [ + { reportId: 1, schemeName: 'Test Scheme', sentDate: report1SentDate }, + { reportId: 2, schemeName: 'Test Scheme', sentDate: report2SentDate } + ] + + db.report.findAll.mockResolvedValue(mockReports) + + const result = await getTodaysReport(schemeName) + + expect(db.report.findAll).toHaveBeenCalledWith({ + where: { + schemeName, + sent: { + [db.Op.between]: [startOfDay, endOfDay], + [db.Op.ne]: null + } + } + }) + + expect(result).toEqual(mockReports) + }) +}) diff --git a/test/unit/reporting/index.test.js b/test/unit/reporting/index.test.js new file mode 100644 index 0000000..f8026b6 --- /dev/null +++ b/test/unit/reporting/index.test.js @@ -0,0 +1,156 @@ +const moment = require('moment') +const config = require('../../../app/config') +const getTodaysReport = require('../../../app/reporting/get-todays-report') +const { sendReport } = require('../../../app/reporting/send-report') +const { start } = require('../../../app/reporting') + +jest.mock('../../../app/config') +jest.mock('../../../app/reporting/get-todays-report') +jest.mock('../../../app/reporting/send-report') + +describe('Reporting', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + describe('start', () => { + beforeEach(() => { + jest.useFakeTimers().setSystemTime(new Date(2022, 7, 5, 15, 30, 10, 120)) + jest.spyOn(global, 'setTimeout') + }) + + afterEach(() => { + jest.useRealTimers() + if (global?.setTimeout?.mockRestore) { + global.setTimeout.mockRestore() + } + }) + + test('should process schemes and call startSchemeReport if report is due today', async () => { + config.reportConfig = { + schemes: [ + { + schemeName: 'scheme1', + schedule: { intervalNumber: 0, intervalType: 'days' }, + dateRange: { durationNumber: 1, durationType: 'days' } + } + ] + } + const startDate = moment().startOf('day').subtract(1, 'days').toDate() + const endDate = moment().endOf('day').toDate() + getTodaysReport.mockResolvedValue([]) + + await start() + + expect(getTodaysReport).toHaveBeenCalledWith('scheme1') + expect(sendReport).toHaveBeenCalledWith('scheme1', startDate, endDate) + }) + + test('should not call startSchemeReport if report is not due today', async () => { + config.reportConfig = { + schemes: [ + { + schemeName: 'scheme1', + schedule: { intervalNumber: 1, intervalType: 'months' }, + dateRange: { durationNumber: 1, durationType: 'days' } + } + ] + } + + await start() + + expect(getTodaysReport).not.toHaveBeenCalled() + expect(sendReport).not.toHaveBeenCalled() + }) + + test('should handle errors and continue processing', async () => { + config.reportConfig = { + schemes: [ + { + schemeName: 'scheme1', + schedule: { intervalNumber: 0, intervalType: 'days' }, + dateRange: { durationNumber: 1, durationType: 'days' } + } + ] + } + getTodaysReport.mockRejectedValue(new Error('error')) + + await start() + + expect(getTodaysReport).toHaveBeenCalledWith('scheme1') + expect(sendReport).not.toHaveBeenCalled() + }) + + test('should schedule the next run', async () => { + config.reportingCheckInterval = 1000 + config.reportConfig = { schemes: [] } + + await start() + + expect(setTimeout).toHaveBeenCalledWith(expect.any(Function), 1000) + }) + + test('should pass correct dates to startSchemeReport for daily schedule', async () => { + config.reportConfig = { + schemes: [ + { + schemeName: 'dailyScheme', + schedule: { intervalNumber: 0, intervalType: 'days' }, + dateRange: { durationNumber: 1, durationType: 'days' } + } + ] + } + const startDate = moment().startOf('day').subtract(1, 'days').toDate() + const endDate = moment().endOf('day').toDate() + getTodaysReport.mockResolvedValue([]) + + await start() + + expect(sendReport).toHaveBeenCalledWith('dailyScheme', startDate, endDate) + }) + + test('should pass correct dates to startSchemeReport for monthly schedule', async () => { + const dayOfMonth = 2 + jest.useFakeTimers().setSystemTime(new Date(2022, 7, dayOfMonth, 15, 30, 10, 120)) + config.reportConfig = { + schemes: [ + { + schemeName: 'monthlyScheme', + schedule: { intervalNumber: 0, intervalType: 'months', dayOfMonth }, + dateRange: { durationNumber: 1, durationType: 'months' } + } + ] + } + const startDate = moment().subtract(1, 'months').date(dayOfMonth).startOf('day').toDate() + const endDate = moment().date(dayOfMonth).endOf('day').toDate() + getTodaysReport.mockResolvedValue([]) + + await start() + + expect(sendReport).toHaveBeenCalledWith('monthlyScheme', startDate, endDate) + }) + + test('should pass correct dates to startSchemeReport for yearly schedule', async () => { + const monthOfYear = 6 + const dayOfYear = 15 + jest.useFakeTimers().setSystemTime(new Date(2022, monthOfYear - 1, dayOfYear, 15, 30, 10, 120)) + config.reportConfig = { + schemes: [ + { + schemeName: 'yearlyScheme', + schedule: { intervalNumber: 0, intervalType: 'years', dayOfYear, monthOfYear }, + dateRange: { durationNumber: 1, durationType: 'years' } + } + ] + } + const startDate = moment().month(monthOfYear - 1).date(dayOfYear).startOf('day').subtract(1, 'years').toDate() + const endDate = moment().month(monthOfYear - 1).date(dayOfYear).endOf('day').toDate() + + getTodaysReport.mockResolvedValue([]) + + await start() + + expect(sendReport).toHaveBeenCalledWith('yearlyScheme', startDate, endDate) + }) + }) +}) diff --git a/test/unit/reporting/send-report.test.js b/test/unit/reporting/send-report.test.js new file mode 100644 index 0000000..3993e52 --- /dev/null +++ b/test/unit/reporting/send-report.test.js @@ -0,0 +1,360 @@ +const { sendReport, getDataRow } = require('../../../app/reporting/send-report') +const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') +const createReport = require('../../../app/reporting/create-report') +const completeReport = require('../../../app/reporting/complete-report') +const { saveReportFile } = require('../../../app/storage') +const db = require('../../../app/data') +const { PassThrough } = require('stream') + +jest.mock('../../../app/reporting/get-deliveries-for-report') +jest.mock('../../../app/reporting/create-report') +jest.mock('../../../app/reporting/complete-report') +jest.mock('../../../app/publishing/publish-by-email') +jest.mock('../../../app/storage') +jest.mock('../../../app/data') + +describe('sendReport', () => { + let transaction + let mockStream + + beforeAll(() => { + console.log = jest.fn() + console.error = jest.fn() + console.debug = jest.fn() + }) + + beforeEach(() => { + transaction = { commit: jest.fn(), rollback: jest.fn() } + db.sequelize.transaction.mockResolvedValue(transaction) + + mockStream = { + on: jest.fn(), + end: jest.fn(), + write: jest.fn() + } + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + test('should create and send report when deliveries are found', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + const mockDeliveries = [ + { deliveryId: 1, statementId: 101, method: 'email', reference: '123e4567-e89b-12d3-a456-426614174000', requested: new Date('2022-07-01T10:00:00Z'), completed: new Date('2022-07-02T10:00:00Z') }, + { deliveryId: 2, statementId: 102, method: 'sms', reference: '123e4567-e89b-12d3-a456-426614174001', requested: new Date('2022-07-03T10:00:00Z'), completed: new Date('2022-07-04T10:00:00Z') }, + { deliveryId: 3, statementId: 103, method: 'letter', reference: '123e4567-e89b-12d3-a456-426614174002', requested: new Date('2022-07-03T10:00:00Z'), completed: new Date('2022-07-04T10:00:00Z'), statusCode: 500, reason: 'Server Error', error: 'Internal Server Error', message: 'Failed to deliver', failed: new Date('2022-07-04T12:00:00Z') } + ] + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'data') { + mockDeliveries.forEach(delivery => callback(delivery)) + } + if (event === 'end') { + callback() + } + return mockStream + }) + } + + getDeliveriesForReport.mockResolvedValue(mockStream) + createReport.mockResolvedValue({ reportId: 1 }) + saveReportFile.mockImplementation((filename, stream) => { + const passThrough = new PassThrough() + stream.pipe(passThrough) + let data = '' + passThrough.on('data', chunk => { + data += chunk.toString() + }) + passThrough.on('end', () => { + expect(data).toMatchSnapshot() + }) + }) + completeReport.mockResolvedValue() + + await sendReport(schemeName, startDate, endDate) + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).toHaveBeenCalledWith(schemeName, 3, startDate, endDate, expect.any(Date), transaction) + expect(saveReportFile).toHaveBeenCalledWith(expect.stringContaining('test-'), expect.any(Object)) + expect(completeReport).toHaveBeenCalledWith(1, expect.any(Object)) + expect(transaction.commit).toHaveBeenCalled() + }) + + test('should handle no deliveries found', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + const mockStream = { + on: jest.fn((event, callback) => { + if (event === 'end') { + callback() + } + return mockStream + }) + } + + getDeliveriesForReport.mockResolvedValue(mockStream) + + await sendReport(schemeName, startDate, endDate) + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).not.toHaveBeenCalled() + expect(saveReportFile).not.toHaveBeenCalled() + expect(completeReport).not.toHaveBeenCalled() + expect(transaction.rollback).toHaveBeenCalled() + }) + + test('should handle errors and rollback transaction', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + getDeliveriesForReport.mockRejectedValue(new Error('Test error')) + + await expect(sendReport(schemeName, startDate, endDate)).rejects.toThrow('Test error') + + expect(getDeliveriesForReport).toHaveBeenCalledWith(schemeName, startDate, endDate, expect.any(Object)) + expect(createReport).not.toHaveBeenCalled() + expect(saveReportFile).not.toHaveBeenCalled() + expect(completeReport).not.toHaveBeenCalled() + expect(transaction.rollback).toHaveBeenCalled() + }) + + test('should rollback transaction and throw error on stream error', async () => { + const error = new Error('Stream error') + const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') + getDeliveriesForReport.mockImplementation(() => { + const stream = mockStream + process.nextTick(() => stream.on.mock.calls.find(x => x[0] === 'error')[1](error)) + return stream + }) + + await expect(sendReport('TEST', new Date(), new Date())) + .rejects.toThrow('Stream error') + expect(transaction.rollback).toHaveBeenCalled() + }) + + test('should rollback transaction when no data is received', async () => { + const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') + getDeliveriesForReport.mockImplementation(() => { + const stream = mockStream + process.nextTick(() => stream.on.mock.calls.find(x => x[0] === 'end')[1]()) + return stream + }) + + await sendReport('TEST', new Date(), new Date()) + expect(transaction.rollback).toHaveBeenCalled() + expect(transaction.commit).not.toHaveBeenCalled() + }) + + test('should rollback transaction when no data received', async () => { + const schemeName = 'TEST' + const startDate = new Date('2022-07-01T00:00:00Z') + const endDate = new Date('2022-07-31T23:59:59Z') + + const getDeliveriesForReport = require('../../../app/reporting/get-deliveries-for-report') + getDeliveriesForReport.mockImplementation(() => { + const stream = mockStream + process.nextTick(() => { + stream.on.mock.calls.find(x => x[0] === 'end')[1]() + }) + return stream + }) + + const { sendReport } = require('../../../app/reporting/send-report') + await sendReport(schemeName, startDate, endDate) + + expect(transaction.rollback).toHaveBeenCalled() + expect(transaction.commit).not.toHaveBeenCalled() + }) +}) + +describe('getDataRow', () => { + test('should return row data with FAILED status', () => { + const data = { + failureId: 1, + frn: 1234567890, + sbi: 123456789, + PaymentReference: 'PR123', + schemeName: 'Scheme Name', + schemeShortName: 'Scheme Short Name', + schemeYear: 2022, + method: 'Email', + businessName: 'Business Name', + addressLine1: 'Address Line 1', + addressLine2: 'Address Line 2', + addressLine3: 'Address Line 3', + addressLine4: 'Address Line 4', + addressLine5: 'Address Line 5', + postcode: 'AB12 3CD', + email: 'email@example.com', + filename: 'filename.pdf', + deliveryId: '12345', + received: '2022-07-01T00:00:00Z', + requested: '2022-07-01T01:00:00Z', + failed: '2022-07-01T02:00:00Z', + completed: '2022-07-01T03:00:00Z', + statusCode: 400, + reason: 'Bad Request', + error: 'Invalid data', + message: 'Data validation failed' + } + const status = 'FAILED' + const address = 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD' + const errors = 'Status Code: 400, Reason: Bad Request, Error: Invalid data, Message: Data validation failed' + const row = getDataRow(data, status, address, errors) + expect(row).toEqual({ + Status: 'FAILED', + 'Error(s)': 'Status Code: 400, Reason: Bad Request, Error: Invalid data, Message: Data validation failed', + FRN: '1234567890', + SBI: '123456789', + 'Payment Reference': 'PR123', + 'Scheme Name': 'Scheme Name', + 'Scheme Short Name': 'Scheme Short Name', + 'Scheme Year': '2022', + 'Delivery Method': 'Email', + 'Business Name': 'Business Name', + 'Business Address': 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD', + Email: 'email@example.com', + Filename: 'filename.pdf', + 'Document DB ID': '12345', + 'Statement Data Received': '2022-07-01 00:00:00', + 'Notify Email Requested': '2022-07-01 01:00:00', + 'Statement Failure Notification': '2022-07-01 02:00:00', + 'Statement Delivery Notification': '2022-07-01 03:00:00' + }) + }) + + test('should return row data with SUCCESS status', () => { + const data = { + completed: '2022-07-01T03:00:00Z', + frn: 1234567890, + sbi: 123456789, + PaymentReference: 'PR123', + schemeName: 'Scheme Name', + schemeShortName: 'Scheme Short Name', + schemeYear: 2022, + method: 'Email', + businessName: 'Business Name', + addressLine1: 'Address Line 1', + addressLine2: 'Address Line 2', + addressLine3: 'Address Line 3', + addressLine4: 'Address Line 4', + addressLine5: 'Address Line 5', + postcode: 'AB12 3CD', + email: 'email@example.com', + filename: 'filename.pdf', + deliveryId: '12345', + received: '2022-07-01T00:00:00Z', + requested: '2022-07-01T01:00:00Z', + failed: '2022-07-01T02:00:00Z' + } + const status = 'SUCCESS' + const address = 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD' + const errors = '' + const row = getDataRow(data, status, address, errors) + expect(row).toEqual({ + Status: 'SUCCESS', + 'Error(s)': '', + FRN: '1234567890', + SBI: '123456789', + 'Payment Reference': 'PR123', + 'Scheme Name': 'Scheme Name', + 'Scheme Short Name': 'Scheme Short Name', + 'Scheme Year': '2022', + 'Delivery Method': 'Email', + 'Business Name': 'Business Name', + 'Business Address': 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD', + Email: 'email@example.com', + Filename: 'filename.pdf', + 'Document DB ID': '12345', + 'Statement Data Received': '2022-07-01 00:00:00', + 'Notify Email Requested': '2022-07-01 01:00:00', + 'Statement Failure Notification': '2022-07-01 02:00:00', + 'Statement Delivery Notification': '2022-07-01 03:00:00' + }) + }) + + test('should return row data with PENDING status', () => { + const data = { + frn: 1234567890, + sbi: 123456789, + PaymentReference: 'PR123', + schemeName: 'Scheme Name', + schemeShortName: 'Scheme Short Name', + schemeYear: 2022, + method: 'Email', + businessName: 'Business Name', + addressLine1: 'Address Line 1', + addressLine2: 'Address Line 2', + addressLine3: 'Address Line 3', + addressLine4: 'Address Line 4', + addressLine5: 'Address Line 5', + postcode: 'AB12 3CD', + email: 'email@example.com', + filename: 'filename.pdf', + deliveryId: '12345', + received: '2022-07-01T00:00:00Z', + requested: '2022-07-01T01:00:00Z', + failed: '2022-07-01T02:00:00Z' + } + const status = 'PENDING' + const address = 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD' + const errors = '' + const row = getDataRow(data, status, address, errors) + expect(row).toEqual({ + Status: 'PENDING', + 'Error(s)': '', + FRN: '1234567890', + SBI: '123456789', + 'Payment Reference': 'PR123', + 'Scheme Name': 'Scheme Name', + 'Scheme Short Name': 'Scheme Short Name', + 'Scheme Year': '2022', + 'Delivery Method': 'Email', + 'Business Name': 'Business Name', + 'Business Address': 'Address Line 1, Address Line 2, Address Line 3, Address Line 4, Address Line 5, AB12 3CD', + Email: 'email@example.com', + Filename: 'filename.pdf', + 'Document DB ID': '12345', + 'Statement Data Received': '2022-07-01 00:00:00', + 'Notify Email Requested': '2022-07-01 01:00:00', + 'Statement Failure Notification': '2022-07-01 02:00:00', + 'Statement Delivery Notification': '' + }) + }) + + test('should return row data with missing fields', () => { + const data = {} + const status = 'PENDING' + const address = '' + const errors = '' + const row = getDataRow(data, status, address, errors) + expect(row).toEqual({ + Status: 'PENDING', + 'Error(s)': '', + FRN: '', + SBI: '', + 'Payment Reference': '', + 'Scheme Name': '', + 'Scheme Short Name': '', + 'Scheme Year': '', + 'Delivery Method': '', + 'Business Name': '', + 'Business Address': '', + Email: '', + Filename: '', + 'Document DB ID': '', + 'Statement Data Received': '', + 'Notify Email Requested': '', + 'Statement Failure Notification': '', + 'Statement Delivery Notification': '' + }) + }) +}) diff --git a/test/unit/storage.test.js b/test/unit/storage.test.js new file mode 100644 index 0000000..522c8e5 --- /dev/null +++ b/test/unit/storage.test.js @@ -0,0 +1,179 @@ +const { DefaultAzureCredential } = require('@azure/identity') +const { BlobServiceClient } = require('@azure/storage-blob') +const { Readable } = require('stream') + +jest.mock('@azure/identity') +jest.mock('@azure/storage-blob') + +describe('storage', () => { + let mockBlobServiceClient + let mockContainerClient + let mockBlockBlobClient + + beforeAll(() => { + console.log = jest.fn() + console.error = jest.fn() + console.debug = jest.fn() + }) + + beforeEach(() => { + jest.clearAllMocks() + + mockBlockBlobClient = { + upload: jest.fn(), + uploadStream: jest.fn(), + downloadToBuffer: jest.fn() + } + + mockContainerClient = { + getBlockBlobClient: jest.fn().mockReturnValue(mockBlockBlobClient), + createIfNotExists: jest.fn() + } + + mockBlobServiceClient = { + getContainerClient: jest.fn().mockReturnValue(mockContainerClient) + } + + BlobServiceClient.fromConnectionString.mockReturnValue(mockBlobServiceClient) + BlobServiceClient.mockImplementation(() => mockBlobServiceClient) + DefaultAzureCredential.mockImplementation(() => ({})) + }) + + test('should initialize BlobServiceClient with connection string', () => { + jest.isolateModules(() => { + const config = require('../../app/config').storageConfig + config.useConnectionStr = true + require('../../app/storage') + expect(BlobServiceClient.fromConnectionString).toHaveBeenCalledWith(config.connectionStr) + }) + }) + + test('should initialize BlobServiceClient using DefaultAzureCredential', () => { + jest.isolateModules(() => { + const config = require('../../app/config').storageConfig + config.storageAccount = 'test' + config.useConnectionStr = false + const uri = `https://${config.storageAccount}.blob.core.windows.net` + const mockCredential = {} + DefaultAzureCredential.mockImplementation(() => mockCredential) + require('../../app/storage') + expect(BlobServiceClient).toHaveBeenCalledWith(uri, mockCredential) + }) + }) + + test('should create containers if not exist', async () => { + jest.isolateModules(async () => { + const config = require('../../app/config').storageConfig + config.createContainers = true + const storage = require('../../app/storage') + await storage.initialiseContainers() + expect(mockContainerClient.createIfNotExists).toHaveBeenCalled() + expect(mockBlockBlobClient.upload).toHaveBeenCalledTimes(2) + }) + }) + + test('should get blob client', async () => { + jest.isolateModules(async () => { + const config = require('../../app/config').storageConfig + const filename = 'test.txt' + const storage = require('../../app/storage') + const blobClient = await storage.getBlob(filename) + expect(mockContainerClient.getBlockBlobClient).toHaveBeenCalledWith(`${config.folder}/${filename}`) + expect(blobClient).toBe(mockBlockBlobClient) + }) + }) + + test('should get file content', async () => { + jest.isolateModules(async () => { + const filename = 'test.txt' + const buffer = Buffer.from('test content') + mockBlockBlobClient.downloadToBuffer.mockResolvedValue(buffer) + const storage = require('../../app/storage') + const fileContent = await storage.getFile(filename) + expect(mockBlockBlobClient.downloadToBuffer).toHaveBeenCalled() + expect(fileContent).toBe(buffer) + }) + }) + + test('should save report file successfully', async () => { + await jest.isolateModules(async () => { + const config = require('../../app/config').storageConfig + const filename = 'test.csv' + const mockStream = new Readable({ + read () { + this.push('test data') + this.push(null) + } + }) + + mockBlockBlobClient.uploadStream.mockResolvedValue() + + const storage = require('../../app/storage') + await storage.saveReportFile(filename, mockStream) + + expect(mockContainerClient.getBlockBlobClient) + .toHaveBeenCalledWith(`${config.reportFolder}/${filename}`) + expect(mockBlockBlobClient.uploadStream).toHaveBeenCalledWith( + mockStream, + 4 * 1024 * 1024, + 5, + expect.objectContaining({ + blobHTTPHeaders: { + blobContentType: 'text/csv' + } + }) + ) + }) + }) + + test('should handle stream error when saving report file', async () => { + mockBlockBlobClient.uploadStream.mockRejectedValue(new Error('Stream failed')) + await jest.isolateModules(async () => { + const filename = 'test.csv' + const mockStream = new Readable({ + read () { + this.emit('error', new Error('Stream failed')) + } + }) + + const storage = require('../../app/storage') + await expect(storage.saveReportFile(filename, mockStream)) + .rejects.toThrow('Stream failed') + }) + }) + + test('should initialize containers before saving report file', async () => { + await jest.isolateModules(async () => { + const filename = 'test.csv' + const mockStream = new Readable({ + read () { + this.push(null) + } + }) + + mockBlockBlobClient.uploadStream.mockResolvedValue() + + const storage = require('../../app/storage') + await storage.saveReportFile(filename, mockStream) + + expect(mockContainerClient.createIfNotExists).toHaveBeenCalled() + }) + }) + + test('should get report file content', async () => { + await jest.isolateModules(async () => { + const config = require('../../app/config').storageConfig + const filename = 'test.csv' + const buffer = Buffer.from('test content') + mockBlockBlobClient.downloadToBuffer.mockResolvedValue(buffer) + + const storage = require('../../app/storage') + const fileContent = await storage.getReportFile(filename) + + expect(mockContainerClient.getBlockBlobClient) + .toHaveBeenCalledWith(`${config.reportFolder}/${filename}`) + expect(mockBlockBlobClient.downloadToBuffer).toHaveBeenCalled() + expect(fileContent).toBe(buffer) + }) + }) +})