DOC-1658: reporting scheduled per scheme based on config

Dockerfile

@@ -8,6 +8,10 @@ LABEL uk.gov.defra.ffc.parent-image=defradigital/node-development:${PARENT_VERSI
 
 ARG PORT_DEBUG
 EXPOSE ${PORT_DEBUG}
+USER root
+RUN apk add --update --no-cache openjdk17-jre
+
+USER node
 
 COPY --chown=node:node package*.json ./
 RUN npm install

app/config/index.js

@@ -2,21 +2,24 @@ const Joi = require('joi')
 const mqConfig = require('./message')
 const dbConfig = require('./database')
 const storageConfig = require('./storage')
+const reportConfig = require('./report')
 
 const schema = Joi.object({
   env: Joi.string().valid('development', 'test', 'production').default('development'),
-  deliveryCheckInterval: Joi.number().default(30000),
+  deliveryCheckInterval: Joi.number().required(),
+  reportingCheckInterval: Joi.number().required(),
   notifyApiKey: Joi.string().required(),
   notifyApiKeyLetter: Joi.string().required(),
   notifyEmailTemplateKey: Joi.string().required(),
-  retentionPeriodInWeeks: Joi.number().default(78),
+  retentionPeriodInWeeks: Joi.number().required(),
   statementReceiverApiVersion: Joi.string().required(),
   statementReceiverEndpoint: Joi.string().required()
 })
 
 const config = {
   env: process.env.NODE_ENV,
   deliveryCheckInterval: process.env.DELIVERY_CHECK_INTERVAL,
+  reportingCheckInterval: process.env.REPORTING_CHECK_INTERVAL,
   notifyApiKey: process.env.NOTIFY_API_KEY,
   notifyApiKeyLetter: process.env.NOTIFY_API_KEY_LETTER,
   notifyEmailTemplateKey: process.env.NOTIFY_EMAIL_TEMPLATE_KEY,
@@ -42,5 +45,6 @@ value.publishSubscription = mqConfig.publishSubscription
 value.crmTopic = mqConfig.crmTopic
 value.dbConfig = dbConfig
 value.storageConfig = storageConfig
+value.reportConfig = reportConfig
 
 module.exports = value

app/config/report.js

@@ -0,0 +1,67 @@
+const Joi = require('joi')
+const { DELINKED: DELINKED_SCHEME_NAME, SFI } = require('../constants/scheme-names').LONG_NAMES
+
+// Define config schema
+const schema = Joi.object({
+  schemes: Joi.array().items(
+    Joi.object({
+      schemeName: Joi.string().required(),
+      schedule: Joi.object({
+        dayOfMonth: Joi.number().integer(),
+        dayOfYear: Joi.number().integer(),
+        monthOfYear: Joi.number().integer(),
+        hour: Joi.number().integer(),
+        minute: Joi.number().integer(),
+        second: Joi.number().integer(),
+        intervalNumber: Joi.number().integer().min(1).required(),
+        intervalType: Joi.string().valid('days', 'weeks', 'months', 'years').required()
+      }).required(),
+      dateRange: Joi.object({
+        durationNumber: Joi.number().integer().min(1).required(),
+        durationType: Joi.string().valid('days', 'weeks', 'months', 'years').required()
+      }).required()
+    })
+  ).required()
+})
+
+// Build config
+const config = {
+  schemes: [
+    {
+      schemeName: process.env.DELINKED_SCHEME_NAME || DELINKED_SCHEME_NAME,
+      schedule: {
+        intervalNumber: process.env.DELINKED_INTERVAL_NUMBER,
+        intervalType: process.env.DELINKED_INTERVAL_TYPE,
+        dayOfMonth: process.env.DELINKED_DAY_OF_MONTH
+      },
+      dateRange: {
+        durationNumber: process.env.DELINKED_DURATION_NUMBER,
+        durationType: process.env.DELINKED_DURATION_TYPE
+      }
+    },
+    {
+      schemeName: process.env.SFI_SCHEME_NAME || SFI,
+      schedule: {
+        intervalNumber: process.env.SFI_INTERVAL_NUMBER,
+        intervalType: process.env.SFI_INTERVAL_TYPE,
+        dayOfMonth: process.env.SFI_DAY_OF_MONTH
+      },
+      dateRange: {
+        durationNumber: process.env.SFI_DURATION_NUMBER,
+        durationType: process.env.SFI_DURATION_TYPE
+      }
+    }
+  ]
+}
+
+// Validate config
+const result = schema.validate(config, {
+  abortEarly: false
+})
+
+// Throw if config is invalid
+if (result.error) {
+  throw new Error(`The report config is invalid. ${result.error.message}`)
+}
+
+module.exports = result.value

app/config/storage.js

@@ -6,6 +6,7 @@ const schema = Joi.object({
   storageAccount: Joi.string().required(),
   container: Joi.string().default('statements'),
   folder: Joi.string().default('outbound'),
+  reportFolder: Joi.string().default('reports'),
   useConnectionStr: Joi.boolean().default(false),
   createContainers: Joi.boolean().default(true)
 })
@@ -16,6 +17,7 @@ const config = {
   storageAccount: process.env.AZURE_STORAGE_ACCOUNT_NAME,
   container: process.env.AZURE_STORAGE_CONTAINER,
   folder: process.env.AZURE_STORAGE_FOLDER,
+  reportFolder: process.AZURE_STORAGE_REPORT_FOLDER,
   useConnectionStr: process.env.AZURE_STORAGE_USE_CONNECTION_STRING,
   createContainers: process.env.AZURE_STORAGE_CREATE_CONTAINERS
 }

app/constants/publish.js

@@ -0,0 +1,4 @@
+module.exports = {
+  RETRIES: 3,
+  RETRY_INTERVAL: 100
+}

app/constants/report-headers.js

@@ -0,0 +1,20 @@
+module.exports = [
+  'Status',
+  'Error(s)',
+  'FRN',
+  'SBI',
+  'Payment Reference',
+  'Scheme Name',
+  'Scheme Short Name',
+  'Scheme Year',
+  'Delivery Method',
+  'Business Name',
+  'Address',
+  'Email',
+  'Filename',
+  'Document DB ID',
+  'Statement Data Received',
+  'Notify Email Requested',
+  'Statement Failure Notification',
+  'Statement Delivery Notification'
+]

app/constants/report.js

@@ -0,0 +1,5 @@
+module.exports = {
+  FAILED: 'Failed',
+  PENDING: 'Pending',
+  SUCCESS: 'Success'
+}

app/data/index.js

@@ -1,6 +1,6 @@
 const fs = require('fs')
 const path = require('path')
-const { Sequelize, DataTypes } = require('sequelize')
+const { Sequelize, DataTypes, Op } = require('sequelize')
 const config = require('../config')
 const dbConfig = config.dbConfig[config.env]
 const modelPath = path.join(__dirname, 'models')
@@ -25,5 +25,6 @@ Object.keys(db).forEach(modelName => {
 
 db.sequelize = sequelize
 db.Sequelize = Sequelize
+db.Op = Op
 
 module.exports = db

app/data/models/report.js

@@ -0,0 +1,17 @@
+module.exports = (sequelize, DataTypes) => {
+  const report = sequelize.define('report', {
+    reportId: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
+    lastDeliveryId: DataTypes.INTEGER,
+    schemeName: DataTypes.STRING,
+    reportStartDate: DataTypes.DATE,
+    reportEndDate: DataTypes.DATE,
+    requested: DataTypes.DATE,
+    sent: DataTypes.DATE
+  },
+  {
+    tableName: 'reports',
+    freezeTableName: true,
+    timestamps: false
+  })
+  return report
+}

app/data/models/statement.js

@@ -18,7 +18,8 @@ module.exports = (sequelize, DataTypes) => {
     schemeFrequency: DataTypes.STRING,
     received: DataTypes.DATE,
     documentReference: DataTypes.INTEGER,
-    emailTemplate: DataTypes.STRING
+    emailTemplate: DataTypes.STRING,
+    paymentReference: DataTypes.STRING
   },
   {
     tableName: 'statements',

app/index.js

@@ -2,6 +2,7 @@ require('./insights').setup()
 require('log-timestamp')
 const messaging = require('./messaging')
 const monitoring = require('./monitoring')
+const reporting = require('./reporting')
 const { initialiseContainers } = require('./storage')
 
 process.on('SIGTERM', async () => {
@@ -18,4 +19,5 @@ module.exports = (async () => {
   await initialiseContainers()
   await messaging.start()
   await monitoring.start()
+  await reporting.start()
 })()

app/publishing/publish-by-email.js

@@ -2,6 +2,7 @@ const moment = require('moment')
 const config = require('../config')
 const { NotifyClient } = require('notifications-node-client')
 const { retry } = require('../retry')
+const { RETRIES, RETRY_INTERVAL } = require('../constants/publish')
 
 function setupEmailRequest (template, email, linkToFile, personalisation, notifyClient) {
   const latestDownloadDate = moment(new Date()).add(config.retentionPeriodInWeeks, 'weeks').format('LL')
@@ -16,18 +17,22 @@ function setupEmailRequest (template, email, linkToFile, personalisation, notify
   }
 }
 
-const publishByEmail = async (template, email, file, personalisation) => {
+const publishByEmail = async (template, email, file, personalisation, filename = null) => {
   moment.locale('en-gb')
   const notifyClient = new NotifyClient(config.notifyApiKey)
+  const fileOptions = {
+    confirmEmailBeforeDownload: true,
+    retentionPeriod: `${config.retentionPeriodInWeeks} weeks`
+  }
+  if (filename) {
+    fileOptions.filename = filename
+  }
   const linkToFile = await notifyClient.prepareUpload(
     file,
-    {
-      confirmEmailBeforeDownload: true,
-      retentionPeriod: `${config.retentionPeriodInWeeks} weeks`
-    }
+    fileOptions
   )
   const emailRequest = setupEmailRequest(template, email, linkToFile, personalisation, notifyClient)
-  return retry(emailRequest, 3, 100, true)
+  return retry(emailRequest, RETRIES, RETRY_INTERVAL, true)
     .then(result => {
       return result
     })

app/reporting/complete-report.js

@@ -0,0 +1,10 @@
+const db = require('../data')
+
+const completeReport = async (reportId, transaction) => {
+  await db.report.update(
+    { sent: new Date() },
+    { where: { reportId } },
+    { transaction })
+}
+
+module.exports = completeReport

app/reporting/create-report.js

@@ -0,0 +1,13 @@
+const db = require('../data')
+
+const createReport = async (schemeName, lastDeliveryId, reportStartDate, reportEndDate, requested, transaction) => {
+  return db.report.create({
+    lastDeliveryId,
+    schemeName,
+    reportStartDate,
+    reportEndDate,
+    requested
+  }, { transaction })
+}
+
+module.exports = createReport

app/reporting/get-deliveries-for-report.js

@@ -0,0 +1,27 @@
+const db = require('../data')
+const QueryStream = require('pg-query-stream')
+
+const getDeliveriesForReport = async (schemeName, start, end) => {
+  console.log('get deliveries', {
+    schemeName,
+    start,
+    end
+  })
+
+  const query = `
+    SELECT d.*, s.*, f.*
+    FROM deliveries d
+    INNER JOIN statements s ON d."statementId" = s."statementId"
+    LEFT JOIN failures f ON d."deliveryId" = f."deliveryId"
+    WHERE s."schemeName" = $1 AND d.requested BETWEEN $2 AND $3
+    ORDER BY d."deliveryId" ASC, d."method"
+  `
+
+  const client = await db.sequelize.connectionManager.getConnection()
+  const stream = new QueryStream(query, [schemeName, start, end])
+  const queryStream = client.query(stream)
+
+  return queryStream
+}
+
+module.exports = getDeliveriesForReport

app/reporting/get-todays-report.js

@@ -0,0 +1,23 @@
+const db = require('../data')
+const hour = 23
+const minute = 59
+const second = 59
+const millisecond = 999
+
+const getTodaysReport = async (schemeName) => {
+  const today = new Date()
+  const startOfDay = new Date(today.setHours(0, 0, 0, 0))
+  const endOfDay = new Date(today.setHours(hour, minute, second, millisecond))
+
+  return db.report.findAll({
+    where: {
+      schemeName,
+      sent: {
+        [db.Op.between]: [startOfDay, endOfDay],
+        [db.Op.ne]: null
+      }
+    }
+  })
+}
+
+module.exports = getTodaysReport

app/reporting/index.js

@@ -0,0 +1,67 @@
+const config = require('../config')
+const getTodaysReport = require('./get-todays-report')
+const { sendReport } = require('./send-report')
+const moment = require('moment')
+
+const startSchemeReport = async (schemeName, startDate, endDate) => {
+  console.log('[REPORTING] Starting report for scheme: ', schemeName)
+  const existingReport = await getTodaysReport(schemeName)
+  if (!existingReport?.length) {
+    await sendReport(schemeName, startDate, endDate)
+  } else {
+    console.log('[REPORTING] A report has already run today for scheme: ', schemeName)
+  }
+}
+
+const isToday = (date) => {
+  return moment(date).isSame(moment(), 'day')
+}
+
+const getRunDate = (schedule) => {
+  const { intervalType, dayOfMonth, dayOfYear, monthOfYear, hour, minute, second } = schedule
+  const baseDate = moment().hour(hour || 0).minute(minute || 0).second(second || 0).startOf('day')
+
+  if (intervalType === 'months') {
+    return baseDate.date(dayOfMonth || 1)
+  } else if (intervalType === 'years') {
+    return baseDate.month((monthOfYear || 1) - 1).date(dayOfYear || 1)
+  } else {
+    return baseDate
+  }
+}
+
+const processScheme = async (scheme) => {
+  const { schemeName, schedule, dateRange } = scheme
+  const runDate = getRunDate(schedule)
+
+  if (isToday(runDate)) {
+    console.log('[REPORTING] A report is due to run today for scheme: ', schemeName)
+    const startDate = moment().subtract(dateRange.durationNumber, dateRange.durationType).startOf('day').toDate()
+    const endDate = moment().endOf('day').toDate()
+    await startSchemeReport(schemeName, startDate, endDate)
+  } else {
+    console.log('[REPORTING] No report is due to run today for scheme: ', schemeName)
+  }
+}
+
+const start = async () => {
+  try {
+    console.log('[REPORTING] Starting reporting')
+    const schemes = config.reportConfig.schemes
+    for (const scheme of schemes) {
+      try {
+        await processScheme(scheme)
+      } catch (error) {
+        console.error('Error processing scheme:', scheme.schemeName, error)
+      }
+    }
+  } catch (err) {
+    console.error(err)
+  } finally {
+    setTimeout(start, config.reportingCheckInterval)
+  }
+}
+
+module.exports = {
+  start
+}