feat(bff): add health checks (#1099)

packages/bff/src/azure/HealthChecks.ts

@@ -0,0 +1,144 @@
+import { setTimeout } from 'node:timers/promises';
+import { logger } from '@digdir/dialogporten-node-logger';
+import axios from 'axios';
+import type { FastifyPluginAsync } from 'fastify';
+import fp from 'fastify-plugin';
+import config from '../config.ts';
+import { dataSource } from '../db.ts';
+import redisClient from '../redisClient.ts';
+
+/**
+ * Health Check System
+ *
+ * - Provides a '/api/health' endpoint that:
+ *   1. Runs all health checks concurrently with timeouts
+ *   2. Calculates overall status ('ok', 'error', 'degraded')
+ *   3. Measures total latency
+ *   4. Returns JSON with overall status, individual check results, and latency
+ * - Handles errors and returns 503 status if checks fail
+ */
+
+interface Props {
+  version: string;
+}
+
+interface HealthCheckResult {
+  status: 'ok' | 'error' | 'timeout';
+  detail?: string;
+  latency: number;
+}
+
+interface HealthChecksResponse {
+  status: 'ok' | 'error' | 'degraded';
+  healthChecks: Record<string, HealthCheckResult>;
+  latency: number;
+}
+
+interface HealthCheck {
+  name: string;
+  checkFn: () => Promise<{ status: 'ok' | 'error'; detail?: string }>;
+}
+
+const HEALTH_CHECK_TIMEOUT = 60000;
+
+const healthCheckList: HealthCheck[] = [
+  {
+    name: 'postgresql',
+    checkFn: async () => {
+      try {
+        if (!dataSource!.isInitialized) {
+          return { status: 'error', detail: 'PostgreSQL not connected' };
+        }
+        await dataSource!.query('SELECT 1');
+        return { status: 'ok' };
+      } catch (error) {
+        logger.error(error, 'PostgreSQL health check failed');
+        return { status: 'error', detail: 'PostgreSQL connection failed' };
+      }
+    },
+  },
+  {
+    name: 'redis',
+    checkFn: async () => {
+      try {
+        await redisClient.ping();
+        return { status: 'ok' };
+      } catch (error) {
+        logger.error(error, 'Redis health check failed');
+        return { status: 'error', detail: 'Redis connection failed' };
+      }
+    },
+  },
+  {
+    name: 'oidc',
+    checkFn: async () => {
+      try {
+        // todo: change to a URL we can use to check secret id and secret key
+        await axios.get(`https://${config.oidc_url}/.well-known/openid-configuration`);
+        return { status: 'ok' };
+      } catch (error) {
+        logger.error(error, 'OIDC health check failed');
+        return { status: 'error', detail: 'OIDC URL unreachable' };
+      }
+    },
+  },
+  // ... add more health checks here ...
+];
+
+const performCheck = async (
+  name: string,
+  checkFn: () => Promise<{ status: 'ok' | 'error'; detail?: string }>,
+): Promise<HealthCheckResult> => {
+  const start = Date.now();
+  return Promise.race([
+    checkFn().then(
+      (result): HealthCheckResult => ({
+        ...result,
+        latency: Date.now() - start,
+      }),
+    ),
+    setTimeout(HEALTH_CHECK_TIMEOUT).then(
+      (): HealthCheckResult => ({
+        status: 'timeout',
+        detail: `${name} timed out`,
+        latency: HEALTH_CHECK_TIMEOUT,
+      }),
+    ),
+  ]);
+};
+
+const plugin: FastifyPluginAsync<Props> = async (fastify) => {
+  fastify.get('/api/health', async (req, reply) => {
+    const overallStart = Date.now();
+
+    try {
+      const healthChecks: Record<string, HealthCheckResult> = await Promise.all(
+        healthCheckList.map(async ({ name, checkFn }) => {
+          const result = await performCheck(name, checkFn);
+          return [name, result] as const;
+        }),
+      ).then((results) => Object.fromEntries(results));
+
+      const overallStatus: HealthChecksResponse['status'] = Object.values(healthChecks).every(
+        (check) => check.status === 'ok',
+      )
+        ? 'ok'
+        : Object.values(healthChecks).some((check) => check.status === 'error' || check.status === 'timeout')
+          ? 'error'
+          : 'degraded';
+
+      const latency = Date.now() - overallStart;
+
+      reply.status(200).send({ status: overallStatus, healthChecks, latency });
+    } catch (error) {
+      const errorMsg = 'Health check endpoint failed';
+      logger.error(error, errorMsg);
+      reply.status(503).send({ error: errorMsg });
+    }
+  });
+};
+
+export default fp(plugin, {
+  fastify: '4.x',
+  name: 'azure-healthprobs',
+});

packages/bff/src/azure/HealthProbes.ts

@@ -8,16 +8,16 @@ interface Props {
 
 const plugin: FastifyPluginAsync<Props> = async (fastify, options) => {
   const { version } = options;
-  const startTimeStamp = new Date();
-  const secondsAfterStart = (new Date().getTime() - startTimeStamp.getTime()) / 1000;
+  const startTimeStamp = Date.now();
+  const secondsAfterStart = (Date.now() - startTimeStamp) / 1000;
 
   logger.info(`${version} starting /api/readiness probe after ${secondsAfterStart} seconds`);
-  fastify.get('/api/readiness', (req: FastifyRequest, reply: FastifyReply) => {
+  fastify.get('/api/readiness', async (req, reply) => {
     reply.status(200).send();
   });
 
   logger.info(`${version} starting /api/liveness probe after ${secondsAfterStart} seconds`);
-  fastify.get('/api/liveness', (req: FastifyRequest, reply: FastifyReply) => {
+  fastify.get('/api/liveness', async (req, reply) => {
     reply.status(200).send();
   });
 };

packages/bff/src/db.ts

@@ -4,9 +4,13 @@ import { ProfileTable, SavedSearch } from './entities.ts';
 export let ProfileRepository: Repository<ProfileTable> | undefined = undefined;
 export let SavedSearchRepository: Repository<SavedSearch> | undefined = undefined;
 
+export let dataSource: DataSource | undefined = undefined;
+
 export const connectToDB = async () => {
+  // we don't want to import data-source.ts in the initial import, because it loads config.ts
+  // which is not needed for example for generating bff-types
   const { connectionOptions } = await import('./data-source.ts');
-  const dataSource = await new DataSource(connectionOptions).initialize();
+  dataSource = await new DataSource(connectionOptions).initialize();
 
   ProfileRepository = dataSource.getRepository(ProfileTable);
   SavedSearchRepository = dataSource.getRepository(SavedSearch);

packages/bff/src/redisClient.ts

@@ -0,0 +1,29 @@
+import { logger } from '@digdir/dialogporten-node-logger';
+import Redis from 'ioredis';
+import config from './config.ts';
+
+const redisClient = new Redis.default(config.redisConnectionString, {
+  enableAutoPipelining: true,
+});
+
+redisClient.on('error', (err) => {
+  logger.error(err, 'Redis Client Error');
+});
+
+redisClient.on('connect', () => {
+  logger.info('Redis Client Connected');
+});
+
+redisClient.on('ready', () => {
+  logger.info('Redis Client Ready');
+});
+
+redisClient.on('close', () => {
+  logger.info('Redis Client Closed');
+});
+
+redisClient.on('reconnecting', () => {
+  logger.info('Redis Client Reconnecting');
+});
+
+export default redisClient;

packages/bff/src/server.ts

@@ -7,13 +7,14 @@ import type { FastifySessionOptions } from '@fastify/session';
 import RedisStore from 'connect-redis';
 import Fastify from 'fastify';
 import fastifyGraphiql from 'fastify-graphiql';
-import { default as Redis } from 'ioredis';
 import { oidc, userApi, verifyToken } from './auth/index.ts';
+import healthChecks from './azure/HealthChecks.ts';
 import healthProbes from './azure/HealthProbes.ts';
 import config from './config.ts';
 import { connectToDB } from './db.ts';
 import graphqlApi from './graphql/api.ts';
 import graphqlStream from './graphql/subscription.ts';
+import redisClient from './redisClient.ts';
 
 const { version, port, host, oidc_url, hostname, client_id, client_secret, redisConnectionString } = config;
 
@@ -23,7 +24,7 @@ const startServer = async (): Promise<void> => {
     ignoreDuplicateSlashes: true,
   });
 
-  await connectToDB();
+  const { dataSource } = await connectToDB();
   /* CORS configuration for local env, needs to be applied before routes are defined */
   const corsOptions = {
     origin: ['http://app.localhost', 'http://localhost:3000'],
@@ -50,9 +51,7 @@ const startServer = async (): Promise<void> => {
 
   if (redisConnectionString) {
     const store = new RedisStore({
-      client: new Redis.default(redisConnectionString, {
-        enableAutoPipelining: true,
-      }),
+      client: redisClient,
     });
 
     logger.info('Setting up fastify-session with a Redis store');
@@ -64,6 +63,7 @@ const startServer = async (): Promise<void> => {
 
   server.register(verifyToken);
   server.register(healthProbes, { version });
+  server.register(healthChecks, { version });
   server.register(oidc, {
     oidc_url,
     hostname,
@@ -85,6 +85,36 @@ const startServer = async (): Promise<void> => {
     }
     logger.info(`Server ${version} is running on ${address}`);
   });
+
+  // Graceful Shutdown
+  const gracefulShutdown = async () => {
+    try {
+      logger.info('Initiating graceful shutdown...');
+
+      // Stop accepting new connections
+      await server.close();
+      logger.info('Closed Fastify server.');
+
+      // Disconnect Redis
+      await redisClient.quit();
+      logger.info('Disconnected Redis client.');
+
+      // Disconnect Database
+      if (dataSource?.isInitialized) {
+        await dataSource.destroy();
+        logger.info('Disconnected from PostgreSQL.');
+      }
+
+      process.exit(0);
+    } catch (err) {
+      logger.error(err, 'Error during graceful shutdown');
+      process.exit(1);
+    }
+  };
+
+  // Handle termination signals
+  process.on('SIGINT', gracefulShutdown);
+  process.on('SIGTERM', gracefulShutdown);
 };
 
 export default startServer;