fix: catch uncaught exceptions & gc handles request aborts (#102)

* fix: catch uncaught exceptions & gc handles request aborts

* feat: allow configurable recovery errors

* chore: remove duplicate gc call

src/constants.ts

@@ -55,3 +55,28 @@ export const USE_DELEGATED_ROUTING = process.env.USE_DELEGATED_ROUTING !== 'fals
  * If not set, we will default delegated routing to `https://delegated-ipfs.dev`
  */
 export const DELEGATED_ROUTING_V1_HOST = process.env.DELEGATED_ROUTING_V1_HOST ?? 'https://delegated-ipfs.dev'
+
+/**
+ * You can set `RECOVERABLE_ERRORS` to a comma delimited list of errors to recover from.
+ * If you want to recover from all errors, set `RECOVERABLE_ERRORS` to 'all'.
+ * If you want to recover from no errors, set `RECOVERABLE_ERRORS` to ''.
+ */
+export const RECOVERABLE_ERRORS = (() => {
+  if (process.env.RECOVERABLE_ERRORS === 'all') {
+    return 'all'
+  }
+  if (process.env.RECOVERABLE_ERRORS === '') {
+    return ''
+  }
+  return process.env.RECOVERABLE_ERRORS?.split(',') ?? 'all'
+})()
+
+export const ALLOW_UNHANDLED_ERROR_RECOVERY = (() => {
+  if (RECOVERABLE_ERRORS === 'all') {
+    return true
+  }
+  if (RECOVERABLE_ERRORS === '') {
+    return false
+  }
+  return RECOVERABLE_ERRORS.length > 0
+})()

src/get-custom-helia.ts

@@ -24,12 +24,12 @@ export async function getCustomHelia (): Promise<Helia> {
   }
 
   let blockstore: HeliaInit['blockstore'] | undefined
-  if (FILE_BLOCKSTORE_PATH != null) {
+  if (FILE_BLOCKSTORE_PATH != null && FILE_BLOCKSTORE_PATH !== '') {
     blockstore = new LevelBlockstore(FILE_BLOCKSTORE_PATH)
   }
 
   let datastore: HeliaInit['datastore'] | undefined
-  if (FILE_DATASTORE_PATH != null) {
+  if (FILE_DATASTORE_PATH != null && FILE_DATASTORE_PATH !== '') {
     datastore = new LevelDatastore(FILE_DATASTORE_PATH)
   }
 

src/helia-server.ts

@@ -242,13 +242,7 @@ export class HeliaServer {
     }
   }
 
-  /**
-   * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia.
-   */
-  async fetch ({ request, reply }: RouteHandler): Promise<void> {
-    const url = this.#getFullUrlFromFastifyRequest(request)
-    this.log('fetching url "%s" with @helia/verified-fetch', url)
-
+  #getRequestAwareSignal (request: FastifyRequest, url = this.#getFullUrlFromFastifyRequest(request), timeout?: number): AbortSignal {
     const opController = new AbortController()
     setMaxListeners(Infinity, opController.signal)
     const cleanupFn = (): void => {
@@ -272,8 +266,26 @@ export class HeliaServer {
      */
     request.raw.on('close', cleanupFn)
 
+    if (timeout != null) {
+      setTimeout(() => {
+        this.log.trace('request timed out for url "%s"', url)
+        opController.abort()
+      }, timeout)
+    }
+    return opController.signal
+  }
+
+  /**
+   * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia.
+   */
+  async fetch ({ request, reply }: RouteHandler): Promise<void> {
+    const url = this.#getFullUrlFromFastifyRequest(request)
+    this.log('fetching url "%s" with @helia/verified-fetch', url)
+
+    const signal = this.#getRequestAwareSignal(request, url)
+
     await this.isReady
-    const resp = await this.heliaFetch(url, { signal: opController.signal, redirect: 'manual' })
+    const resp = await this.heliaFetch(url, { signal, redirect: 'manual' })
     await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply)
   }
 
@@ -319,10 +331,11 @@ export class HeliaServer {
   /**
    * GC the node
    */
-  async gc ({ reply }: RouteHandler): Promise<void> {
+  async gc ({ reply, request }: RouteHandler): Promise<void> {
     await this.isReady
     this.log('running `gc` on Helia node')
-    await this.heliaNode?.gc({ signal: AbortSignal.timeout(20000) })
+    const signal = this.#getRequestAwareSignal(request, undefined, 20000)
+    await this.heliaNode?.gc({ signal })
     await reply.code(200).send('OK')
   }
 

src/index.ts

@@ -68,6 +68,7 @@
  * | `ECHO_HEADERS`              | A debug flag to indicate whether you want to output request and response headers                                                               | `false`                                                                                                                 |
  * | `USE_DELEGATED_ROUTING`     | Whether to use the delegated routing v1 API                                                                                                    | `true`                                                                                                                  |
  * | `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1                                                                                                       | `https://delegated-ipfs.dev`                                                                                            |
+ * | `RECOVERABLE_ERRORS`        | A comma delimited list of errors to recover from. These errors are checked in `uncaughtException` and `unhandledRejection` callbacks           | `all`                                                                                            |
  *
  * <!--
  * TODO: currently broken when used in docker, but they work when running locally (you can cache datastore and blockstore locally to speed things up if you want)
@@ -159,7 +160,7 @@ import compress from '@fastify/compress'
 import cors from '@fastify/cors'
 import Fastify from 'fastify'
 import metricsPlugin from 'fastify-metrics'
-import { HOST, PORT, METRICS, ECHO_HEADERS, FASTIFY_DEBUG } from './constants.js'
+import { HOST, PORT, METRICS, ECHO_HEADERS, FASTIFY_DEBUG, RECOVERABLE_ERRORS, ALLOW_UNHANDLED_ERROR_RECOVERY } from './constants.js'
 import { HeliaServer, type RouteEntry } from './helia-server.js'
 import { logger } from './logger.js'
 
@@ -249,7 +250,7 @@ const stopWebServer = async (): Promise<void> => {
 }
 
 let shutdownRequested = false
-async function closeGracefully (signal: number): Promise<void> {
+async function closeGracefully (signal: number | string): Promise<void> {
   log(`Received signal to terminate: ${signal}`)
   if (shutdownRequested) {
     log('closeGracefully: shutdown already requested, exiting callback.')
@@ -268,3 +269,15 @@ async function closeGracefully (signal: number): Promise<void> {
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
   process.once(signal, closeGracefully)
 })
+
+const uncaughtHandler = (error: any): void => {
+  log.error('Uncaught Exception:', error)
+  if (ALLOW_UNHANDLED_ERROR_RECOVERY && (RECOVERABLE_ERRORS === 'all' || RECOVERABLE_ERRORS.includes(error?.code) || RECOVERABLE_ERRORS.includes(error?.name))) {
+    log.trace('Ignoring error')
+    return
+  }
+  void closeGracefully('SIGTERM')
+}
+
+process.on('uncaughtException', uncaughtHandler)
+process.on('unhandledRejection', uncaughtHandler)