Fix functions build failure with esm.

Making this work with

  * firebase functions
  * esm modules, esp with "type":"modules" in package.json
  * typescript

is hilariously hard.  Firebase functions do not understand esm modules. This
causes their preprocessor to blow up if you use them. There is a workaround
here:

  firebase/firebase-tools#2994 (comment)

that uses esbuild to bundle all of the code into one huge index.js that
papers over all the preprocessing issues. It also does some magic with
package.json and the npm-run-all package to ensure all of esbuild, tsc,
and firebase run in parallel so that changing a typescript source file
results in an actual behavior change to a running function emulator.

However, this is still not sufficient because esbuild has decided to
bundle everything in devDependencies and dependencies into the index.js.
This will pull in old commonJS packages but attempt to run that
syntax in an ESM environment which causes a cryptic

  "Dynamic require of 'something' is not supported"

This will send you down rabbit holes of trying to fix tsc config because
it looks like you are exporting the wrong module loading type..which
will lead you to try setting tsconfig's module to "nodenext"...which
confusing *requires* you to put .js after all your relative path imports
even if the source file is .ts.  This is confusing as heck.

But the real problem is you need to exclude packages from esbuild:

  evanw/esbuild#3324 (comment)

And now everything will run. Howeever, it means you have to ensure the
deploy picks up your node_modules.... ugh.

Oh...and jest's setup requires you to NOT use verbatimModuleSyntax otherwise
you cannot import typescript types since its config file is a CommonJs
setup and the import syntax is ESM.

common/storage.ts

@@ -1,8 +1,8 @@
 import * as Constants from 'config/constants';
 import * as fs from 'node:fs';
-import * as streamBuffers from 'stream-buffers';
 import { Storage } from '@google-cloud/storage';
 import { Stream } from "stream";
+import { arrayBuffer } from 'node:stream/consumers';
 import { createGzip, createGunzip } from 'zlib';
 import { pipeline } from 'node:stream/promises';
 
@@ -99,8 +99,6 @@ export class CloudStorageAccessor implements StorageAccessor {
   }
 
   async readBytesWithFilter(path: string, filter: any) {
-    const buffer = new streamBuffers.WritableStreamBuffer({ initialSize: 150 * 1024 });
-    await pipeline(this.bucket.file(path).createReadStream(), filter, buffer);
-    return buffer;
+    return await arrayBuffer(this.bucket.file(path).createReadStream().pipe(filter));
   }
 }

functions/jest.config.ts

@@ -3,8 +3,8 @@
  * https://jestjs.io/docs/configuration
  */
 
-const { pathsToModuleNameMapper } = require('ts-jest');
-const { compilerOptions } = require('./tsconfig');
+import { pathsToModuleNameMapper } from 'ts-jest';
+import { compilerOptions } from './tsconfig.json';
 import type { Config } from 'jest';
 
 const config: Config = {

functions/package.json

@@ -3,10 +3,9 @@
   "type": "module",
   "scripts": {
     "lint": "eslint --ext .js,.ts .",
-    "build": "esbuild src/index.ts --bundle --platform=node --outfile=lib/index.js --format=esm --external:./node_modules/*",
-    "build:watch": "tsc --watch",
+    "build": "esbuild src/index.ts --bundle --platform=node --outfile=lib/index.js --format=esm --external:./node_modules/* --packages=external",
     "dev:tsc": "tsc --watch --preserveWatchOutput",
-    "dev:node": "firebase emulators:start --only functions,storage,database",
+    "dev:node": "firebase emulators:start --only functions",
     "dev:esbuild": "npm run build --watch",
     "dev": "run-p dev:*",
     "serve": "npm run build && firebase emulators:start --only functions,storage,database",
@@ -31,7 +30,6 @@
     "lodash.isequal": "^4.5.0",
     "lzma-native": "^8.0.6",
     "sentence-splitter": "^5.0.0",
-    "stream-buffers": "^3.0.2",
     "youtubei.js": "^9.4.0"
   },
   "devDependencies": {
@@ -46,7 +44,7 @@
     "npm-run-all": "^4.1.5",
     "source-map-support": "^0.5.21",
     "ts-jest": "^29.1.4",
-    "typescript": "^4.9.0"
+    "typescript": "^5.5.2"
   },
   "private": true
 }

functions/src/index.ts

@@ -1,11 +1,9 @@
-import "source-map-support/register";
+import 'source-map-support/register.js';
 
-import { speakerinfo } from "./speakerinfo";
-import { metadata, transcript } from "./transcript";
-import { video_queue, vast } from "./video_queue";
+export { speakerinfo } from "./speakerinfo";
+export { metadata, transcript } from "./transcript";
+export { video_queue, vast } from "./video_queue";
 
 import { initializeFirebase } from "./utils/firebase";
 
 initializeFirebase({});
-
-export {speakerinfo, metadata, transcript, video_queue, vast};

functions/tsconfig.json

@@ -7,18 +7,19 @@
     },
     "noImplicitReturns": true,
     "noUnusedLocals": true,
-    "skipLibCheck": true,
-    "incremental": true,
     "esModuleInterop": true,
-    "module": "esnext",
-    "moduleResolution": "node",
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "moduleDetection": "force",
     "resolveJsonModule": true,
     "isolatedModules": true,
     "outDir": "lib",
     "sourceMap": true,
     "strict": false,
     "strictNullChecks": true,
-    "target": "esnext"
+    "target": "esnext",
+    "lib": ["esnext"]
   },
   "compileOnSave": true,
   "include": [

package.json

@@ -62,7 +62,6 @@
     "react-youtube": "^10.1.0",
     "sass": "^1.77.2",
     "sentence-splitter": "^5.0.0",
-    "stream-buffers": "^3.0.3",
     "tailwindcss": "^3.4.3",
     "typescript": "^5.4.5"
   },