Integration tests in docker

This PR widens test coverage for `integration` tests, adding running
application in container (which proves for PRs that Docker environment
in actually correct), and by extension, uses webhooks as tip trigger
(thus, covering GitHub interaction)

.dockerignore

@@ -10,3 +10,5 @@
 **/dist
 **/*.pem
 Dockerfile
+**/*.e2e.ts
+**/*.integration.ts

.github/workflows/test-integration.yml

@@ -15,7 +15,11 @@ jobs:
         with:
           node-version: 22.5.1
       - run: yarn --immutable
+      - run: yarn build:docker
       - run: yarn test:integration --verbose
+      - name: Debug application container logs
+        if: failure()
+        run: cat integration_tests/containter_logs/application.log
       - name: Debug rococo container logs
         if: failure()
         run: cat integration_tests/containter_logs/rococo.log

.papi/polkadot-api-integration.json

@@ -0,0 +1,44 @@
+{
+  "version": 0,
+  "descriptorPath": ".papi/descriptors",
+  "entries": {
+    "localpolkadot": {
+      "wsUrl": "ws://127.0.0.1:9900",
+      "chain": "polkadot",
+      "metadata": ".papi/metadata/polkadot.scale"
+    },
+    "localkusama": {
+      "wsUrl": "ws://127.0.0.1:9901",
+      "chain": "ksmcc3",
+      "metadata": ".papi/metadata/kusama.scale"
+    },
+    "localwestend": {
+      "wsUrl": "ws://localwestend:9945",
+      "metadata": ".papi/metadata/localwestend.scale"
+    },
+    "localrococo": {
+      "wsUrl": "ws://localrococo:9945",
+      "metadata": ".papi/metadata/localrococo.scale"
+    },
+    "polkadot": {
+      "wsUrl": "wss://rpc.polkadot.io",
+      "chain": "polkadot",
+      "metadata": ".papi/metadata/polkadot.scale"
+    },
+    "kusama": {
+      "wsUrl": "wss://kusama-rpc.polkadot.io",
+      "chain": "ksmcc3",
+      "metadata": ".papi/metadata/kusama.scale"
+    },
+    "westend": {
+      "wsUrl": "wss://westend-rpc.polkadot.io",
+      "chain": "westend2",
+      "metadata": ".papi/metadata/westend.scale"
+    },
+    "rococo": {
+      "wsUrl": "wss://rococo-rpc.polkadot.io",
+      "chain": "rococo_v2_2",
+      "metadata": ".papi/metadata/rococo.scale"
+    }
+  }
+}

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22.5.1-alpine as builder
+FROM node:22.5.1-alpine AS builder
 
 WORKDIR /usr/src/app
 
@@ -32,8 +32,8 @@ WORKDIR /usr/src/app
 COPY --from=builder /usr/src/app/package.json ./
 COPY --from=builder /usr/src/app/dist ./dist/
 COPY --from=builder /usr/src/app/node_modules ./node_modules/
-COPY --from=builder /usr/src/app/.papi ./papi/
+COPY --from=builder /usr/src/app/.papi ./.papi/
 
 ENV NODE_ENV="production"
 
-CMD [ "node", "dist/bot.js" ]
+CMD [ "node", "--enable-source-maps", "dist/src/bot.js" ]

package.json

@@ -11,19 +11,22 @@
     "github",
     "probot-app"
   ],
+  "main": "dist/src/bot.js",
   "scripts": {
-    "typecheck": "tsc --noEmit",
-    "lint": "npx eslint --quiet '{*,**/*}.{js,ts}' && npx prettier --check '{*,**/*}.json' && yarn typecheck",
+    "build": "rimraf dist; tsc",
+    "build:docker": "docker build -t substrate-tip-bot .",
+    "fix": "yarn fix:eslint '{*,**/*}.{js,ts}' && yarn fix:prettier '{*,**/*}.json'",
     "fix:eslint": "npx eslint --fix",
     "fix:prettier": "npx prettier --write",
-    "fix": "yarn fix:eslint '{*,**/*}.{js,ts}' && yarn fix:prettier '{*,**/*}.json'",
-    "start": "concurrently \"tsc -w\" \"node --watch dist/bot.js\"",
-    "build": "rimraf dist; tsc",
+    "lint": "npx eslint --quiet '{*,**/*}.{js,ts}' && npx prettier --check '{*,**/*}.json' && yarn typecheck",
+    "papi": "papi generate",
+    "postinstall": "yarn papi",
+    "dev": "concurrently \"tsc -w\" \"node --watch --enable-source-maps dist/src/bot.js\"",
+    "start": "node --enable-source-maps dist/src/bot.js",
     "test": "jest",
-    "test:integration": "jest -c jest.integration.config.js",
     "test:e2e": "jest -c jest.e2e.config.js",
-    "papi": "papi generate",
-    "postinstall": "yarn papi"
+    "test:integration": "jest -c jest.integration.config.js",
+    "typecheck": "tsc --noEmit"
   },
   "imports": {
     "#src/*": "./src/*.js"
@@ -43,6 +46,7 @@
   },
   "devDependencies": {
     "@eng-automation/js-style": "^3.1.0",
+    "@eng-automation/testing": "^1.5.0",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.14.2",
     "dotenv": "^16.0.1",

src/bot-handle-comment.ts

@@ -25,19 +25,34 @@ export const handleIssueCommentCreated = async (state: State, event: IssueCommen
     return;
   }
 
+  // The "Unsafe assignment of an error typed value" error here goes deep into octokit types, that are full of `any`s
+  // I wasn't able to get around it
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+  const appOctokitInstance: GitHubInstance = await github.getInstance({
+    authType: "app",
+    appId: envVar("GITHUB_APP_ID"),
+    privateKey: envVar("GITHUB_PRIVATE_KEY"),
+    ...(process.env.GITHUB_BASE_URL && { apiEndpoint: envVar("GITHUB_BASE_URL") }),
+  });
+
   const tipRequester = event.comment.user.login;
   const installationId = (
-    await github.getRepoInstallation({ owner: event.repository.owner.login, repo: event.repository.name })
+    await github.getRepoInstallation(
+      {
+        owner: event.repository.owner.login,
+        repo: event.repository.name,
+      },
+      { octokitInstance: appOctokitInstance },
+    )
   ).id;
 
-  // The "Unsafe assignment of an error typed value" error here goes deep into octokit types, that are full of `any`s
-  // I wasn't able to get around it
   // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
   const octokitInstance: GitHubInstance = await github.getInstance({
     authType: "installation",
     appId: envVar("GITHUB_APP_ID"),
     installationId: String(installationId),
     privateKey: envVar("GITHUB_PRIVATE_KEY"),
+    ...(process.env.GITHUB_BASE_URL && { apiEndpoint: envVar("GITHUB_BASE_URL") }),
   });
 
   const respondParams = {
@@ -88,10 +103,10 @@ export const handleIssueCommentCreated = async (state: State, event: IssueCommen
       });
       await githubComment(`The referendum has appeared on [Polkassembly](${url}).`);
     } catch (e) {
-      state.bot.log.error("Failed to update the Polkasssembly metadata", {
-        referendumId: result.tipResult.referendumNumber,
-        tipRequest: JSON.stringify(result.tipRequest),
-      });
+      state.bot.log.error(
+        `Failed to update the Polkasssembly metadata; referendumId: ${result.tipResult.referendumNumber}`,
+        result.tipRequest,
+      );
       state.bot.log.error(e.message);
       await matrixNotifyOnFailure(state.matrix, event, {
         tagMaintainers: true,

src/bot-initialize.ts

@@ -2,6 +2,7 @@ import { envVar } from "@eng-automation/js";
 import { sr25519CreateDerive } from "@polkadot-labs/hdkd";
 import { entropyToMiniSecret, mnemonicToEntropy, parseSuri, ss58Address } from "@polkadot-labs/hdkd-helpers";
 import { createClient } from "matrix-js-sdk";
+import * as process from "node:process";
 import { PolkadotSigner } from "polkadot-api";
 import { getPolkadotSigner } from "polkadot-api/signer";
 import { ApplicationFunction, Context, Probot } from "probot";
@@ -41,19 +42,33 @@ export const botInitialize: AsyncApplicationFunction = async (bot: Probot, { get
     allowedGitHubOrg: envVar("APPROVERS_GH_ORG"),
     allowedGitHubTeam: envVar("APPROVERS_GH_TEAM"),
     botTipAccount,
-    polkassembly: new Polkassembly(
-      envVar("POLKASSEMBLY_ENDPOINT"),
-      { type: "polkadot", keyringPair: botTipAccount },
-      bot.log,
-    ),
-    matrix: {
-      client: createClient({
-        accessToken: envVar("MATRIX_ACCESS_TOKEN"),
-        baseUrl: envVar("MATRIX_SERVER_URL"),
-        localTimeoutMs: 10000,
-      }),
-      roomId: envVar("MATRIX_ROOM_ID"),
-    },
+    polkassembly: (() => {
+      if (!process.env.POLKASSEMBLY_ENDPOINT) {
+        // convenient for local development, and tests
+        bot.log.warn("POLKASSEMBLY_ENDPOINT is not set; polkassembly integration is disabled");
+        return undefined;
+      }
+      return new Polkassembly(
+        envVar("POLKASSEMBLY_ENDPOINT"),
+        { type: "polkadot", keyringPair: botTipAccount },
+        bot.log,
+      );
+    })(),
+    matrix: (() => {
+      if (!process.env.MATRIX_ACCESS_TOKEN) {
+        // convenient for local development, and tests
+        bot.log.warn("MATRIX_ACCESS_TOKEN is not set; matrix notifications are disabled");
+        return undefined;
+      }
+      return {
+        client: createClient({
+          accessToken: envVar("MATRIX_ACCESS_TOKEN"),
+          baseUrl: envVar("MATRIX_SERVER_URL"),
+          localTimeoutMs: 10000,
+        }),
+        roomId: envVar("MATRIX_ROOM_ID"),
+      };
+    })(),
   };
 
   bot.log.info("Tip bot was loaded!");

src/chain-config.ts

@@ -12,7 +12,12 @@ import { readFileSync } from "fs";
 
 import { ChainConfig, TipNetwork } from "./types";
 
-export const papiConfig = JSON.parse(readFileSync(".papi/polkadot-api.json", "utf-8")) as {
+export const papiConfig = JSON.parse(
+  readFileSync(
+    process.env.INTEGRATION_TEST ? ".papi/polkadot-api-integration.json" : ".papi/polkadot-api.json",
+    "utf-8",
+  ),
+) as {
   entries: {
     [p in TipNetwork]: {
       wsUrl: string;