feat: Zinnia Runtime and Peer Checker module (#92)

* feat: install Zinnia runtime Add another post-install step to download `zinniad` for running Zinnia modules. Signed-off-by: Miroslav Bajtoš <[email protected]> * feat: install mod-peer-checker Signed-off-by: Miroslav Bajtoš <[email protected]> * feat: run peer checker module via Zinnia Signed-off-by: Miroslav Bajtoš <[email protected]> * fix formatting Signed-off-by: Miroslav Bajtoš <[email protected]> * fixup! windows zinniad extension Signed-off-by: Miroslav Bajtoš <[email protected]> * test: fix never ending test Signed-off-by: Miroslav Bajtoš <[email protected]> * upgrade zinniad to v0.7.0 * Update commands/station.js Co-authored-by: Julian Gruber <[email protected]> * update tests Signed-off-by: Miroslav Bajtoš <[email protected]> * ci: execute Docker image and test activity Signed-off-by: Miroslav Bajtoš <[email protected]> * fixup! fix failing tests Signed-off-by: Miroslav Bajtoš <[email protected]> * fixup! docker CI workflow Signed-off-by: Miroslav Bajtoš <[email protected]> * troubleshoot docker CI not running Signed-off-by: Miroslav Bajtoš <[email protected]> * enable Docker workflow for all pushes * Revert "troubleshoot docker CI not running" This reverts commit 11a56e7. * more docker tweaks Signed-off-by: Miroslav Bajtoš <[email protected]> * feat: switch base Docker image to full `node:18` We cannot use `node:18-alpine` because Zinnia does not support that yet. I tried to use `node:18-slim`, but Saturn L2 was not able to start in that system. The Node.js project recommends to use `node:18`, so we should be fine. See https://github.com/nodejs/docker-node#image-variants Signed-off-by: Miroslav Bajtoš <[email protected]> * fixup! standard coding style Signed-off-by: Miroslav Bajtoš <[email protected]> * fix zip assets on windows * fixup! remove github from zinnia module repo name Signed-off-by: Miroslav Bajtoš <[email protected]> * bump zinnia to 0.8.0 Signed-off-by: Miroslav Bajtoš <[email protected]> * rename Module Runtime to Zinnia Signed-off-by: Miroslav Bajtoš <[email protected]> * move mocha config to config file and add --exit Signed-off-by: Miroslav Bajtoš <[email protected]> * make test/station easier to troubleshoot Signed-off-by: Miroslav Bajtoš <[email protected]> * fix expected Saturn messages Signed-off-by: Miroslav Bajtoš <[email protected]> --------- Signed-off-by: Miroslav Bajtoš <[email protected]> Co-authored-by: Julian Gruber <[email protected]>
CheckerNetwork · May 4, 2023 · 53d4912 · 53d4912
1 parent a2a27f2
commit 53d4912
Show file tree

Hide file tree

Showing 12 changed files with 262 additions and 38 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -93,3 +93,6 @@ jobs:
 
       - name: Check | Saturn started
         run: docker exec station bin/station.js activity | grep "Saturn module started"
+
+      - name: Check | Zinnia started
+        run: docker exec station bin/station.js activity | grep "Zinnia started"
diff --git a/.mocharc.yaml b/.mocharc.yaml
@@ -0,0 +1,2 @@
+timeout: 15000
+exit: true
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-alpine
+FROM node:18
 LABEL org.opencontainers.image.source https://github.com/filecoin-station/core
 USER node
 WORKDIR /usr/src/app

diff --git a/bin/station.js b/bin/station.js
@@ -21,6 +21,7 @@ Sentry.init({
 
 const core = new Core(getDefaultRootDirs())
 const modules = [
+  'zinnia',
   'saturn-L2-node',
   'bacalhau'
 ]

diff --git a/commands/station.js b/commands/station.js
@@ -1,5 +1,6 @@
 import { join } from 'node:path'
 import * as saturnNode from '../lib/saturn-node.js'
+import * as zinniaRuntime from '../lib/zinnia.js'
 import { formatActivityObject } from '../lib/activity.js'
 import lockfile from 'proper-lockfile'
 import { maybeCreateFile } from '../lib/util.js'
@@ -36,6 +37,16 @@ export const station = async ({ core, json, experimental }) => {
       logStream: core.logs.createWriteStream(
         join(core.paths.moduleLogs, 'saturn-L2-node.log')
       )
+    }),
+    zinniaRuntime.start({
+      FIL_WALLET_ADDRESS,
+      STATE_ROOT: join(core.paths.moduleState, 'zinnia'),
+      CACHE_ROOT: join(core.paths.moduleCache, 'zinnia'),
+      metricsStream: await core.metrics.createWriteStream('zinnia'),
+      activityStream: core.activity.createWriteStream('Zinnia'),
+      logStream: core.logs.createWriteStream(
+        join(core.paths.moduleLogs, 'zinnia.log')
+      )
     })
   ]
 

diff --git a/lib/modules.js b/lib/modules.js
@@ -21,10 +21,14 @@ export const getBinaryModuleExecutable = ({
   return join(
     moduleBinaries,
     module,
-    `${executable}${os.platform() === 'win32' ? '.exe' : ''}`
+    getExecutableFileName(executable)
   )
 }
 
+const getExecutableFileName = executable => {
+  return `${executable}${os.platform() === 'win32' ? '.exe' : ''}`
+}
+
 export const installBinaryModule = async ({
   module,
   repo,
@@ -79,8 +83,9 @@ export const installBinaryModule = async ({
           const [entry] =
             /** @type {[UnzipStreamEntry]} */
             (await once(parser, 'entry'))
-          if (entry.path === executable) {
-            const outPath = join(moduleBinaries, module, executable)
+          const executableFileName = getExecutableFileName(executable)
+          if (entry.path === executableFileName) {
+            const outPath = join(moduleBinaries, module, executableFileName)
             await pipeline(entry, createWriteStream(outPath))
             await chmod(outPath, 0o755)
             return
@@ -92,3 +97,35 @@ export const installBinaryModule = async ({
   }
   console.log(`[${module}] ✓ ${outFile}`)
 }
+
+export async function downloadSourceFiles ({ module, repo, distTag }) {
+  await mkdir(moduleBinaries, { recursive: true })
+  const outDir = join(moduleBinaries, module)
+
+  console.log(`[${module}]  ⇣ downloading source files`)
+
+  const url = `https://github.com/${repo}/archive/refs/tags/${distTag}.tar.gz`
+  const res = await fetch(url, {
+    headers: {
+      ...(authorization ? { authorization } : {})
+    },
+    redirect: 'follow'
+  })
+
+  if (res.status >= 300) {
+    throw new Error(
+      `[${module}] Cannot fetch ${module} archive for tag ${distTag}: ${res.status}\n` +
+      await res.text()
+    )
+  }
+
+  if (!res.body) {
+    throw new Error(
+      `[${module}] Cannot fetch ${module} archive for tag ${distTag}: no response body`
+    )
+  }
+
+  // `{ strip: 1}` tells tar to remove the top-level directory (e.g. `mod-peer-checker-v1.0.0`)
+  await pipeline(res.body, gunzip(), tar.extract(outDir, { strip: 1 }))
+  console.log(`[${module}]  ✓ ${outDir}`)
+}
diff --git a/lib/zinnia.js b/lib/zinnia.js
@@ -0,0 +1,144 @@
+import timers from 'node:timers/promises'
+import { execa } from 'execa'
+import * as Sentry from '@sentry/node'
+import { installBinaryModule, downloadSourceFiles, getBinaryModuleExecutable } from './modules.js'
+import { moduleBinaries } from './paths.js'
+
+const ZINNIA_DIST_TAG = 'v0.8.0'
+const ZINNIA_MODULES = [
+  {
+    module: 'peer-checker',
+    repo: 'filecoin-station/mod-peer-checker',
+    distTag: 'v1.0.0'
+  }
+]
+
+export async function install () {
+  await Promise.all([
+    installBinaryModule({
+      module: 'zinnia',
+      repo: 'filecoin-station/zinnia',
+      distTag: ZINNIA_DIST_TAG,
+      executable: 'zinniad',
+      targets: [
+        { platform: 'darwin', arch: 'arm64', asset: 'zinniad-macos-arm64.zip' },
+        { platform: 'darwin', arch: 'x64', asset: 'zinniad-macos-x64.zip' },
+        { platform: 'linux', arch: 'arm64', asset: 'zinniad-linux-arm64.tar.gz' },
+        { platform: 'linux', arch: 'x64', asset: 'zinniad-linux-x64.tar.gz' },
+        { platform: 'win32', arch: 'x64', asset: 'zinniad-windows-x64.zip' }
+      ]
+    }),
+
+    ...Object.values(ZINNIA_MODULES).map(downloadSourceFiles)
+  ])
+}
+
+async function start ({
+  FIL_WALLET_ADDRESS,
+  STATE_ROOT,
+  CACHE_ROOT,
+  metricsStream,
+  activityStream,
+  logStream
+}) {
+  logStream.write('Starting Zinnia')
+
+  const zinniadExe = getBinaryModuleExecutable({ module: 'zinnia', executable: 'zinniad' })
+  const modules = [
+    // all paths are relative to `moduleBinaries`
+    'peer-checker/peer-checker.js'
+  ]
+  const childProcess = execa(zinniadExe, modules, {
+    cwd: moduleBinaries,
+    env: {
+      FIL_WALLET_ADDRESS,
+      STATE_ROOT,
+      CACHE_ROOT
+    }
+  })
+
+  const readyPromise = new Promise((resolve, reject) => {
+    childProcess.stdout.setEncoding('utf-8')
+    childProcess.stdout.on('data', data => {
+      logStream.write(data)
+      handleEvents({ activityStream, metricsStream }, data)
+    })
+
+    childProcess.stderr.setEncoding('utf-8')
+    childProcess.stderr.on('data', data => {
+      logStream.write(data)
+    })
+
+    childProcess.stdout.once('data', _data => {
+      // This is based on an implicit assumption that zinniad reports an info activity
+      // after it starts
+      resolve()
+    })
+    childProcess.catch(reject)
+  })
+
+  childProcess.on('close', code => {
+    logStream.write(`Zinnia closed all stdio with code ${code ?? '<no code>'}`)
+    childProcess.stderr.removeAllListeners()
+    childProcess.stdout.removeAllListeners()
+    Sentry.captureException('Zinnia exited')
+  })
+
+  childProcess.on('exit', (code, signal) => {
+    const reason = signal ? `via signal ${signal}` : `with code: ${code}`
+    const msg = `Zinnia exited ${reason}`
+    logStream.write(msg)
+    activityStream.write({ type: 'info', message: msg })
+  })
+
+  try {
+    await Promise.race([
+      readyPromise,
+      timers.setTimeout(500)
+    ])
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : '' + err
+    const message = `Cannot start Zinnia: ${errorMsg}`
+    logStream.write(message)
+    activityStream.write({ type: 'error', message })
+  }
+}
+
+function handleEvents ({ activityStream, metricsStream }, text) {
+  text
+    .trimEnd()
+    .split(/\n/g)
+    .forEach(line => {
+      try {
+        const event = JSON.parse(line)
+        switch (event.type) {
+          case 'activity:info':
+            activityStream.write({
+              type: 'info',
+              message: event.message.replace(/Module Runtime/, 'Zinnia'),
+              source: event.module ?? 'Zinnia'
+            })
+            break
+
+          case 'activity:error':
+            activityStream.write({
+              type: 'error',
+              message: event.message.replace(/Module Runtime/, 'Zinnia'),
+              source: event.module ?? 'Zinnia'
+            })
+            break
+
+          case 'jobs-completed':
+            metricsStream.write({ totalJobsCompleted: event.total, totalEarnings: '0' })
+            break
+
+          default:
+            console.error('Ignoring Zinnia event of unknown type:', event)
+        }
+      } catch (err) {
+        console.error('Ignoring malformed Zinnia event:', line)
+      }
+    })
+}
+
+export { start }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -15,9 +15,10 @@
   "scripts": {
     "build": "tsc",
     "format": "prettier --write .",
+    "start": "node ./bin/station.js",
     "test": "npm run build && npm run test:lint && npm run test:unit",
     "test:lint": "prettier --check . && standard",
-    "test:unit": "mocha --timeout 15000",
+    "test:unit": "mocha",
     "version": "npm run build && node ./scripts/version.js",
     "postinstall": "node ./scripts/post-install.js",
     "postpublish": "node ./scripts/post-publish.js",
@@ -29,6 +30,7 @@
     ]
   },
   "devDependencies": {
+    "get-stream": "^6.0.1",
     "mocha": "^10.2.0",
     "np": "^7.6.3",
     "prettier": "^2.8.4",

diff --git a/scripts/post-install.js b/scripts/post-install.js
@@ -2,8 +2,10 @@
 
 import { install as installSaturn } from '../lib/saturn-node.js'
 import { install as installBacalhau } from '../lib/bacalhau.js'
+import { install as installZinnia } from '../lib/zinnia.js'
 
 await Promise.all([
   installSaturn(),
-  installBacalhau()
+  installBacalhau(),
+  installZinnia()
 ])