feat(explorer): cluster testing

.github/actions/setup-all/action.yml

@@ -6,7 +6,7 @@ inputs:
   node_version:
     description: Node.js version
     required: false
-    default: '20.10.0'
+    default: '22.12.0'
   go_version:
     description: Go version
     required: false

.github/actions/setup-js/action.yml

@@ -6,7 +6,7 @@ inputs:
   node_version:
     description: Node.js version
     required: false
-    default: '20.10.0'
+    default: '22.12.0'
   base_ref:
     description: Determines which base SHA to use ('base' for a specific SHA/branch, 'last_successful' for the last successful workflow)
     required: false

.gitignore

@@ -47,6 +47,7 @@ deploy.log
 
 # Next.js
 .next
+.next-*
 
 # builds
 dist

apps/explorer-e2e/playwright.config.ts

@@ -1,10 +1,9 @@
 import { defineConfig, devices } from '@playwright/test'
 import { nxE2EPreset } from '@nx/playwright/preset'
-
 import { workspaceRoot } from '@nx/devkit'
 
-// For CI, you may want to set BASE_URL to the deployed application.
-const baseURL = process.env['BASE_URL'] || 'http://localhost:3005'
+// // For CI, you may want to set BASE_URL to the deployed application.
+// const baseURL = process.env['BASE_URL'] || 'http://localhost:3005'
 
 /**
  * Read environment variables from file.
@@ -20,13 +19,13 @@ export default defineConfig({
   reporter: process.env.CI ? 'blob' : 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
-    baseURL,
+    // baseURL,
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
     video: 'on-first-retry',
   },
   // Timeout per test.
-  timeout: 60_000,
+  timeout: 180_000,
   expect: {
     // Raise the timeout because it is running against next dev mode
     // which requires compilation the first to a page is visited.
@@ -36,7 +35,7 @@ export default defineConfig({
   /* Run your local dev server before starting the tests */
   webServer: {
     command: 'npx nx run explorer:serve:development-testnet-zen',
-    url: baseURL,
+    // url: baseURL,
     reuseExistingServer: !process.env.CI,
     cwd: workspaceRoot,
   },

apps/explorer-e2e/src/fixtures/ExplorerApp.ts

@@ -1,11 +1,18 @@
 import { Locator, Page } from 'playwright'
 import { SEARCHBAR } from './constants'
+import path from 'path'
 
 export class ExplorerApp {
+  // If no baseUrl is provided, use the default address and port for:
+  // explorer:serve:development-testnet-zen.
+  public readonly baseUrl: string = 'http://localhost:3005'
   private readonly searchBar: Locator
 
-  constructor(public readonly page: Page) {
+  constructor(public readonly page: Page, customBaseUrl?: string) {
     this.searchBar = this.page.locator(SEARCHBAR)
+    if (customBaseUrl) {
+      this.baseUrl = customBaseUrl
+    }
   }
 
   async navigateBySearchBar(searchTerm: string) {
@@ -15,6 +22,6 @@ export class ExplorerApp {
   }
 
   async goTo(url: string) {
-    await this.page.goto(url)
+    await this.page.goto(path.join(this.baseUrl, url))
   }
 }

apps/explorer-e2e/src/fixtures/cluster.ts

@@ -0,0 +1,82 @@
+import { Explored } from '@siafoundation/explored-js'
+import { Hostd } from '@siafoundation/hostd-js'
+import { Bus } from '@siafoundation/renterd-js'
+import { Walletd } from '@siafoundation/walletd-js'
+import { startWebServerCluster, stopWebServer } from './webServerCluster'
+import {
+  clusterd,
+  setupCluster,
+  teardownCluster,
+} from '@siafoundation/clusterd'
+
+export type Cluster = Awaited<ReturnType<typeof startCluster>>
+
+export async function startCluster() {
+  const exploredCount = 1
+  const renterdCount = 1
+  const walletdCount = 1
+  const hostdCount = 3
+  await setupCluster({
+    exploredCount,
+    renterdCount,
+    walletdCount,
+    hostdCount,
+  })
+  const renterd = clusterd.nodes.find((n) => n.type === 'renterd')
+  const explored = clusterd.nodes.find((n) => n.type === 'explored')
+  const hostds = clusterd.nodes.filter((n) => n.type === 'hostd')
+  const walletds = clusterd.nodes.filter((n) => n.type === 'walletd')
+  if (!renterd || !explored || !hostds || !walletds) {
+    throw new Error('Failed to start cluster')
+  }
+  const daemons = {
+    renterd,
+    explored,
+    hostds,
+    walletds,
+  }
+  const apis = {
+    renterd: Bus({
+      api: `${renterd.apiAddress}/api`,
+      password: renterd.password,
+    }),
+    explored: Explored({
+      api: `${explored.apiAddress}/api`,
+      password: explored.password,
+    }),
+    hostds: hostds.map((h) =>
+      Hostd({
+        api: `${h.apiAddress}/api`,
+        password: h.password,
+      })
+    ),
+    walletds: walletds.map((w) =>
+      Walletd({
+        api: `${w.apiAddress}/api`,
+        password: w.password,
+      })
+    ),
+  }
+  const { baseUrl } = await startWebServerCluster({
+    exploredAddress: daemons.explored.apiAddress,
+  })
+  console.log(`
+    webServerUrl: ${baseUrl}
+    clusterd: http://localhost:${clusterd.managementPort}
+    explored: ${daemons.explored.apiAddress}
+    renterd: ${daemons.renterd.apiAddress}
+    hostds: ${daemons.hostds.map((h) => h.apiAddress)}
+    walletds: ${daemons.walletds.map((w) => w.apiAddress)}
+  `)
+  return {
+    webServerUrl: baseUrl,
+    clusterd,
+    apis,
+    daemons,
+  }
+}
+
+export async function stopCluster() {
+  stopWebServer()
+  teardownCluster()
+}

apps/explorer-e2e/src/fixtures/constants.ts

@@ -56,15 +56,6 @@ export const TEST_TX_1 = {
   },
 }
 
-export const TEST_ADDRESS_1 = {
-  id: '68bf48e81536f2221f3809aa9d1c89c1c869a17c6f186a088e49fd2605e4bfaaa24f26e4c42c',
-  display: {
-    title: 'Address 68bf48e81536f22...',
-    transactionNumber: '500 events',
-    transactionID: 'c0b92135ca06...',
-  },
-}
-
 export const TEST_CONTRACT_1 = {
   id: '25c94822bf7bd86a92d28a148d9d30151949f3599bf93af0df7b4f1e1b3c990d',
   renewedFromTitle: 'Contract 494d147a8028217...',

apps/explorer-e2e/src/fixtures/utils.ts

@@ -0,0 +1,3 @@
+export function keys<T extends object>(obj: T) {
+  return Object.keys(obj) as Array<keyof T>
+}

apps/explorer-e2e/src/fixtures/walletd.ts

@@ -0,0 +1,96 @@
+import { Walletd } from '@siafoundation/walletd-js'
+import { blake2bHex } from 'blakejs'
+import { WalletAddressMetadata } from '@siafoundation/walletd-types'
+import { to } from '@siafoundation/request'
+import { clusterd, mine } from '@siafoundation/clusterd'
+import { Bus } from '@siafoundation/renterd-js'
+import { humanSiacoin } from '@siafoundation/units'
+
+export async function addWalletToWalletd(walletd: ReturnType<typeof Walletd>) {
+  // For some reason when this code runs on GitHub Actions it throws an ESM
+  // import error for the SDK. Running locally it works fine.
+  // Error: require() of ES Module sdk/index.esm.js from /fixtures/walletd.ts not supported.
+  // This dynamic import is a workaround.
+  const { initSDK, getSDK } = await import('@siafoundation/sdk')
+  await initSDK()
+  const sdk = getSDK()
+  const { phrase: mnemonic } = sdk.wallet.generateSeedPhrase()
+  const mnemonicHash = blake2bHex(mnemonic)
+  const [wallet, walletError] = await to(
+    walletd.walletAdd({
+      data: {
+        name: 'test',
+        description: 'test',
+        metadata: {
+          type: 'seed',
+          mnemonicHash,
+        },
+      },
+    })
+  )
+  if (!wallet || walletError) {
+    throw new Error(`Failed to add wallet: ${walletError}`)
+  }
+  const kp = sdk.wallet.keyPairFromSeedPhrase(mnemonic, 0)
+  const suh = sdk.wallet.standardUnlockHash(kp.publicKey)
+  const uc = sdk.wallet.standardUnlockConditions(kp.publicKey)
+  const metadata: WalletAddressMetadata = {
+    index: 0,
+  }
+  const [, addressError] = await to(
+    walletd.walletAddressAdd({
+      params: {
+        id: wallet.id,
+      },
+      data: {
+        address: suh.address,
+        description: '',
+        spendPolicy: {
+          type: 'uc',
+          policy: uc.unlockConditions,
+        },
+        metadata,
+      },
+    })
+  )
+  if (addressError) {
+    throw new Error(`Failed to add address: ${addressError}`)
+  }
+  return { wallet, address: suh.address }
+}
+
+// The renterd node receives all the initial funds so we can use it to fund
+// other wallets.
+export async function sendSiacoinFromRenterd(address: string, amount: string) {
+  console.log(`Sending ${humanSiacoin(amount)} from renterd to:`, address)
+  const renterdNode = clusterd.nodes.find((n) => n.type === 'renterd')
+  if (!renterdNode) {
+    throw new Error('Renterd node not found')
+  }
+  const bus = Bus({
+    api: renterdNode.apiAddress + '/api',
+    password: renterdNode.password,
+  })
+
+  try {
+    // Send some funds to the wallet.
+    await bus.walletSend({
+      data: {
+        address,
+        amount,
+        subtractMinerFee: false,
+      },
+    })
+    await mine(1)
+  } catch (e) {
+    console.log('error sending siacoin', e)
+  }
+}
+
+export async function getRenterdAddress() {
+  const renterdNode = clusterd.nodes.find((n) => n.type === 'renterd')
+  if (!renterdNode) {
+    throw new Error('Renterd node not found')
+  }
+  return renterdNode.walletAddress
+}

apps/explorer-e2e/src/fixtures/webServerCluster.ts

@@ -0,0 +1,80 @@
+import { ChildProcess, spawn } from 'child_process'
+import { workspaceRoot } from '@nx/devkit'
+import net from 'net'
+
+let server: ChildProcess
+let baseUrl: string
+
+// Starts the explorer app webserver configured to run against the testnet
+// cluster provided via the NEXT_PUBLIC_EXPLORED_ADDRESS environment variable.
+export async function startWebServerCluster({
+  exploredAddress,
+}: {
+  exploredAddress: string
+}) {
+  const port = await findFreePort()
+  server = spawn(
+    'npx',
+    [
+      'nx',
+      'run',
+      'explorer:serve:development-testnet-cluster',
+      '--port',
+      port.toString(),
+    ],
+    {
+      cwd: workspaceRoot,
+      shell: true,
+      env: {
+        ...process.env,
+        NEXT_PUBLIC_EXPLORED_ADDRESS: exploredAddress,
+      },
+    }
+  )
+
+  server.stdout?.on('data', (data) => {
+    console.log(data.toString())
+  })
+
+  server.stderr?.on('data', (data) => {
+    console.error(data.toString())
+  })
+
+  // Wait until stdout prints "Ready", eg:
+  // ✓ Starting...
+  // ✓ Ready in 1606ms
+  await new Promise((resolve) => {
+    server.stdout?.on('data', (data) => {
+      if (data.toString().includes('Ready')) {
+        console.log('Server ready')
+        resolve(true)
+      }
+    })
+  })
+  baseUrl = `http://localhost:${port}`
+  return {
+    baseUrl,
+  }
+}
+
+export function stopWebServer() {
+  console.log('Stopping webserver: ', baseUrl)
+  server.kill() // Kill the server after each test
+}
+
+async function findFreePort(): Promise<number> {
+  return new Promise((res) => {
+    const srv = net.createServer()
+    srv.listen(0, () => {
+      const addr = srv.address()
+      if (typeof addr === 'string') {
+        throw new Error('Address is a string')
+      }
+      const port = addr?.port
+      if (!port) {
+        throw new Error('Port is undefined')
+      }
+      srv.close(() => res(port))
+    })
+  })
+}