E2E Smoke test

.github/actions/setup_test_project/setup_test_project.mjs

@@ -3,7 +3,6 @@ import path from 'node:path'
 
 import { exec } from '@actions/exec'
 import * as core from '@actions/core'
-import * as io from '@actions/io'
 
 const test_project_path = path.join(
   os.tmpdir(),
@@ -18,26 +17,5 @@ console.log({
 
 core.setOutput('test_project_path', test_project_path)
 
-// See https://github.com/actions/toolkit/tree/main/packages/io#cpmv.
-console.log('Copying test project fixture')
-await io.cp(
-  './__fixtures__/test-project',
-  test_project_path,
-  {
-    recursive: true,
-    copySourceDirectory: false
-  }
-)
-
-await exec(`ls ${test_project_path}`)
-
-console.log('Project deps')
-await exec(`yarn project:deps ${test_project_path}`)
-
-console.log('Install')
-await exec('yarn install', null, {
-  cwd: test_project_path,
-})
 
-console.log('Project copy')
-await exec(`yarn project:copy ${test_project_path}`)
+await exec(`yarn build:test-project --ts --link ${test_project_path}`)

.github/workflows/cli.yaml → .github/workflows/smoke-test.yaml

@@ -1,11 +1,11 @@
-name: CLI Checks and Telemetry Benchmarks
+name: Smoke test, CLI Checks and Telemetry Benchmarks
 
 on:
   pull_request:
     types: [opened, synchronize, reopened]
 
 jobs:
-  cli-checks-and-telemetry-benchmarks:
+  smoke-test:
     if: github.repository == 'redwoodjs/redwood'
     strategy:
       matrix:
@@ -18,6 +18,7 @@ jobs:
     env:
       REDWOOD_CI: 1
       REDWOOD_VERBOSE_TELEMETRY: 1
+      PLAYWRIGHT_BROWSERS_PATH: 0
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -40,24 +41,32 @@ jobs:
           restore-keys: |
             yarn-
 
-      - name: Install dependencies and Build Framework
+      - name: Install framework dependencies
         run: |
           yarn install --immutable
-          yarn build:clean || echo "Project already clean"
-          yarn build:js
 
       - name: Setup test project
         id: setup_test_project
         uses: ./.github/actions/setup_test_project
 
-      - name: Run `rw info`
+      - name: Install Playwright deps
+        run: npx playwright install --with-deps chromium
+
+      - name: Run `rw build`
         run: |
-          yarn rw info
+          yarn rw build
         working-directory: ${{ steps.setup_test_project.outputs.test_project_path }}
 
-      - name: Run `rw build` without prerender
+      # We run prisma reset, seed, rw dev, rw build and rw serve
+      - name: Run smoke tests
+        working-directory: ./tasks/smoke-test
+        run: npx playwright test
+        env:
+          PROJECT_PATH: ${{ steps.setup_test_project.outputs.test_project_path }}
+
+      - name: Run `rw info`
         run: |
-          yarn rw build --no-prerender
+          yarn rw info
         working-directory: ${{ steps.setup_test_project.outputs.test_project_path }}
 
       - name: Run `rw prerender`

package.json

@@ -24,6 +24,7 @@
     "project:sync": "node ./tasks/framework-tools/frameworkSyncToProject.mjs",
     "publish:canary": "lerna publish --force-publish --canary --include-merged-tags --preid canary --dist-tag canary --yes --loglevel verbose",
     "release": "node ./tasks/release/cli.mjs",
+    "smoke-test": "cd ./tasks/smoke-test && npx playwright install && npx playwright test",
     "test": "lerna run test --stream -- --colors --maxWorkers=4",
     "test:release-script": "NODE_OPTIONS=--experimental-vm-modules ./node_modules/.bin/jest --config ./tasks/release/jest.config.mjs"
   },
@@ -51,9 +52,11 @@
     "@babel/preset-react": "7.16.7",
     "@babel/preset-typescript": "7.16.7",
     "@babel/runtime-corejs3": "7.16.7",
+    "@playwright/test": "1.19.1",
     "@testing-library/jest-dom": "5.16.2",
     "@testing-library/react": "12.1.3",
     "@testing-library/user-event": "13.5.0",
+    "@types/fs-extra": "9.0.13",
     "@types/jest": "27.4.0",
     "@types/jscodeshift": "0.11.3",
     "@types/lodash.template": "4.5.0",
@@ -70,6 +73,8 @@
     "cypress-wait-until": "1.7.2",
     "eslint": "8.9.0",
     "fast-glob": "3.2.11",
+    "fs-extra": "10.0.0",
+    "is-port-reachable": "3.1.0",
     "jest": "27.5.1",
     "jscodeshift": "0.13.1",
     "lerna": "4.0.0",

tasks/smoke-test/playwright-fixtures/devServer.fixture.ts

@@ -0,0 +1,78 @@
+/* eslint-disable no-empty-pattern */
+import { test as base } from '@playwright/test'
+import execa from 'execa'
+
+import { waitForServer } from '../util'
+
+// Declare worker fixtures.
+type DevServerFixtures = {
+  webServerPort: number
+  apiServerPort: number
+  server: any
+}
+
+// Note that we did not provide an test-scoped fixtures, so we pass {}.
+const test = base.extend<any, DevServerFixtures>({
+  webServerPort: [
+    async ({}, use, workerInfo) => {
+      // "port" fixture uses a unique value of the worker process index.
+      await use(9000 + workerInfo.workerIndex)
+    },
+    { scope: 'worker' },
+  ],
+  apiServerPort: [
+    async ({}, use, workerInfo) => {
+      // "port" fixture uses a unique value of the worker process index.
+      await use(9001 + workerInfo.workerIndex)
+    },
+    { scope: 'worker' },
+  ],
+
+  // "server" fixture starts automatically for every worker - we pass "auto" for that.
+  server: [
+    async ({ webServerPort, apiServerPort }, use) => {
+      console.log('Starting dev server.....')
+
+      const projectPath = process.env.PROJECT_PATH
+
+      if (!projectPath) {
+        throw new Error(
+          'PROJECT_PATH env var not defined. Please build a test project, and re-run with PROJECT_PATH defined'
+        )
+      }
+
+      console.log(`Launching dev server at ${projectPath}`)
+
+      // Don't wait for this to finish, because it doens't
+      const devServerHandler = execa.command(
+        `yarn rw dev --fwd="--no-open" --no-generate`,
+        {
+          cwd: projectPath,
+          shell: true,
+          env: {
+            WEB_DEV_PORT: webServerPort,
+            API_DEV_PORT: apiServerPort,
+          },
+        }
+      )
+
+      // Pipe out logs so we can debug, when required
+      devServerHandler.stdout.on('data', (data) => {
+        console.log(
+          '[devServer-fixture] ',
+          Buffer.from(data, 'utf-8').toString()
+        )
+      })
+
+      console.log('Waiting for dev servers.....')
+      await waitForServer(webServerPort, 1000)
+      await waitForServer(apiServerPort, 1000)
+
+      console.log('Starting tests!')
+      await use()
+    },
+    { scope: 'worker', auto: true },
+  ],
+})
+
+export default test

tasks/smoke-test/playwright-fixtures/rwServe.fixture.ts

@@ -0,0 +1,80 @@
+/* eslint-disable no-empty-pattern */
+import { test as base } from '@playwright/test'
+import execa from 'execa'
+import { pathExistsSync } from 'fs-extra'
+import path from 'node:path'
+
+import { waitForServer } from '../util'
+
+// Declare worker fixtures.
+type DevServerFixtures = {
+  port: number
+  server: any
+}
+
+// Note that we did not provide an test-scoped fixtures, so we pass {}.
+const test = base.extend<any, DevServerFixtures>({
+  port: [
+    async ({}, use, workerInfo) => {
+      // "port" fixture uses a unique value of the worker process index.
+      await use(8899 + workerInfo.workerIndex)
+    },
+    { scope: 'worker' },
+  ],
+
+  // "server" fixture starts automatically for every worker - we pass "auto" for that.
+  server: [
+    async ({ port }, use) => {
+      console.log('Starting rw server.....')
+
+      const projectPath = process.env.PROJECT_PATH
+
+      if (!projectPath) {
+        throw new Error(
+          'PROJECT_PATH env var not defined. Please build a test project, and re-run with PROJECT_PATH defined'
+        )
+      }
+
+      console.log(`Running rw serve at ${projectPath}`)
+
+      if (projectNeedsBuilding(projectPath)) {
+        // skip rw build if its already done
+        execa.sync(`yarn rw build`, {
+          cwd: projectPath,
+          shell: true,
+        })
+      }
+
+      // Don't wait for this to finish, because it doens't
+      const rwServeHandler = execa.command(`yarn rw serve -p ${port}`, {
+        cwd: projectPath,
+        shell: true,
+      })
+
+      // Pipe out logs so we can debug, when required
+      rwServeHandler.stdout.on('data', (data) => {
+        console.log(
+          '[rw-serve-fixture] ',
+          Buffer.from(data, 'utf-8').toString()
+        )
+      })
+
+      console.log('Waiting for server.....')
+      await waitForServer(port, 1000)
+
+      console.log('Starting tests!')
+      await use()
+    },
+    { scope: 'worker', auto: true },
+  ],
+})
+
+const projectNeedsBuilding = (
+  projectPath: string = process.env.PROJECT_PATH
+) => {
+  const webDist = path.join(projectPath, 'web/dist')
+  const apiDist = path.join(projectPath, 'api/dist')
+  return !(pathExistsSync(webDist) && pathExistsSync(apiDist))
+}
+
+export default test

tasks/smoke-test/playwright.config.ts

@@ -0,0 +1,8 @@
+import type { PlaywrightTestConfig } from '@playwright/test'
+
+// See https://playwright.dev/docs/test-configuration#global-configuration
+const config: PlaywrightTestConfig = {
+  timeout: 60_000,
+}
+
+export default config

tasks/smoke-test/tests/common.ts

@@ -0,0 +1,27 @@
+import { expect } from '@playwright/test'
+
+export const smokeTest = async ({ page, webServerPort }) => {
+  // Go to http://localhost:8910/
+  await page.goto(`http://localhost:${webServerPort}/`)
+
+  // Check that the blog posts are being loaded
+  await page.textContent('text=Welcome to the blog!')
+  await page.textContent('text=What is the meaning of life?')
+  await page.textContent('text=A little more about me')
+
+  // Click text=About
+  await page.click('text=About')
+
+  expect(page.url()).toBe(`http://localhost:${webServerPort}/about`)
+
+  await page.textContent(
+    'text=This site was created to demonstrate my mastery of Redwood: Look on my works, ye'
+  )
+  // Click text=Contact
+  await page.click('text=Contact')
+  expect(page.url()).toBe(`http://localhost:${webServerPort}/contact`)
+
+  // Click text=Admin
+  await page.click('text=Admin')
+  expect(page.url()).toBe(`http://localhost:${webServerPort}/posts`)
+}

tasks/smoke-test/tests/rwDev.spec.ts

@@ -0,0 +1,5 @@
+import devServerTest from '../playwright-fixtures/devServer.fixture'
+
+import { smokeTest } from './common'
+
+devServerTest('Smoke test with dev server', smokeTest)

tasks/smoke-test/tests/rwServe.spec.ts

@@ -0,0 +1,21 @@
+import rwServeTest from '../playwright-fixtures/rwServe.fixture'
+
+import { smokeTest } from './common'
+
+rwServeTest('Smoke test with rw serve', ({ port, page }) =>
+  smokeTest({ webServerPort: port, page })
+)
+
+// @TODO it might be useful to have this visual check
+// But gitpod screenshot doesnt seem to match up with github runners
+// Possibly due to fonts
+// rwServeTest('Visual check', async ({ page, port }) => {
+//   await page.goto(`http://localhost:${port}/`)
+
+//   // Wait till cell has finished rendering
+//   await page.textContent('text=Welcome to the blog!')
+
+//   expect(await page.screenshot()).toMatchSnapshot('landing.png', {
+//     threshold: 0.2, // reduce sensitivity of visual comparison
+//   })
+// })

tasks/smoke-test/util.ts

@@ -0,0 +1,10 @@
+import isPortReachable from 'is-port-reachable'
+
+export function waitForServer(port, interval) {
+  return new Promise((resolve) => {
+    setInterval(async () => {
+      const isServerUp = await isPortReachable(port, { host: 'localhost' })
+      isServerUp && resolve(true)
+    }, interval)
+  })
+}

tasks/test-project/tasks.js

@@ -248,14 +248,12 @@ async function apiTasks(outputPath, { verbose }) {
         },
       },
       {
-        title: 'Seeding database',
+        title: 'Adding seed script',
         task: async () => {
           await applyCodemod(
             'seed.js',
             fullPath('scripts/seed.ts', { addExtension: false })
           )
-
-          return execa('yarn rw prisma db seed', [], execaOptionsForProject)
         },
       },
       {