Fixes end-to-end tests (#871)

- Fixes the broken end-to-end tests
- Changes the playwright test configuration to produce
   test coverage
- Adds end-to-end tests of client to github actions.
  But these tests fail of non-availability of gitlab instance.

.github/workflows/client.yml

@@ -15,6 +15,9 @@ jobs:
   client:
     name: Test react website
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: client/
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -26,29 +29,49 @@ jobs:
           cache: "yarn"
           cache-dependency-path: "**/yarn.lock"
 
+      - name: Install dependencies
+        run: |
+          yarn install
+          yarn playwright install --with-deps
+
       - name: Run the linting checks
         run: |
-          cd client/	
-          yarn install	
           yarn syntax
-      - name: Build the React SPA website
+
+      - name: Build the React SPA website and setup env
         run: |
-          cd client/	
-          yarn install	
           yarn build
+          yarn config:test
+
       - name: Run client unit and integration tests
         run: |
-          cd client/	
-          yarn install	
-          yarn config:test
           yarn test:int
           yarn test:unit
 
       - name: Upload unit and integration test coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           files: client/coverage/clover.xml
-          flags: client-unit-integration-tests
+          flags: client-unit-integration-tests 
+
+      - name: Run e2e tests
+        continue-on-error: true
+        run: |
+          yarn test:e2e
+
+      - name: Upload Playwright Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: client/playwright-report/**
+          retention-days: 30
+
+      - name: Upload e2e test coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: client/coverage/e2e/codecov.json
+          flags: client-e2e-tests
 
   get_version:
     name: Get version

.gitignore

@@ -121,7 +121,7 @@ build/
 # Test reports
 coverage/
 playwright-report/
-playwright/
+client/playwright/
 test_results/
 
 # Mac temp files

client/.eslintignore

@@ -2,4 +2,5 @@ api/
 build/
 config/
 node_modules/
-script/
+script/
+coverage/

client/.gitignore

@@ -1 +1,2 @@
-playwright/.auth/
+playwright/.auth/
+test-results/

client/DEVELOPER.md

@@ -39,9 +39,6 @@ yarn test:e2e   #run end-to-end tests
 yarn test:all   #run all tests
 ```
 
-:bug: The end-to-end tests are extremely brittle.
-They are not being used at present.
-
 ## Authorization
 
 The react client website uses OAuth authorization.

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@into-cps-association/dtaas-web",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Web client for Digital Twin as a Service (DTaaS)",
   "main": "index.tsx",
   "author": "prasadtalasila <[email protected]> (http://prasad.talasila.in/)",
@@ -30,7 +30,8 @@
     "stop": "npx kill-port 4000",
     "syntax": "npx eslint . --fix",
     "test:all": "yarn test:unit && yarn test:int && yarn test:e2e",
-    "test:e2e": "yarn build && yarn config:test && npx kill-port 4000 && yarn start >/dev/null & playwright test && npx kill-port 4000",
+    "test:e2e:ext": "cross-env ext=true yarn test:e2e",
+    "test:e2e": "playwright test -c ./playwright.config.ts",
     "test:int": "jest -c ./jest.config.json ../test/integration --setupFilesAfterEnv ./test/integration/jest.setup.ts",
     "test:unit": "jest -c ./jest.config.json ../test/unit --setupFilesAfterEnv ./test/unit/jest.setup.ts"
   },
@@ -53,6 +54,7 @@
     "@types/styled-components": "^5.1.32",
     "@typescript-eslint/eslint-plugin": "^6.12.0",
     "@typescript-eslint/parser": "^6.12.0",
+    "cross-env": "^7.0.3",
     "dotenv": "^16.1.4",
     "eslint": "8.54.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -92,6 +94,7 @@
     "@types/react-dom": "^18.2.17",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
+    "monocart-coverage-reports": "^2.10.2",
     "playwright": "^1.32.1",
     "prettier": "^3.2.4",
     "shx": "^0.3.4",

client/playwright.config.ts

@@ -5,26 +5,50 @@
 import { defineConfig, devices } from '@playwright/test';
 import * as dotenv from 'dotenv';
 
+// Check if playwright was called with 'ext' flag.
+const useExtServer = process.env.ext === 'true';
+
 dotenv.config({ path: './test/.env' });
 // import fs from 'fs';
 // import path from 'path';
 
 // const storeState = JSON.parse(fs.readFileSync(path.resolve('./playwright/.auth/user.json'), 'utf-8'));
-const BASE_URI = process.env.REACT_APP_URL.toString() ?? '';
+const BASE_URI = process.env.REACT_APP_URL ?? 'http://localhost:4000/';
 
 export default defineConfig({
-  timeout: 45000,
-  globalTimeout: 600000,
-  testDir: './test/e2e',
+  webServer: useExtServer
+    ? undefined
+    : {
+        command: 'yarn start',
+      },
+  timeout: 60 * 1000,
+  globalTimeout: 10 * 60 * 1000,
+  testDir: './test/e2e/tests',
   testMatch: /.*\.test\.ts/,
   reporter: [
-    ['html', { outputFile: 'playwright-report/index.html' }],
+    [
+      'html',
+      {
+        outputFile: 'playwright-report/index.html',
+      },
+    ],
     ['list'],
-    ['junit', { outputFile: 'playwright-report/results.xml' }],
-    ['json', { outputFile: 'playwright-report/results.json' }],
-  ],
+    [
+      'junit',
+      {
+        outputFile: 'playwright-report/results.xml',
+      },
+    ],
+    [
+      'json',
+      {
+        outputFile: 'playwright-report/results.json',
+      },
+    ],
+  ], // Codecov handled through Monocart-Reporter https://github.com/cenfun/monocart-reporter
   use: {
     baseURL: BASE_URI,
+    trace: 'retain-on-failure',
   },
   projects: [
     // Setup project
@@ -51,4 +75,6 @@ export default defineConfig({
       dependencies: ['setup'],
     },
   ],
+  globalSetup: 'test/e2e/setup/global.setup.ts',
+  globalTeardown: 'test/e2e/setup/global-teardown.ts',
 });

client/test/README.md

@@ -1,24 +1,57 @@
 # End-to-End (E2E) Tests
 
-The E2E tests require a functional gitlab oauth setup, traefik gateway and a
-live react client website. The E2E tests do not launch the react client website.
-So it is important to launch the website using `yarn start`. Keep this server
-running while performing the E2E tests with `yarn test:e2e` command.
+The E2E tests require playwright test runner, an on-premise GitLab OAuth setup and
+configured `config/test.js` and `test/.env` files. When everything is set up, you
+can run the tests by running `yarn test:e2e`.
+
+It is also possible to test the hosted DTaaS applications hosted at a URL,
+say `https://foo.com` using `yarn test:e2e:ext`. Remember to set the environment
+variable in `test/.env` to the URL of the hosted DTaaS application. An example
+is shown below:
+
+```js
+if (typeof window !== 'undefined') {
+  window.env = {
+    REACT_APP_ENVIRONMENT: 'test',
+    REACT_APP_URL: 'https://foo.com/',
+    REACT_APP_URL_BASENAME: '',
+    REACT_APP_URL_DTLINK: '/lab',
+    REACT_APP_URL_LIBLINK: '',
+    REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword',
+    REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/',
+    REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab',
+    REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '',
+
+    REACT_APP_CLIENT_ID: '1be55736756190b3ace4c2c4fb19bde386d1dcc748d20b47ea8cfb5935b8446c',
+    REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/',
+    REACT_APP_REDIRECT_URI: 'https://foo.com/Library',
+    REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/',
+    REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api',
+  };
+};
+```
+
+The `yarn install` and `yarn config:test` need to be run before `yarn test:e2e:ext`
+can be run successfully. Also note that if you are deploying the client
+application with Traeffik forward authorization, the tests will fail due to
+the additionally required Gitlab authorization.
+
+## Playwright
 
 The E2E tests use playwright test runner. You also need to have the software
-installed. If it is not installed, you can install with the following command.
+installed. If it is not installed, you can install it with the following command.
 
 ```bash
-sudo npx playwright install-deps
+yarn playwright install --with-deps
 ```
 
 ## OAuth Setup
 
-You can follow the instructions in
-[authorization page](../../docs/admin/client/auth.md) to setup oauth for the
-react client website. Remember to add the `http://localhost:4000` as callback URL
-in the oauth application. The gitlab will still be running on a remote machine.
-It is not possible to run both the gitlab and react client website on localhost.
+You can follow the instructions in [authorization page
+](../../docs/admin/client/auth.md)to setup OAuth for the react client website.
+Remember to add the `http://localhost:4000` as callback URL in the OAuth
+application. The GitLab will still be running on a remote machine.
+It is not possible to run both the GitLab and react client website on localhost.
 
 ## config/test.js file
 
@@ -48,11 +81,14 @@ REACT_APP_REDIRECT_URI="http://localhost:4000/Library"
 REACT_APP_LOGOUT_REDIRECT_URI="http://localhost:4000"
 ```
 
+Finally, run `yarn config:test` to copy the config file into the `build`
+and `public` folders.
+
 ## env file
 
 You need to create a `test/.env` file where you will store the GitLab user
-credentials. These credentials will be used by playwright to simulate real
-user interactions during the E2E tests.
+credentials and application URL for the website. The credentials will be
+used by playwright to simulate real user interactions during the E2E tests.
 
 A template for `test/.env` is given here:
 
@@ -77,7 +113,8 @@ REACT_APP_URL='http://localhost:4000'
 
 ## Testing on localhost
 
-There are two possible testing setups you can create.
+If you want to handle starting the react client server yourself, there are two
+possible testing setups you can create.
 
 1. Host website on the developer computer and test from developer computer
 1. Host website on the integration server and test from the integration server
@@ -127,7 +164,7 @@ credentials on `gitlab.foo.com`.
 ## Testing on the integration server
 
 In this test setup, the DTaaS application is running at `https://foo.com` and
-the gitlab instance is running at `https://gitlab.foo.com`. The E2E test shall
+the GitLab instance is running at `https://gitlab.foo.com`. The E2E test shall
 be run from the developer computer. The codebase commit should be the same on
 both the developer computer and integration server.
 
@@ -150,7 +187,7 @@ window.env = {
   REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/',
   REACT_APP_REDIRECT_URI: 'https://foo.com/Library',
   REACT_APP_LOGOUT_REDIRECT_UR: 'https://foo.com/',
-  REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api',
+  REACT_APP_GitLab_SCOPES: 'openid profile read_user read_repository api',
 };
 ```
 
@@ -171,12 +208,21 @@ gateway and the client website hosted behind traefik.
 
 ## Running the Tests
 
-Once you've properly set up your .env file, you can now run the end-to-end tests.
+Once you've properly set up your .env file, you can run the end-to-end tests as follows:
 
 ```bash
 yarn test:e2e
 ```
 
-This command launches the test runner and executes all end-to-end tests.
+Or with manual website launch:
+
+```bash
+yarn test:e2e:ext
+```
+
+These commands launch the test runner and execute all end-to-end tests. The first
+command also runs the `yarn start` command to start the client website
+and terminates it after testing.
+
 Make sure you have an active internet connection while running these tests,
 as they simulate real user interactions with your GitLab account.

client/test/e2e/setup/fixtures.ts

@@ -0,0 +1,47 @@
+import { test as testBase } from '@playwright/test';
+import MCR from 'monocart-coverage-reports';
+import coverageOptions from './mcr.config';
+
+// fixtures
+const test = testBase.extend<{
+  autoTestFixture: string;
+}>({
+  autoTestFixture: [
+    async ({ page }, use) => {
+      const isChromium = test.info().project.name === 'chromium';
+
+      // console.log('autoTestFixture setup...');
+      // coverage API is chromium only
+      if (isChromium) {
+        await Promise.all([
+          page.coverage.startJSCoverage({
+            resetOnNavigation: false,
+          }),
+          page.coverage.startCSSCoverage({
+            resetOnNavigation: false,
+          }),
+        ]);
+      }
+
+      await use('autoTestFixture');
+
+      // console.log('autoTestFixture teardown...');
+      if (isChromium) {
+        const [jsCoverage, cssCoverage] = await Promise.all([
+          page.coverage.stopJSCoverage(),
+          page.coverage.stopCSSCoverage(),
+        ]);
+        const coverageList = [...jsCoverage, ...cssCoverage];
+        // console.log(coverageList.map((item) => item.url));
+        const mcr = MCR(coverageOptions);
+        await mcr.add(coverageList);
+      }
+    },
+    {
+      scope: 'test',
+      auto: true,
+    },
+  ],
+});
+
+export default test;

client/test/e2e/setup/global-teardown.ts

@@ -0,0 +1,9 @@
+import MCR from 'monocart-coverage-reports';
+import coverageOptions from './mcr.config';
+
+async function globalTeardown() {
+  const mcr = MCR(coverageOptions);
+  await mcr.generate();
+}
+
+export default globalTeardown;