Rework ci and add e2e tests scaffolding

.github/workflows/phpunit-tests.yml → .github/workflows/automatic-dispatch.yml

@@ -1,10 +1,6 @@
-name: PHPUnit Tests
+name: Automatic Tests Dispatch
 
-on:
-  push:
-    branches:
-      - trunk
-  pull_request:
+on: [push, pull_request]
 
 jobs:
   test:
@@ -20,6 +16,7 @@ jobs:
         php: [ '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3' ]
 
     with:
-      os: ${{ matrix.os }}
-      php: ${{ matrix.php }}
-      phpunit-config: ${{ 'phpunit.xml.dist' }}
+      OS: ${{ matrix.os }}
+      PHP: ${{ matrix.php }}
+      TEST_SUITE: all
+      FAIL_FAST: true

.github/workflows/manual-dispatch.yml

@@ -0,0 +1,60 @@
+name: Manual Tests Dispatch
+
+on:
+  workflow_dispatch:
+    inputs:
+      PHP:
+        description: 'pick which PHP version to use'
+        type: choice
+        options:
+          - all
+          - '7.2'
+          - '7.3'
+          - '7.4'
+          - '8.0'
+          - '8.1'
+          - '8.2'
+          - '8.3'
+        required: false
+        default: all
+      OS:
+        description: 'pick which OS to run on'
+        type: choice
+        options:
+          - all
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        required: false
+        default: all
+      TEST_SUITE:
+        description: 'pick which PHPUnit test suite to run'
+        type: choice
+        options:
+          - all
+          - unit
+          - e2e
+        required: false
+        default: all
+
+run-name: >-
+  PHP: ${{ github.event.inputs.PHP }} OS: ${{ github.event.inputs.OS }} TEST SUITE: ${{ github.event.inputs.TEST_SUITE }}
+
+jobs:
+  test:
+    name: PHP ${{ matrix.php }}
+    uses: ./.github/workflows/phpunit-tests-run.yml
+    permissions:
+      contents: read
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ ( github.event.inputs.OS == 'all' && fromJSON( '["ubuntu-latest", "macos-latest", "windows-latest"]' ) ) || fromJSON( format( '["{0}"]', github.event.inputs.OS ) ) }}
+        php: ${{ ( github.event.inputs.PHP == 'all' && fromJSON( '["7.2", "7.3", "7.4", "8.0", "8.1", "8.2", "8.3"]' ) ) || fromJSON( format( '["{0}"]', github.event.inputs.PHP ) ) }}
+
+    with:
+      OS: ${{ matrix.os }}
+      PHP: ${{ matrix.php }}
+      TEST_SUITE: ${{ github.event.inputs.TEST_SUITE }}
+      FAIL_FAST: false

.github/workflows/phpunit-tests-run.yml

@@ -3,29 +3,37 @@ name: Run PHPUnit tests
 on:
   workflow_call:
     inputs:
-      os:
-        description: 'Operating system to run tests on'
+      OS:
+        description: 'The operating system to run tests on'
         required: false
         type: 'string'
-        default: 'ubuntu-latest'
-      php:
+        default: ubuntu-latest
+      PHP:
         description: 'The version of PHP to use, in the format of X.Y'
-        required: true
+        required: false
         type: 'string'
-      phpunit-config:
-        description: 'The PHPUnit configuration file to use'
+        default: '7.2'
+      TEST_SUITE:
+        description: 'The PHPUnit test suite to use'
         required: false
         type: 'string'
-        default: 'phpunit.xml.dist'
+        default: unit
+      FAIL_FAST:
+        description: 'If test job should fail at first defect'
+        required: false
+        type: boolean
+        default: true
+
 env:
-  LOCAL_PHP: ${{ inputs.php }}-fpm
-  PHPUNIT_CONFIG: ${{ inputs.phpunit-config }}
+  LOCAL_PHP: ${{ inputs.PHP }}-fpm
+  PHPUNIT_CONFIG: phpunit.xml.dist
+  STRICT_FLAGS: ${{ ( inputs.FAIL_FAST && '--stop-on-defect --fail-on-warning --fail-on-risky' ) || '' }}
 
 jobs:
   phpunit-tests:
-    name: ${{ inputs.os }}
-    runs-on: ${{ inputs.os }}
-    timeout-minutes: 20
+    name: ${{ inputs.OS }} ${{ inputs.TEST_SUITE }} tests
+    runs-on: ${{ inputs.OS }}
+    timeout-minutes: 5
 
     steps:
       - name: Checkout repository
@@ -34,15 +42,21 @@ jobs:
       - name: Set up PHP
         uses: shivammathur/setup-php@v2
         with:
-          php-version: '${{ inputs.php }}'
+          php-version: '${{ inputs.PHP }}'
           tools: phpunit-polyfills
-          extensions: zip
+          extensions: zip, pdo_sqlite, sqlite3
 
       - name: Install Composer dependencies
         uses: ramsey/composer-install@v3
         with:
           ignore-cache: "yes"
           composer-options: "--optimize-autoloader"
 
-      - name: Run PHPUnit tests
-        run: phpunit tests --testdox
+      - name: Run PHPUnit unit tests
+        id: unit
+        if: contains( fromJSON( '["unit", "all"]' ), inputs.TEST_SUITE )
+        run: phpunit --testsuite unit ${{ env.STRICT_FLAGS }}
+
+      - name: Run PHPUnit e2e tests
+        if: inputs.TEST_SUITE == 'e2e' || ( inputs.TEST_SUITE == 'all' && ( ! inputs.FAIL_FAST || steps.unit.outcome == 'success' ) )
+        run: phpunit --testsuite e2e ${{ env.STRICT_FLAGS }}

README.md

@@ -68,10 +68,21 @@ Install [composer](https://getcomposer.org/). Once you have it, install the requ
 composer install
 ```
 
-### Run tests with
+## Tests
+### Run all tests with
+```shell
+vendor/bin/phpunit
+```
+
+### Run unit tests with
+
+```shell
+vendor/bin/phpunit --testsuite unit
+```
 
+### Run e2e tests with
 ```shell
-vendor/bin/phpunit --testdox
+vendor/bin/phpunit --testsuite e2e
 ```
 
 ### Run Blueprints in a variety of ways
@@ -82,7 +93,7 @@ vendor/bin/phpunit --testdox
  php examples/blueprint_compiling.php
 ```
 
-#### using a string containg a Blueprint (in JSON):
+#### using a string containing a Blueprint (in JSON):
 
 ```shell
  php examples/json_string_compiling.php

phpunit.xml.dist

@@ -4,10 +4,18 @@
         bootstrap="vendor/autoload.php"
         colors="true"
         beStrictAboutOutputDuringTests="true"
+        defaultTestSuite="all"
+        testdox="true"
 >
     <testsuites>
-        <testsuite name="default">
+        <testsuite name="all">
             <directory>tests</directory>
         </testsuite>
+        <testsuite name="unit">
+            <directory>tests/unit</directory>
+        </testsuite>
+        <testsuite name="e2e">
+            <directory>tests/e2e</directory>
+        </testsuite>
     </testsuites>
 </phpunit>

tests/e2e/configuration/E2ETestCase.php

@@ -0,0 +1,5 @@
+<?php
+
+use Yoast\PHPUnitPolyfills\TestCases\XTestCase as TestCase;
+
+abstract class E2ETestCase extends TestCase {}