From b19038299c6bff0802d2d9bc4f9b65477d615ec2 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 20 Jan 2025 08:08:57 +0700 Subject: [PATCH] [ci] Tests run on Linux arm64 hosted runners (#2592) Signed-off-by: Viet Nguyen Duc --- .circleci/{config.yml => config.bak} | 26 ++-- .../actions/get-latest-upstream/action.yml | 12 +- .github/workflows/docker-test.yml | 122 +++++++++++++++++- .github/workflows/helm-chart-test.yml | 33 +++-- .github/workflows/k8s-scaling-test.yml | 12 +- generate_chart_changelog.sh | 6 +- 6 files changed, 166 insertions(+), 45 deletions(-) rename .circleci/{config.yml => config.bak} (98%) diff --git a/.circleci/config.yml b/.circleci/config.bak similarity index 98% rename from .circleci/config.yml rename to .circleci/config.bak index bc2145577c..cd5dec0cc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.bak @@ -3,16 +3,6 @@ version: 2.1 workflows: build-and-test-multi-arch: jobs: - - kubernetes-test: - name: "K8s test - Playwright Connect Autoscaling Grid" - platforms: linux/arm64 - machine-type: ubuntu2204arm64large - k8s-version: 'v1.25.16' - test-strategy: playwright_connect_grid - cluster: 'minikube' - helm-version: 'v3.11.3' - docker-version: '24.0.9' - test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling disabled" platforms: linux/arm64 @@ -47,7 +37,7 @@ workflows: name: "K8s test - Autoscaling Jobs - HTTPS" platforms: linux/arm64 machine-type: ubuntu2204arm64large - k8s-version: 'v1.29.12' + k8s-version: 'v1.29.13' test-strategy: job_https cluster: 'minikube' helm-version: 'v3.14.3' @@ -57,7 +47,7 @@ workflows: name: "K8s test - Autoscaling Jobs - Ingress hostname" platforms: linux/arm64 machine-type: ubuntu2204arm64large - k8s-version: 'v1.30.8' + k8s-version: 'v1.30.9' test-strategy: job_hostname cluster: 'minikube' helm-version: 'v3.15.4' @@ -67,12 +57,22 @@ workflows: name: "K8s test - Autoscaling Deployments - HTTPS" platforms: linux/arm64 machine-type: ubuntu2204arm64large - k8s-version: 'v1.31.4' + k8s-version: 'v1.31.5' test-strategy: deployment_https cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' test-upgrade: true + - kubernetes-test: + name: "K8s test - Playwright Connect Autoscaling Grid" + platforms: linux/arm64 + machine-type: ubuntu2204arm64large + k8s-version: 'v1.32.1' + test-strategy: playwright_connect_grid + cluster: 'minikube' + helm-version: 'v3.17.0' + docker-version: '26.1.4' + test-upgrade: true - docker-test: name: "Docker test - Use random user (true)" test-strategy: test diff --git a/.github/actions/get-latest-upstream/action.yml b/.github/actions/get-latest-upstream/action.yml index 7ed58233c6..1cbb7a73c4 100644 --- a/.github/actions/get-latest-upstream/action.yml +++ b/.github/actions/get-latest-upstream/action.yml @@ -18,20 +18,20 @@ runs: shell: bash run: | sudo apt update - sudo apt install gh - echo "${{ inputs.gh_cli_token }}" | gh auth login --with-token + sudo apt install jq + AUTH_HEADER="Authorization: token ${{ inputs.gh_cli_token }}" if [ "${{ inputs.release }}" = "true" ]; then echo "Getting the latest stable release." - RELEASE=$(gh release list -R SeleniumHQ/selenium | grep -v nightly | awk '{ print $4 }' | head -1) + RELEASE=$(curl -s -H "$AUTH_HEADER" https://api.github.com/repos/SeleniumHQ/selenium/releases | jq -r '[.[] | select(.prerelease == false)] | .[0].tag_name') else echo "Getting the latest Nightly release." - RELEASE=$(gh release list -R SeleniumHQ/selenium | grep nightly | awk '{ print $3 }' | head -1) + RELEASE=$(curl -s -H "$AUTH_HEADER" https://api.github.com/repos/SeleniumHQ/selenium/releases | jq -r '[.[] | select(.prerelease == true)] | .[0].tag_name') if [ -z "${RELEASE}" ]; then echo "Nightly release not found, getting the latest stable release." - RELEASE=$(gh release list -R SeleniumHQ/selenium | grep -v nightly | awk '{ print $4 }' | head -1) + RELEASE=$(curl -s -H "$AUTH_HEADER" https://api.github.com/repos/SeleniumHQ/selenium/releases | jq -r '[.[] | select(.prerelease == false)] | .[0].tag_name') fi fi - jar_file=$(gh release view -R SeleniumHQ/selenium ${RELEASE} | grep jar | awk '{ print $2 }' | tail -n 1) + jar_file=$(curl -s -H "$AUTH_HEADER" https://api.github.com/repos/SeleniumHQ/selenium/releases/tags/${RELEASE} | jq -r '.assets[] | select(.name | endswith(".jar")) | .name' | tail -n 1) echo "Server package: ${jar_file}" VERSION=$(echo $jar_file | sed 's/selenium-server-//;s/\.jar//') echo "BASE_RELEASE=${RELEASE} | BASE_VERSION=${VERSION} | VERSION=${VERSION}" diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 7568426748..f4e4c6b6d6 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -29,7 +29,7 @@ permissions: jobs: build-and-test: name: Test Docker Selenium - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -38,38 +38,129 @@ jobs: use-random-user: true test-video: false build-all: true + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test use-random-user: false test-video: false build-all: true + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_video use-random-user: false test-video: true build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_video_dynamic_name use-random-user: false test-video: true build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_video_standalone use-random-user: false test-video: true build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_node_docker use-random-user: false test-video: true build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_standalone_docker use-random-user: false test-video: true build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_parallel use-random-user: false test-video: false build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: - test-strategy: test_node_relay use-random-user: false test-video: false build-all: false + os: ubuntu-24.04 + firefox-install-lang-package: + enable-managed-downloads: + ### + - test-strategy: test + use-random-user: true + test-video: false + build-all: true + os: ubuntu-24.04-arm + firefox-install-lang-package: false + enable-managed-downloads: false + - test-strategy: test + use-random-user: false + test-video: false + build-all: true + os: ubuntu-24.04-arm + firefox-install-lang-package: false + enable-managed-downloads: false + - test-strategy: test_video + use-random-user: false + test-video: true + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: true + - test-strategy: test_video_dynamic_name + use-random-user: false + test-video: true + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: true + - test-strategy: test_video_standalone + use-random-user: false + test-video: true + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: true + - test-strategy: test_node_docker + use-random-user: false + test-video: true + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: false + - test-strategy: test_standalone_docker + use-random-user: false + test-video: true + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: true + - test-strategy: test_parallel + use-random-user: false + test-video: false + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: false + enable-managed-downloads: true + - test-strategy: test_node_relay + use-random-user: false + test-video: false + build-all: false + os: ubuntu-24.04-arm + firefox-install-lang-package: true + enable-managed-downloads: true steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -96,7 +187,7 @@ jobs: python-version: '3.11' check-latest: true - name: Enable KVM - if: matrix.test-strategy == 'test_node_relay' + if: matrix.test-strategy == 'test_node_relay' && contains(matrix.os, 'arm') == false run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules @@ -143,7 +234,19 @@ jobs: LOG_LEVEL: ${{ github.event.inputs.log-level || 'INFO' }} TEST_PARALLEL_HARDENING: ${{ github.event.inputs.parallel-hardening || 'true' }} REQUEST_TIMEOUT: ${{ github.event.inputs.request-timeout || '400' }} - - name: Run Docker Compose to ${{ matrix.test-strategy }} + - name: Set environment variables + run: | + if [ -n "${TEST_FIREFOX_INSTALL_LANG_PACKAGE}" ]; then + echo "TEST_FIREFOX_INSTALL_LANG_PACKAGE=${TEST_FIREFOX_INSTALL_LANG_PACKAGE}" >> $GITHUB_ENV + fi + if [ -n "${SELENIUM_ENABLE_MANAGED_DOWNLOADS}" ]; then + echo "SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS}" >> $GITHUB_ENV + fi + env: + TEST_FIREFOX_INSTALL_LANG_PACKAGE: ${{ matrix.firefox-install-lang-package }} + SELENIUM_ENABLE_MANAGED_DOWNLOADS: ${{ matrix.enable-managed-downloads }} + - name: Run Docker Compose to ${{ matrix.test-strategy }} on AMD64 + if: contains(matrix.os, 'arm') == false uses: nick-invision/retry@master with: timeout_minutes: 40 @@ -151,11 +254,22 @@ jobs: retry_wait_seconds: 60 command: | USE_RANDOM_USER_ID=${{ matrix.use-random-user }} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${{ matrix.test-strategy }} + - name: Run Docker Compose to ${{ matrix.test-strategy }} on ARM64 + if: contains(matrix.os, 'arm') == true + uses: nick-invision/retry@master + with: + timeout_minutes: 40 + max_attempts: 2 + retry_wait_seconds: 60 + command: | + USE_RANDOM_USER_ID=${{ matrix.use-random-user }} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \ + TEST_FIREFOX_INSTALL_LANG_PACKAGE=${TEST_FIREFOX_INSTALL_LANG_PACKAGE} SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} \ + make ${{ matrix.test-strategy }} - name: Upload recorded video if: matrix.test-video == true uses: actions/upload-artifact@main with: - name: "${{ matrix.test-strategy }}_artifacts" + name: "${{ matrix.test-strategy }}_artifacts_${{ matrix.os }}" path: ./tests/videos/ - name: Clean up Docker if: always() diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 21bac15a31..d77ad16fc5 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -34,19 +34,11 @@ permissions: jobs: build-and-test: name: Test K8s - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - - k8s-version: 'v1.25.16' - test-strategy: playwright_connect_grid - cluster: 'minikube' - helm-version: 'v3.11.3' - docker-version: '26.1.4' - python-version: '3.8' - test-upgrade: true - service-mesh: true - k8s-version: 'v1.26.15' test-strategy: disabled cluster: 'minikube' @@ -55,6 +47,7 @@ jobs: python-version: '3.8' test-upgrade: true service-mesh: false + os: ubuntu-24.04 - k8s-version: 'v1.27.16' test-strategy: job cluster: 'minikube' @@ -63,6 +56,7 @@ jobs: python-version: '3.9' test-upgrade: true service-mesh: true + os: ubuntu-24.04 - k8s-version: 'v1.28.15' test-strategy: deployment cluster: 'minikube' @@ -71,7 +65,8 @@ jobs: python-version: '3.10' test-upgrade: true service-mesh: true - - k8s-version: 'v1.29.12' + os: ubuntu-24.04 + - k8s-version: 'v1.29.13' test-strategy: job_https cluster: 'minikube' helm-version: 'v3.14.3' @@ -79,7 +74,8 @@ jobs: python-version: '3.11' test-upgrade: true service-mesh: false - - k8s-version: 'v1.30.8' + os: ubuntu-24.04 + - k8s-version: 'v1.30.9' test-strategy: job_hostname cluster: 'minikube' helm-version: 'v3.15.4' @@ -87,7 +83,8 @@ jobs: python-version: '3.12' test-upgrade: true service-mesh: false - - k8s-version: 'v1.31.4' + os: ubuntu-24.04 + - k8s-version: 'v1.31.5' test-strategy: deployment_https cluster: 'minikube' helm-version: 'v3.16.4' @@ -95,6 +92,16 @@ jobs: python-version: '3.13' test-upgrade: true service-mesh: false + os: ubuntu-24.04 + - k8s-version: 'v1.32.1' + test-strategy: playwright_connect_grid + cluster: 'minikube' + helm-version: 'v3.17.0' + docker-version: '26.1.4' + python-version: '3.10' + test-upgrade: true + service-mesh: true + os: ubuntu-24.04 env: CLUSTER: ${{ matrix.cluster }} KUBERNETES_VERSION: ${{ matrix.k8s-version }} @@ -199,7 +206,7 @@ jobs: if: always() uses: actions/upload-artifact@main with: - name: "${{ env.ARTIFACT_NAME }}_${{ env.CHART_FILE_NAME }}" + name: "${{ env.ARTIFACT_NAME }}_${{ env.CHART_FILE_NAME }}_${{ matrix.os }}" path: ${{ env.CHART_PACKAGE_PATH }} - name: Upload chart test artifacts if: always() diff --git a/.github/workflows/k8s-scaling-test.yml b/.github/workflows/k8s-scaling-test.yml index 7bf7b0d9e5..d0e5aaad49 100644 --- a/.github/workflows/k8s-scaling-test.yml +++ b/.github/workflows/k8s-scaling-test.yml @@ -61,37 +61,37 @@ jobs: fail-fast: false matrix: include: - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_job_count_strategy_default_in_chaos cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_job_count_strategy_default cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_deployment_count_in_chaos cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_deployment_count_with_node_max_sessions cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' - - k8s-version: 'v1.31.4' + - k8s-version: 'v1.31.5' test-strategy: test_k8s_autoscaling_deployment_count cluster: 'minikube' helm-version: 'v3.16.4' diff --git a/generate_chart_changelog.sh b/generate_chart_changelog.sh index eaa025f869..29854aa4d2 100755 --- a/generate_chart_changelog.sh +++ b/generate_chart_changelog.sh @@ -45,9 +45,9 @@ generate_changelog() { # Get the changes for each section (Added, Removed, Fixed, Changed) image_tag_changes=$(echo "Chart is using image tag $CHART_APP_VERSION" | sed -e 's/^/- /') - k8s_versions_tested=$(echo "Chart is tested on Kubernetes versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "k8s-version: '\Kv.*(?=')" | tr '\n' ',' | sed s/,/,\ /g)" | sed -e 's/^/- /') - docker_versions_tested=$(echo "Chart is tested on container runtime Docker versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "docker-version: '\K.*(?=')" | tr '\n' ',' | sed s/,/,\ /g)" | sed -e 's/^/- /') - helm_versions_tested=$(echo "Chart is tested on Helm versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "helm-version: '\Kv.*(?=')" | tr '\n' ',' | sed s/,/,\ /g)" | sed -e 's/^/- /') + k8s_versions_tested=$(echo "Chart is tested on Kubernetes versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "k8s-version: '\Kv.*(?=')" | sort -u | tr '\n' ',' | sed 's/,$//')" | sed -e 's/^/- /') + docker_versions_tested=$(echo "Chart is tested on container runtime Docker versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "docker-version: '\K.*(?=')" | sort -u | tr '\n' ',' | sed 's/,$//')" | sed -e 's/^/- /') + helm_versions_tested=$(echo "Chart is tested on Helm versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "helm-version: '\Kv.*(?=')" | sort -u | tr '\n' ',' | sed 's/,$//')" | sed -e 's/^/- /') added_changes=$(git log --pretty=format:"[\`%h\`](http://github.com/seleniumhq/docker-selenium/commit/%H) - %s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "\- feat|\- add" | sed -e 's/^/- /') removed_changes=$(git log --pretty=format:"[\`%h\`](http://github.com/seleniumhq/docker-selenium/commit/%H) - %s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "\- remove|\- deprecate|\- delete" | sed -e 's/^/- /') fixed_changes=$(git log --pretty=format:"[\`%h\`](http://github.com/seleniumhq/docker-selenium/commit/%H) - %s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "\- fix|\- bug" | sed -e 's/^/- /')