Rework test targets and artifacts (#4720)

Move test targets to its own home in config json. Improve test artifacts packaging to support android. b/386420709
youtube · Jan 22, 2025 · 79df06d · 79df06d
1 parent e649afc
commit 79df06d
Show file tree

Hide file tree

Showing 27 changed files with 502 additions and 244 deletions.
diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml
@@ -4,6 +4,15 @@ inputs:
   targets:
     description: "List of ninja targets for Cobalt build."
     required: true
+  upload_on_host_test_artifacts:
+    description: "Indicates if on-host test artifacts should be uploaded."
+    required: true
+  upload_on_device_test_artifacts:
+    description: "Indicates if on-device test artifacts should be uploaded."
+    required: true
+  test_artifacts_key:
+    description: "Artifact key used to store on-host test artifacts."
+    required: true
 runs:
   using: "composite"
   steps:
@@ -38,13 +47,23 @@ runs:
           cd src
           gn args --list --short --overrides-only out/${{ matrix.platform }}_${{ matrix.config }}
         shell: bash
+      - name: Ninja build test targets
+        if: inputs.upload_on_host_test_artifacts == 'true' || inputs.upload_on_device_test_artifacts == 'true'
+        env:
+          # TODO(b/382508397): Replace hardcoded list with dynamically generated one.
+          TEST_TARGETS_JSON_FILE: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json
+        run: |
+          set -x
+          cd src
+          time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(cat "${TEST_TARGETS_JSON_FILE}" | jq -cr '.test_targets | join(" ")')
+        shell: bash
       - name: Ninja build
         env:
           TARGETS: ${{ inputs.targets }}
         run: |
           set -x
           cd src
-          time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(echo "${TARGETS}" | tr -d '"')
+          time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} ${TARGETS}
         shell: bash
       - name: Archive Android APKs
         if: startsWith(matrix.platform, 'android') && matrix.config == 'qa'
@@ -55,3 +74,12 @@ runs:
             src/out/${{ matrix.platform }}_qa/apks/*.apk
             src/out/${{ matrix.platform }}_qa/*_apk/*.apk
             src/out/${{ matrix.platform }}_qa/gen/build_info.json
+      - name: Upload Test Artifacts
+        if: inputs.upload_on_host_test_artifacts == 'true' || inputs.upload_on_device_test_artifacts == 'true'
+        uses: ./src/.github/actions/upload_test_artifacts
+        with:
+          test_artifacts_key: ${{ inputs.test_artifacts_key }}
+          upload_on_host_test_artifacts: ${{ inputs.upload_on_host_test_artifacts }}
+          upload_on_device_test_artifacts: ${{ inputs.upload_on_device_test_artifacts }}
+          # TODO(b/382508397): Replace hardcoded list with dynamically generated one.
+          test_targets_json_file: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json
diff --git a/.github/actions/on_host_tests/action.yaml b/.github/actions/on_host_tests/action.yaml
@@ -27,7 +27,7 @@ runs:
 
         mkdir -p ${test_dir}
         cd ${test_dir}
-        tar xvf ../test_artifacts.tar.xz
+        tar xvf ../test_artifacts.tar.gz
     - name: Run Tests
       id: run-tests
       shell: bash
@@ -38,17 +38,19 @@ runs:
         env
 
         # Explicitly point to libraries in extracted dir.
-        LD_LIBRARY_PATH="${test_dir}/starboard"
-        LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${test_dir}"
+        LD_LIBRARY_PATH="${test_dir}/out/${{ matrix.platform}}_${{ matrix.config }}/starboard"
+        LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${test_dir}/out/${{ matrix.platform}}_${{ matrix.config }}"
         export LD_LIBRARY_PATH
 
         # Make results dir available to the archiving step below.
         results_dir="${GITHUB_WORKSPACE}/results"
         echo "results_dir=${results_dir}" >> $GITHUB_ENV
 
         failed_suites=""
-        cd ${test_dir}
-        for test_binary in $(ls {*tests,nplb}); do
+        cd "${test_dir}"
+
+        for test_binary_path in $(cat out/${{ matrix.platform}}_${{ matrix.config }}/test_targets.json | jq -cr '.executables | join(" ")'); do
+          test_binary=$(basename "${test_binary_path}")
           echo "Running tests for suite: ${test_binary}"
 
           test_filter="*"
@@ -60,7 +62,13 @@ runs:
           echo "Test filter evaluated to: ${test_filter}"
 
           xml_path="${results_dir}/shard_${{ matrix.shard }}/${test_binary}_testoutput.xml"
-          /usr/bin/xvfb-run -a --server-args="${XVFB_SERVER_ARGS}" "./${test_binary}" --test-launcher-shard-index=${{ matrix.shard }} --test-launcher-total-shards=${{ inputs.num_gtest_shards }} --gtest_filter="${test_filter}" --gtest_output="xml:${xml_path}" || {
+          # TODO: Investigate test_runner.py
+          /usr/bin/xvfb-run -a --server-args="${XVFB_SERVER_ARGS}" \
+              ./"${test_binary_path}" \
+              --test-launcher-shard-index=${{ matrix.shard }} \
+              --test-launcher-total-shards=${{ inputs.num_gtest_shards }} \
+              --gtest_output="xml:${xml_path}" \
+              --gtest_filter="${test_filter}" || {
             # Set exit code on failure.
             failed_suites="${failed_suites} ${test_binary}"
           }

diff --git a/.github/actions/upload_test_artifacts/action.yaml b/.github/actions/upload_test_artifacts/action.yaml
@@ -1,23 +1,26 @@
 name: Test Artifact Upload
 description: Uploads test archives for on-device and on-host tests.
 inputs:
-  on_host:
+  upload_on_host_test_artifacts:
     description: "Indicates if on-host test artifacts should be uploaded."
     required: true
-  on_device:
+  upload_on_device_test_artifacts:
     description: "Indicates if on-device test artifacts should be uploaded."
     required: true
   test_artifacts_key:
     description: "Artifact key used to store on-host test artifacts."
     required: true
+  test_targets_json_file:
+    description: "File containing the test target json."
+    required: true
 runs:
   using: "composite"
   steps:
     - name: Set up Cloud SDK
-      if: inputs.on_device == 'true'
+      if: inputs.upload_on_device_test_artifacts == 'true'
       uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
     - name: Upload Android Test Artifacts to GCS
-      if: inputs.on_device == 'true'
+      if: inputs.upload_on_device_test_artifacts == 'true'
       env:
         WORKFLOW: ${{ github.workflow }}
       run: |
@@ -28,23 +31,21 @@ runs:
       shell: bash
 
     - name: Create On-Host Test Artifacts Archive
-      if: inputs.on_host == 'true'
+      if: inputs.upload_on_host_test_artifacts == 'true'
       run: |
         set -x
-        cd src/out/${{ matrix.platform }}_${{ matrix.config }}
-        test_deps_file="test.deps"
-        for test_binary in $(ls {nplb,*tests}); do
-          echo $test_binary
-          if [ -f "${test_binary}.runtime_deps" ]; then
-            cat "${test_binary}.runtime_deps" >> "${test_deps_file}"
-          fi
-        done
-        tar cvf - -T "${test_deps_file}" | xz -T0 -1 -z - > "${GITHUB_WORKSPACE}/test_artifacts.tar.xz"
+        mkdir ${GITHUB_WORKSPACE}/artifacts
+        cd src/
+        ./cobalt/build/archive_test_artifacts.py \
+          --source out/${{ matrix.platform }}_${{ matrix.config }}/ \
+          --destination ${GITHUB_WORKSPACE}/artifacts \
+          --platform ${{ matrix.platform }} \
+          --targets $(cat "${{ inputs.test_targets_json_file }}" | jq -cr '.test_targets | join(",")')
       shell: bash
     - name: Upload On-Host Test Artifacts Archive
-      if: inputs.on_host == 'true'
+      if: inputs.upload_on_host_test_artifacts == 'true'
       uses: actions/upload-artifact@v4
       with:
         name: ${{ inputs.test_artifacts_key }}
-        path: test_artifacts.tar.xz
+        path: artifacts/*
         retention-days: 3
diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json
@@ -1,28 +1,19 @@
 {
-    "docker_service": "linux",
-    "platforms": [
-        "android-arm"
-    ],
-    "targets": [
-        "base_unittests",
-        "sql_unittests",
-        "net_unittests",
-        "url_unittests",
-        "ipc_tests",
-        "mojo_unittests",
-        "gpu_unittests",
-        "gin_unittests",
-        "blink_unittests",
-        "media_unittests",
-        "content_shell",
-        "system_webview_apk",
-        "system_webview_shell_apk",
-        "cobalt:gn_all"
-    ],
-    "includes": [
-        {
-        "name":"arm",
-        "platform":"android-arm"
-        }
-    ]
+  "docker_service": "linux",
+  "platforms": [
+    "android-arm"
+  ],
+  "test_on_device": true,
+  "targets": [
+    "content_shell",
+    "system_webview_apk",
+    "system_webview_shell_apk",
+    "cobalt:gn_all"
+  ],
+  "includes": [
+    {
+      "name": "arm",
+      "platform": "android-arm"
+    }
+  ]
 }
diff --git a/.github/config/android-arm64.json b/.github/config/android-arm64.json
@@ -1,28 +1,19 @@
 {
-    "docker_service": "linux",
-    "platforms": [
-        "android-arm64"
-    ],
-    "targets": [
-        "base_unittests",
-        "sql_unittests",
-        "net_unittests",
-        "url_unittests",
-        "ipc_tests",
-        "mojo_unittests",
-        "gpu_unittests",
-        "gin_unittests",
-        "blink_unittests",
-        "media_unittests",
-        "content_shell",
-        "system_webview_apk",
-        "system_webview_shell_apk",
-        "cobalt:gn_all"
-    ],
-    "includes": [
-        {
-        "name":"arm64",
-        "platform":"android-arm64"
-        }
-    ]
+  "docker_service": "linux",
+  "platforms": [
+    "android-arm64"
+  ],
+  "test_on_device": true,
+  "targets": [
+    "content_shell",
+    "system_webview_apk",
+    "system_webview_shell_apk",
+    "cobalt:gn_all"
+  ],
+  "includes": [
+    {
+      "name": "arm64",
+      "platform": "android-arm64"
+    }
+  ]
 }
diff --git a/.github/config/android-x86.json b/.github/config/android-x86.json
@@ -1,28 +1,18 @@
 {
-    "docker_service": "linux",
-    "platforms": [
-        "android-x86"
-    ],
-    "targets": [
-        "base_unittests",
-        "sql_unittests",
-        "net_unittests",
-        "url_unittests",
-        "ipc_tests",
-        "mojo_unittests",
-        "gpu_unittests",
-        "gin_unittests",
-        "blink_unittests",
-        "media_unittests",
-        "content_shell",
-        "system_webview_apk",
-        "system_webview_shell_apk",
-        "cobalt:gn_all"
-    ],
-    "includes": [
-        {
-        "name":"x86",
-        "platform":"android-x86"
-        }
-    ]
+  "docker_service": "linux",
+  "platforms": [
+    "android-x86"
+  ],
+  "targets": [
+    "content_shell",
+    "system_webview_apk",
+    "system_webview_shell_apk",
+    "cobalt:gn_all"
+  ],
+  "includes": [
+    {
+      "name": "x86",
+      "platform": "android-x86"
+    }
+  ]
 }
diff --git a/.github/config/chromium_android-arm.json b/.github/config/chromium_android-arm.json
@@ -1,27 +1,27 @@
 {
-    "docker_service": "linux",
-    "platforms": [
-        "chromium_android-arm"
-    ],
-    "targets": [
-        "base_unittests",
-        "sql_unittests",
-        "net_unittests",
-        "url_unittests",
-        "ipc_tests",
-        "mojo_unittests",
-        "gpu_unittests",
-        "gin_unittests",
-        "blink_unittests",
-        "media_unittests",
-        "content_shell",
-        "system_webview_apk",
-        "system_webview_shell_apk"
-    ],
-    "includes": [
-        {
-        "name":"arm",
-        "platform":"chromium_android-arm"
-        }
-    ]
+  "docker_service": "linux",
+  "platforms": [
+    "chromium_android-arm"
+  ],
+  "targets": [
+    "base_unittests",
+    "sql_unittests",
+    "net_unittests",
+    "url_unittests",
+    "ipc_tests",
+    "mojo_unittests",
+    "gpu_unittests",
+    "gin_unittests",
+    "blink_unittests",
+    "media_unittests",
+    "content_shell",
+    "system_webview_apk",
+    "system_webview_shell_apk"
+  ],
+  "includes": [
+    {
+      "name": "arm",
+      "platform": "chromium_android-arm"
+    }
+  ]
 }