ci: test unified app with all backends (#938)

* Build unified test app * Add env BROWSERSTACK_DEVICES * Add android-test-browserstack.yml * Move test-android-apk-unified * Fix test-android-apk-unified * Retry step trigger app automate * Add expected throughput and accuracy for _kPixel9Pro * Update expected throughput for _kS24 * Add expected throughput and accuracy for _kS10Plus * Add expected throughput and accuracy for _kS24Ultra * Add expected throughput and accuracy for _kM32 * Update retry@v2 to retry@v3 * Update expected_throughput * Remove single backend APK test * Fix issue with not showing error message * Display device status * Change run order * Update expected values * Update browserstack-app-automate.sh
mlcommons · Dec 31, 2024 · f04e397 · f04e397
1 parent 06666e1
commit f04e397
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 130 deletions.
diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml
@@ -167,6 +167,20 @@ jobs:
           make flutter/android/release flutter/android/apk flutter/android/test-apk
           gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-samsung.apk
           gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-samsung.apk
+      - name: Build and upload test APK for all backends
+        env:
+          OFFICIAL_BUILD: false
+          PERF_TEST: true
+          WITH_TFLITE: 1
+          WITH_PIXEL: 1
+          WITH_MEDIATEK: 1
+          WITH_QTI: 1
+          WITH_SAMSUNG: 1
+          WITH_APPLE: 0
+        run: |
+          make flutter/android/release flutter/android/apk flutter/android/test-apk
+          gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-unified.apk
+          gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-unified.apk
       - name: Build Android release APK with all backends
         env:
           OFFICIAL_BUILD: true
@@ -231,113 +245,13 @@ jobs:
           retention-days: 28
           if-no-files-found: error
 
-  test-android-apk-tflite:
-    needs: build-android-apk
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up authentication for Google Cloud SDK
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '>= 363.0.0'
-          project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole)
-        run: |
-          gcloud firebase test android run \
-            --type instrumentation \
-            --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \
-            --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \
-            --timeout 30m \
-            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
-            --device model=oriole,version=31,locale=en,orientation=portrait \
-            --client-details=buildNumber=${{ github.run_number }}
-
-  test-android-apk-pixel:
-    needs: build-android-apk
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up authentication for Google Cloud SDK
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '>= 363.0.0'
-          project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole)
-        run: |
-          gcloud firebase test android run \
-            --type instrumentation \
-            --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \
-            --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \
-            --timeout 30m \
-            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
-            --device model=oriole,version=32,locale=en,orientation=portrait \
-            --client-details=buildNumber=${{ github.run_number }}
-
-  test-android-apk-qti:
-    needs: build-android-apk
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up authentication for Google Cloud SDK
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '>= 363.0.0'
-          project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [Samsung Galaxy S22 Ultra](b0q)
-        run: |
-          gcloud firebase test android run \
-            --type instrumentation \
-            --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \
-            --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \
-            --timeout 30m \
-            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
-            --device model=b0q,version=33,locale=en,orientation=portrait \
-            --client-details=buildNumber=${{ github.run_number }}
-
-  test-android-apk-mtk:
-    needs: build-android-apk
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up authentication for Google Cloud SDK
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
-        with:
-          version: '>= 363.0.0'
-          project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [OnePlus DN2103](OP515BL1)
-        run: |
-          gcloud firebase test android run \
-            --type instrumentation \
-            --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \
-            --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \
-            --timeout 30m \
-            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
-            --device model=OP515BL1,version=33,locale=en,orientation=portrait \
-            --client-details=buildNumber=${{ github.run_number }}
-
-  test-android-apk-samsung:
+  test-android-apk-unified:
     needs: build-android-apk
     runs-on: ubuntu-22.04
-    timeout-minutes: 30
+    timeout-minutes: 60
     env:
-      MAIN_APK_NAME: test-main-samsung-${{ github.run_number }}.apk
-      HELPER_APK_NAME: test-helper-samsung-${{ github.run_number }}.apk
+      MAIN_APK_NAME: test-main-unified-${{ github.run_number }}.apk
+      HELPER_APK_NAME: test-helper-unified-${{ github.run_number }}.apk
     steps:
       - uses: actions/checkout@v4
       - name: Set up authentication for Google Cloud SDK
@@ -350,8 +264,8 @@ jobs:
           version: '>= 363.0.0'
       - name: Download test APK
         run: |
-          gsutil cp $GCLOUD_BUCKET_PATH/test-main-samsung.apk /tmp/$MAIN_APK_NAME
-          gsutil cp $GCLOUD_BUCKET_PATH/test-helper-samsung.apk /tmp/$HELPER_APK_NAME
+          gsutil cp $GCLOUD_BUCKET_PATH/test-main-unified.apk /tmp/$MAIN_APK_NAME
+          gsutil cp $GCLOUD_BUCKET_PATH/test-helper-unified.apk /tmp/$HELPER_APK_NAME
       - name: Upload main app
         run: |
           curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \
@@ -364,19 +278,25 @@ jobs:
           -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \
           -F "file=@/tmp/$HELPER_APK_NAME" \
           -F "custom_id=$HELPER_APK_NAME"
-      - name: Trigger App Automate for [Samsung Galaxy S24]
+      - name: Trigger App Automate
         env:
           BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }}
           BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }}
           BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }}
           BROWSERSTACK_BUILD_TAG: ${{ github.run_number }}
+          BROWSERSTACK_DEVICES: >-
+            ["Samsung Galaxy M32-11.0",
+            "Samsung Galaxy S24-14.0",
+            "Samsung Galaxy S24 Ultra-14.0",
+            "Samsung Galaxy Tab S10 Plus-14.0",
+            "Google Pixel 9 Pro-15.0"]
         run: |
           bash .github/workflows/scripts/browserstack-app-automate.sh
 
   deploy-playstore:
     needs:
       - build-android-apk
-      - test-android-apk-tflite
+      - test-android-apk-unified
     runs-on: ubuntu-22.04
     # if: github.ref == 'refs/heads/master'
     steps:

diff --git a/.github/workflows/ios-build-test.yml b/.github/workflows/ios-build-test.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Build iOS integration tests
         run: |
           cd flutter && flutter --no-version-check build ios --simulator integration_test/first_test.dart
-      - uses: nick-fields/retry@v2
+      - uses: nick-fields/retry@v3
         name: Run iOS integration tests
         with:
           timeout_minutes: 45

diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh
@@ -2,10 +2,7 @@
 
 # This script is used to trigger a build on BrowserStack App Automate and monitor its status.
 
-set -e
-
 # Build parameters
-DEVICES='["Samsung Galaxy S24-14.0"]'
 PROJECT="mobile-app-build-290400"
 DEVICE_LOGS=true
 RETRY_INTERVAL=10
@@ -19,15 +16,18 @@ CREDENTIALS="${BROWSERSTACK_CREDENTIALS:-}"
 APP="${BROWSERSTACK_APP:-}"
 TEST_SUITE="${BROWSERSTACK_TEST_SUITE:-}"
 BUILD_TAG="${BROWSERSTACK_BUILD_TAG:-}"
+DEVICES="${BROWSERSTACK_DEVICES:-}"
 
 # Validate required environment variables
 if [[ -z "$CREDENTIALS" ]]; then
   echo "Error: Environment variable BROWSERSTACK_CREDENTIALS must be set in the format 'user_name:access_key'."
   exit 1
 fi
 
-if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" ]]; then
-  echo "Error: Environment variables BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE and BROWSERSTACK_BUILD_TAG must be set."
+if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" || -z "$DEVICES" ]]; then
+  echo "Error: Environment variables"\
+  "BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE, BROWSERSTACK_BUILD_TAG and BROWSERSTACK_DEVICES"\
+  "must be set."
   exit 1
 fi
 
@@ -49,18 +49,33 @@ trigger_build() {
 
   if [[ "$build_id" == "null" || -z "$build_id" ]]; then
     echo "Failed to trigger the build. Response: $response"
-    exit 1
+    return 1
+  else
+    echo "$build_id"
+    return 0
   fi
-
-  echo "$build_id"
 }
 
 # Function to check build status
 check_build_status() {
   local build_id=$1
   local response=$(curl -s -u "$CREDENTIALS" -X GET "$STATUS_URL/$build_id")
   local status=$(echo "$response" | jq -r '.status')
+
   echo "$(date +'%Y-%m-%d %H:%M:%S') Build Status: $status"
+
+  # Display device status
+  if [[ "$status" != "running" ]]; then
+    echo "$response" | jq -r '
+      .devices[] |
+      "Device: " + .device +
+      ", OS Version: " + .os_version +
+      ", Duration: " + (.sessions[0].duration | tostring) + "s" +
+      ", Status: " + (.sessions[0].status)
+    '
+  fi
+
+  # Display build status
   if [[ "$status" == "passed" ]]; then
     echo "Build completed successfully!"
     exit 0
@@ -71,7 +86,11 @@ check_build_status() {
 }
 
 # Main
-BUILD_ID=$(trigger_build)
+if ! BUILD_ID=$(trigger_build); then
+  echo "Trigger build failed. Message: $BUILD_ID"
+  exit 1
+fi
+
 echo "Build triggered successfully. Build ID: $BUILD_ID"
 echo "See the build status at: https://app-automate.browserstack.com/dashboard/v2/builds/$BUILD_ID"
 

diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart
@@ -19,13 +19,13 @@ key: <accelerator> OR <accelerator>|<backendName>
 
 const Map<String, Interval> _imageClassificationV2 = {
   'cpu': Interval(min: 0.88, max: 0.91),
-  'npu': Interval(min: 0.88, max: 0.91),
+  'npu': Interval(min: 0.69, max: 0.91),
   'tpu': Interval(min: 0.88, max: 0.91),
   'ane': Interval(min: 0.69, max: 0.91),
   'cpu&gpu&ane': Interval(min: 0.69, max: 0.91),
   'snpe_dsp': Interval(min: 0.88, max: 0.91),
   'psnpe_dsp': Interval(min: 0.88, max: 0.91),
-  'neuron-mdla': Interval(min: 0.79, max: 0.81),
+  'neuron-mdla': Interval(min: 0.79, max: 0.91),
   'samsung_npu': Interval(min: 0.99, max: 1.0),
 };
 
@@ -38,7 +38,7 @@ const Map<String, Interval> _objectDetection = {
   'cpu&gpu&ane': Interval(min: 0.45, max: 0.46),
   'snpe_dsp': Interval(min: 0.32, max: 0.35),
   'psnpe_dsp': Interval(min: 0.32, max: 0.35),
-  'neuron': Interval(min: 0.28, max: 0.31),
+  'neuron': Interval(min: 0.28, max: 0.35),
   'samsung_npu': Interval(min: 0.36, max: 0.39),
 };
 
@@ -78,7 +78,7 @@ const Map<String, Interval> _superResolution = {
   'snpe_dsp': Interval(min: 0.32, max: 0.35),
   'psnpe_dsp': Interval(min: 0.32, max: 0.35),
   'neuron': Interval(min: 0.32, max: 0.35),
-  'samsung_npu': Interval(min: 0.08, max: 0.11),
+  'samsung_npu': Interval(min: 0.08, max: 0.12),
 };
 
 // TODO (anhappdev): update expected accuracy for stable diffusion