Merge branch 'submission-v4.1' into features/timestamp-embedding-parser

* submission-v4.1:
  feat: add icon and description for Stable Diffusion benchmark (#917)
  enable stable diffusion in Pixel backend (#936)
  Update tflite_settings_mtk_mt6989.pbtxt
  Update QTI backend for submission v4.1 (#13)
  Applying linter changes
  Ran make format
  Update seed and num_steps for TFLite SD task (#16)
  Addressing review comments
  Final Submission for code for Qualcomm
  Add a caption_id to coco_gen dataset (#918)
  Enable stable_diffusion tests

# Conflicts:
#	flutter/cpp/datasets/coco_gen.cc
#	mobile_back_apple/dev-utils/Makefile
#	mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt
#	mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h

.github/workflows/android-build-test.yml

@@ -100,15 +100,22 @@ jobs:
           rm /tmp/${SAMSUNG_LIB}.zip && \
           mkdir -p mobile_back_samsung/samsung/lib/internal && \
           mv /tmp/${SAMSUNG_LIB}/* mobile_back_samsung/samsung/lib/internal/
+      - name: Download QTI SDK
+        env:
+          QTI_SDK: qairt-2.25.0.240728-linux
+        run: |
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v4.1/${QTI_SDK}.zip /tmp/ && \
+          unzip /tmp/${QTI_SDK}.zip -d /tmp/${QTI_SDK} && \
+          rm /tmp/${QTI_SDK}.zip && \
+          mv /tmp/${QTI_SDK}/* mobile_back_qti/
       - name: Download QTI libraries
         env:
-          QTI_LIB: qaisw-2.20.0.240223_linux
+          QTI_LIB: StableDiffusionShared
         run: |
-          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v4.0/${QTI_LIB}.zip /tmp/ && \
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v4.1/${QTI_LIB}.zip /tmp/ && \
           unzip /tmp/${QTI_LIB}.zip -d /tmp/${QTI_LIB} && \
           rm /tmp/${QTI_LIB}.zip && \
-          mkdir -p mobile_back_qti/${QTI_LIB} && \
-          mv /tmp/${QTI_LIB}/* mobile_back_qti/${QTI_LIB}/
+          mv /tmp/${QTI_LIB}/* mobile_back_qti/cpp/backend_qti/StableDiffusionShared/
       - name: Cache bazel
         uses: actions/cache@v4
         with:
@@ -143,6 +150,7 @@ jobs:
           PERF_TEST: true
           WITH_TFLITE: 0
           WITH_QTI: 1
+          WITH_STABLEDIFFUSION: 1
         run: |
           make flutter/android/release flutter/android/apk flutter/android/test-apk
           gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-qti.apk
@@ -177,6 +185,7 @@ jobs:
           WITH_QTI: 1
           WITH_SAMSUNG: 1
           WITH_APPLE: 0
+          WITH_STABLEDIFFUSION: 1
         run: |
           make flutter/android/release
           gsutil cp flutter/build/app/outputs/bundle/release/app-release.aab $GCLOUD_BUCKET_PATH/app-release.aab

.gitignore

@@ -10,8 +10,6 @@
 /output_logs
 /datasets/output
 /datasets/downloads
-/mobile_back_qti/snpe-*
-/mobile_back_qti/qaisw-*
 *.so
 * .apk
 * .tflite
@@ -23,3 +21,7 @@ __pycache__/
 *.log
 *.iml
 *.env
+
+*.g.dart
+*.gen.dart
+*.gen.h

WORKSPACE

@@ -107,3 +107,10 @@ snpe_version_loader(
     name = "snpe_version_loader",
     workspace_dir = __workspace_dir__,
 )
+
+load("//mobile_back_qti/cpp/backend_qti/StableDiffusionShared:variables.bzl", "stable_diffusion_external_deps_shared")
+
+stable_diffusion_external_deps_shared(
+    name = "stable_diffusion_external_deps_shared",
+    workspace_dir = __workspace_dir__,
+)

flutter/android/android-docker.mk

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 The MLPerf Authors. All Rights Reserved.
+# Copyright 2020-2024 The MLPerf Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,6 +42,7 @@ flutter_common_docker_flags= \
 		--env WITH_PIXEL=${WITH_PIXEL} \
 		--env WITH_MEDIATEK=${WITH_MEDIATEK} \
 		--env proxy_bazel_args=${proxy_bazel_args} \
+		--env BAZEL_OUTPUT_ROOT_ARG="--output_user_root=/image-workdir/cache/bazel" \
 		--env OFFICIAL_BUILD=${OFFICIAL_BUILD} \
 		--env FIREBASE_CRASHLYTICS_ENABLED=${FIREBASE_CRASHLYTICS_ENABLED} \
 		--env FLUTTER_BUILD_NUMBER=${FLUTTER_BUILD_NUMBER} \

flutter/android/android.mk

@@ -20,7 +20,7 @@ ANDROID_NDK_API_LEVEL?=33
 
 flutter/android: flutter/android/libs
 flutter/android/release: flutter/check-release-env flutter/android flutter/prepare flutter/android/apk flutter/android/appbundle
-flutter/android/libs: flutter/android/libs/checksum flutter/android/libs/build flutter/android/libs/copy
+flutter/android/libs: flutter/android/libs/deps flutter/android/libs/checksum flutter/android/libs/build flutter/android/libs/copy
 # run `make flutter/android/apk` before `flutter/android/test-apk`
 flutter/android/test-apk: flutter/android/test-apk/main flutter/android/test-apk/helper
 
@@ -35,6 +35,10 @@ else
 	@echo "Skip checksum validation"
 endif
 
+.PHONY: flutter/android/libs/deps
+flutter/android/libs/deps:
+	${backend_qti_libs_deps}
+
 .PHONY: flutter/android/libs/build
 flutter/android/libs/build:
 	bazel ${BAZEL_OUTPUT_ROOT_ARG} ${proxy_bazel_args} ${sonar_bazel_startup_options} \

flutter/android/docker/Dockerfile

@@ -30,6 +30,7 @@ RUN curl --proto '=https' -L https://github.com/bazelbuild/bazelisk/releases/dow
     chmod +x /usr/local/bin/bazel
 
 ENV ANDROID_SDK_ROOT=/opt/android
+ENV ANDROID_HOME=/opt/android
 WORKDIR $ANDROID_SDK_ROOT/cmdline-tools
 # sdkmanager expects to be placed into `$ANDROID_SDK_ROOT/cmdline-tools/tools`
 RUN curl --proto '=https' -L https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip | jar x && \
@@ -73,3 +74,94 @@ RUN mkdir $ANDROID_SDK_HOME && \
 
 # Git repo will be mounted at '/image-workdir/project'
 RUN git config --global --add safe.directory /image-workdir/project
+
+# Add 32-bit support since the adb in our sdk's are 32-bit binaries
+RUN dpkg --add-architecture i386
+ARG DEBIAN_FRONTEND=noninteractive
+ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn
+
+# build-essential: for rest-kit(rest-kit needed for crad-repo)
+# file : used by ndk to determine if host is 32 or 64 bit
+# libtbb-dev: used for parallelization of postprocessing for pose-estimation
+# libcanberra-gtk-module, libgtk2.0-dev: required by opencv runtime
+# libstdc++6:i386: to add support for 32 bit binaries.
+# libxtst6 : required to run ide(e.g intellij) inside docker
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential \
+    cmake \
+    curl \
+    git \
+    emacs \
+    file \
+    less \
+    libcanberra-gtk-module \
+    libgtk2.0-dev \
+    libtbb-dev \
+    libstdc++6:i386 \
+    libxtst6
+
+# pkg-config: required by opencv runtime
+# software-properties-common: for curl
+# xterm : to make resize available for avoiding line wrapping
+# zip : required when sourcing snpe envsetup
+RUN apt-get update && apt-get install --no-install-recommends -y mc \
+    meld \
+    pkg-config \
+    python3-dev \
+    software-properties-common \
+    sudo \
+    tmux \
+    tree \
+    unzip \
+    vim \
+    wget \
+    xterm \
+    zip \
+    ffmpeg \
+    libjpeg-dev \
+    zlib1g-dev && \
+    apt-get clean autoclean
+
+# Get cmake-3.19.3
+# apt has 3.10.2 as latest version, so remove it
+RUN apt-get remove --no-install-recommends -y cmake
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-x86_64.sh -O /opt/cmake-3.19.3-Linux-x86_64.sh && \
+    chmod +x /opt/cmake-3.19.3-Linux-x86_64.sh && mkdir -p /opt/cmake && \
+    bash /opt/cmake-3.19.3-Linux-x86_64.sh --skip-license --prefix=/opt/cmake && \
+    rm -rf /opt/cmake-3.19.3-Linux-x86_64.sh
+# Add CMAKE into PATH
+ENV PATH "/opt/cmake/bin:${PATH}"
+
+# OpenCV
+ENV CMAKE_TOOLCHAIN_FILE "${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake"
+ENV ANDROID_ABI "arm64-v8a"
+ENV API_LEVEL "31"
+ENV ANDROID_TOOLCHAIN_NAME "aarch64-linux-android-4.9"
+ARG COMMON_CMAKE_OPTIONS="-DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=release \
+                          -DBUILD_ZLIB=ON -DWITH_FFMPEG=ON -DBUILD_TESTS=OFF \
+                          -DWITH_TBB=ON -DBUILD_PERF_TESTS=OFF -DWITH_IPP=OFF \
+                          -DWITH_OPENEXR=ON -DWITH_JASPER=ON -WITH_PNG=ON -DBUILD_FAT_JAVA_LIB=OFF WITH_IMGCODEC=ON"
+# Ref: https://docs.opencv.org/3.4.0/d7/d9f/tutorial_linux_install.html
+RUN wget -q https://github.com/opencv/opencv/archive/3.4.7/opencv-3.4.7.tar.gz -O /tmp/3.4.7.tar.gz && \
+    tar -C /tmp -xvf /tmp/3.4.7.tar.gz  && \
+    # First build for arm-android
+    cd /tmp/opencv-3.4.7 && mkdir -p /opt/opencv-3.4.7_android/ release_android && cd release_android && \
+    cmake -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" \
+          -DANDROID_NDK="${ANDROID_NDK}" \
+          -DANDROID_HOME="${ANDROID_HOME}" \
+          -DANDROID_STL=c++_shared  \
+          -DBUILD_ANDROID_PROJECTS=OFF \
+          -DANDROID_NATIVE_API_LEVEL="${API_LEVEL}" \
+          -DANDROID_ABI="${ANDROID_ABI}" \
+          -DWITH_CUDA=ON -DWITH_OPENCL=ON \
+          $COMMON_CMAKE_OPTIONS \
+          -D WITH_ITT=OFF \
+          -DCMAKE_INSTALL_PREFIX=/opt/opencv-3.4.7_android/ .. && \
+    make -j16  && \
+    sudo make -j16 install  && cp -rf ./3rdparty/ /opt/opencv-3.4.7_android/
+
+# Set the variables to be used for actual app development/build
+ENV ANDROID_SYSROOT "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot"
+ENV ANDROID_PLATFORM "${API_LEVEL}"
+ENV ANDROID_PLATFORM_TOOLS "${ANDROID_HOME}/platform-tools"
+ENV PATH "${ANDROID_PLATFORM_TOOLS}:${PATH}"
+ENV LD_LIBRARY_PATH "/usr/local/lib/:${LD_LIBRARY_PATH}"

flutter/assets/tasks.pbtxt

@@ -229,18 +229,18 @@ task {
     type: COCOGEN
     full {
       name: "COCO validation set for Stable Diffusion"
-      input_path: "https://github.com/anhappdev/tmp/releases/download/5/coco_gen_test.tfrecord"
+      input_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-dataset/coco_gen_test.tfrecord"
       groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512.tflite"
     }
     lite {
       name: "COCO validation set for Stable Diffusion"
-      input_path: "https://github.com/anhappdev/tmp/releases/download/5/coco_gen_full.tfrecord"
+      input_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-dataset/coco_gen_full.tfrecord"
       groundtruth_path: ""
     }
     tiny {
       name: "COCO validation set for Stable Diffusion"
-      input_path: "https://github.com/anhappdev/tmp/releases/download/2/coco_gen.tfrecord"
-      groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512.tflite"
+      input_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-dataset/coco_gen_test.tfrecord"
+      groundtruth_path: ""
     }
   }
   model {

flutter/cpp/binary/cmdline.mk

@@ -15,7 +15,11 @@
 
 include flutter/cpp/binary/cmdline-docker.mk
 
-cmdline/android/bins/release: cmdline/android/bins/build cmdline/android/bins/copy
+cmdline/android/bins/release: cmdline/android/libs/deps cmdline/android/bins/build cmdline/android/bins/copy
+
+.PHONY: cmdline/android/libs/deps
+cmdline/android/libs/deps:
+	${backend_qti_libs_deps}
 
 .PHONY: cmdline/android/bins/build
 cmdline/android/bins/build:

flutter/cpp/datasets/coco_gen.cc

@@ -81,8 +81,6 @@ void CocoGen::UnloadSamplesFromRam(
   }
 }
 
-#define OUTPUT_WIDTH 512
-#define OUTPUT_HEIGHT 512
 #define OUTPUT_SIZE 512 * 512 * 3
 std::vector<uint8_t> CocoGen::ProcessOutput(const int sample_idx,
                                             const std::vector<void*>& outputs) {
@@ -102,22 +100,20 @@ std::vector<uint8_t> CocoGen::ProcessOutput(const int sample_idx,
     }
   }
 
-  auto total_byte = output_format_[0].size * GetByte(output_format_[0]);
-  backend_->ConvertOutputs(total_byte, OUTPUT_WIDTH, OUTPUT_HEIGHT,
-                           output_pixels.data());
-
-  std::string raw_output_filename =
-      raw_output_dir_ + "/output_" + std::to_string(sample_idx) + ".rgb8";
-  dump_output_pixels(output_pixels, raw_output_filename);
-
   if (!output_pixels.empty()) {
     sample_ids_.insert(sample_idx);
     CaptionRecord* record = samples_.at(sample_idx).get();
-    LOG(INFO) << "caption: " << record->get_caption();
-    caption_map[sample_idx] = record->get_caption();
+    LOG(INFO) << "caption_id: " << record->get_caption_id()
+              << " caption_text: " << record->get_caption_text();
+    caption_id_map[sample_idx] = record->get_caption_id();
+    caption_text_map[sample_idx] = record->get_caption_text();
     output_pixels_map[sample_idx] = output_pixels;
     attention_mask_map[sample_idx] = record->get_attention_mask_vector();
     input_ids_map[sample_idx] = record->get_input_ids_vector();
+    std::string raw_output_filename = raw_output_dir_ + "/caption_id_" +
+                                      std::to_string(record->get_caption_id()) +
+                                      ".rgb8";
+    dump_output_pixels(output_pixels, raw_output_filename);
     return output_pixels;
   } else {
     return std::vector<uint8_t>();
@@ -130,7 +126,8 @@ float CocoGen::ComputeAccuracy() {
   float total_score = 0.0f;
   float total_samples = static_cast<float>(sample_ids_.size());
   for (int sample_idx : sample_ids_) {
-    std::string caption = caption_map[sample_idx];
+    int caption_id = caption_id_map[sample_idx];
+    std::string caption_text = caption_text_map[sample_idx];
     std::vector<int32_t> input_ids = input_ids_map[sample_idx];
     std::vector<int32_t> attention_mask = attention_mask_map[sample_idx];
     std::vector<uint8_t> output_pixels = output_pixels_map[sample_idx];
@@ -140,8 +137,8 @@ float CocoGen::ComputeAccuracy() {
     }
     float score =
         score_predictor_.predict(attention_mask, input_ids, pixel_values);
-    LOG(INFO) << "sample_idx: " << sample_idx << " caption: " << caption
-              << " score: " << score;
+    LOG(INFO) << "sample_idx: " << sample_idx << " caption_id: " << caption_id
+              << " caption_text: " << caption_text << " score: " << score;
     total_score += score;
   }
   float avg_score = total_score / total_samples;

flutter/cpp/datasets/coco_gen.h

@@ -83,7 +83,8 @@ class CocoGen : public Dataset {
   std::set<int> sample_ids_;
   bool isModelFound;
   std::string raw_output_dir_;
-  std::unordered_map<int, std::string> caption_map;
+  std::unordered_map<int, int> caption_id_map;
+  std::unordered_map<int, std::string> caption_text_map;
   std::unordered_map<int, std::vector<uint8_t>> output_pixels_map;
   std::unordered_map<int, std::vector<int32_t>> attention_mask_map;
   std::unordered_map<int, std::vector<int32_t>> input_ids_map;

flutter/cpp/datasets/coco_gen_utils/CLIP_Model_to_TFLite.ipynb

@@ -46,7 +46,7 @@
       "source": [
         "SAVED_MODEL_DIR = './clip_model'\n",
         "TFLITE_MODEL_PATH = './clip_model.tflite'\n",
-        "MODEL_NAME = \"openai/clip-vit-base-patch32\""
+        "MODEL_NAME = \"openai/clip-vit-large-patch14\""
       ],
       "metadata": {
         "id": "eOxB3zL_33tq"

flutter/cpp/datasets/coco_gen_utils/generate_tfrecords.py

@@ -56,9 +56,10 @@ def download_image(url, file_path):
     print(f"Downloaded image to {file_path}")
 
 
-def serialize_example(caption, input_ids, attention_mask, file_name, clip_score):
+def serialize_example(caption_id, caption, input_ids, attention_mask, file_name, clip_score):
   """Creates a tf.train.Example message ready to be written to a file."""
   feature = {
+    'caption_id': tf.train.Feature(int64_list=tf.train.Int64List(value=caption_id)),
     'caption': tf.train.Feature(bytes_list=tf.train.BytesList(value=[caption.encode()])),
     'input_ids': tf.train.Feature(int64_list=tf.train.Int64List(value=input_ids)),
     'attention_mask': tf.train.Feature(int64_list=tf.train.Int64List(value=attention_mask)),
@@ -87,6 +88,7 @@ def main():
   with tf.io.TFRecordWriter(args.output_tfrecord, options='ZLIB') as writer:
     total = len(df)
     for idx, row in df.iterrows():
+      caption_id = row['id']
       caption = row['caption']
       file_name = row['file_name']
       coco_url = row['coco_url']
@@ -104,6 +106,7 @@ def main():
       clip_score = outputs.logits_per_image.numpy().flatten().tolist()
 
       example = serialize_example(
+        caption_id=[int(caption_id)],
         caption=caption,
         input_ids=input_ids,
         attention_mask=attention_mask,