Implement GHA performance tests in nightly validation to compare read_model and ovc.convert

.github/workflows/job_tensorflow_hub_performance_models_tests.yml

@@ -0,0 +1,116 @@
+name: TensorFlow Hub Performance Models tests
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        description: 'Machine on which the tests would run'
+        type: string
+        required: true
+      container:
+        description: 'JSON to be converted to the value of the "container" configuration for the job'
+        type: string
+        required: false
+        default: '{"image": null}'
+      event:
+        description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs'
+        type: string
+        required: true
+
+jobs:
+  TensorFlow_Hub_Performance_Models_Tests:
+    name: TensorFlow Hub Performance Models tests
+    timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 25 }}
+    runs-on: ${{ inputs.runner }}
+    container: ${{ fromJSON(inputs.container) }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
+      OPENVINO_REPO: ${{ github.workspace }}/openvino
+      INSTALL_DIR: ${{ github.workspace }}/install
+      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
+      MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
+    steps:
+
+      - name: Check sudo
+        if: ${{ runner.os == 'Linux' }}
+        run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi
+
+      - name: Download OpenVINO package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_package
+          path: ${{ env.INSTALL_DIR }}
+
+      - name: Download OpenVINO tests package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tests
+          path: ${{ env.INSTALL_TEST_DIR }}
+
+      # Needed as ${{ github.workspace }} is not working correctly when using Docker
+      - name: Setup Variables
+        run: |
+          echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
+          echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
+          echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
+          echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV"
+
+      - name: Extract OpenVINO packages
+        run: |
+          pushd ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+          popd
+
+          pushd ${INSTALL_TEST_DIR}
+            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+          popd
+
+      - name: Fetch setup_python action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/actions/setup_python/action.yml
+          sparse-checkout-cone-mode: false
+          path: 'openvino'
+
+      - name: Install dependencies
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          # install git (required to build pip deps from the sources)
+          sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates
+
+      - name: Setup Python 3.11
+        uses: ./openvino/.github/actions/setup_python
+        with:
+          version: '3.11'
+          should-setup-pip-paths: 'false'
+          self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}
+
+      - name: Install OpenVINO Python wheels
+        run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
+
+      - name: Install TF Hub tests requirements
+        run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt
+
+      - name: Install Hub Performance tests requirements
+        run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/requirements.txt
+
+      - name: Performance Hub Tests
+        run: |
+          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_performance.html --self-contained-html -v
+        env:
+          TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'nightly'}}
+          TEST_DEVICE: CPU
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: test-results-tensorflow-hub-performance-models
+          path: |
+            ${{ env.INSTALL_TEST_DIR }}/TEST*.html
+          if-no-files-found: 'error'

.github/workflows/linux.yml

@@ -730,7 +730,7 @@ jobs:
   Overall_Status:
     name: ci/gha_overall_status
     needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests,
-            CPU_Functional_Tests, TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers]
+            CPU_Functional_Tests, TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers, TensorFlow_Hub_Performance_Models_Tests]
     if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:
@@ -741,3 +741,18 @@ jobs:
             contains(needs.*.result, 'cancelled')
           }}
         run: exit 1
+
+  # TODO: Switch back to self-hosted runners
+  # container:
+  #   image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+  #   volumes:
+  #     - /mount:/mount
+  TensorFlow_Hub_Performance_Models_Tests:
+    name: TensorFlow Hub Performance Models tests
+    if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test ||
+      fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test
+    needs: [ Build, Smart_CI ]
+    uses: ./.github/workflows/job_tensorflow_hub_performance_models_tests.yml
+    with:
+      runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }}
+      event: ${{ github.event_name }}

tests/model_hub_tests/models_hub_common/constants.py

@@ -10,11 +10,13 @@
 loading and heating and includes measurement only one of 2 models - got through convert and read_model.
 Both "converted" and "read_model" modes will be 2 * runtime_measure_duration 
 '''
-runtime_measure_duration = os.environ.get('RUNTIME_MEASURE_DURATION', '60')
+precommit_runtime_measure_duration = os.environ.get('PRECOMMIT_RUNTIME_MEASURE_DURATION', '60')
+nightly_runtime_measure_duration = os.environ.get('NIGHTLY_RUNTIME_MEASURE_DURATION', '15')
 '''
 @brief Time in seconds of heating before measurement
 '''
-runtime_heat_duration = os.environ.get('RUNTIME_HEAT_DURATION', '5')
+precommit_runtime_heat_duration = os.environ.get('PRECOMMIT_RUNTIME_HEAT_DURATION', '5')
+nigtly_runtime_heat_duration = os.environ.get('NIGHTLY_RUNTIME_HEAT_DURATION', '5')
 
 
 tf_hub_cache_dir = os.environ.get('TFHUB_CACHE_DIR',

tests/model_hub_tests/models_hub_common/test_performance_model.py

@@ -120,14 +120,12 @@ def get_read_model(self, model_path: str):
         core = ov.Core()
         return core.read_model(model=model_path)
 
-    def heat_hardware(self, ov_model, inputs) -> None:
-        _, heat_n_repeats, _ = utils.measure(utils.nano_secs(const.runtime_heat_duration), ov_model, (inputs,))
+    def heat_hardware(self, ov_model, inputs, conf) -> None:
+        _, heat_n_repeats, _ = utils.measure(conf.runtime_heat_duration, ov_model, (inputs,))
         print('heat done in {} repeats'.format(heat_n_repeats))
 
-    def measure_inference(self, ov_model, inputs) -> ModelResults:
-        time_slices, infer_n_repeats, real_runtime = utils.measure(utils.nano_secs(const.runtime_measure_duration),
-                                                                   ov_model,
-                                                                   (inputs,))
+    def measure_inference(self, ov_model, inputs, conf) -> ModelResults:
+        time_slices, infer_n_repeats, real_runtime = utils.measure(conf.runtime_measure_duration, ov_model, (inputs,))
         print('measurement done in {} repeats'.format(infer_n_repeats))
         infer_throughput = float(infer_n_repeats * (10 ** 9)) / real_runtime
         infer_mean_time_ns = np.mean(time_slices)
@@ -141,15 +139,15 @@ def measure_inference(self, ov_model, inputs) -> ModelResults:
         results.infer_variance = infer_variance
         return results
 
-    def infer_model(self, ov_model, inputs) -> ModelResults:
-        self.heat_hardware(ov_model, inputs)
-        return self.measure_inference(ov_model, inputs)
+    def infer_model(self, ov_model, inputs, conf) -> ModelResults:
+        self.heat_hardware(ov_model, inputs, conf)
+        return self.measure_inference(ov_model, inputs, conf)
 
     def compile_model(self, model, ie_device):
         core = ov.Core()
         return core.compile_model(model, ie_device)
 
-    def __run(self, model_name, model_link, ie_device):
+    def __run(self, model_name, model_link, ie_device, conf):
         results = Results()
         results.status = None
         try:
@@ -168,11 +166,11 @@ def __run(self, model_name, model_link, ie_device):
             results.status = Status.INFER_CONVERTED_MODEL
             results.converted_model_results = utils.call_with_timer('Infer converted model',
                                                                     self.infer_model,
-                                                                    (converted_model, inputs))
+                                                                    (converted_model, inputs, conf))
             results.status = Status.INFER_READ_MODEL
             results.read_model_results = utils.call_with_timer('Infer read model',
                                                                self.infer_model,
-                                                               (read_model, inputs))
+                                                               (read_model, inputs, conf))
 
             infer_time_ratio = (results.converted_model_results.infer_mean_time /
                                 results.read_model_results.infer_mean_time)
@@ -196,10 +194,10 @@ def __run(self, model_name, model_link, ie_device):
                                                                          ex_type=ex_type.__name__, ex_value=ex_value)
         return results
 
-    def run(self, model_name, model_link, ie_device):
+    def run(self, model_name, model_link, ie_device, conf):
         self.result = Results()
         t0 = time.time()
-        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device], model_name,
+        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device, conf], model_name,
                                           self.infer_timeout)
         t1 = time.time()
         utils.print_stat('test run time {} secs', (t1 - t0))

tests/model_hub_tests/models_hub_common/utils.py

@@ -32,6 +32,28 @@ def get_models_list(file_name: str):
     return models
 
 
+def get_skipped_model_links(filename: str):
+    links = set()
+    if not os.path.exists(filename):
+        return links
+    with open(filename) as f:
+        for model_info in f:
+            model_info = model_info.strip()
+            model_name, model_link = model_info.split(',')
+            links.add(model_link)
+    return links
+
+
+def get_models_list_not_skipped(model_list_file: str, skip_list_file: str):
+    skipped_links = get_skipped_model_links(skip_list_file)
+    not_skipped_models = []
+    for model_name, model_link, mark, reason in get_models_list(model_list_file):
+        if model_link in skipped_links:
+            continue
+        not_skipped_models.append((model_name, model_link, mark, reason))
+    return not_skipped_models
+
+
 def compare_two_tensors(ov_res, fw_res, eps):
     is_ok = True
     if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):