From 5309430141a1c7b283564a37a399781866d531f5 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Wed, 4 Oct 2023 21:47:35 +0200 Subject: [PATCH 1/5] Fix results validation --- .../model_hub_tests/models_hub_common/test_convert_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/model_hub_tests/models_hub_common/test_convert_model.py b/tests/model_hub_tests/models_hub_common/test_convert_model.py index 56a402bced99fc..50b63e0200957e 100644 --- a/tests/model_hub_tests/models_hub_common/test_convert_model.py +++ b/tests/model_hub_tests/models_hub_common/test_convert_model.py @@ -61,7 +61,7 @@ def infer_ov_model(self, ov_model, inputs, ie_device): def compare_results(self, fw_outputs, ov_outputs): assert len(fw_outputs) == len(ov_outputs), \ - "Different number of outputs between TensorFlow and OpenVINO:" \ + "Different number of outputs between framework and OpenVINO:" \ " {} vs. {}".format(len(fw_outputs), len(ov_outputs)) fw_eps = 5e-2 @@ -73,13 +73,13 @@ def compare_results(self, fw_outputs, ov_outputs): "OpenVINO outputs does not contain tensor with name {}".format(out_name) cur_ov_res = ov_outputs[out_name] print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}") - is_ok = compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) + is_ok = is_ok or compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) else: for i in range(len(ov_outputs)): cur_fw_res = fw_outputs[i] cur_ov_res = ov_outputs[i] print(f"fw_res: {cur_fw_res};\n ov_res: {cur_ov_res}") - is_ok = compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) + is_ok = is_ok or compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) assert is_ok, "Accuracy validation failed" def teardown_method(self): From e65ee445b513377c7040211233ba0699918b579b Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Wed, 4 Oct 2023 22:21:34 +0200 Subject: [PATCH 2/5] Remove pt quantization --- tests/model_hub_tests/torch_tests/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/model_hub_tests/torch_tests/requirements.txt b/tests/model_hub_tests/torch_tests/requirements.txt index f0a013faadbcc3..406607b69d2d89 100644 --- a/tests/model_hub_tests/torch_tests/requirements.txt +++ b/tests/model_hub_tests/torch_tests/requirements.txt @@ -11,7 +11,6 @@ sacremoses sentencepiece datasets pyctcdecode -pytorch-quantization<=2.1.3 --extra-index-url https://pypi.ngc.nvidia.com protobuf soundfile pandas From 68fef6243c739e7d36203bcef98682767c557346 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Wed, 4 Oct 2023 22:55:11 +0200 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Roman Kazantsev --- tests/model_hub_tests/models_hub_common/test_convert_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/model_hub_tests/models_hub_common/test_convert_model.py b/tests/model_hub_tests/models_hub_common/test_convert_model.py index 50b63e0200957e..68e58a0658defc 100644 --- a/tests/model_hub_tests/models_hub_common/test_convert_model.py +++ b/tests/model_hub_tests/models_hub_common/test_convert_model.py @@ -73,13 +73,13 @@ def compare_results(self, fw_outputs, ov_outputs): "OpenVINO outputs does not contain tensor with name {}".format(out_name) cur_ov_res = ov_outputs[out_name] print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}") - is_ok = is_ok or compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) + is_ok = is_ok and compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) else: for i in range(len(ov_outputs)): cur_fw_res = fw_outputs[i] cur_ov_res = ov_outputs[i] print(f"fw_res: {cur_fw_res};\n ov_res: {cur_ov_res}") - is_ok = is_ok or compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) + is_ok = is_ok and compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps) assert is_ok, "Accuracy validation failed" def teardown_method(self): From f78fe3d774c8aa7f5e4d5de0a520f9125991dcfa Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Wed, 4 Oct 2023 23:38:51 +0200 Subject: [PATCH 4/5] Refactor requirements --- .github/workflows/linux.yml | 2 +- tests/model_hub_tests/torch_tests/requirements.txt | 2 ++ tests/model_hub_tests/torch_tests/requirements_secondary.txt | 5 ----- 3 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 tests/model_hub_tests/torch_tests/requirements_secondary.txt diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 052f8ebfcd6081..a0a924a5a7719d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1156,8 +1156,8 @@ jobs: - name: Install PyTorch tests requirements run: | + python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt echo "Available storage:" df -h env: diff --git a/tests/model_hub_tests/torch_tests/requirements.txt b/tests/model_hub_tests/torch_tests/requirements.txt index 406607b69d2d89..41d25874a47572 100644 --- a/tests/model_hub_tests/torch_tests/requirements.txt +++ b/tests/model_hub_tests/torch_tests/requirements.txt @@ -14,3 +14,5 @@ pyctcdecode protobuf soundfile pandas +git+https://github.com/facebookresearch/detectron2.git@1a4df4d954089534daebbd7a0a45f08ece87056e +natten diff --git a/tests/model_hub_tests/torch_tests/requirements_secondary.txt b/tests/model_hub_tests/torch_tests/requirements_secondary.txt deleted file mode 100644 index 803c3c4b839006..00000000000000 --- a/tests/model_hub_tests/torch_tests/requirements_secondary.txt +++ /dev/null @@ -1,5 +0,0 @@ --c ../../constraints.txt -# This file contains requirements dependednt from modules in requirements.txt -# get immutable commit to be able to use pip cache -git+https://github.com/facebookresearch/detectron2.git@1a4df4d954089534daebbd7a0a45f08ece87056e -natten From 3289e9251a13939d8aa5497b08f74b77fdd3aaff Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Thu, 5 Oct 2023 10:30:47 +0200 Subject: [PATCH 5/5] Revert "Refactor requirements" This reverts commit f78fe3d774c8aa7f5e4d5de0a520f9125991dcfa. --- .github/workflows/linux.yml | 2 +- tests/model_hub_tests/torch_tests/requirements.txt | 2 -- tests/model_hub_tests/torch_tests/requirements_secondary.txt | 5 +++++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 tests/model_hub_tests/torch_tests/requirements_secondary.txt diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a0a924a5a7719d..052f8ebfcd6081 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1156,8 +1156,8 @@ jobs: - name: Install PyTorch tests requirements run: | - python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt echo "Available storage:" df -h env: diff --git a/tests/model_hub_tests/torch_tests/requirements.txt b/tests/model_hub_tests/torch_tests/requirements.txt index 41d25874a47572..406607b69d2d89 100644 --- a/tests/model_hub_tests/torch_tests/requirements.txt +++ b/tests/model_hub_tests/torch_tests/requirements.txt @@ -14,5 +14,3 @@ pyctcdecode protobuf soundfile pandas -git+https://github.com/facebookresearch/detectron2.git@1a4df4d954089534daebbd7a0a45f08ece87056e -natten diff --git a/tests/model_hub_tests/torch_tests/requirements_secondary.txt b/tests/model_hub_tests/torch_tests/requirements_secondary.txt new file mode 100644 index 00000000000000..803c3c4b839006 --- /dev/null +++ b/tests/model_hub_tests/torch_tests/requirements_secondary.txt @@ -0,0 +1,5 @@ +-c ../../constraints.txt +# This file contains requirements dependednt from modules in requirements.txt +# get immutable commit to be able to use pip cache +git+https://github.com/facebookresearch/detectron2.git@1a4df4d954089534daebbd7a0a45f08ece87056e +natten