diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 40ee93dd08b9f2..2959728e39dee7 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -105,8 +105,7 @@ jobs: python3 -m pip install ./openvino_tokenizers-* working-directory: ${{ env.INSTALL_WHEELS_DIR }} - - name: Install PyTorch tests requirements for precommit - if: ${{ inputs.model_scope == 'precommit' }} + - name: Install PyTorch tests requirements run: | python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_pytorch env: @@ -116,7 +115,7 @@ jobs: if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope1' }} run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_${{ inputs.model_scope }}_tests.html --self-contained-html -v -n 2 -k "TestTimmConvertModel or TestTorchHubConvertModel or TestEdsrConvertModel" + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_${{ inputs.model_scope }}_tests.html --self-contained-html -v -n 4 -k "TestTimmConvertModel or TestTorchHubConvertModel or TestEdsrConvertModel" env: TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }} TEST_DEVICE: CPU diff --git a/tests/constraints.txt b/tests/constraints.txt index 616aea79c82153..775d3287c061a1 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -27,11 +27,6 @@ kornia==0.7.0 networkx<=3.3 flax<=0.9.0 -# PyTorch tests -timm==1.0.8 -transformers~=4.44.0 -easyocr==1.7.2 - --extra-index-url https://download.pytorch.org/whl/cpu torch~=2.4.1; platform_system != "Darwin" or platform_machine != "x86_64" torch~=2.2.0; platform_system == "Darwin" and platform_machine == "x86_64" diff --git a/tests/model_hub_tests/pytorch/test_timm.py b/tests/model_hub_tests/pytorch/test_timm.py index 8c11d32517c4d6..0c151e804720ca 100644 --- a/tests/model_hub_tests/pytorch/test_timm.py +++ b/tests/model_hub_tests/pytorch/test_timm.py @@ -14,10 +14,10 @@ def filter_timm(timm_list: list) -> list: unique_models = dict() filtered_list = [] - ignore_list = ["base", "atto", "femto", "xxtiny", "xxsmall", "xxs", "pico", - "xtiny", "xmall", "xs", "nano", "tiny", "s", "mini", "small", - "lite", "medium", "m", "big", "large", "l", "xlarge", "xl", - "huge", "xxlarge", "gigantic", "giant", "enormous"] + ignore_list = ["base", "zepto", "atto", "femto", "xxtiny", "xxsmall", "xxs", + "pico", "xtiny", "xmall", "xs", "nano", "tiny", "s", "mini", + "small", "lite", "medium", "m", "big", "large", "l", "xlarge", + "xl", "huge", "xxlarge", "gigantic", "giant", "enormous"] ignore_set = set(ignore_list) for name in sorted(timm_list): if "x_" in name: @@ -47,7 +47,7 @@ def filter_timm(timm_list: list) -> list: class TestTimmConvertModel(TestTorchConvertModel): - @retry(3, exceptions=(OSError,), delay=1) + @retry(3, exceptions=(OSError,), delay=5) def load_model(self, model_name, model_link): m = timm.create_model(model_name, pretrained=True) cfg = timm.get_pretrained_cfg(model_name) diff --git a/tests/model_hub_tests/pytorch/timm_models b/tests/model_hub_tests/pytorch/timm_models index 42119a2c2f80d2..9732569b648245 100644 --- a/tests/model_hub_tests/pytorch/timm_models +++ b/tests/model_hub_tests/pytorch/timm_models @@ -13,7 +13,7 @@ cait_s36_384.fb_dist_in1k,None cait_xs24_384.fb_dist_in1k,None cait_xxs24_224.fb_dist_in1k,None cait_xxs36_224.fb_dist_in1k,None -coat_tiny.in1k,None,xfail_export,Requested None inlined input +coat_tiny.in1k,None coatnet_bn_0_rw_224.sw_in1k,None coatnet_nano_rw_224.sw_in1k,None coatnet_rmlp_1_rw2_224.sw_in12k,None @@ -30,9 +30,11 @@ convnext_atto_ols.a2_in1k,None convnext_base.clip_laion2b,None convnext_large_mlp.clip_laion2b_augreg,None convnext_tiny_hnf.a2h_in1k,None +convnext_zepto_rms.ra4_e3600_r224_in1k,None +convnext_zepto_rms_ols.ra4_e3600_r224_in1k,None convnextv2_base.fcmae,None -crossvit_15_dagger_240.in1k,None,xfail_export,Unsupported on torch 2.4 -crossvit_base_240.in1k,None,xfail_export,Unsupported on torch 2.4 +crossvit_15_dagger_240.in1k,None +crossvit_base_240.in1k,None cs3darknet_focus_m.c2ns_in1k,None cs3darknet_m.c2ns_in1k,None cs3darknet_x.c2ns_in1k,None @@ -103,10 +105,10 @@ edgenext_xx_small.in1k,None efficientformer_l1.snap_dist_in1k,None efficientformer_l3.snap_dist_in1k,None efficientformer_l7.snap_dist_in1k,None -efficientformerv2_l.snap_dist_in1k,None,xfail_export,Requested None inlined input -efficientformerv2_s0.snap_dist_in1k,None,xfail_export,Requested None inlined input -efficientformerv2_s1.snap_dist_in1k,None,xfail_export,Requested None inlined input -efficientformerv2_s2.snap_dist_in1k,None,xfail_export,Requested None inlined input +efficientformerv2_l.snap_dist_in1k,None +efficientformerv2_s0.snap_dist_in1k,None +efficientformerv2_s1.snap_dist_in1k,None +efficientformerv2_s2.snap_dist_in1k,None efficientnet_b0.ra4_e3600_r224_in1k,None efficientnet_b1.ft_in1k,None efficientnet_b1_pruned.in1k,None @@ -147,7 +149,7 @@ eva02_large_patch14_clip_224.merged2b,None fastvit_ma36.apple_dist_in1k,None fastvit_mci0.apple_mclip,None fastvit_mci1.apple_mclip,None -fastvit_mci2.apple_mclip,None,xfail_trace,Accuracy validation failed +fastvit_mci2.apple_mclip,None,xfail,Accuracy validation failed fastvit_s12.apple_dist_in1k,None fastvit_sa12.apple_dist_in1k,None fastvit_sa24.apple_dist_in1k,None @@ -171,7 +173,7 @@ gcresnext50ts.ch_in1k,None gcvit_base.in1k,None gernet_s.idstcv_in1k,None ghostnet_100.in1k,None -ghostnetv2_100.in1k,None,xfail_export,Requested None inlined input +ghostnetv2_100.in1k,None gmixer_24_224.ra3_in1k,None gmlp_s16_224.ra3_in1k,None halo2botnet50ts_256.a1h_in1k,None @@ -194,16 +196,17 @@ hgnetv2_b5.ssld_stage1_in22k_in1k,None hgnetv2_b6.ssld_stage1_in22k_in1k,None hiera_base_224.mae,None hiera_base_plus_224.mae,None -hrnet_w18_small.gluon_in1k,None,xfail_export,Requested None inlined input -hrnet_w18_small_v2.gluon_in1k,None,xfail_export,Requested None inlined input -hrnet_w18_ssld.paddle_in1k,None,xfail_export,Requested None inlined input -hrnet_w30.ms_in1k,None,xfail_export,Requested None inlined input -hrnet_w32.ms_in1k,None,xfail_export,Requested None inlined input -hrnet_w40.ms_in1k,None,xfail_export,Requested None inlined input -hrnet_w44.ms_in1k,None,xfail_export,Requested None inlined input -hrnet_w48.ms_in1k,None,xfail_export,Requested None inlined input -hrnet_w48_ssld.paddle_in1k,None,xfail_export,Requested None inlined input -hrnet_w64.ms_in1k,None,xfail_export,Requested None inlined input +hiera_small_abswin_256.sbb2_e200_in12k,None,xfail_export,Unsupported aten._upsample_bicubic2d_aa.default +hrnet_w18_small.gluon_in1k,None +hrnet_w18_small_v2.gluon_in1k,None +hrnet_w18_ssld.paddle_in1k,None +hrnet_w30.ms_in1k,None +hrnet_w32.ms_in1k,None +hrnet_w40.ms_in1k,None +hrnet_w44.ms_in1k,None +hrnet_w48.ms_in1k,None +hrnet_w48_ssld.paddle_in1k,None +hrnet_w64.ms_in1k,None inception_next_base.sail_in1k,None inception_resnet_v2.tf_ens_adv_in1k,None inception_v3.gluon_in1k,None @@ -223,10 +226,17 @@ legacy_seresnext101_32x4d.in1k,None legacy_seresnext26_32x4d.in1k,None legacy_seresnext50_32x4d.in1k,None legacy_xception.tf_in1k,None -levit_128.fb_dist_in1k,None -levit_128s.fb_dist_in1k,None +levit_128.fb_dist_in1k,None,xfail,Accuracy validation failed +levit_128s.fb_dist_in1k,None,xfail,Accuracy validation failed levit_conv_128.fb_dist_in1k,None levit_conv_128s.fb_dist_in1k,None +mambaout_base.in1k,None +mambaout_base_plus_rw.sw_e150_in12k,None +mambaout_base_short_rw.sw_e500_in1k,None +mambaout_base_tall_rw.sw_e500_in1k,None +mambaout_base_wide_rw.sw_e500_in1k,None +mambaout_kobe.in1k,None +mambaout_small_rw.sw_e450_in1k,None maxvit_base_tf_224.in1k,None maxvit_nano_rw_256.sw_in1k,None maxvit_rmlp_base_rw_224.sw_in12k,None @@ -243,6 +253,7 @@ mobilenetv1_100h.ra4_e3600_r224_in1k,None mobilenetv2_050.lamb_in1k,None mobilenetv2_110d.ra_in1k,None mobilenetv2_120d.ra_in1k,None +mobilenetv3_large_150d.ra4_e3600_r256_in1k,None mobilenetv3_rw.rmsp_in1k,None mobilenetv3_small_050.lamb_in1k,None mobilenetv4_conv_aa_large.e230_r384_in12k,None @@ -288,6 +299,7 @@ pvt_v2_b2_li.in1k,None pvt_v2_b3.in1k,None pvt_v2_b4.in1k,None pvt_v2_b5.in1k,None +rdnet_base.nv_in1k,None regnetv_040.ra3_in1k,None regnetx_002.pycls_in1k,None regnetx_004_tv.tv2_in1k,None @@ -361,7 +373,7 @@ resnet34d.ra2_in1k,None resnet50.a1_in1k,None resnet50_clip.openai,None resnet50_clip_gap.openai,None -resnet50_gn.a1h_in1k,None,xfail_export,Unsupported on torch 2.4 +resnet50_gn.a1h_in1k,None resnet50c.gluon_in1k,None resnet50d.a1_in1k,None resnet50s.gluon_in1k,None @@ -403,6 +415,8 @@ resnext50_32x4d.a1_in1k,None resnext50d_32x4d.bt_in1k,None rexnet_100.nav_in1k,None rexnetr_200.sw_in12k,None +sam2_hiera_base_plus,None +sam2_hiera_tiny.r224,None samvit_base_patch16.sa1b,None sebotnet33ts_256.a1h_in1k,None sehalonet33ts.ra2_in1k,None @@ -440,8 +454,18 @@ swinv2_base_window8_256.ms_in1k,None swinv2_cr_small_224.sw_in1k,None swinv2_cr_tiny_ns_224.sw_in1k,None test_byobnet.r160_in1k,None +test_convnext.r160_in1k,None +test_convnext2.r160_in1k,None +test_convnext3.r160_in1k,None test_efficientnet.r160_in1k,None +test_efficientnet_evos.r160_in1k,None +test_efficientnet_gn.r160_in1k,None +test_efficientnet_ln.r160_in1k,None +test_nfnet.r160_in1k,None +test_resnet.r160_in1k,None test_vit.r160_in1k,None +test_vit2.r160_in1k,None +test_vit3.r160_in1k,None tf_efficientnet_b0.aa_in1k,None tf_efficientnet_b1.aa_in1k,None tf_efficientnet_b2.aa_in1k,None @@ -451,9 +475,9 @@ tf_efficientnet_b5.aa_in1k,None tf_efficientnet_b6.aa_in1k,None tf_efficientnet_b7.aa_in1k,None tf_efficientnet_b8.ap_in1k,None -tf_efficientnet_cc_b0_4e.in1k,None,xfail_trace,Unsupported dynamic weights shape -tf_efficientnet_cc_b0_8e.in1k,None,xfail_trace,Unsupported dynamic weights shape -tf_efficientnet_cc_b1_8e.in1k,None,xfail_trace,Unsupported dynamic weights shape +tf_efficientnet_cc_b0_4e.in1k,None,xfail,Unsupported dynamic weights shape +tf_efficientnet_cc_b0_8e.in1k,None,xfail,Unsupported dynamic weights shape +tf_efficientnet_cc_b1_8e.in1k,None,xfail,Unsupported dynamic weights shape tf_efficientnet_el.in1k,None tf_efficientnet_em.in1k,None tf_efficientnet_es.in1k,None @@ -479,7 +503,7 @@ tinynet_b.in1k,None tinynet_c.in1k,None tinynet_d.in1k,None tinynet_e.in1k,None -tnt_s_patch16_224,None,xfail_export,Requested None inlined input +tnt_s_patch16_224,None tresnet_m.miil_in1k,None tresnet_v2_l.miil_in21k,None twins_pcpvt_base.in1k,None @@ -508,22 +532,23 @@ vit_base_patch32_224.augreg_in1k,None vit_base_patch32_clip_224.datacompxl,None vit_base_patch32_clip_quickgelu_224.metaclip_2pt5b,None vit_base_patch8_224.augreg2_in21k_ft_in1k,None -vit_base_r50_s16_224.orig_in21k,None,xfail_export,Unsupported on torch 2.4 +vit_base_r50_s16_224.orig_in21k,None vit_betwixt_patch16_reg1_gap_256.sbb_in1k,None -vit_betwixt_patch16_reg4_gap_256.sbb_in12k,None +vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12k,None vit_betwixt_patch16_rope_reg4_gap_256.sbb_in1k,None,xfail_trace,Argument shapes are inconsistent vit_betwixt_patch32_clip_224.tinyclip_laion400m,None vit_huge_patch14_224.mae,None vit_huge_patch14_gap_224.in1k_ijepa,None +vit_intern300m_patch14_448.ogvl_dist,None vit_large_patch14_clip_224.datacompxl,None vit_large_patch14_clip_quickgelu_224.dfn2b,None -vit_large_r50_s32_224.augreg_in21k,None,xfail_export,Unsupported on torch 2.4 +vit_large_r50_s32_224.augreg_in21k,None vit_little_patch16_reg1_gap_256.sbb_in12k,None vit_little_patch16_reg4_gap_256.sbb_in1k,None vit_medium_patch16_gap_240.sw_in12k,None vit_medium_patch16_reg1_gap_256.sbb_in1k,None vit_medium_patch16_reg4_gap_256.sbb_in12k,None -vit_mediumd_patch16_reg4_gap_256.sbb_in12k,None +vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12k,None vit_mediumd_patch16_rope_reg1_gap_256.sbb_in1k,None,xfail_trace,Argument shapes are inconsistent vit_pwee_patch16_reg1_gap_256.sbb_in1k,None vit_relpos_base_patch16_224.sw_in1k,None @@ -531,20 +556,22 @@ vit_relpos_base_patch16_clsgap_224.sw_in1k,None vit_relpos_base_patch32_plus_rpn_256.sw_in1k,None vit_relpos_medium_patch16_cls_224.sw_in1k,None vit_relpos_medium_patch16_rpn_224.sw_in1k,None -vit_small_r26_s32_224.augreg_in21k,None,xfail_export,Unsupported on torch 2.4 +vit_small_r26_s32_224.augreg_in21k,None vit_so400m_patch14_siglip_224.webli,None vit_so400m_patch14_siglip_gap_224.pali_mix,None,skip,Access to model google/paligemma-3b-mix-224-jax is restricted +vit_so400m_patch16_siglip_256.webli_i18n,None +vit_so400m_patch16_siglip_gap_256.webli_i18n,None vit_srelpos_small_patch16_224.sw_in1k,None vit_tiny_r_s16_p8_224.augreg_in21k,None vit_wee_patch16_reg1_gap_256.sbb_in1k,None vit_xsmall_patch16_clip_224.tinyclip_yfcc15m,None vitamin_base_224.datacomp1b_clip,None,xfail,RuntimeError Error in loading state_dict for VisionTransformer vitamin_large2_224.datacomp1b_clip,None -volo_d1_224.sail_in1k,None,xfail_export,Requested None inlined input for op aten.index.Tensor -volo_d2_224.sail_in1k,None,xfail_export,Requested None inlined input for op aten.index.Tensor -volo_d3_224.sail_in1k,None,xfail_export,Requested None inlined input for op aten.index.Tensor -volo_d4_224.sail_in1k,None,xfail_export,Requested None inlined input for op aten.index.Tensor -volo_d5_224.sail_in1k,None,xfail_export,Requested None inlined input for op aten.index.Tensor +volo_d1_224.sail_in1k,None +volo_d2_224.sail_in1k,None +volo_d3_224.sail_in1k,None +volo_d4_224.sail_in1k,None +volo_d5_224.sail_in1k,None wide_resnet101_2.tv2_in1k,None wide_resnet50_2.racm_in1k,None xception41.tf_in1k,None diff --git a/tests/model_hub_tests/pytorch/torchvision_models b/tests/model_hub_tests/pytorch/torchvision_models index 3bb2d9423446bd..a67d47d333dc08 100644 --- a/tests/model_hub_tests/pytorch/torchvision_models +++ b/tests/model_hub_tests/pytorch/torchvision_models @@ -3,9 +3,9 @@ convnext_base,none convnext_large,none convnext_small,none convnext_tiny,none -deeplabv3_mobilenet_v3_large,none,xfail_export,Requested None inlined input -deeplabv3_resnet101,none,xfail_export,Requested None inlined input -deeplabv3_resnet50,none,xfail_export,Requested None inlined input +deeplabv3_mobilenet_v3_large,none +deeplabv3_resnet101,none +deeplabv3_resnet50,none densenet121,none densenet161,none densenet169,none @@ -25,13 +25,13 @@ fasterrcnn_mobilenet_v3_large_320_fpn,none,skip,Not supported fasterrcnn_mobilenet_v3_large_fpn,none,skip,Not supported fasterrcnn_resnet50_fpn,none,skip,Not supported fasterrcnn_resnet50_fpn_v2,none,skip,Not supported -fcn_resnet101,none,xfail_export,Requested None inlined input -fcn_resnet50,none,xfail_export,Requested None inlined input +fcn_resnet101,none +fcn_resnet50,none fcos_resnet50_fpn,none,skip,Not supported googlenet,none inception_v3,none keypointrcnn_resnet50_fpn,none,skip,Not supported -lraspp_mobilenet_v3_large,none,xfail_export,Requested None inlined input +lraspp_mobilenet_v3_large,none maskrcnn_resnet50_fpn,none,skip,Not supported maskrcnn_resnet50_fpn_v2,none,skip,Not supported maxvit_t,none diff --git a/tests/requirements_pytorch b/tests/requirements_pytorch index 40e1f6f66f52e8..d4e59c40f56f61 100644 --- a/tests/requirements_pytorch +++ b/tests/requirements_pytorch @@ -32,7 +32,7 @@ sacremoses==0.1.1 sentencepiece==0.2.0 soundfile==0.12.1 super-image==0.1.7; python_version < "3.12" -timm==1.0.8 +timm==1.0.11 torchaudio==2.4.1; platform_system != "Darwin" or platform_machine != "x86_64" torchaudio==2.2.2; platform_system == "Darwin" and platform_machine == "x86_64" wheel==0.44.0