Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

upgrade protobuf to 3.20.2 and onnx to 1.13 #14279

Merged
merged 33 commits into from
Jan 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
79c6d37
upgrade protobuf
mszhanyi Jan 13, 2023
a926c22
gcmanifest.json
mszhanyi Jan 13, 2023
488b7f4
update
mszhanyi Jan 13, 2023
6e488ad
add one log
mszhanyi Jan 13, 2023
79e9f3b
update artifacts
mszhanyi Jan 13, 2023
b5a7f28
update protobuf 3.20.2
mszhanyi Jan 13, 2023
d73d925
minor update
mszhanyi Jan 13, 2023
3b45872
update
mszhanyi Jan 16, 2023
56f7d8d
update
mszhanyi Jan 16, 2023
aa0ea4a
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
mszhanyi Jan 17, 2023
a92be8e
update cmake.patch
mszhanyi Jan 18, 2023
0dd4358
revert eigen
mszhanyi Jan 18, 2023
0c1f723
fix tvm
mszhanyi Jan 18, 2023
0fb4e5d
fix rocm
mszhanyi Jan 19, 2023
42c2f85
-Wno-unused-parameter for eager
mszhanyi Jan 19, 2023
c47f71f
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
mszhanyi Jan 19, 2023
bb09daa
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
mszhanyi Jan 24, 2023
a5d45bf
c-api-cpu
mszhanyi Jan 24, 2023
a05dc54
update cmakelist
mszhanyi Jan 25, 2023
1b845e9
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
mszhanyi Jan 25, 2023
a7e1ad3
update one 3.20.1 to 3.20.2
mszhanyi Jan 25, 2023
9c503d4
minor fix
mszhanyi Jan 25, 2023
9f42643
diables test
mszhanyi Jan 26, 2023
b07d385
merge with main
mszhanyi Jan 26, 2023
544524a
Revert "fix tvm"
mszhanyi Jan 26, 2023
1845b4e
Revert "fix rocm"
mszhanyi Jan 26, 2023
9043929
Revert "-Wno-unused-parameter for eager"
mszhanyi Jan 26, 2023
48270e4
Revert "c-api-cpu"
mszhanyi Jan 27, 2023
39efc46
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
mszhanyi Jan 27, 2023
a14f672
revert change in cmakelist
mszhanyi Jan 28, 2023
96843a4
update protobuf patch
mszhanyi Jan 29, 2023
1d21267
update onnx.patch
mszhanyi Jan 29, 2023
70d671b
update onnx.patch
mszhanyi Jan 29, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion cgmanifests/generated/cgmanifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "5a5f8a5935762397aa68429b5493084ff970f774",
"commitHash": "9b7bca2a723ff94edcd007d93b5d0cf1838591dc",
"repositoryUrl": "https://github.com/onnx/onnx.git"
},
"comments": "git submodule at cmake/external/onnx"
Expand Down Expand Up @@ -318,6 +318,16 @@
"comments": "onnx_tensorrt"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "a20c65f2cd549445fda907f7b83894c8eb7427d6",
"repositoryUrl": "https://github.com/protocolbuffers/protobuf.git"
},
"comments": "protobuf"
}
},
{
"component": {
"type": "git",
Expand Down
4 changes: 2 additions & 2 deletions cmake/deps.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf36
microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becdd918fcc872039ba.zip;fd119887d0d17c37adf1fc227b054befa28158ad
mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5
onnx;https://github.com/onnx/onnx/archive/1ba785612a79fe749aa1e478336e534743372639.zip;6c22c32fabafac81107186ea6bedb31741a3142d
onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.13.0.zip;8dda5079cdb5a134b08b0c73f4592a6404fc2dc6
snnn marked this conversation as resolved.
Show resolved Hide resolved
#use the commit where it's several commits after 8.5-GA branch (https://github.com/onnx/onnx-tensorrt/commit/369d6676423c2a6dbf4a5665c4b5010240d99d3c)
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/369d6676423c2a6dbf4a5665c4b5010240d99d3c.zip;62119892edfb78689061790140c439b111491275
protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd
protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.20.2.zip;9f71dad95fb83438e88822a9969fc93773fd8c48
psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013
pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101
pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13
Expand Down
2 changes: 1 addition & 1 deletion cmake/external/onnxruntime_external_deps.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ FetchContent_Declare(
URL_HASH SHA1=${DEP_SHA1_protobuf}
SOURCE_SUBDIR cmake
PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND}
FIND_PACKAGE_ARGS 3.18.0 NAMES Protobuf
FIND_PACKAGE_ARGS 3.20.2 NAMES Protobuf
)
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
if (CMAKE_SYSTEM_NAME STREQUAL "Android")
Expand Down
2 changes: 1 addition & 1 deletion cmake/external/protobuf
Submodule protobuf updated 894 files
2 changes: 1 addition & 1 deletion cmake/onnxruntime_unittests.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -763,7 +763,7 @@ if (MSVC)
# The warning means the type of two integral values around a binary operator is narrow than their result.
# If we promote the two input values first, it could be more tolerant to integer overflow.
# However, this is test code. We are less concerned.
target_compile_options(onnxruntime_test_all PRIVATE "/wd26451")
target_compile_options(onnxruntime_test_all PRIVATE "/wd26451" "/wd4244")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

4244 is a critical warning that we should not disable.
https://github.com/microsoft/onnxruntime/blob/main/docs/Coding_Conventions_and_Standards.md#other

It is required by BinSkim. However, this is test code. We don't ship it. So it is kind of ok.

else()
target_compile_options(onnxruntime_test_all PRIVATE "-Wno-parentheses")
endif()
Expand Down
38 changes: 36 additions & 2 deletions cmake/patches/onnx/onnx.patch
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8e595855..789ec80a 100644
index 4dd56b6e..018da488 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -545,20 +545,9 @@ endif()
@@ -397,6 +397,7 @@ if (MSVC)
endif()
else()
# On non-Windows, hide all symbols we don't need
+ set(EXTRA_FLAGS "-Wno-unused-parameter")
set(ONNX_API_DEFINE "-DONNX_API=__attribute__\(\(__visibility__\(\"default\"\)\)\)")
set_target_properties(onnx_proto PROPERTIES CXX_VISIBILITY_PRESET hidden)
set_target_properties(onnx_proto PROPERTIES VISIBILITY_INLINES_HIDDEN 1)
@@ -548,20 +549,9 @@ endif()
if(MSVC)
target_compile_options(onnx_proto
PRIVATE /MP
Expand All @@ -23,3 +31,29 @@ index 8e595855..789ec80a 100644
${EXTRA_FLAGS})
if(ONNX_USE_PROTOBUF_SHARED_LIBS)
target_compile_options(onnx_proto
diff --git a/onnx/onnx_pb.h b/onnx/onnx_pb.h
index 0aab3e26..0f859267 100644
--- a/onnx/onnx_pb.h
+++ b/onnx/onnx_pb.h
@@ -47,10 +47,21 @@
#define ONNX_API ONNX_IMPORT
#endif

+// onnx/onnx-operators-ml.pb.h:1178:25: required from here
+// protobuf/src/google/protobuf/repeated_ptr_field.h:752:66: error: unused parameter ‘prototype’ [-Werror=unused-parameter]
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
#ifdef ONNX_ML
#include "onnx/onnx-ml.pb.h"
#else
#include "onnx/onnx.pb.h"
#endif

+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
#endif // ! ONNX_ONNX_PB_H
18 changes: 12 additions & 6 deletions cmake/patches/protobuf/protobuf_cmake.patch
Original file line number Diff line number Diff line change
@@ -1,25 +1,31 @@
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index 61a5c3ded..68d7c5598 100644
index ac92442a1..e930cbd2e 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -199,12 +199,8 @@ if (MSVC)
endif()
@@ -240,9 +240,7 @@ if (MSVC)
# MSVC warning suppressions
add_definitions(
- /wd4018 # 'expression' : signed/unsigned mismatch
/wd4065 # switch statement contains 'default' but no 'case' labels
- /wd4146 # unary minus operator applied to unsigned type, result still unsigned
- /wd4244 # 'conversion' conversion from 'type1' to 'type2', possible loss of data
/wd4251 # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'
- /wd4267 # 'var' : conversion from 'size_t' to 'type', possible loss of data
/wd4305 # 'identifier' : truncation from 'type1' to 'type2'
/wd4307 # 'operator' : integral constant overflow
/wd4309 # 'conversion' : truncation of constant value
@@ -212,7 +208,6 @@ if (MSVC)
@@ -250,7 +248,6 @@ if (MSVC)
/wd4355 # 'this' : used in base member initializer list
/wd4506 # no definition for inline function 'function'
/wd4800 # 'type' : forcing value to bool 'true' or 'false' (performance warning)
- /wd4996 # The compiler encountered a deprecated declaration.
)
# Allow big object
add_definitions(/bigobj)
@@ -272,6 +269,8 @@ if (MSVC)
endif()

configure_file(version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY)
+else (MSVC)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
endif (MSVC)


3 changes: 3 additions & 0 deletions onnxruntime/test/python/onnx_backend_test_series.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ def create_backend_test(test_name=None):
if backend.supports_device("OPENVINO_CPU_FP16"):
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP16")

if backend.supports_device("OPENVINO"):
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_opset18")

if backend.supports_device("MIGRAPHX"):
current_failing_tests += apply_filters(filters, "current_failing_tests_MIGRAPHX")

Expand Down
77 changes: 77 additions & 0 deletions onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,83 @@
"^test_training_dropout_mask", // Runs but there's accuracy mismatch
"^test_training_dropout_default" // Runs but there's accuracy mismatch
],
"current_failing_tests_OPENVINO_opset18": [
// pending opset 18 support, RUNTIME_EXCEPTION : Encountered unknown exception in Initialize()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/ pending opset 18 support, RUNTIME_EXCEPTION : Encountered unknown exception in Initialize()

Does it mean that we are trying to update to a version that fails to initialize and run for a good number of tests in the same opset?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Those tests are introduced in the Latest ONNX(1.13). They are not running in the old OpenVINO pipeline.
Scott did a similar thing in packaging pipeline #14259
Maybe they could be removed after the next upgrade of OpenVINO.

ONNX 1.12 used the protobuf 2.18.3, which doesn't meet the compliance.

"^test_center_crop_pad_crop_axes_chw",
"^test_center_crop_pad_crop_axes_chw_expanded",
"^test_center_crop_pad_crop_axes_hwc",
"^test_center_crop_pad_crop_axes_hwc_expanded",
"^test_group_normalization_epsilon",
"^test_group_normalization_example",
"^test_layer_normalization_2d_axis1_expanded_ver18",
"^test_layer_normalization_2d_axis_negative_1_expanded_ver18",
"^test_layer_normalization_3d_axis1_epsilon_expanded_ver18",
"^test_layer_normalization_3d_axis2_epsilon_expanded_ver18",
"^test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18",
"^test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18",
"^test_layer_normalization_4d_axis1_expanded_ver18",
"^test_layer_normalization_4d_axis2_expanded_ver18",
"^test_layer_normalization_4d_axis3_expanded_ver18",
"^test_layer_normalization_4d_axis_negative_1_expanded_ver18",
"^test_layer_normalization_4d_axis_negative_2_expanded_ver18",
"^test_layer_normalization_4d_axis_negative_3_expanded_ver18",
"^test_layer_normalization_default_axis_expanded_ver18",
"^test_reduce_l1_do_not_keepdims_example_expanded",
"^test_reduce_l1_do_not_keepdims_random_expanded",
"^test_reduce_log_sum_asc_axes_expanded",
"^test_reduce_log_sum_desc_axes_expanded",
"^test_reduce_sum_square_do_not_keepdims_example_expanded",
"^test_reduce_sum_square_do_not_keepdims_random_expanded",
"^test_scatter_elements_with_reduction_max",
"^test_scatter_elements_with_reduction_min",
"^test_scatternd_max",
"^test_scatternd_min",
// pending opset 18 support, test failures
"^test_group_normalization_epsilon_expanded",
"^test_group_normalization_example_expanded",
"^test_logsoftmax_large_number_expanded_ver18",
"^test_mvn_expanded_ver18_cpu",
"^test_reduce_l1_do_not_keepdims_example",
"^test_reduce_l1_do_not_keepdims_random",
"^test_reduce_l1_keep_dims_example",
"^test_reduce_l1_keep_dims_random",
"^test_reduce_l1_negative_axes_keep_dims_example",
"^test_reduce_l1_negative_axes_keep_dims_random",
"^test_reduce_l2_do_not_keepdims_example",
"^test_reduce_l2_do_not_keepdims_random",
"^test_reduce_l2_keep_dims_example",
"^test_reduce_l2_keep_dims_random",
"^test_reduce_l2_negative_axes_keep_dims_example",
"^test_reduce_l2_negative_axes_keep_dims_random",
"^test_reduce_log_sum_asc_axes",
"^test_reduce_log_sum_desc_axes",
"^test_reduce_log_sum_negative_axes",
"^test_reduce_max_do_not_keepdims_example",
"^test_reduce_max_do_not_keepdims_random",
"^test_reduce_max_keepdims_example",
"^test_reduce_max_keepdims_random",
"^test_reduce_max_negative_axes_keepdims_example",
"^test_reduce_max_negative_axes_keepdims_random",
"^test_reduce_mean_do_not_keepdims_example",
"^test_reduce_mean_do_not_keepdims_random",
"^test_reduce_mean_keepdims_example",
"^test_reduce_mean_keepdims_random",
"^test_reduce_mean_negative_axes_keepdims_example",
"^test_reduce_mean_negative_axes_keepdims_random",
"^test_reduce_prod_do_not_keepdims_example",
"^test_reduce_prod_do_not_keepdims_random",
"^test_reduce_prod_keepdims_example",
"^test_reduce_prod_keepdims_random",
"^test_reduce_prod_negative_axes_keepdims_example",
"^test_reduce_prod_negative_axes_keepdims_random",
"^test_reduce_sum_square_do_not_keepdims_example",
"^test_reduce_sum_square_do_not_keepdims_random",
"^test_reduce_sum_square_keepdims_example",
"^test_reduce_sum_square_keepdims_random",
"^test_reduce_sum_square_negative_axes_keepdims_example",
"^test_reduce_sum_square_negative_axes_keepdims_random",
"^test_softmax_large_number_expanded_ver18"
],
"current_failing_tests_MIGRAPHX": [
"^test_constant_pad_cpu",
"^test_round_cpu",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ steps:
packageType: upack
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0'
version: 1.0.24
version: 1.0.25
downloadPath: $(Build.BinariesDirectory)/deps

# The private ADO project
Expand All @@ -22,7 +22,7 @@ steps:
packageType: upack
feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325'
definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a'
version: 1.0.24
version: 1.0.25
downloadPath: $(Build.BinariesDirectory)/deps

# You can add more ADO accounts at here.
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ pytest
setuptools>=41.4.0
wheel
git+http://github.com/onnx/onnx.git@5a5f8a5935762397aa68429b5493084ff970f774#egg=onnx
protobuf==3.18.3
protobuf==3.20.2
sympy==1.10.1
flatbuffers
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ mypy
pytest
setuptools>=41.4.0
wheel
onnx==1.12.0
protobuf==3.18.3
onnx==1.13.0
protobuf==3.20.2
sympy==1.10.1
flatbuffers
6 changes: 3 additions & 3 deletions tools/ci_build/github/linux/docker/scripts/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ mypy
pytest
setuptools>=41.4.0
wheel>=0.35.1
onnx==1.12.0
onnx==1.13.0
argparse
sympy==1.10.1
flatbuffers
protobuf==3.18.1
packaging
protobuf==3.20.2
packaging
4 changes: 2 additions & 2 deletions tools/ci_build/github/linux/tvm/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ scipy
wheel
setuptools
onnx==1.11.0
protobuf==3.18.3
protobuf==3.20.2
flatbuffers
tornado
psutil
xgboost
cloudpickle
pytest
pytest
1 change: 1 addition & 0 deletions tools/ci_build/github/windows/helpers.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ function Install-Protobuf {

pushd .
$url=Get-DownloadURL -name protobuf -src_root $src_root
Write-Host "Downloading protobuf from $url"
$temp_dir = Get-TempDirectory
$protobuf_src_dir = Join-Path $temp_dir "protobuf"
$download_finished = DownloadAndExtract -Uri $url -InstallDirectory $protobuf_src_dir -Force
Expand Down
2 changes: 1 addition & 1 deletion tools/ci_build/github/windows/install_third_party_deps.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $bu
Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args

# TODO: parse it from deps.txt
$protobuf_version="3.18.3"
$protobuf_version="3.20.2"

# ONNX doesn't allow us to specify CMake's path
Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version
2 changes: 1 addition & 1 deletion tools/ci_build/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# packages used by transformers tool test
packaging
protobuf==3.20.1
protobuf==3.20.2
numpy==1.24.0
coloredlogs==15.0
transformers==4.24.0
Expand Down