From 7ee8aca1bff1eae633eacf16b4526171ae13f149 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 19:53:15 -0700 Subject: [PATCH] Avoid downloading test data into C:\ (#1562) --- .../azure-pipelines-py-packaging.yml | 4 +- .../c-api-packaging-pipelines.yml | 2 +- .../linux-gpu-ci-pipeline-cuda9.yml | 2 +- .../linux-gpu-tensorrt-ci-pipeline.yml | 2 +- .../linux-ngraph-ci-pipeline.yml | 2 +- .../linux-nocontribops-ci-pipeline.yml | 2 +- .../linux-openvino-ci-pipeline.yml | 2 +- .../linux-ort-srv-ci-pipeline.yml | 2 +- .../linux-ort-srv-nightly-pipeline.yml | 2 +- .../linux-x86-nocontribops-ci-pipeline.yml | 2 +- .../mac-nocontribops-ci-pipeline.yml | 2 +- .../nuget/templates/cpu-mklml.yml | 2 +- .../templates/cpu-nocontribops-arm64.yml | 6 +- .../nuget/templates/cpu-nocontribops.yml | 4 +- .../azure-pipelines/nuget/templates/cpu.yml | 4 +- .../azure-pipelines/nuget/templates/gpu.yml | 2 +- .../nuget/templates/test_win.yml | 2 +- .../linux-set-variables-and-download.yml | 2 +- .../mac-set-variables-and-download.yml | 2 +- .../windows-build-and-test-steps.yml | 4 +- .../windows-build-tools-setup-steps.yml | 41 ++++++------- .../azure-pipelines/win-ci-pipeline.yml | 2 +- .../azure-pipelines/win-gpu-ci-pipeline.yml | 2 +- .../win-gpu-tensorrt-ci-pipeline.yml | 6 +- .../azure-pipelines/win-mklml-ci-pipeline.yml | 6 +- .../win-ngraph-ci-pipeline.yml | 6 +- .../win-nocontribops-ci-pipeline.yml | 2 +- .../azure-pipelines/win-x86-ci-pipeline.yml | 2 +- .../win-x86-nocontribops-ci-pipeline.yml | 2 +- tools/ci_build/github/download_test_data.py | 58 +++++++++++++++---- 30 files changed, 105 insertions(+), 74 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 3ac6d3bc198b7..8721078422c21 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -24,7 +24,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) @@ -86,7 +86,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 9fafeea239f6c..d8222370b9433 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -156,7 +156,7 @@ jobs: displayName: 'Build and Test OnnxRuntime' inputs: script: | - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" workingDirectory: '$(Build.BinariesDirectory)' - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml index aa262490ee54f..1f6de3aef300d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml @@ -16,7 +16,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 688303d94f313..8b5fe3a3cd9eb 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -16,7 +16,7 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' # There are some tests in 20190130.zip that TensorRT can't run. Instead here use 20181210 opset8 for TensorRT test. - arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip + arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml index b6ab3fb42e7e7..131c4cc06e83f 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml @@ -27,7 +27,7 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' # nGraph provider fails on the latest 20190729.zip test. revert back to previous zip file until failures can be investigated - arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip + arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml index 40e05acbd0d66..d518ec3285071 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml @@ -31,7 +31,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) + arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 94f8550542ac2..07ba00ff3bd71 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index ae44e6dacf3d2..2786df3c4e905 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml index b037dd78efa30..0000e6c7097cf 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml index ef570a3cb6872..71b5d5fc94117 100644 --- a/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml @@ -32,7 +32,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) + arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml index 74a004d89fe21..39aca7ec08596 100644 --- a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus + arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/local/bin/python3' workingDirectory: $(Build.BinariesDirectory) - script: | diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index f942734521f22..700f7b2b4c200 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index ae463bfcd8ff7..8cdcbadece4e8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -26,7 +26,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -41,7 +41,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_Arm64_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64 --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml index 125aa2e8eabe4..659b5dad12e0d 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -26,7 +26,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 92ce0195a4ce1..72377b20683ef 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -13,7 +13,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: ${{ parameters.DoCompliance }} @@ -27,7 +27,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 26d12e9c7f8f0..ebe56764e8b80 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -13,7 +13,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_GPU_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index ce445baa6977e..4336c4279c599 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -25,7 +25,7 @@ jobs: - script: | @echo "Running build.py --update" - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data displayName: 'Download Test Data' diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml index 0242ac0a65187..405dcc9418aa1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml @@ -24,7 +24,7 @@ steps: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml index 829ef22b97612..233fab77222d2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml @@ -13,6 +13,6 @@ steps: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) --azure_region centralus + arguments: --test_data_url $(TestDataUrl) --azure_region centralus --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/local/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml index a846bb8dcb78f..a6a4c4d06206e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml @@ -11,7 +11,7 @@ steps: displayName: 'Download test data and generate cmake config' inputs: script: | - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update ${{parameters.buildAdditionalParams}} + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --update ${{parameters.buildAdditionalParams}} workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -29,7 +29,7 @@ steps: displayName: 'Test ${{parameters.buildConfig}}' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test ${{parameters.buildAdditionalParams}}' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test ${{parameters.buildAdditionalParams}}' workingFolder: '$(Build.BinariesDirectory)' - task: PublishTestResults@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index d149775a78317..d9fda65add37d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -26,19 +26,25 @@ steps: # downloadDirectory: '$(Build.BinariesDirectory)\python' # Temporary bypass of artifacts permission issue - - task: PowerShell@2 + - task: CmdLine@1 + displayName: 'Download azcopy' + inputs: + filename: 'AzCopy.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe' + + - task: CmdLine@1 displayName: 'Download python' inputs: - targetType: 'inline' - script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' - workingDirectory: '$(Build.BinariesDirectory)' - + filename: 'AzCopy.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/Miniconda3-4.7.10-Windows-x86_64.exe /Dest:$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe' + - task: CmdLine@1 displayName: 'Run python installer' inputs: - filename: '$(Build.BinariesDirectory)\installer.exe' + filename: '$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe' arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' timeoutInMinutes: 10 + - task: BatchScript@1 displayName: 'setup env' inputs: @@ -52,24 +58,13 @@ steps: arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy' timeoutInMinutes: 10 - - task: CmdLine@1 - displayName: 'Download cmake' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' - - - task: PowerShell@2 - displayName: 'Download OpenCppCoverage installer' - continueOnError: true + - task: PythonScript@0 + displayName: 'Download test data' inputs: - targetType: 'inline' - script: ' - New-Item -Path "$(Build.BinariesDirectory)\installer" -ItemType "directory" - - New-Item -Path "$(Build.BinariesDirectory)\installer\opencppcoverage" -ItemType "directory" - - Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe https://onnxruntimeinstaller.blob.core.windows.net/opencppcovergae-installer/OpenCppCoverageSetup-x64-0.9.7.0.exe - ' + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' + workingDirectory: $(Build.BinariesDirectory) - task: CmdLine@1 continueOnError: true diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 22aef163727e3..bdaa8f0cf5289 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --gen_doc' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index d4930faf193e7..a2c42de7d7367 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-GPU-CUDA10' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11' JobName: 'Windows_CI_GPU_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 1711ce9c0ac92..a31ad59615489 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -28,7 +28,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --update --msvc_toolset=14.11' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 @@ -45,7 +45,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build C# Debug' @@ -81,7 +81,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test' workingFolder: "$(Build.BinariesDirectory)" - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml index e8580f8fd495d..7b539c4c6b24d 100644 --- a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --update' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 @@ -31,7 +31,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build C# Debug' @@ -66,7 +66,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test' workingFolder: "$(Build.BinariesDirectory)" - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml index 46008a8733f12..b45c0bf74c44b 100644 --- a/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml @@ -17,7 +17,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --update' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 displayName: 'Build Debug' @@ -33,7 +33,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build Release' @@ -49,7 +49,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test' workingFolder: "$(Build.BinariesDirectory)" - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml index 7339ecab0000d..b41400e0bd7ca 100644 --- a/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml @@ -8,7 +8,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml index 87e7b6eec4fca..3d3ced7591015 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86' JobName: 'Windows_CI_Dev_x86' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml index b309f33a50d14..96a62acee69b6 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml @@ -9,7 +9,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --disable_contrib_ops --enable_msvc_static_runtime --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --disable_contrib_ops --enable_msvc_static_runtime --x86' JobName: 'Windows_CI_Dev_x86' DoNugetPack: 'false' NuPackScript : '' \ No newline at end of file diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index dc475071265fc..821b6054eca5e 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -1,12 +1,16 @@ #!/usr/bin/python3 - import urllib.request import json import subprocess import os +import sys +import shutil import argparse from urllib.parse import urlparse +def is_windows(): + return sys.platform.startswith("win") + def get_azure_region(): req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01') req.add_header('Metadata', 'true') @@ -18,28 +22,60 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.") parser.add_argument("--test_data_url", help="Test data URL.") parser.add_argument("--azure_region", help="Azure region") + parser.add_argument("--build_dir", required=True, help="Path to the build directory.") return parser.parse_args() def get_server_hostname(azure_location): if azure_location is None: #should be northcentralus or centralus - azure_location=get_azure_region() + azure_location = get_azure_region() print("This VM is in azure location: %s" % azure_location) if azure_location == 'centralus': - hostname='onnxruntimetestdata' + hostname = 'onnxruntimetestdata' elif azure_location == 'northcentralus': - hostname='onnxruntimetestdata2' + hostname = 'onnxruntimetestdata2' else: print('warning: no local data cache for azure region %s' % azure_location) - hostname='onnxruntimetestdata2' + hostname = 'onnxruntimetestdata2' return hostname + +def download_and_unzip(build_dir, url, dest_folder): + print("Downloading %s" % url) + dest_folder = os.path.join(build_dir, dest_folder) + subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) + os.makedirs(dest_folder,exist_ok=True) + local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path)) + if is_windows(): + print("unzip %s" % local_file_name) + if shutil.which('7z'): # 7-Zip + subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True) + elif shutil.which('7za'): # 7-Zip standalone + subprocess.run(['7za', 'x', local_file_name, '-y', '-o' + dest_folder], check=True) + else: + log.error("No unzip tool for use") + sys.exit(1) + else: + subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) + os.unlink(local_file_name) + args = parse_arguments() -hostname=get_server_hostname(args.azure_region) -url=args.test_data_url.replace('onnxruntimetestdata', hostname) +hostname = get_server_hostname(args.azure_region) +url = args.test_data_url.replace('onnxruntimetestdata', hostname) print('data url=%s' % url) -subprocess.run(['./azcopy','cp', '--log-level','ERROR', url,'.'],check=True) -os.makedirs('models',exist_ok=True) -local_file_name = os.path.basename(urlparse(url).path) -subprocess.run(['unzip', '-qd','models',local_file_name]) +download_and_unzip(args.build_dir, url, 'models') +if is_windows(): + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' + url = url.replace('onnxruntimetestdata', hostname) + download_and_unzip(args.build_dir, url, 'cmake_temp') + dest_dir = os.path.join(args.build_dir,'cmake') + if os.path.exists(dest_dir): + print('deleting %s' % dest_dir) + shutil.rmtree(dest_dir) + shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' + url = url.replace('onnxruntimetestdata', hostname) + dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') + os.makedirs(dest_folder,exist_ok=True) + subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)