From 14db8fee0e55de263274c2ef27d349f73b587a6b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 11:07:25 -0700 Subject: [PATCH 01/21] a --- .../templates/windows-build-tools-setup-steps.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index d149775a78317..52b71a7b41bce 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -30,9 +30,21 @@ steps: displayName: 'Download python' inputs: targetType: 'inline' - script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' + script: 'Invoke-WebRequest --MaximumRetryCount 10 -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' workingDirectory: '$(Build.BinariesDirectory)' + - task: PowerShell@2 + displayName: 'Download azcopy' + inputs: + targetType: 'inline' + script: 'Invoke-WebRequest --MaximumRetryCount 10 -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\azcopy.zip' + destinationFolder: $(Build.BinariesDirectory)\azcopy + - task: CmdLine@1 displayName: 'Run python installer' inputs: From ecf136a62ecce05d2dbe1bdc497bd3f2867885a2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 11:22:11 -0700 Subject: [PATCH 02/21] update --- .../templates/windows-build-tools-setup-steps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 52b71a7b41bce..3716f0e747281 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -30,14 +30,14 @@ steps: displayName: 'Download python' inputs: targetType: 'inline' - script: 'Invoke-WebRequest --MaximumRetryCount 10 -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' + script: 'Invoke-WebRequest -MaximumRetryCount 10 -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' workingDirectory: '$(Build.BinariesDirectory)' - task: PowerShell@2 displayName: 'Download azcopy' inputs: targetType: 'inline' - script: 'Invoke-WebRequest --MaximumRetryCount 10 -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' + script: 'Invoke-WebRequest -MaximumRetryCount 10 -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' workingDirectory: '$(Build.BinariesDirectory)' - task: ExtractFiles@1 From 01ccbcde643a547d852ae8e0317054097ecbf18f Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 11:39:12 -0700 Subject: [PATCH 03/21] update --- .../windows-build-tools-setup-steps.yml | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 3716f0e747281..2d244822eedcc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -30,21 +30,27 @@ steps: displayName: 'Download python' inputs: targetType: 'inline' - script: 'Invoke-WebRequest -MaximumRetryCount 10 -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' + script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' workingDirectory: '$(Build.BinariesDirectory)' - task: PowerShell@2 displayName: 'Download azcopy' inputs: targetType: 'inline' - script: 'Invoke-WebRequest -MaximumRetryCount 10 -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' + script: 'Invoke-WebRequest -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' workingDirectory: '$(Build.BinariesDirectory)' - task: ExtractFiles@1 inputs: archiveFilePatterns: '$(Build.BinariesDirectory)\azcopy.zip' - destinationFolder: $(Build.BinariesDirectory)\azcopy - + destinationFolder: '$(Build.BinariesDirectory)\azcopy' + + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.BinariesDirectory)\azcopy' + contents: '**\*.exe' + destinationFolder: '$(Build.BinariesDirectory)' + - task: CmdLine@1 displayName: 'Run python installer' inputs: @@ -64,11 +70,21 @@ steps: arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy' timeoutInMinutes: 10 - - task: CmdLine@1 + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 displayName: 'Download cmake' inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' + pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py' + arguments: --build_dir $(Build.BinariesDirectory) + - task: PowerShell@2 displayName: 'Download OpenCppCoverage installer' From add3dac4324c5377a89c4ecafdee789042ce54a9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:09:59 -0700 Subject: [PATCH 04/21] update --- .../windows-build-tools-setup-steps.yml | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 2d244822eedcc..9e5ac3160c539 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -26,30 +26,17 @@ steps: # downloadDirectory: '$(Build.BinariesDirectory)\python' # Temporary bypass of artifacts permission issue + - task: CmdLine@1 + displayName: 'Run python installer' + filename: 'AzCopy.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe' + - task: PowerShell@2 displayName: 'Download python' inputs: targetType: 'inline' script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' - workingDirectory: '$(Build.BinariesDirectory)' - - - task: PowerShell@2 - displayName: 'Download azcopy' - inputs: - targetType: 'inline' - script: 'Invoke-WebRequest -OutFile azcopy.zip https://aka.ms/downloadazcopy-v10-windows' - workingDirectory: '$(Build.BinariesDirectory)' - - - task: ExtractFiles@1 - inputs: - archiveFilePatterns: '$(Build.BinariesDirectory)\azcopy.zip' - destinationFolder: '$(Build.BinariesDirectory)\azcopy' - - - task: CopyFiles@2 - inputs: - sourceFolder: '$(Build.BinariesDirectory)\azcopy' - contents: '**\*.exe' - destinationFolder: '$(Build.BinariesDirectory)' + workingDirectory: '$(Build.BinariesDirectory)' - task: CmdLine@1 displayName: 'Run python installer' From eeea76fd8fa15b059439fe46ded00d5741b23394 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:10:58 -0700 Subject: [PATCH 05/21] update --- .../templates/windows-build-tools-setup-steps.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 9e5ac3160c539..6d5114be4077a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -28,8 +28,9 @@ steps: # Temporary bypass of artifacts permission issue - task: CmdLine@1 displayName: 'Run python installer' - filename: 'AzCopy.exe' - arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe' + inputs: + filename: 'AzCopy.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe' - task: PowerShell@2 displayName: 'Download python' @@ -44,6 +45,7 @@ steps: filename: '$(Build.BinariesDirectory)\installer.exe' arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' timeoutInMinutes: 10 + - task: BatchScript@1 displayName: 'setup env' inputs: From c5180fd0e76acfea5782e9c7ca8e0ec8aae630f8 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:22:12 -0700 Subject: [PATCH 06/21] update --- tools/ci_build/github/download_test_data.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index dc475071265fc..3c6011852261c 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -39,7 +39,17 @@ def get_server_hostname(azure_location): hostname=get_server_hostname(args.azure_region) url=args.test_data_url.replace('onnxruntimetestdata', hostname) print('data url=%s' % url) -subprocess.run(['./azcopy','cp', '--log-level','ERROR', url,'.'],check=True) +BUILD_BINARIESDIRECTORY = os.environ.get('BUILD_BINARIESDIRECTORY') +subprocess.run([os.path.join(BUILD_BINARIESDIRECTORY,'azcopy'),'cp', '--log-level','ERROR', url,'.'],check=True) os.makedirs('models',exist_ok=True) local_file_name = os.path.basename(urlparse(url).path) -subprocess.run(['unzip', '-qd','models',local_file_name]) +if is_windows(): + if shutil.which('7z'): # 7-Zip + run_subprocess(['7z','x', local_file_name, '-y', '-o models']) + elif shutil.which('7za'): # 7-Zip standalone + run_subprocess(['7za', 'x', local_file_name, '-y', '-o models']) + else: + log.error("No unzip tool for use") + sys.exit(1) +else: + subprocess.run(['unzip','-qd', 'models',local_file_name], check=True) From dc6c845424fa4ffcaea00253ef8cebf040c4ff86 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:39:18 -0700 Subject: [PATCH 07/21] a --- .../linux-set-variables-and-download.yml | 2 +- .../windows-build-tools-setup-steps.yml | 12 +--- tools/ci_build/github/download_test_data.py | 57 ++++++++++++------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml index 0242ac0a65187..405dcc9418aa1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml @@ -24,7 +24,7 @@ steps: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 6d5114be4077a..c89aa0b3b5ccb 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -63,17 +63,9 @@ steps: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' - workingDirectory: $(Build.BinariesDirectory) - - - task: PythonScript@0 - displayName: 'Download cmake' - inputs: - pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py' - arguments: --build_dir $(Build.BinariesDirectory) - + workingDirectory: $(Build.BinariesDirectory) - task: PowerShell@2 displayName: 'Download OpenCppCoverage installer' diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 3c6011852261c..78ee0039e1f3d 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -1,5 +1,4 @@ #!/usr/bin/python3 - import urllib.request import json import subprocess @@ -7,6 +6,9 @@ import argparse from urllib.parse import urlparse +def is_windows(): + return sys.platform.startswith("win") + def get_azure_region(): req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01') req.add_header('Metadata', 'true') @@ -18,38 +20,53 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.") parser.add_argument("--test_data_url", help="Test data URL.") parser.add_argument("--azure_region", help="Azure region") + parser.add_argument("--build_dir", required=True, help="Path to the build directory.") return parser.parse_args() def get_server_hostname(azure_location): if azure_location is None: #should be northcentralus or centralus - azure_location=get_azure_region() + azure_location = get_azure_region() print("This VM is in azure location: %s" % azure_location) if azure_location == 'centralus': - hostname='onnxruntimetestdata' + hostname = 'onnxruntimetestdata' elif azure_location == 'northcentralus': - hostname='onnxruntimetestdata2' + hostname = 'onnxruntimetestdata2' else: print('warning: no local data cache for azure region %s' % azure_location) - hostname='onnxruntimetestdata2' + hostname = 'onnxruntimetestdata2' return hostname + +def download_and_unzip(build_dir, url, dest_folder): + subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) + os.makedirs(dest_folder,exist_ok=True) + local_file_name = os.path.basename(urlparse(url).path) + if is_windows(): + if shutil.which('7z'): # 7-Zip + run_subprocess(['7z','x', local_file_name, '-y', '-o', dest_folder]) + elif shutil.which('7za'): # 7-Zip standalone + run_subprocess(['7za', 'x', local_file_name, '-y', '-o', dest_folder]) + else: + log.error("No unzip tool for use") + sys.exit(1) + else: + subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) + args = parse_arguments() -hostname=get_server_hostname(args.azure_region) -url=args.test_data_url.replace('onnxruntimetestdata', hostname) +hostname = get_server_hostname(args.azure_region) +url = args.test_data_url.replace('onnxruntimetestdata', hostname) print('data url=%s' % url) -BUILD_BINARIESDIRECTORY = os.environ.get('BUILD_BINARIESDIRECTORY') -subprocess.run([os.path.join(BUILD_BINARIESDIRECTORY,'azcopy'),'cp', '--log-level','ERROR', url,'.'],check=True) -os.makedirs('models',exist_ok=True) -local_file_name = os.path.basename(urlparse(url).path) +download_and_unzip(args.build_dir, url, 'models') if is_windows(): - if shutil.which('7z'): # 7-Zip - run_subprocess(['7z','x', local_file_name, '-y', '-o models']) - elif shutil.which('7za'): # 7-Zip standalone - run_subprocess(['7za', 'x', local_file_name, '-y', '-o models']) - else: - log.error("No unzip tool for use") - sys.exit(1) -else: - subprocess.run(['unzip','-qd', 'models',local_file_name], check=True) + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' + url = args.test_data_url.replace('onnxruntimetestdata', hostname) + download_and_unzip(args.build_dir, url, 'cmake_temp') + dest_dir = os.path.join(args.build_dir,'cmake') + if os.path.exists(dest_dir): + print('deleting %s' % dest_dir) + shutil.rmtree(dest_dir) + shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.13.2-win64-x64'),dest_dir) + + From 92407168e0a9ecfa87733bc8ffae9addb0e6686d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:48:05 -0700 Subject: [PATCH 08/21] update --- .../templates/windows-build-tools-setup-steps.yml | 13 ++++++------- .../github/azure-pipelines/win-ci-pipeline.yml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index c89aa0b3b5ccb..bdaa21a1b2e54 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -27,22 +27,21 @@ steps: # Temporary bypass of artifacts permission issue - task: CmdLine@1 - displayName: 'Run python installer' + displayName: 'Download azcopy' inputs: filename: 'AzCopy.exe' arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe' - - - task: PowerShell@2 + + - task: CmdLine@1 displayName: 'Download python' inputs: - targetType: 'inline' - script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' - workingDirectory: '$(Build.BinariesDirectory)' + filename: 'AzCopy.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/Anaconda3-2019.07-Windows-x86_64.exe /Dest:$(Build.BinariesDirectory)\Anaconda3-2019.07-Windows-x86_64.exe' - task: CmdLine@1 displayName: 'Run python installer' inputs: - filename: '$(Build.BinariesDirectory)\installer.exe' + filename: '$(Build.BinariesDirectory)\Anaconda3-2019.07-Windows-x86_64.exe' arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' timeoutInMinutes: 10 diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 22aef163727e3..bdaa8f0cf5289 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --gen_doc' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' From 6fbab4d3f3278317cd2e4c78f14ff32ec0dab17f Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 12:50:47 -0700 Subject: [PATCH 09/21] update --- tools/ci_build/github/download_test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 78ee0039e1f3d..d7c59c9f3de05 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -3,6 +3,7 @@ import json import subprocess import os +import sys import argparse from urllib.parse import urlparse From d854597c8df894100e1d14992f369a69cd510df0 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 13:04:35 -0700 Subject: [PATCH 10/21] a --- tools/ci_build/github/download_test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index d7c59c9f3de05..b71b6e66718f0 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -4,6 +4,7 @@ import subprocess import os import sys +import shutil import argparse from urllib.parse import urlparse From 525d3e28aed1b1eeda79b0b009caef59494d45ce Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 13:12:01 -0700 Subject: [PATCH 11/21] update --- .../templates/windows-build-tools-setup-steps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index bdaa21a1b2e54..eae045f3ba252 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -36,12 +36,12 @@ steps: displayName: 'Download python' inputs: filename: 'AzCopy.exe' - arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/Anaconda3-2019.07-Windows-x86_64.exe /Dest:$(Build.BinariesDirectory)\Anaconda3-2019.07-Windows-x86_64.exe' + arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/Miniconda3-4.7.10-Windows-x86_64.exe /Dest:$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe' - task: CmdLine@1 displayName: 'Run python installer' inputs: - filename: '$(Build.BinariesDirectory)\Anaconda3-2019.07-Windows-x86_64.exe' + filename: '$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe' arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' timeoutInMinutes: 10 From 46127351138bb987111bcf2abbfddbc1cc1d0ed4 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 13:19:46 -0700 Subject: [PATCH 12/21] update --- tools/ci_build/github/download_test_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index b71b6e66718f0..f1ac2ee159f03 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -47,9 +47,9 @@ def download_and_unzip(build_dir, url, dest_folder): local_file_name = os.path.basename(urlparse(url).path) if is_windows(): if shutil.which('7z'): # 7-Zip - run_subprocess(['7z','x', local_file_name, '-y', '-o', dest_folder]) + subprocess.run(['7z','x', local_file_name, '-y', '-o', dest_folder], check=True) elif shutil.which('7za'): # 7-Zip standalone - run_subprocess(['7za', 'x', local_file_name, '-y', '-o', dest_folder]) + subprocess.run(['7za', 'x', local_file_name, '-y', '-o', dest_folder], check=True) else: log.error("No unzip tool for use") sys.exit(1) From ca56a16aed162c989bba0ca6b643f8364fb8d7b9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 13:47:43 -0700 Subject: [PATCH 13/21] update --- tools/ci_build/github/download_test_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index f1ac2ee159f03..de622368db3e4 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -47,9 +47,9 @@ def download_and_unzip(build_dir, url, dest_folder): local_file_name = os.path.basename(urlparse(url).path) if is_windows(): if shutil.which('7z'): # 7-Zip - subprocess.run(['7z','x', local_file_name, '-y', '-o', dest_folder], check=True) + subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True) elif shutil.which('7za'): # 7-Zip standalone - subprocess.run(['7za', 'x', local_file_name, '-y', '-o', dest_folder], check=True) + subprocess.run(['7za', 'x', local_file_name, '-y', '-o'+ dest_folder], check=True) else: log.error("No unzip tool for use") sys.exit(1) From a6ee6dac561180a48f952aef811b4fe8b71f431c Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 14:40:57 -0700 Subject: [PATCH 14/21] update --- tools/ci_build/github/download_test_data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index de622368db3e4..d899f1965c62f 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -44,7 +44,7 @@ def get_server_hostname(azure_location): def download_and_unzip(build_dir, url, dest_folder): subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) os.makedirs(dest_folder,exist_ok=True) - local_file_name = os.path.basename(urlparse(url).path) + local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path)) if is_windows(): if shutil.which('7z'): # 7-Zip subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True) @@ -55,6 +55,7 @@ def download_and_unzip(build_dir, url, dest_folder): sys.exit(1) else: subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) + os.unlink(local_file_name) args = parse_arguments() hostname = get_server_hostname(args.azure_region) @@ -69,6 +70,6 @@ def download_and_unzip(build_dir, url, dest_folder): if os.path.exists(dest_dir): print('deleting %s' % dest_dir) shutil.rmtree(dest_dir) - shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.13.2-win64-x64'),dest_dir) + shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) From ac9697206909f334a56a2348d38a69b5217ad4de Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 15:01:21 -0700 Subject: [PATCH 15/21] update --- tools/ci_build/github/download_test_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index d899f1965c62f..8a66334c741bc 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -42,10 +42,13 @@ def get_server_hostname(azure_location): def download_and_unzip(build_dir, url, dest_folder): + print("Downloading %s" % url) + dest_folder = os.path.join(build_dir, dest_folder) subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) os.makedirs(dest_folder,exist_ok=True) local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path)) if is_windows(): + print("unzip %s" % local_file_name) if shutil.which('7z'): # 7-Zip subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True) elif shutil.which('7za'): # 7-Zip standalone From 889ea8be0d96bd0ff2e93c3b1eb2a51ea4c7653d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 15:16:48 -0700 Subject: [PATCH 16/21] update --- tools/ci_build/github/download_test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 8a66334c741bc..a6ad6d585a6cc 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -67,7 +67,7 @@ def download_and_unzip(build_dir, url, dest_folder): download_and_unzip(args.build_dir, url, 'models') if is_windows(): url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' - url = args.test_data_url.replace('onnxruntimetestdata', hostname) + url = url.replace('onnxruntimetestdata', hostname) download_and_unzip(args.build_dir, url, 'cmake_temp') dest_dir = os.path.join(args.build_dir,'cmake') if os.path.exists(dest_dir): From 68efe8b39565842d75104ece6e114c6a048bf734 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 15:32:41 -0700 Subject: [PATCH 17/21] update --- .../templates/windows-build-tools-setup-steps.yml | 15 +-------------- tools/ci_build/github/download_test_data.py | 5 +++++ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index eae045f3ba252..d9fda65add37d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -64,20 +64,7 @@ steps: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' - workingDirectory: $(Build.BinariesDirectory) - - - task: PowerShell@2 - displayName: 'Download OpenCppCoverage installer' - continueOnError: true - inputs: - targetType: 'inline' - script: ' - New-Item -Path "$(Build.BinariesDirectory)\installer" -ItemType "directory" - - New-Item -Path "$(Build.BinariesDirectory)\installer\opencppcoverage" -ItemType "directory" - - Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe https://onnxruntimeinstaller.blob.core.windows.net/opencppcovergae-installer/OpenCppCoverageSetup-x64-0.9.7.0.exe - ' + workingDirectory: $(Build.BinariesDirectory) - task: CmdLine@1 continueOnError: true diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index a6ad6d585a6cc..976699e13c767 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -74,5 +74,10 @@ def download_and_unzip(build_dir, url, dest_folder): print('deleting %s' % dest_dir) shutil.rmtree(dest_dir) shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) + url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' + url = url.replace('onnxruntimetestdata', hostname) + dest_folder = os.path.join(build_dir, 'installer','opencppcoverage') + os.makedirs(dest_folder,exist_ok=True) + subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) From df8e7dc5196f1db124776e8d9d1a84d12efeba71 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 15:45:54 -0700 Subject: [PATCH 18/21] update --- .../github/azure-pipelines/azure-pipelines-py-packaging.yml | 4 ++-- .../github/azure-pipelines/c-api-packaging-pipelines.yml | 2 +- .../github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml | 2 +- .../azure-pipelines/linux-nocontribops-ci-pipeline.yml | 2 +- .../github/azure-pipelines/linux-openvino-ci-pipeline.yml | 2 +- .../github/azure-pipelines/linux-ort-srv-ci-pipeline.yml | 2 +- .../azure-pipelines/linux-ort-srv-nightly-pipeline.yml | 2 +- .../azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml | 2 +- .../github/azure-pipelines/mac-nocontribops-ci-pipeline.yml | 2 +- .../github/azure-pipelines/nuget/templates/cpu-mklml.yml | 2 +- .../nuget/templates/cpu-nocontribops-arm64.yml | 6 +++--- .../azure-pipelines/nuget/templates/cpu-nocontribops.yml | 4 ++-- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 4 ++-- .../ci_build/github/azure-pipelines/nuget/templates/gpu.yml | 2 +- .../github/azure-pipelines/nuget/templates/test_win.yml | 2 +- .../templates/mac-set-variables-and-download.yml | 2 +- .../templates/windows-build-and-test-steps.yml | 4 ++-- .../ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml | 2 +- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 6 +++--- .../github/azure-pipelines/win-mklml-ci-pipeline.yml | 6 +++--- .../github/azure-pipelines/win-ngraph-ci-pipeline.yml | 6 +++--- .../github/azure-pipelines/win-nocontribops-ci-pipeline.yml | 2 +- .../ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml | 2 +- .../azure-pipelines/win-x86-nocontribops-ci-pipeline.yml | 2 +- tools/ci_build/github/download_test_data.py | 2 +- 25 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 3ac6d3bc198b7..8721078422c21 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -24,7 +24,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) @@ -86,7 +86,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 9fafeea239f6c..d8222370b9433 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -156,7 +156,7 @@ jobs: displayName: 'Build and Test OnnxRuntime' inputs: script: | - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" workingDirectory: '$(Build.BinariesDirectory)' - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml index aa262490ee54f..1f6de3aef300d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml @@ -16,7 +16,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml index 40e05acbd0d66..d518ec3285071 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml @@ -31,7 +31,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) + arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 94f8550542ac2..07ba00ff3bd71 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index ae44e6dacf3d2..2786df3c4e905 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml index b037dd78efa30..0000e6c7097cf 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml index ef570a3cb6872..71b5d5fc94117 100644 --- a/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-x86-nocontribops-ci-pipeline.yml @@ -32,7 +32,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) + arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml index 74a004d89fe21..39aca7ec08596 100644 --- a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus + arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/local/bin/python3' workingDirectory: $(Build.BinariesDirectory) - script: | diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index f942734521f22..700f7b2b4c200 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index ae463bfcd8ff7..8cdcbadece4e8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -26,7 +26,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -41,7 +41,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_Arm64_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64 --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml index 125aa2e8eabe4..659b5dad12e0d 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml @@ -12,7 +12,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -26,7 +26,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 92ce0195a4ce1..72377b20683ef 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -13,7 +13,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: ${{ parameters.DoCompliance }} @@ -27,7 +27,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 26d12e9c7f8f0..ebe56764e8b80 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -13,7 +13,7 @@ jobs: parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_GPU_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index ce445baa6977e..4336c4279c599 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -25,7 +25,7 @@ jobs: - script: | @echo "Running build.py --update" - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data displayName: 'Download Test Data' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml index 829ef22b97612..233fab77222d2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-set-variables-and-download.yml @@ -13,6 +13,6 @@ steps: displayName: 'Download test data' inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) --azure_region centralus + arguments: --test_data_url $(TestDataUrl) --azure_region centralus --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/local/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml index a846bb8dcb78f..a6a4c4d06206e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-and-test-steps.yml @@ -11,7 +11,7 @@ steps: displayName: 'Download test data and generate cmake config' inputs: script: | - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update ${{parameters.buildAdditionalParams}} + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --update ${{parameters.buildAdditionalParams}} workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -29,7 +29,7 @@ steps: displayName: 'Test ${{parameters.buildConfig}}' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test ${{parameters.buildAdditionalParams}}' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test ${{parameters.buildAdditionalParams}}' workingFolder: '$(Build.BinariesDirectory)' - task: PublishTestResults@2 diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index d4930faf193e7..a2c42de7d7367 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-GPU-CUDA10' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11' JobName: 'Windows_CI_GPU_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 1711ce9c0ac92..a31ad59615489 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -28,7 +28,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --update --msvc_toolset=14.11' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 @@ -45,7 +45,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build C# Debug' @@ -81,7 +81,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test' workingFolder: "$(Build.BinariesDirectory)" - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml index e8580f8fd495d..7b539c4c6b24d 100644 --- a/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-mklml-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --update' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 @@ -31,7 +31,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build C# Debug' @@ -66,7 +66,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test' workingFolder: "$(Build.BinariesDirectory)" - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml index 46008a8733f12..b45c0bf74c44b 100644 --- a/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ngraph-ci-pipeline.yml @@ -17,7 +17,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --update' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 displayName: 'Build Debug' @@ -33,7 +33,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build Release' @@ -49,7 +49,7 @@ jobs: displayName: 'Test Release' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test' workingFolder: "$(Build.BinariesDirectory)" - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml index 7339ecab0000d..b41400e0bd7ca 100644 --- a/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-nocontribops-ci-pipeline.yml @@ -8,7 +8,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum)' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml index 87e7b6eec4fca..3d3ced7591015 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86' JobName: 'Windows_CI_Dev_x86' DoNugetPack: 'false' NuPackScript : '' diff --git a/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml index b309f33a50d14..96a62acee69b6 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-nocontribops-ci-pipeline.yml @@ -9,7 +9,7 @@ jobs: AgentPool : 'Win-CPU' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --disable_contrib_ops --enable_msvc_static_runtime --x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --disable_contrib_ops --enable_msvc_static_runtime --x86' JobName: 'Windows_CI_Dev_x86' DoNugetPack: 'false' NuPackScript : '' \ No newline at end of file diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 976699e13c767..81dae302b7195 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -76,7 +76,7 @@ def download_and_unzip(build_dir, url, dest_folder): shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' url = url.replace('onnxruntimetestdata', hostname) - dest_folder = os.path.join(build_dir, 'installer','opencppcoverage') + dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') os.makedirs(dest_folder,exist_ok=True) subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) From 706fe94afbd524dcf7cb0a0670598a31178e03b2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 15:54:50 -0700 Subject: [PATCH 19/21] update --- tools/ci_build/github/download_test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 81dae302b7195..c90f84926e822 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -78,6 +78,6 @@ def download_and_unzip(build_dir, url, dest_folder): url = url.replace('onnxruntimetestdata', hostname) dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') os.makedirs(dest_folder,exist_ok=True) - subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) + subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) From 361b47c01fd80462b9b220153debf10ee33199af Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 16:05:20 -0700 Subject: [PATCH 20/21] update --- .../azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 2 +- .../github/azure-pipelines/linux-ngraph-ci-pipeline.yml | 2 +- tools/ci_build/github/download_test_data.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 688303d94f313..8b5fe3a3cd9eb 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -16,7 +16,7 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' # There are some tests in 20190130.zip that TensorRT can't run. Instead here use 20181210 opset8 for TensorRT test. - arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip + arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml index b6ab3fb42e7e7..131c4cc06e83f 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml @@ -27,7 +27,7 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' # nGraph provider fails on the latest 20190729.zip test. revert back to previous zip file until failures can be investigated - arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip + arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip --build_dir $(Build.BinariesDirectory) pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index c90f84926e822..12e10f4540717 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -52,7 +52,7 @@ def download_and_unzip(build_dir, url, dest_folder): if shutil.which('7z'): # 7-Zip subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True) elif shutil.which('7za'): # 7-Zip standalone - subprocess.run(['7za', 'x', local_file_name, '-y', '-o'+ dest_folder], check=True) + subprocess.run(['7za', 'x', local_file_name, '-y', '-o' + dest_folder], check=True) else: log.error("No unzip tool for use") sys.exit(1) @@ -78,6 +78,4 @@ def download_and_unzip(build_dir, url, dest_folder): url = url.replace('onnxruntimetestdata', hostname) dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') os.makedirs(dest_folder,exist_ok=True) - subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) - - + subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, dest_folder],check=True) From 9ad92a4669c4dc535472b3a93d20e95d1787535e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 5 Aug 2019 16:38:55 -0700 Subject: [PATCH 21/21] a --- tools/ci_build/github/download_test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 12e10f4540717..821b6054eca5e 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -78,4 +78,4 @@ def download_and_unzip(build_dir, url, dest_folder): url = url.replace('onnxruntimetestdata', hostname) dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') os.makedirs(dest_folder,exist_ok=True) - subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, dest_folder],check=True) + subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)