From 48dd7c83ca80a9f92c04707823e136f89cabfb24 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Fri, 21 Jun 2019 17:15:37 -0700 Subject: [PATCH 01/67] commented out part of cpu pipeline, added azure blob upload --- .../azure-pipelines/nuget/templates/cpu.yml | 277 ++++++++++-------- 1 file changed, 150 insertions(+), 127 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 75caa0fc1dd93..4f9e647d3e2de 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -22,131 +22,154 @@ jobs: copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) -- template: ../../templates/win-x86-ci.yml - parameters: - AgentPool : $(AgentPoolWin) - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage - cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - -- template: ../../templates/linux-ci.yml - parameters: - AgentPool : $(AgentPoolLinux) - JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' - DoNugetPack : 'true' - ArtifactName: 'drop-linux' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/linux-x64 - cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 - cd $(Build.BinariesDirectory) - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) - ls -al $(Build.ArtifactStagingDirectory) - -- template: ../../templates/linux-ci.yml - parameters: - AgentPool : $(AgentPoolLinux) - JobName: 'Linux_CI_Dev_x86' - BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' - DoNugetPack : 'true' - ArtifactName: 'drop-linux-x86' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/linux-x86 - cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 - cd $(Build.BinariesDirectory) - zip -r linux-x86.zip linux-x86 - cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) - ls -al $(Build.ArtifactStagingDirectory) - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - cd $cwd - -- job: NuGet_Packaging - pool: $(AgentPoolWin) - dependsOn: - - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Linux_CI_Dev - - Linux_CI_Dev_x86 - - MacOS_CI_Dev - condition: succeeded() - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux-x86' +# - template: ../../templates/win-x86-ci.yml +# parameters: +# AgentPool : $(AgentPoolWin) +# JobName: 'Windows_CI_Dev_x86' +# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' +# DoDebugBuild: 'false' +# DoNugetPack : 'true' +# DoCompliance: 'false' +# DoEsrp: ${{ parameters.DoEsrp }} +# NuPackScript: | +# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage +# cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ +# ren *.nupkg win-x86.zip +# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/linux-ci.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# JobName: 'Linux_CI_Dev' +# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' +# DoNugetPack : 'true' +# ArtifactName: 'drop-linux' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/linux-x64 +# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 +# cd $(Build.BinariesDirectory) +# zip -r linux-x64.zip linux-x64 +# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) +# ls -al $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/linux-ci.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# JobName: 'Linux_CI_Dev_x86' +# BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' +# DoNugetPack : 'true' +# ArtifactName: 'drop-linux-x86' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/linux-x86 +# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 +# cd $(Build.BinariesDirectory) +# zip -r linux-x86.zip linux-x86 +# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) +# ls -al $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/mac-ci.yml +# parameters: +# AgentPool : $(AgentPoolMacOS) +# JobName: 'MacOS_CI_Dev' +# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' +# DoNugetPack : 'true' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/osx-x64 +# find $(Build.BinariesDirectory) +# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ +# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM +# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib +# find $(Build.BinariesDirectory)/osx-x64 -ls +# cwd=`pwd` +# cd $(Build.BinariesDirectory) +# zip -r osx-x64.zip osx-x64 +# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) +# cd $cwd + +# - job: NuGet_Packaging +# pool: $(AgentPoolWin) +# dependsOn: +# - Windows_CI_Dev +# - Windows_CI_Dev_x86 +# - Linux_CI_Dev +# - Linux_CI_Dev_x86 +# - MacOS_CI_Dev +# condition: succeeded() +# steps: +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - NuGet' +# inputs: +# artifactName: 'drop-nuget' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true + +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - Win-x86' +# inputs: +# artifactName: 'drop-win-x86-zip' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true + +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - Linux' +# inputs: +# artifactName: 'drop-linux' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true + +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - Linux-x86' +# inputs: +# artifactName: 'drop-linux-x86' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true + +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - MacOS' +# inputs: +# artifactName: 'drop-osx' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true + +# - template: bundle_dlls.yml + +# - template: ../../templates/esrp_nuget.yml +# parameters: +# DisplayName: 'ESRP - sign NuGet package' +# FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' +# DoEsrp: ${{ parameters.DoEsrp }} + + +# - task: PublishPipelineArtifact@0 +# displayName: 'Publish Pipeline NuGet Artifact' +# inputs: +# artifactName: 'drop-signed-nuget' +# targetPath: '$(Build.ArtifactStagingDirectory)' + + - task: AzureFileCopy@2 + displayName: 'AzureBlob File Copy' inputs: - artifactName: 'drop-linux-x86' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - template: bundle_dlls.yml - - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' - DoEsrp: ${{ parameters.DoEsrp }} - - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' - -- template: test_all_os.yml + SourcePath: '$(Build.ArtifactStagingDirectory)' + azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' + Destination: AzureBlob + storage: ortpackages + ContainerName: ortpackages + + +# - template: test_all_os.yml + + +# - job: Publish package +# pool: $(AgentPoolWin) +# steps: +# - task: AzureFileCopy@2 +# displayName: 'AzureBlob File Copy' +# inputs: +# SourcePath: '$(Build.BinariesDirectory)/nuget-artifact' +# azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' +# Destination: AzureBlob +# storage: ortpackages +# ContainerName: ortpackages From c496a1a3e4eb3605de4040b63eba9b19ccb0de34 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Fri, 21 Jun 2019 17:19:16 -0700 Subject: [PATCH 02/67] fix indentation --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 4f9e647d3e2de..e3badb76b5a3a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -89,8 +89,8 @@ jobs: # cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) # cd $cwd -# - job: NuGet_Packaging -# pool: $(AgentPoolWin) +- job: NuGet_Packaging + pool: $(AgentPoolWin) # dependsOn: # - Windows_CI_Dev # - Windows_CI_Dev_x86 @@ -98,7 +98,7 @@ jobs: # - Linux_CI_Dev_x86 # - MacOS_CI_Dev # condition: succeeded() -# steps: + steps: # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - NuGet' # inputs: From c82cbdd28564e448283c78787143851ab3952bf2 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 25 Jun 2019 17:54:23 -0700 Subject: [PATCH 03/67] dependecy added --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index e3badb76b5a3a..ffccaef68729b 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -91,8 +91,8 @@ jobs: - job: NuGet_Packaging pool: $(AgentPoolWin) -# dependsOn: -# - Windows_CI_Dev + dependsOn: + - Windows_CI_Dev # - Windows_CI_Dev_x86 # - Linux_CI_Dev # - Linux_CI_Dev_x86 @@ -149,7 +149,7 @@ jobs: # artifactName: 'drop-signed-nuget' # targetPath: '$(Build.ArtifactStagingDirectory)' - - task: AzureFileCopy@2 + - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: SourcePath: '$(Build.ArtifactStagingDirectory)' From da5b6c733f67af60515e386e33f42bd3be4413ab Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 25 Jun 2019 19:54:04 -0700 Subject: [PATCH 04/67] use v-2 of the azure file copy task instead of v-3 --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index ffccaef68729b..beb14a5ed707d 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -149,7 +149,7 @@ jobs: # artifactName: 'drop-signed-nuget' # targetPath: '$(Build.ArtifactStagingDirectory)' - - task: AzureFileCopy@3 + - task: AzureFileCopy@2 displayName: 'AzureBlob File Copy' inputs: SourcePath: '$(Build.ArtifactStagingDirectory)' From 1bc927d4d4d798f2d450543e3996154dff3e16b9 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 10:57:28 -0700 Subject: [PATCH 05/67] use hosted agent for the copy task --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index beb14a5ed707d..a11892e9d7dac 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -90,7 +90,9 @@ jobs: # cd $cwd - job: NuGet_Packaging - pool: $(AgentPoolWin) + pool: + name: Hosted + demands: azureps dependsOn: - Windows_CI_Dev # - Windows_CI_Dev_x86 @@ -149,7 +151,7 @@ jobs: # artifactName: 'drop-signed-nuget' # targetPath: '$(Build.ArtifactStagingDirectory)' - - task: AzureFileCopy@2 + - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: SourcePath: '$(Build.ArtifactStagingDirectory)' From ec91daabfc4bd7ecc0146e5992c1430853e88cd0 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 11:58:54 -0700 Subject: [PATCH 06/67] use latest windows hosted agent --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index a11892e9d7dac..115b1391b411f 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -91,7 +91,7 @@ jobs: - job: NuGet_Packaging pool: - name: Hosted + name: Hosted Windows 2019 with VS2019 demands: azureps dependsOn: - Windows_CI_Dev From 3aaf88b8996c412e34a316bf7f430daca6c3cda9 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 12:01:27 -0700 Subject: [PATCH 07/67] do without waiting for build --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 115b1391b411f..716abb2b24e31 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -93,8 +93,8 @@ jobs: pool: name: Hosted Windows 2019 with VS2019 demands: azureps - dependsOn: - - Windows_CI_Dev +# dependsOn: +# - Windows_CI_Dev # - Windows_CI_Dev_x86 # - Linux_CI_Dev # - Linux_CI_Dev_x86 From 38cd7bc8883ebe877ed641e66892087609eddc5d Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 12:04:09 -0700 Subject: [PATCH 08/67] add dependency to build for copy job --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 716abb2b24e31..baa52b4df86e8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -93,8 +93,8 @@ jobs: pool: name: Hosted Windows 2019 with VS2019 demands: azureps -# dependsOn: -# - Windows_CI_Dev + dependsOn: + - Windows_CI_Dev # - Windows_CI_Dev_x86 # - Linux_CI_Dev # - Linux_CI_Dev_x86 From 33fd354d8fdd515d8c455c71deabe5a613b11d0f Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 12:58:32 -0700 Subject: [PATCH 09/67] try a file without build --- .../azure-pipelines/nuget/templates/cpu.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index baa52b4df86e8..786ead21111e7 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -8,18 +8,18 @@ parameters: DoEsrp: 'false' jobs: -- template: ../../templates/win-ci.yml - parameters: - AgentPool : $(AgentPoolWin) - JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) +# - template: ../../templates/win-ci.yml +# parameters: +# AgentPool : $(AgentPoolWin) +# JobName: 'Windows_CI_Dev' +# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' +# DoDebugBuild: 'false' +# DoNugetPack : 'true' +# DoCompliance: 'false' +# DoEsrp: ${{ parameters.DoEsrp }} +# NuPackScript: | +# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage +# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) # - template: ../../templates/win-x86-ci.yml @@ -93,8 +93,8 @@ jobs: pool: name: Hosted Windows 2019 with VS2019 demands: azureps - dependsOn: - - Windows_CI_Dev +# dependsOn: +# - Windows_CI_Dev # - Windows_CI_Dev_x86 # - Linux_CI_Dev # - Linux_CI_Dev_x86 @@ -154,7 +154,7 @@ jobs: - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: - SourcePath: '$(Build.ArtifactStagingDirectory)' + SourcePath: '$(Build.SourcesDirectory)\README.md' azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' Destination: AzureBlob storage: ortpackages From f6202c74951006d0703c8f69a7466fb4d5f60357 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 13:55:35 -0700 Subject: [PATCH 10/67] try some wildcard file --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 786ead21111e7..f72a831535dce 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -154,7 +154,7 @@ jobs: - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: - SourcePath: '$(Build.SourcesDirectory)\README.md' + SourcePath: '$(Build.SourcesDirectory)\.\requirements-*.txt' azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' Destination: AzureBlob storage: ortpackages From e172399a480578a249c2f88505a528e25050cd4f Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 20:48:09 -0700 Subject: [PATCH 11/67] added the whole pipeline --- .../azure-pipelines/nuget/templates/cpu.yml | 259 +++++++++--------- .../set-version-number-variables-step.yml | 7 + 2 files changed, 137 insertions(+), 129 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index f72a831535dce..f6805de22d1c9 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -8,153 +8,154 @@ parameters: DoEsrp: 'false' jobs: -# - template: ../../templates/win-ci.yml -# parameters: -# AgentPool : $(AgentPoolWin) -# JobName: 'Windows_CI_Dev' -# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' -# DoDebugBuild: 'false' -# DoNugetPack : 'true' -# DoCompliance: 'false' -# DoEsrp: ${{ parameters.DoEsrp }} -# NuPackScript: | -# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage -# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - - -# - template: ../../templates/win-x86-ci.yml -# parameters: -# AgentPool : $(AgentPoolWin) -# JobName: 'Windows_CI_Dev_x86' -# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' -# DoDebugBuild: 'false' -# DoNugetPack : 'true' -# DoCompliance: 'false' -# DoEsrp: ${{ parameters.DoEsrp }} -# NuPackScript: | -# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage -# cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ -# ren *.nupkg win-x86.zip -# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/linux-ci.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# JobName: 'Linux_CI_Dev' -# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' -# DoNugetPack : 'true' -# ArtifactName: 'drop-linux' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/linux-x64 -# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 -# cd $(Build.BinariesDirectory) -# zip -r linux-x64.zip linux-x64 -# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) -# ls -al $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/linux-ci.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# JobName: 'Linux_CI_Dev_x86' -# BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' -# DoNugetPack : 'true' -# ArtifactName: 'drop-linux-x86' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/linux-x86 -# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 -# cd $(Build.BinariesDirectory) -# zip -r linux-x86.zip linux-x86 -# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) -# ls -al $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/mac-ci.yml -# parameters: -# AgentPool : $(AgentPoolMacOS) -# JobName: 'MacOS_CI_Dev' -# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' -# DoNugetPack : 'true' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/osx-x64 -# find $(Build.BinariesDirectory) -# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ -# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM -# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -# find $(Build.BinariesDirectory)/osx-x64 -ls -# cwd=`pwd` -# cd $(Build.BinariesDirectory) -# zip -r osx-x64.zip osx-x64 -# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) -# cd $cwd +- template: ../../templates/win-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_Dev' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + + +- template: ../../templates/win-x86-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_Dev_x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage + cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ + ren *.nupkg win-x86.zip + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) + +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' + DoNugetPack : 'true' + ArtifactName: 'drop-linux' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x64 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 + cd $(Build.BinariesDirectory) + zip -r linux-x64.zip linux-x64 + cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) + +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_Dev_x86' + BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' + DoNugetPack : 'true' + ArtifactName: 'drop-linux-x86' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x86 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 + cd $(Build.BinariesDirectory) + zip -r linux-x86.zip linux-x86 + cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) + +- template: ../../templates/mac-ci.yml + parameters: + AgentPool : $(AgentPoolMacOS) + JobName: 'MacOS_CI_Dev' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' + DoNugetPack : 'true' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/osx-x64 + find $(Build.BinariesDirectory) + cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ + dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM + strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib + find $(Build.BinariesDirectory)/osx-x64 -ls + cwd=`pwd` + cd $(Build.BinariesDirectory) + zip -r osx-x64.zip osx-x64 + cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) + cd $cwd - job: NuGet_Packaging pool: name: Hosted Windows 2019 with VS2019 demands: azureps -# dependsOn: -# - Windows_CI_Dev -# - Windows_CI_Dev_x86 -# - Linux_CI_Dev -# - Linux_CI_Dev_x86 -# - MacOS_CI_Dev -# condition: succeeded() + dependsOn: + - Windows_CI_Dev + - Windows_CI_Dev_x86 + - Linux_CI_Dev + - Linux_CI_Dev_x86 + - MacOS_CI_Dev + condition: succeeded() steps: -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - NuGet' -# inputs: -# artifactName: 'drop-nuget' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'drop-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - Win-x86' -# inputs: -# artifactName: 'drop-win-x86-zip' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Win-x86' + inputs: + artifactName: 'drop-win-x86-zip' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - Linux' -# inputs: -# artifactName: 'drop-linux' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - Linux-x86' -# inputs: -# artifactName: 'drop-linux-x86' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux-x86' + inputs: + artifactName: 'drop-linux-x86' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - MacOS' -# inputs: -# artifactName: 'drop-osx' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - MacOS' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - template: bundle_dlls.yml + - template: bundle_dlls.yml -# - template: ../../templates/esrp_nuget.yml -# parameters: -# DisplayName: 'ESRP - sign NuGet package' -# FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' -# DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} -# - task: PublishPipelineArtifact@0 -# displayName: 'Publish Pipeline NuGet Artifact' -# inputs: -# artifactName: 'drop-signed-nuget' -# targetPath: '$(Build.ArtifactStagingDirectory)' + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' + - template: ../../templates/set-version-number-variable-step.yml - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: - SourcePath: '$(Build.SourcesDirectory)\.\requirements-*.txt' + SourcePath: '$(Build.ArtifactStagingDirectory)/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' Destination: AzureBlob storage: ortpackages diff --git a/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml index 007d40b7aef0f..dcbdcffb730d8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml +++ b/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml @@ -12,6 +12,10 @@ steps: FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse HEAD`) DO ( @echo ##vso[task.setvariable variable=OnnxRuntimeGitCommitHash;]%%F ) + + FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( + @echo ##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]%%F + ) workingDirectory: '$(Build.SourcesDirectory)' condition: eq(variables['Agent.OS'], 'Windows_NT') @@ -26,5 +30,8 @@ steps: _OnnxRuntimeGitCommitHash=$(git rev-parse HEAD) echo "##vso[task.setvariable variable=OnnxRuntimeGitCommitHash;]$_OnnxRuntimeGitCommitHash" + _OnnxRuntimeGitCommitHash=$(git rev-parse --short=8 HEAD) + echo "##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]$_OnnxRuntimeGitCommitHash" + workingDirectory: '$(Build.SourcesDirectory)' condition: not(eq(variables['Agent.OS'], 'Windows_NT')) \ No newline at end of file From e9a7ef1011827f56d176da1256b2806e42665260 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 20:55:37 -0700 Subject: [PATCH 12/67] fix syntax error --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index f6805de22d1c9..9f8651047d2e8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -155,7 +155,7 @@ jobs: - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: - SourcePath: '$(Build.ArtifactStagingDirectory)/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + SourcePath: '$(Build.ArtifactStagingDirectory)/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' Destination: AzureBlob storage: ortpackages From 1eafb26155c7ec976e01ca126a068498b4f332e8 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 20:56:52 -0700 Subject: [PATCH 13/67] fix syntax error more --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 9f8651047d2e8..770ff39c11c39 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -151,7 +151,7 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/set-version-number-variable-step.yml + - template: ../../templates/set-version-number-variables-step.yml - task: AzureFileCopy@3 displayName: 'AzureBlob File Copy' inputs: From d55993bf31f38eddfbf01c7c707a31d39500fb69 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 21:45:22 -0700 Subject: [PATCH 14/67] skip linux ci dev --- .../nuget/templates/bundle_dlls.yml | 6 +++--- .../github/azure-pipelines/nuget/templates/cpu.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml index cc8e0fc1c8e57..60b4db609e2a5 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml @@ -11,10 +11,10 @@ steps: unzip %%~ni.zip -d %%~ni del /Q %%~ni.zip unzip win-x86.zip -d win-x86 - unzip linux-x64.zip -d linux-x64 +# unzip linux-x64.zip -d linux-x64 unzip linux-x86.zip -d linux-x86 mkdir %%~ni\runtimes\win-x86 - mkdir %%~ni\runtimes\linux-x64 +# mkdir %%~ni\runtimes\linux-x64 mkdir %%~ni\runtimes\linux-x86 mkdir %%~ni\runtimes\win-x86\native mkdir %%~ni\runtimes\linux-x64\native @@ -22,7 +22,7 @@ steps: move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb - move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so +# move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so move linux-x86\linux-x86\libonnxruntime.so %%~ni\runtimes\linux-x86\native\libonnxruntime.so unzip osx-x64.zip -d osx-x64 dir osx-x64 /s diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 770ff39c11c39..28ae71a8521aa 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -96,7 +96,7 @@ jobs: dependsOn: - Windows_CI_Dev - Windows_CI_Dev_x86 - - Linux_CI_Dev +# - Linux_CI_Dev - Linux_CI_Dev_x86 - MacOS_CI_Dev condition: succeeded() @@ -115,12 +115,12 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Linux' + # inputs: + # artifactName: 'drop-linux' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Linux-x86' From 5a1c42721b1233ef346d06c17a96608ae831b62a Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 26 Jun 2019 22:00:02 -0700 Subject: [PATCH 15/67] remove linux --- .../azure-pipelines/nuget/templates/bundle_dlls.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml index 60b4db609e2a5..2b02a436f0513 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml @@ -11,10 +11,10 @@ steps: unzip %%~ni.zip -d %%~ni del /Q %%~ni.zip unzip win-x86.zip -d win-x86 -# unzip linux-x64.zip -d linux-x64 + unzip linux-x86.zip -d linux-x86 mkdir %%~ni\runtimes\win-x86 -# mkdir %%~ni\runtimes\linux-x64 + mkdir %%~ni\runtimes\linux-x86 mkdir %%~ni\runtimes\win-x86\native mkdir %%~ni\runtimes\linux-x64\native @@ -22,7 +22,7 @@ steps: move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb -# move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so + move linux-x86\linux-x86\libonnxruntime.so %%~ni\runtimes\linux-x86\native\libonnxruntime.so unzip osx-x64.zip -d osx-x64 dir osx-x64 /s @@ -39,3 +39,6 @@ steps: copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) displayName: 'Bundle NuGet and other binaries' +# unzip linux-x64.zip -d linux-x64 +# mkdir %%~ni\runtimes\linux-x64 +# move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so From 5c76212ed341fda5ad1c05dc483bd1397e4d6d1f Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 27 Jun 2019 09:35:16 -0700 Subject: [PATCH 16/67] finalized, cleaned up --- .../nuget/templates/bundle_dlls.yml | 9 ++---- .../azure-pipelines/nuget/templates/cpu.yml | 32 +++++++------------ 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml index 2b02a436f0513..cc8e0fc1c8e57 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/bundle_dlls.yml @@ -11,10 +11,10 @@ steps: unzip %%~ni.zip -d %%~ni del /Q %%~ni.zip unzip win-x86.zip -d win-x86 - + unzip linux-x64.zip -d linux-x64 unzip linux-x86.zip -d linux-x86 mkdir %%~ni\runtimes\win-x86 - + mkdir %%~ni\runtimes\linux-x64 mkdir %%~ni\runtimes\linux-x86 mkdir %%~ni\runtimes\win-x86\native mkdir %%~ni\runtimes\linux-x64\native @@ -22,7 +22,7 @@ steps: move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb - + move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so move linux-x86\linux-x86\libonnxruntime.so %%~ni\runtimes\linux-x86\native\libonnxruntime.so unzip osx-x64.zip -d osx-x64 dir osx-x64 /s @@ -39,6 +39,3 @@ steps: copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) displayName: 'Bundle NuGet and other binaries' -# unzip linux-x64.zip -d linux-x64 -# mkdir %%~ni\runtimes\linux-x64 -# move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 28ae71a8521aa..606e25e0a24d3 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -92,11 +92,15 @@ jobs: - job: NuGet_Packaging pool: name: Hosted Windows 2019 with VS2019 + # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, + # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. + # So, all the copy/publish jobs are being run on hosted agent + # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy demands: azureps dependsOn: - Windows_CI_Dev - Windows_CI_Dev_x86 -# - Linux_CI_Dev + - Linux_CI_Dev - Linux_CI_Dev_x86 - MacOS_CI_Dev condition: succeeded() @@ -115,12 +119,12 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Linux' - # inputs: - # artifactName: 'drop-linux' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Linux-x86' @@ -161,18 +165,6 @@ jobs: storage: ortpackages ContainerName: ortpackages - -# - template: test_all_os.yml +- template: test_all_os.yml -# - job: Publish package -# pool: $(AgentPoolWin) -# steps: -# - task: AzureFileCopy@2 -# displayName: 'AzureBlob File Copy' -# inputs: -# SourcePath: '$(Build.BinariesDirectory)/nuget-artifact' -# azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' -# Destination: AzureBlob -# storage: ortpackages -# ContainerName: ortpackages From 50790704fa4417fc340f665fb0755ca085eb380e Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 27 Jun 2019 11:44:47 -0700 Subject: [PATCH 17/67] make the publish task conditional on signing --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index df57ce672e476..498fc2a0e8dfa 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -158,7 +158,8 @@ jobs: - template: ../../templates/set-version-number-variables-step.yml - task: AzureFileCopy@3 - displayName: 'AzureBlob File Copy' + displayName: 'Copy Signed NuGet Package to Blob Store' + condition: ${{ parameters.DoEsrp }} inputs: SourcePath: '$(Build.ArtifactStagingDirectory)/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' From 23e9342382ff1d3668dcb5c183d061a35b5a13b2 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 18 Jul 2019 17:09:39 -0700 Subject: [PATCH 18/67] added job name parameters --- .../github/azure-pipelines/nuget/templates/test_linux.yml | 3 ++- .../github/azure-pipelines/nuget/templates/test_macos.yml | 3 ++- .../github/azure-pipelines/nuget/templates/test_win.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 1eabf3fdf8408..cdf2a1c06e063 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,8 +1,9 @@ parameters: AgentPool: 'Linux-CPU' + JobName: 'NuGet_Test_Linux' jobs: -- job: NuGet_Test_Linux +- job: ${{ parameters.JobName }} pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 03bb2cf4198b2..6db2d125cab99 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -1,8 +1,9 @@ parameters: AgentPool : 'Hosted macOS High Sierra' + JobName: 'NuGet_Test_MacOS' jobs: -- job: NuGet_Test_MacOS +- job: ${{ parameters.JobName }} pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index ce445baa6977e..d27c4e2515db9 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -1,8 +1,9 @@ parameters: AgentPool : 'Win-CPU' + JobName: 'NuGet_Test_Win' jobs: -- job: NuGet_Test_Win +- job: ${{ parameters.JobName }} pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging From 51a3918b123b192a8c53ab76c037d279fea59b23 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 20 Jul 2019 20:58:29 -0700 Subject: [PATCH 19/67] some changes in cpu yml --- .../azure-pipelines/nuget/templates/cpu.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 498fc2a0e8dfa..6914fbe005037 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -169,4 +169,39 @@ jobs: - template: test_all_os.yml +- job: Publish_NuGet_Package_And_Report + pool: + name: Hosted Windows 2019 with VS2019 + # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, + # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. + # So, all the copy/publish jobs are being run on hosted agent + # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy + demands: azureps + dependsOn: + - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + + - task: CmdLine@1 + displayName: 'Install conda modules mysql-connector-python' + inputs: + filename: '$CONDA\conda.exe' + arguments: 'install -q --insecure -y mysql-connector-python' + timeoutInMinutes: 10 + - template: ../../templates/set-version-number-variables-step.yml + - task: AzureFileCopy@3 + displayName: 'Copy Signed NuGet Package to Blob Store' + condition: ${{ parameters.DoEsrp }} + inputs: + SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' + Destination: AzureBlob + storage: ortpackages + ContainerName: ortpackages From 84475399d8e3518531161b3d54e43cd42347d476 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Mon, 22 Jul 2019 19:08:10 -0700 Subject: [PATCH 20/67] stripped off part of the pipeline for testing --- .../azure-pipelines/nuget/templates/cpu.yml | 226 +++++++++--------- .../nuget/templates/test_all_os.yml | 12 +- 2 files changed, 123 insertions(+), 115 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 8aa434b9c3029..670527f44cb76 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -23,72 +23,72 @@ jobs: copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) -- template: ../../templates/win-x86-ci.yml - parameters: - AgentPool : $(AgentPoolWin) - JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage - cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ - ren *.nupkg win-x86.zip - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - -- template: ../../templates/linux-ci.yml - parameters: - AgentPool : $(AgentPoolLinux) - JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' - DoNugetPack : 'true' - ArtifactName: 'drop-linux' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/linux-x64 - cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 - cd $(Build.BinariesDirectory) - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) - ls -al $(Build.ArtifactStagingDirectory) - -- template: ../../templates/linux-ci.yml - parameters: - AgentPool : $(AgentPoolLinux) - JobName: 'Linux_CI_Dev_x86' - BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' - DoNugetPack : 'true' - ArtifactName: 'drop-linux-x86' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/linux-x86 - cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 - cd $(Build.BinariesDirectory) - zip -r linux-x86.zip linux-x86 - cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) - ls -al $(Build.ArtifactStagingDirectory) - -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - cd $cwd +# - template: ../../templates/win-x86-ci.yml +# parameters: +# AgentPool : $(AgentPoolWin) +# JobName: 'Windows_CI_Dev_x86' +# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' +# DoDebugBuild: 'false' +# DoNugetPack : 'true' +# DoCompliance: 'false' +# DoEsrp: ${{ parameters.DoEsrp }} +# NuPackScript: | +# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage +# cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ +# ren *.nupkg win-x86.zip +# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/linux-ci.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# JobName: 'Linux_CI_Dev' +# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' +# DoNugetPack : 'true' +# ArtifactName: 'drop-linux' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/linux-x64 +# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 +# cd $(Build.BinariesDirectory) +# zip -r linux-x64.zip linux-x64 +# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) +# ls -al $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/linux-ci.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# JobName: 'Linux_CI_Dev_x86' +# BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' +# DoNugetPack : 'true' +# ArtifactName: 'drop-linux-x86' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/linux-x86 +# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 +# cd $(Build.BinariesDirectory) +# zip -r linux-x86.zip linux-x86 +# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) +# ls -al $(Build.ArtifactStagingDirectory) + +# - template: ../../templates/mac-ci.yml +# parameters: +# AgentPool : $(AgentPoolMacOS) +# JobName: 'MacOS_CI_Dev' +# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' +# DoNugetPack : 'true' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/osx-x64 +# find $(Build.BinariesDirectory) +# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ +# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM +# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib +# find $(Build.BinariesDirectory)/osx-x64 -ls +# cwd=`pwd` +# cd $(Build.BinariesDirectory) +# zip -r osx-x64.zip osx-x64 +# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) +# cd $cwd - job: NuGet_Packaging pool: @@ -100,10 +100,10 @@ jobs: demands: azureps dependsOn: - Windows_CI_Dev - - Windows_CI_Dev_x86 - - Linux_CI_Dev - - Linux_CI_Dev_x86 - - MacOS_CI_Dev + # - Windows_CI_Dev_x86 + # - Linux_CI_Dev + # - Linux_CI_Dev_x86 + # - MacOS_CI_Dev condition: succeeded() steps: - task: DownloadPipelineArtifact@0 @@ -113,33 +113,33 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Win-x86' - inputs: - artifactName: 'drop-win-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux-x86' - inputs: - artifactName: 'drop-linux-x86' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Win-x86' + # inputs: + # artifactName: 'drop-win-x86-zip' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true + + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Linux' + # inputs: + # artifactName: 'drop-linux' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true + + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Linux-x86' + # inputs: + # artifactName: 'drop-linux-x86' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true + + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - MacOS' + # inputs: + # artifactName: 'drop-osx' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true - template: bundle_dlls.yml @@ -156,16 +156,6 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' - - template: ../../templates/set-version-number-variables-step.yml - - task: AzureFileCopy@3 - displayName: 'Copy Signed NuGet Package to Blob Store' - condition: ${{ parameters.DoEsrp }} - inputs: - SourcePath: '$(Build.ArtifactStagingDirectory)/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' - Destination: AzureBlob - storage: ortpackages - ContainerName: ortpackages - template: test_all_os.yml @@ -179,8 +169,8 @@ jobs: demands: azureps dependsOn: - NuGet_Test_Win - - NuGet_Test_Linux - - NuGet_Test_MacOS + # - NuGet_Test_Linux + # - NuGet_Test_MacOS steps: - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Signed NuGet Package' @@ -188,12 +178,30 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' + - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: filename: '$CONDA\conda.exe' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 + + - task: BatchScript@1 + displayName: 'Post binary sizes to the dashboard database' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + workingFolder: "$(Build.BinariesDirectory)" + continueOnError: ${{parameters.ContinueOnError}} + condition: ${{parameters.PostToDashBoard}} + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + - template: ../../templates/set-version-number-variables-step.yml - task: AzureFileCopy@3 displayName: 'Copy Signed NuGet Package to Blob Store' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml index 8a60f4a2675ef..65180fe55b6d8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml @@ -3,10 +3,10 @@ jobs: - template: test_win.yml parameters: AgentPool : $(AgentPoolWin) -- template: test_linux.yml - parameters: - AgentPool : $(AgentPoolLinux) -- template: test_macos.yml - parameters: - AgentPool : $(AgentPoolMacOS) +# - template: test_linux.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# - template: test_macos.yml +# parameters: +# AgentPool : $(AgentPoolMacOS) From 13c1ecf4649a731dd9465808150d32c2c035815b Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Mon, 22 Jul 2019 19:13:19 -0700 Subject: [PATCH 21/67] fixes --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 670527f44cb76..43c136c24a47e 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -197,8 +197,7 @@ jobs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' workingFolder: "$(Build.BinariesDirectory)" - continueOnError: ${{parameters.ContinueOnError}} - condition: ${{parameters.PostToDashBoard}} + condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From f663ca29c149f93d5644754fd2c097e8cd5529ae Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 11:17:54 -0700 Subject: [PATCH 22/67] use win-cpu agent pool --- .../github/azure-pipelines/nuget/cpu-esrp-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml index 2bd66515dc072..5e82a8941a805 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml @@ -1,6 +1,6 @@ # Defined as pipeline variables -# variables: -# AgentPoolWin : 'Win-CPU' +variables: + AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' From ddeb62bfe6b4bcd52442bd30c64c5ffef64de3d9 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 11:20:28 -0700 Subject: [PATCH 23/67] revert last --- .../github/azure-pipelines/nuget/cpu-esrp-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml index 5e82a8941a805..2bd66515dc072 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml @@ -1,6 +1,6 @@ # Defined as pipeline variables -variables: - AgentPoolWin : 'Win-CPU' +# variables: +# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' From 49f98278513d6b478f6e7a81aa2f4e1123ecab2e Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 12:31:02 -0700 Subject: [PATCH 24/67] fix agent pool in win-ci --- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 8ce6984dc55b4..bcd1cb8da9b76 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -13,10 +13,12 @@ parameters: MsbuildArguments: '/m' EnvSetupScript: 'setup_env.bat' CudaVersion: '' + AgentPool: 'Win-CPU' jobs: - job: ${{ parameters.JobName }} timeoutInMinutes: 120 + pool: ${{ parameters.AgentPool }} variables: buildDirectory: '$(Build.BinariesDirectory)' BuildCommand: ${{ parameters.BuildCommand }} From 839bf07e4e4d7e636d39e7014adbaa5d012a3fdb Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 14:42:23 -0700 Subject: [PATCH 25/67] disable others, just check conda and python --- .../azure-pipelines/nuget/templates/cpu.yml | 115 ++++++++++-------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 43c136c24a47e..6907c0c57e4c5 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -9,18 +9,18 @@ parameters: DoCompliance: 'false' jobs: -- template: ../../templates/win-ci.yml - parameters: - AgentPool : $(AgentPoolWin) - JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) +# - template: ../../templates/win-ci.yml +# parameters: +# AgentPool : $(AgentPoolWin) +# JobName: 'Windows_CI_Dev' +# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' +# DoDebugBuild: 'false' +# DoNugetPack : 'true' +# DoCompliance: ${{ parameters.DoCompliance }} +# DoEsrp: ${{ parameters.DoEsrp }} +# NuPackScript: | +# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage +# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) # - template: ../../templates/win-x86-ci.yml @@ -90,28 +90,28 @@ jobs: # cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) # cd $cwd -- job: NuGet_Packaging - pool: - name: Hosted Windows 2019 with VS2019 - # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, - # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. - # So, all the copy/publish jobs are being run on hosted agent - # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy - demands: azureps - dependsOn: - - Windows_CI_Dev - # - Windows_CI_Dev_x86 - # - Linux_CI_Dev - # - Linux_CI_Dev_x86 - # - MacOS_CI_Dev - condition: succeeded() - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true +# - job: NuGet_Packaging +# pool: +# name: Hosted Windows 2019 with VS2019 +# # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, +# # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. +# # So, all the copy/publish jobs are being run on hosted agent +# # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy +# demands: azureps +# dependsOn: +# - Windows_CI_Dev +# # - Windows_CI_Dev_x86 +# # - Linux_CI_Dev +# # - Linux_CI_Dev_x86 +# # - MacOS_CI_Dev +# condition: succeeded() +# steps: +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - NuGet' +# inputs: +# artifactName: 'drop-nuget' +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' +# continueOnError: true # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Win-x86' @@ -141,23 +141,23 @@ jobs: # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' # continueOnError: true - - template: bundle_dlls.yml +# - template: bundle_dlls.yml - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' - DoEsrp: ${{ parameters.DoEsrp }} +# - template: ../../templates/esrp_nuget.yml +# parameters: +# DisplayName: 'ESRP - sign NuGet package' +# FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' +# DoEsrp: ${{ parameters.DoEsrp }} - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' +# - task: PublishPipelineArtifact@0 +# displayName: 'Publish Pipeline NuGet Artifact' +# inputs: +# artifactName: 'drop-signed-nuget' +# targetPath: '$(Build.ArtifactStagingDirectory)' -- template: test_all_os.yml +# - template: test_all_os.yml - job: Publish_NuGet_Package_And_Report pool: @@ -168,15 +168,15 @@ jobs: # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy demands: azureps dependsOn: - - NuGet_Test_Win + # - NuGet_Test_Win # - NuGet_Test_Linux # - NuGet_Test_MacOS steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Signed NuGet Package' + # inputs: + # artifactName: 'drop-signed-nuget' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - task: UsePythonVersion@0 inputs: @@ -184,17 +184,24 @@ jobs: addToPath: true architecture: 'x64' + - task: CmdLine@2 + displayName: 'Check environment variables' + inputs: + script: | + echo %PATH% + echo %CONDA% + - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: - filename: '$CONDA\conda.exe' + filename: '%CONDA%\conda.exe' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 - task: BatchScript@1 displayName: 'Post binary sizes to the dashboard database' inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + filename: 'python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' workingFolder: "$(Build.BinariesDirectory)" condition: ${{ parameters.DoEsrp }} From edf631b24d1e2cedf27ae714f1f655913b32a031 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 14:46:52 -0700 Subject: [PATCH 26/67] some more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 6907c0c57e4c5..c40b654ef0c15 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -190,11 +190,12 @@ jobs: script: | echo %PATH% echo %CONDA% + dir %CONDA% /s - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: - filename: '%CONDA%\conda.exe' + filename: 'conda.exe' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 From f7e22119b2c4b5ab63c957f52f6f10e5330656db Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 15:19:14 -0700 Subject: [PATCH 27/67] some more log --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index c40b654ef0c15..acff8fe208570 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -191,11 +191,14 @@ jobs: echo %PATH% echo %CONDA% dir %CONDA% /s + where conda.exe + where conda.bat + where conda - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: - filename: 'conda.exe' + filename: 'conda.bat' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 From f6291c5b92a278dec7f72d77e23d9f840c14509a Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 15:23:45 -0700 Subject: [PATCH 28/67] some more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index acff8fe208570..3807e48e30398 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -190,7 +190,6 @@ jobs: script: | echo %PATH% echo %CONDA% - dir %CONDA% /s where conda.exe where conda.bat where conda From dc6170e536f9b4c6a79eda5d325b769f23ddea4a Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 15:28:26 -0700 Subject: [PATCH 29/67] cleanup --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 3807e48e30398..2b47b4ef68d27 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -190,14 +190,11 @@ jobs: script: | echo %PATH% echo %CONDA% - where conda.exe - where conda.bat - where conda - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: - filename: 'conda.bat' + filename: 'conda' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 From 71de71c9df8a1144ed51e344b0dfe8fa679f5893 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 15:44:12 -0700 Subject: [PATCH 30/67] add condabin path --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 2b47b4ef68d27..e3b253080bf05 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -194,7 +194,7 @@ jobs: - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: - filename: 'conda' + filename: '%CONDA%\condabin\conda.bat' arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 From 23da3e5a936c791f5310d9ba153d80c05bf75874 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 16:29:18 -0700 Subject: [PATCH 31/67] remove workingdir --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index e3b253080bf05..7568294f30c78 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -201,9 +201,8 @@ jobs: - task: BatchScript@1 displayName: 'Post binary sizes to the dashboard database' inputs: - filename: 'python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - workingFolder: "$(Build.BinariesDirectory)" + filename: 'python' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From f20c8e443e6972339ee5b7621c0aac3daeac12a5 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 16:40:43 -0700 Subject: [PATCH 32/67] more log --- .../ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 7568294f30c78..91fbe4fd00631 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -190,7 +190,8 @@ jobs: script: | echo %PATH% echo %CONDA% - + where python.exe + python.exe --version - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: @@ -201,7 +202,7 @@ jobs: - task: BatchScript@1 displayName: 'Post binary sizes to the dashboard database' inputs: - filename: 'python' + filename: 'python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' condition: ${{ parameters.DoEsrp }} env: From f5a10d6123182282971322b2fb8983734b7c789a Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:14:41 -0700 Subject: [PATCH 33/67] more log --- .../azure-pipelines/nuget/templates/cpu.yml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 91fbe4fd00631..240c67afb90e1 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -192,6 +192,8 @@ jobs: echo %CONDA% where python.exe python.exe --version + echo $(pythonLocation) + echo $(.pythonLocation) - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: @@ -199,15 +201,25 @@ jobs: arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 - - task: BatchScript@1 - displayName: 'Post binary sizes to the dashboard database' + - task: CmdLine@2 + displayName: 'Post binary sizes to the dashboard database using command line' inputs: - filename: 'python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + script: | + python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + # - task: BatchScript@1 + # displayName: 'Post binary sizes to the dashboard database' + # inputs: + # filename: 'python.exe' + # arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + # condition: ${{ parameters.DoEsrp }} + # env: + # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + - template: ../../templates/set-version-number-variables-step.yml - task: AzureFileCopy@3 displayName: 'Copy Signed NuGet Package to Blob Store' From fe2b57087230857b086411fe0cd264237ae49af5 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:16:28 -0700 Subject: [PATCH 34/67] more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 240c67afb90e1..efd7d00980f9a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -206,7 +206,7 @@ jobs: inputs: script: | python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg - $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + rem $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From 65f06381980d9635fe40f98b054b5a59a63c4596 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:17:31 -0700 Subject: [PATCH 35/67] more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index efd7d00980f9a..31c6ece64b081 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -205,8 +205,8 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg - rem $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From cdc47648dca914e37eafd07707d7764d0b779665 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:23:24 -0700 Subject: [PATCH 36/67] update --- .../azure-pipelines/nuget/templates/cpu.yml | 2 +- .../windows/post_binary_sizes_to_dashboard.py | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 31c6ece64b081..d0c0de31dece5 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -206,7 +206,7 @@ jobs: inputs: script: | python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg - $(pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + $(.pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py new file mode 100644 index 0000000000000..eb08c097c19f6 --- /dev/null +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +# command line arguments +# --package_file= +# --commit_hash= + +import argparse +import mysql.connector +import xml.etree.ElementTree as ET +import sys +import os + +def parse_arguments(): + parser = argparse.ArgumentParser(description="ONNXRuntime test coverge report uploader for dashboard") + parser.add_argument("--report_url", help="URL to the Cobertura XML report") + parser.add_argument("--report_file", help="Path to the local cobertura XML report") + parser.add_argument("--commit_hash", help="Full Git commit hash") + return parser.parse_args() + +def parse_xml_report(report_file): + tree = ET.parse(report_file) # may throw exception + root = tree.getroot() + result = {} + + result['coverage'] = float(root.get('line-rate')) + result['lines_covered'] = int(root.get('lines-covered')) + result['lines_valid'] = int(root.get('lines-valid')) + return result + +def write_to_db(coverage_data, args): + # connect to database + + cnx = mysql.connector.connect( + user='ort@onnxruntimedashboard', + password=os.environ.get('DASHBOARD_MYSQL_ORT_PASSWORD'), + host='onnxruntimedashboard.mysql.database.azure.com', + database='onnxruntime') + + try: + cursor = cnx.cursor() + + #delete old records + delete_query = ('DELETE FROM onnxruntime.test_coverage ' + 'WHERE UploadTime < DATE_SUB(Now(), INTERVAL 30 DAY);' + ) + + cursor.execute(delete_query) + + #insert current record + insert_query = ('INSERT INTO onnxruntime.test_coverage ' + '(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, ReportURL) ' + 'VALUES (Now(), "%s", %f, %d, %d, "%s") ' + 'ON DUPLICATE KEY UPDATE ' + 'UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, ReportURL="%s";' + ) % (args.commit_hash, + coverage_data['coverage'], + coverage_data['lines_covered'], + coverage_data['lines_valid'], + args.report_url, + coverage_data['coverage'], + coverage_data['lines_covered'], + coverage_data['lines_valid'], + args.report_url + ) + + cursor.execute(insert_query) + cnx.commit() + + # # Use below for debugging: + # cursor.execute('select * from onnxruntime.test_coverage') + # for r in cursor: + # print(r) + + cursor.close() + cnx.close() + except BaseException as e: + cnx.close() + raise e + + +if __name__ == "__main__": + try: + args = parse_arguments() + print(args) + # coverage_data = parse_xml_report(args.report_file) + # write_to_db(coverage_data, args) + except BaseException as e: + print(str(e)) + sys.exit(1) + + + From df94bcc4710f6d821bc0992212d405b5fea088ff Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:28:03 -0700 Subject: [PATCH 37/67] update --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index d0c0de31dece5..269ad975d4689 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -205,8 +205,7 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg - $(.pythonLocation)\python $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From 12c3b21c7d768b3c102b1face12051ffecd950e9 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 17:37:22 -0700 Subject: [PATCH 38/67] update py script --- .../github/windows/post_binary_sizes_to_dashboard.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index eb08c097c19f6..043ce5beead2d 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -14,9 +14,8 @@ import os def parse_arguments(): - parser = argparse.ArgumentParser(description="ONNXRuntime test coverge report uploader for dashboard") - parser.add_argument("--report_url", help="URL to the Cobertura XML report") - parser.add_argument("--report_file", help="Path to the local cobertura XML report") + parser = argparse.ArgumentParser(description="ONNXRuntime binary size uploader for dashboard") + parser.add_argument("--package_file", help="Path to the local nuget package") parser.add_argument("--commit_hash", help="Full Git commit hash") return parser.parse_args() From a403dca9b8794d35127f820b3c644c05662ffcd9 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 18:52:48 -0700 Subject: [PATCH 39/67] more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 269ad975d4689..f3744eea4fb56 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -194,6 +194,7 @@ jobs: python.exe --version echo $(pythonLocation) echo $(.pythonLocation) + where 7z.exe - task: CmdLine@1 displayName: 'Install conda modules mysql-connector-python' inputs: From 2f4ecd98027d9fd6815b618db924ee0bfed103b8 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 18:53:20 -0700 Subject: [PATCH 40/67] more log --- .../azure-pipelines/nuget/templates/cpu.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index f3744eea4fb56..266407ed5bfe2 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -221,13 +221,13 @@ jobs: # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) - template: ../../templates/set-version-number-variables-step.yml - - task: AzureFileCopy@3 - displayName: 'Copy Signed NuGet Package to Blob Store' - condition: ${{ parameters.DoEsrp }} - inputs: - SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' - Destination: AzureBlob - storage: ortpackages - ContainerName: ortpackages + # - task: AzureFileCopy@3 + # displayName: 'Copy Signed NuGet Package to Blob Store' + # condition: ${{ parameters.DoEsrp }} + # inputs: + # SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + # azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' + # Destination: AzureBlob + # storage: ortpackages + # ContainerName: ortpackages From 48964d93831a5c810e83bd77f7b118187497f344 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 19:55:21 -0700 Subject: [PATCH 41/67] try all packaging with 7zip --- .../azure-pipelines/nuget/templates/cpu.yml | 204 +++++++++--------- 1 file changed, 106 insertions(+), 98 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 266407ed5bfe2..cbb9c4cc99a0a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -9,18 +9,18 @@ parameters: DoCompliance: 'false' jobs: -# - template: ../../templates/win-ci.yml -# parameters: -# AgentPool : $(AgentPoolWin) -# JobName: 'Windows_CI_Dev' -# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' -# DoDebugBuild: 'false' -# DoNugetPack : 'true' -# DoCompliance: ${{ parameters.DoCompliance }} -# DoEsrp: ${{ parameters.DoEsrp }} -# NuPackScript: | -# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage -# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) +- template: ../../templates/win-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_Dev' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) # - template: ../../templates/win-x86-ci.yml @@ -38,21 +38,21 @@ jobs: # ren *.nupkg win-x86.zip # copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) -# - template: ../../templates/linux-ci.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# JobName: 'Linux_CI_Dev' -# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' -# DoNugetPack : 'true' -# ArtifactName: 'drop-linux' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/linux-x64 -# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 -# cd $(Build.BinariesDirectory) -# zip -r linux-x64.zip linux-x64 -# cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) -# ls -al $(Build.ArtifactStagingDirectory) +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' + DoNugetPack : 'true' + ArtifactName: 'drop-linux' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x64 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 + cd $(Build.BinariesDirectory) + zip -r linux-x64.zip linux-x64 + cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) # - template: ../../templates/linux-ci.yml # parameters: @@ -70,48 +70,48 @@ jobs: # cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) # ls -al $(Build.ArtifactStagingDirectory) -# - template: ../../templates/mac-ci.yml -# parameters: -# AgentPool : $(AgentPoolMacOS) -# JobName: 'MacOS_CI_Dev' -# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' -# DoNugetPack : 'true' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/osx-x64 -# find $(Build.BinariesDirectory) -# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ -# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM -# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -# find $(Build.BinariesDirectory)/osx-x64 -ls -# cwd=`pwd` -# cd $(Build.BinariesDirectory) -# zip -r osx-x64.zip osx-x64 -# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) -# cd $cwd - -# - job: NuGet_Packaging -# pool: -# name: Hosted Windows 2019 with VS2019 -# # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, -# # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. -# # So, all the copy/publish jobs are being run on hosted agent -# # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy -# demands: azureps -# dependsOn: -# - Windows_CI_Dev +- template: ../../templates/mac-ci.yml + parameters: + AgentPool : $(AgentPoolMacOS) + JobName: 'MacOS_CI_Dev' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' + DoNugetPack : 'true' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/osx-x64 + find $(Build.BinariesDirectory) + cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ + dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM + strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib + find $(Build.BinariesDirectory)/osx-x64 -ls + cwd=`pwd` + cd $(Build.BinariesDirectory) + zip -r osx-x64.zip osx-x64 + cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) + cd $cwd + +- job: NuGet_Packaging + pool: + name: Hosted Windows 2019 with VS2019 + # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, + # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. + # So, all the copy/publish jobs are being run on hosted agent + # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy + demands: azureps + dependsOn: + - Windows_CI_Dev # # - Windows_CI_Dev_x86 -# # - Linux_CI_Dev + - Linux_CI_Dev # # - Linux_CI_Dev_x86 -# # - MacOS_CI_Dev -# condition: succeeded() -# steps: -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - NuGet' -# inputs: -# artifactName: 'drop-nuget' -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact' -# continueOnError: true + - MacOS_CI_Dev + condition: succeeded() + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'drop-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Win-x86' @@ -120,12 +120,12 @@ jobs: # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' # continueOnError: true - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Linux' - # inputs: - # artifactName: 'drop-linux' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Linux-x86' @@ -134,27 +134,27 @@ jobs: # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' # continueOnError: true - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - MacOS' - # inputs: - # artifactName: 'drop-osx' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - MacOS' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true -# - template: bundle_dlls.yml + - template: bundle_dlls.yml -# - template: ../../templates/esrp_nuget.yml -# parameters: -# DisplayName: 'ESRP - sign NuGet package' -# FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' -# DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} -# - task: PublishPipelineArtifact@0 -# displayName: 'Publish Pipeline NuGet Artifact' -# inputs: -# artifactName: 'drop-signed-nuget' -# targetPath: '$(Build.ArtifactStagingDirectory)' + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' # - template: test_all_os.yml @@ -168,15 +168,15 @@ jobs: # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy demands: azureps dependsOn: - # - NuGet_Test_Win - # - NuGet_Test_Linux - # - NuGet_Test_MacOS + - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS steps: - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Signed NuGet Package' - # inputs: - # artifactName: 'drop-signed-nuget' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - task: UsePythonVersion@0 inputs: @@ -206,6 +206,14 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | + set PACKAGE_PATH_PREFIX = $(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort) + copy %PACKAGE_PATH_PREFIX%.nupkg %PACKAGE_PATH_PREFIX%.zip + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x64\native\libonnxruntime.so | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x86\native\libonnxruntime.so | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\osx-x64\native\libonnxruntime.dylib | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x64\native\onnxruntime.dll | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x86\native\onnxruntime.dll | grep "Size" + %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg condition: ${{ parameters.DoEsrp }} env: From f14f404f6ce59e638e006fef124a817a736de3dc Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 19:57:42 -0700 Subject: [PATCH 42/67] add all tests too --- .../azure-pipelines/nuget/templates/test_all_os.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml index 65180fe55b6d8..8a60f4a2675ef 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml @@ -3,10 +3,10 @@ jobs: - template: test_win.yml parameters: AgentPool : $(AgentPoolWin) -# - template: test_linux.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# - template: test_macos.yml -# parameters: -# AgentPool : $(AgentPoolMacOS) +- template: test_linux.yml + parameters: + AgentPool : $(AgentPoolLinux) +- template: test_macos.yml + parameters: + AgentPool : $(AgentPoolMacOS) From e7de1d090c9b85f5ba71df1aa286ef3b56e9fd20 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 19:59:24 -0700 Subject: [PATCH 43/67] enable tests --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index cbb9c4cc99a0a..d46becdd7cab2 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -157,7 +157,7 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)' -# - template: test_all_os.yml + - template: test_all_os.yml - job: Publish_NuGet_Package_And_Report pool: From a7c22dd6171927b23cc9c2349642238d66ba027b Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 20:00:18 -0700 Subject: [PATCH 44/67] enable tests --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index d46becdd7cab2..584645739559e 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -157,7 +157,7 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)' - - template: test_all_os.yml +- template: test_all_os.yml - job: Publish_NuGet_Package_And_Report pool: From 2a4db29116fe3c5dd56e1147c0f63e003fed0c51 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 21:34:32 -0700 Subject: [PATCH 45/67] some more log --- .../github/azure-pipelines/nuget/templates/cpu.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 584645739559e..d6204b31412bc 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -206,12 +206,17 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - set PACKAGE_PATH_PREFIX = $(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort) + set PACKAGE_DIR = $(Build.BinariesDirectory)\nuget-artifact\final-package + dir %PACKAGE_DIR% + set PACKAGE_PATH_PREFIX = $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort) copy %PACKAGE_PATH_PREFIX%.nupkg %PACKAGE_PATH_PREFIX%.zip - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x64\native\libonnxruntime.so | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x86\native\libonnxruntime.so | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x64\native\libonnxruntime.so + rem | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x86\native\libonnxruntime.so + rem | grep "Size" 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\osx-x64\native\libonnxruntime.dylib | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x64\native\onnxruntime.dll | grep "Size" + 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x64\native\onnxruntime.dll + rem | grep "Size" 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x86\native\onnxruntime.dll | grep "Size" %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg From ea30689f24d599338c736ec58774df8681de2861 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 23 Jul 2019 21:35:19 -0700 Subject: [PATCH 46/67] some more log --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index d6204b31412bc..5d1e652bd7042 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -202,6 +202,8 @@ jobs: arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 + - template: ../../templates/set-version-number-variables-step.yml + - task: CmdLine@2 displayName: 'Post binary sizes to the dashboard database using command line' inputs: @@ -233,7 +235,6 @@ jobs: # env: # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) - - template: ../../templates/set-version-number-variables-step.yml # - task: AzureFileCopy@3 # displayName: 'Copy Signed NuGet Package to Blob Store' # condition: ${{ parameters.DoEsrp }} From ee8c842e8883e55ce5ee95590cf4410c2e095f35 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 24 Jul 2019 09:17:59 -0700 Subject: [PATCH 47/67] some more log --- .../github/azure-pipelines/nuget/templates/cpu.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 5d1e652bd7042..78c3c0972af4a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -209,9 +209,12 @@ jobs: inputs: script: | set PACKAGE_DIR = $(Build.BinariesDirectory)\nuget-artifact\final-package - dir %PACKAGE_DIR% + echo dir %PACKAGE_DIR% + dir $(Build.BinariesDirectory)\nuget-artifact\final-package set PACKAGE_PATH_PREFIX = $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort) - copy %PACKAGE_PATH_PREFIX%.nupkg %PACKAGE_PATH_PREFIX%.zip + echo copy %PACKAGE_PATH_PREFIX%.nupkg %PACKAGE_PATH_PREFIX%.zip + copy $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).zip + where 7z.exe 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x64\native\libonnxruntime.so rem | grep "Size" 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x86\native\libonnxruntime.so From 1a35e78d700e2ee17b9a53406950bdabf6902691 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 24 Jul 2019 10:47:34 -0700 Subject: [PATCH 48/67] use for /R loop --- .../azure-pipelines/nuget/templates/cpu.yml | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 78c3c0972af4a..21836f55dbf4a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -208,23 +208,20 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - set PACKAGE_DIR = $(Build.BinariesDirectory)\nuget-artifact\final-package - echo dir %PACKAGE_DIR% - dir $(Build.BinariesDirectory)\nuget-artifact\final-package - set PACKAGE_PATH_PREFIX = $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort) - echo copy %PACKAGE_PATH_PREFIX%.nupkg %PACKAGE_PATH_PREFIX%.zip - copy $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg $(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).zip - where 7z.exe - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x64\native\libonnxruntime.so - rem | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\linux-x86\native\libonnxruntime.so - rem | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\osx-x64\native\libonnxruntime.dylib | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x64\native\onnxruntime.dll - rem | grep "Size" - 7z.exe l -slt %PACKAGE_PATH_PREFIX%.zip runtimes\win-x86\native\onnxruntime.dll | grep "Size" - - %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg + pushd $(Build.BinariesDirectory)\nuget-artifact\final-package + FOR /R %%i IN (*.nupkg) do ( + copy %%~ni.nupkg %%~ni.zip + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so + rem | grep "Size" + 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so + rem | grep "Size" + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | grep "Size" + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll + rem | grep "Size" + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | grep "Size" + %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file=%%~ni.nupkg + ) + condition: ${{ parameters.DoEsrp }} env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From 1ecde0a5295a508067276952f5ad4de41a1f7ad5 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 24 Jul 2019 21:50:50 -0700 Subject: [PATCH 49/67] updated the python script. templatized the solution --- .../azure-pipelines/nuget/templates/cpu.yml | 73 ++------------ ...upload-binary-sizes-from-nuget-package.yml | 44 +++++++++ .../windows/post_binary_sizes_to_dashboard.py | 97 +++++++++++-------- 3 files changed, 109 insertions(+), 105 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 21836f55dbf4a..e357135a6aa7f 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -91,13 +91,7 @@ jobs: cd $cwd - job: NuGet_Packaging - pool: - name: Hosted Windows 2019 with VS2019 - # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, - # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. - # So, all the copy/publish jobs are being run on hosted agent - # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy - demands: azureps + pool: $(AgentPoolWin) dependsOn: - Windows_CI_Dev # # - Windows_CI_Dev_x86 @@ -172,68 +166,13 @@ jobs: - NuGet_Test_Linux - NuGet_Test_MacOS steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - - - task: CmdLine@2 - displayName: 'Check environment variables' - inputs: - script: | - echo %PATH% - echo %CONDA% - where python.exe - python.exe --version - echo $(pythonLocation) - echo $(.pythonLocation) - where 7z.exe - - task: CmdLine@1 - displayName: 'Install conda modules mysql-connector-python' - inputs: - filename: '%CONDA%\condabin\conda.bat' - arguments: 'install -q --insecure -y mysql-connector-python' - timeoutInMinutes: 10 - template: ../../templates/set-version-number-variables-step.yml - - - task: CmdLine@2 - displayName: 'Post binary sizes to the dashboard database using command line' - inputs: - script: | - pushd $(Build.BinariesDirectory)\nuget-artifact\final-package - FOR /R %%i IN (*.nupkg) do ( - copy %%~ni.nupkg %%~ni.zip - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so - rem | grep "Size" - 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so - rem | grep "Size" - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | grep "Size" - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll - rem | grep "Size" - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | grep "Size" - %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file=%%~ni.nupkg - ) - - condition: ${{ parameters.DoEsrp }} - env: - DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) - - # - task: BatchScript@1 - # displayName: 'Post binary sizes to the dashboard database' - # inputs: - # filename: 'python.exe' - # arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --package_file="$(Build.BinariesDirectory)\nuget-artifact\final-package\Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - # condition: ${{ parameters.DoEsrp }} - # env: - # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + - template: upload-binary-sizes-from-nuget-package.yml + parameters: + nugetArtifactName: 'drop-signed-nuget' + downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + gitCommitHash: $(OnnxRuntimeGitCommitHashShort) # - task: AzureFileCopy@3 # displayName: 'Copy Signed NuGet Package to Blob Store' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml new file mode 100644 index 0000000000000..3c6b5c844420a --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -0,0 +1,44 @@ +parameters: + nugetArtifactName: 'drop-signed-nuget' + downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + gitCommitHash: '' +steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + inputs: + artifactName: {{ parameters.nugetArtifactName }} + targetPath: {{ parameters.downloadPath }} + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' + + - task: CmdLine@1 + displayName: 'Install conda modules mysql-connector-python' + inputs: + filename: '%CONDA%\condabin\conda.bat' + arguments: 'install -q --insecure -y mysql-connector-python' + timeoutInMinutes: 10 + + - task: CmdLine@2 + displayName: 'Post binary sizes to the dashboard database using command line' + inputs: + script: | + pushd {{ parameters.downloadPath }} + FOR /R %%i IN (*.nupkg) do ( + copy %%~ni.nupkg %%~ni.zip + REM use a single .csv file to put the data + echo "os,arch,build_config,size" > binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x86,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "osx,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x86,openmp,%%a" >> binary_size_data.txt + + %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + ) + + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index 043ce5beead2d..daeac9c328ea0 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -3,10 +3,6 @@ # Licensed under the MIT License. -# command line arguments -# --package_file= -# --commit_hash= - import argparse import mysql.connector import xml.etree.ElementTree as ET @@ -15,21 +11,38 @@ def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime binary size uploader for dashboard") - parser.add_argument("--package_file", help="Path to the local nuget package") parser.add_argument("--commit_hash", help="Full Git commit hash") - return parser.parse_args() + parser.add_argument("--build_project", default='Lotus', choices=['Lotus','onnxruntime'], help="Lotus or onnxruntime build project, to construct the build URL") + parser.add_argument("--build_id", help="Build Id") + parser.add_argument("--size_data_file", help="Path to file that contains the binary size data") -def parse_xml_report(report_file): - tree = ET.parse(report_file) # may throw exception - root = tree.getroot() - result = {} - - result['coverage'] = float(root.get('line-rate')) - result['lines_covered'] = int(root.get('lines-covered')) - result['lines_valid'] = int(root.get('lines-valid')) - return result + return parser.parse_args() -def write_to_db(coverage_data, args): +# Assumes size_data_file is a csv file with a header line, containing binary sizes and other attributes +# CSV fields are: +# os,arch,build_config,size +# No empty line or space between fields expected +def get_binary_sizes(size_data_file): + binary_size = [] + with open(size_data_file, 'r') as f: + line = f.readline() + headers = line.split(',') + while line: + line = f.readline() + if not line: + break; + linedata = line.split(',') + tablerow = {} + for i in range(1,len(headers)): + if headers[i] == 'size': + tablerow[headers[i]] = int(linedata[i]) + else: + tablerow[headers[i]] = linedata[i] + binary_size.append(tablerow) + return binary_size + + +def write_to_db(binary_size_data, args): # connect to database cnx = mysql.connector.connect( @@ -42,30 +55,38 @@ def write_to_db(coverage_data, args): cursor = cnx.cursor() #delete old records - delete_query = ('DELETE FROM onnxruntime.test_coverage ' - 'WHERE UploadTime < DATE_SUB(Now(), INTERVAL 30 DAY);' + delete_query = ('DELETE FROM onnxruntime.binary_size ' + 'WHERE build_time < DATE_SUB(Now(), INTERVAL 30 DAY);' ) cursor.execute(delete_query) - #insert current record - insert_query = ('INSERT INTO onnxruntime.test_coverage ' - '(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, ReportURL) ' - 'VALUES (Now(), "%s", %f, %d, %d, "%s") ' - 'ON DUPLICATE KEY UPDATE ' - 'UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, ReportURL="%s";' - ) % (args.commit_hash, - coverage_data['coverage'], - coverage_data['lines_covered'], - coverage_data['lines_valid'], - args.report_url, - coverage_data['coverage'], - coverage_data['lines_covered'], - coverage_data['lines_valid'], - args.report_url - ) - - cursor.execute(insert_query) + #insert current records + for row in binary_size_data: + insert_query = ('INSERT INTO onnxruntime.binary_size ' + '(build_time, build_project, build_id, commit_id, os, arch, build_config, size) ' + 'VALUES (Now(), "%s", "%s", "%s", "%s", "%s", "%s", %d) ' + 'ON DUPLICATE KEY UPDATE ' + 'build_time=Now(), build_project="%s", build_id="%s", commit_id="%s", os="%s", arch="%s", build_config="%s", size=%d;' + ) % ( + args.commit_hash, + args.build_project, + args.build_id, + row['os'], + row['arch'], + row['build_config'], + row['size'], + + args.build_project, + args.build_id, + row['os'], + row['arch'], + row['build_config'], + row['size'] + ) + + cursor.execute(insert_query) + cnx.commit() # # Use below for debugging: @@ -84,8 +105,8 @@ def write_to_db(coverage_data, args): try: args = parse_arguments() print(args) - # coverage_data = parse_xml_report(args.report_file) - # write_to_db(coverage_data, args) + binary_size_data = get_binary_sizes(args.size_data_file) + write_to_db(binary_size_data, args) except BaseException as e: print(str(e)) sys.exit(1) From 5711b33125a50d02c858733d7eaa188a3f6d27ef Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Wed, 24 Jul 2019 21:55:14 -0700 Subject: [PATCH 50/67] format fix --- ...upload-binary-sizes-from-nuget-package.yml | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 3c6b5c844420a..9288c5e800cde 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -1,44 +1,45 @@ parameters: - nugetArtifactName: 'drop-signed-nuget' - downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - gitCommitHash: '' + nugetArtifactName: 'drop-signed-nuget' + downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + gitCommitHash: '' + steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: {{ parameters.nugetArtifactName }} - targetPath: {{ parameters.downloadPath }} +- task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + inputs: + artifactName: {{ parameters.nugetArtifactName }} + targetPath: {{ parameters.downloadPath }} + +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - - - task: CmdLine@1 - displayName: 'Install conda modules mysql-connector-python' - inputs: - filename: '%CONDA%\condabin\conda.bat' - arguments: 'install -q --insecure -y mysql-connector-python' - timeoutInMinutes: 10 +- task: CmdLine@1 + displayName: 'Install conda modules mysql-connector-python' + inputs: + filename: '%CONDA%\condabin\conda.bat' + arguments: 'install -q --insecure -y mysql-connector-python' + timeoutInMinutes: 10 - - task: CmdLine@2 - displayName: 'Post binary sizes to the dashboard database using command line' - inputs: - script: | - pushd {{ parameters.downloadPath }} - FOR /R %%i IN (*.nupkg) do ( - copy %%~ni.nupkg %%~ni.zip - REM use a single .csv file to put the data - echo "os,arch,build_config,size" > binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x86,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "osx,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x86,openmp,%%a" >> binary_size_data.txt - - %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) - ) +- task: CmdLine@2 + displayName: 'Post binary sizes to the dashboard database using command line' + inputs: + script: | + pushd {{ parameters.downloadPath }} + FOR /R %%i IN (*.nupkg) do ( + copy %%~ni.nupkg %%~ni.zip + REM use a single .csv file to put the data + echo "os,arch,build_config,size" > binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x86,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "osx,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x86,openmp,%%a" >> binary_size_data.txt + + %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + ) - env: - DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) From 64e7e0cddffdd536d52ae4a0ec39f2b9bddab8b7 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 25 Jul 2019 09:21:59 -0700 Subject: [PATCH 51/67] format fix --- .../nuget/templates/upload-binary-sizes-from-nuget-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 9288c5e800cde..92ada3ec3da68 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -8,7 +8,8 @@ steps: displayName: 'Download Pipeline Artifact - Signed NuGet Package' inputs: artifactName: {{ parameters.nugetArtifactName }} - targetPath: {{ parameters.downloadPath }} + targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + #{{ parameters.downloadPath }} - task: UsePythonVersion@0 inputs: From a72b957ac16986971fa64e164ad98a07437f96a7 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 25 Jul 2019 09:23:58 -0700 Subject: [PATCH 52/67] try debug --- .../upload-binary-sizes-from-nuget-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 92ada3ec3da68..addc7886172c5 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -4,11 +4,11 @@ parameters: gitCommitHash: '' steps: -- task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: {{ parameters.nugetArtifactName }} - targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' +# - task: DownloadPipelineArtifact@0 +# displayName: 'Download Pipeline Artifact - Signed NuGet Package' +# inputs: +# artifactName: {{ parameters.nugetArtifactName }} +# targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' #{{ parameters.downloadPath }} - task: UsePythonVersion@0 From 2df53dcc8532a480dd912137f16544649632ba35 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 25 Jul 2019 09:35:15 -0700 Subject: [PATCH 53/67] try debug --- .../upload-binary-sizes-from-nuget-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index addc7886172c5..6d0c9f3f03ef0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -4,11 +4,11 @@ parameters: gitCommitHash: '' steps: -# - task: DownloadPipelineArtifact@0 -# displayName: 'Download Pipeline Artifact - Signed NuGet Package' -# inputs: -# artifactName: {{ parameters.nugetArtifactName }} -# targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' +- task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' #{{ parameters.downloadPath }} - task: UsePythonVersion@0 From ae1ca4c74487abedcb5337a563ff148a4b374fca Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Thu, 25 Jul 2019 12:43:09 -0700 Subject: [PATCH 54/67] added more log --- ...upload-binary-sizes-from-nuget-package.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 6d0c9f3f03ef0..fa5e36ddf9deb 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -28,17 +28,22 @@ steps: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - pushd {{ parameters.downloadPath }} + echo changing directory to artifact download path + pushd $(Build.BinariesDirectory)\nuget-artifact\final-package + echo processing nupkg FOR /R %%i IN (*.nupkg) do ( + echo processing %%~ni.nupkg copy %%~ni.nupkg %%~ni.zip + echo copied to zip + echo listing lib files in the zip REM use a single .csv file to put the data echo "os,arch,build_config,size" > binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "linux,x86,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "osx,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do echo "win,x86,openmp,%%a" >> binary_size_data.txt - + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "linux,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "linux,x86,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "osx,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "win,x64,openmp,%%a" >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "win,x86,openmp,%%a" >> binary_size_data.txt + echo calling python script to post to database %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) ) From 681cfc602b1a4954fa1d70ac6a6b9aa230fa6ff4 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Fri, 26 Jul 2019 17:42:30 -0700 Subject: [PATCH 55/67] fixed bugs in the py script --- .../windows/post_binary_sizes_to_dashboard.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index daeac9c328ea0..dbd697dc91f4c 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -26,19 +26,20 @@ def get_binary_sizes(size_data_file): binary_size = [] with open(size_data_file, 'r') as f: line = f.readline() - headers = line.split(',') + headers = line.strip().split(',') while line: line = f.readline() if not line: break; - linedata = line.split(',') + linedata = line.strip().split(',') tablerow = {} - for i in range(1,len(headers)): + for i in range(0,len(headers)): if headers[i] == 'size': tablerow[headers[i]] = int(linedata[i]) else: tablerow[headers[i]] = linedata[i] binary_size.append(tablerow) + print(binary_size) return binary_size @@ -63,15 +64,16 @@ def write_to_db(binary_size_data, args): #insert current records for row in binary_size_data: + print (row) insert_query = ('INSERT INTO onnxruntime.binary_size ' '(build_time, build_project, build_id, commit_id, os, arch, build_config, size) ' 'VALUES (Now(), "%s", "%s", "%s", "%s", "%s", "%s", %d) ' 'ON DUPLICATE KEY UPDATE ' - 'build_time=Now(), build_project="%s", build_id="%s", commit_id="%s", os="%s", arch="%s", build_config="%s", size=%d;' + 'build_time=Now(), build_project="%s", build_id="%s", size=%d;' ) % ( - args.commit_hash, args.build_project, args.build_id, + args.commit_hash, row['os'], row['arch'], row['build_config'], @@ -79,20 +81,17 @@ def write_to_db(binary_size_data, args): args.build_project, args.build_id, - row['os'], - row['arch'], - row['build_config'], row['size'] ) - + print (insert_query) cursor.execute(insert_query) - + cnx.commit() # # Use below for debugging: - # cursor.execute('select * from onnxruntime.test_coverage') - # for r in cursor: - # print(r) + cursor.execute('select * from onnxruntime.binary_size') + for r in cursor: + print(r) cursor.close() cnx.close() From f4c38c5b15df2a6afd70b63a26f77795ac807dff Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 11:47:33 -0700 Subject: [PATCH 56/67] check env on win ci machine --- .../azure-pipelines/nuget/templates/cpu.yml | 112 ++++++++++-------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 74adeecff2c13..21aad48bc3b82 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -9,18 +9,18 @@ parameters: DoCompliance: 'false' jobs: -- template: ../../templates/win-ci.yml - parameters: - AgentPool : $(AgentPoolWin) - JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' - DoDebugBuild: 'false' - DoNugetPack : 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) +# - template: ../../templates/win-ci.yml +# parameters: +# AgentPool : $(AgentPoolWin) +# JobName: 'Windows_CI_Dev' +# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' +# DoDebugBuild: 'false' +# DoNugetPack : 'true' +# DoCompliance: ${{ parameters.DoCompliance }} +# DoEsrp: ${{ parameters.DoEsrp }} +# NuPackScript: | +# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage +# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) # - template: ../../templates/win-x86-ci.yml @@ -38,21 +38,21 @@ jobs: # ren *.nupkg win-x86.zip # copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) -- template: ../../templates/linux-ci.yml - parameters: - AgentPool : $(AgentPoolLinux) - JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' - DoNugetPack : 'true' - ArtifactName: 'drop-linux' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/linux-x64 - cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 - cd $(Build.BinariesDirectory) - zip -r linux-x64.zip linux-x64 - cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) - ls -al $(Build.ArtifactStagingDirectory) +# - template: ../../templates/linux-ci.yml +# parameters: +# AgentPool : $(AgentPoolLinux) +# JobName: 'Linux_CI_Dev' +# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' +# DoNugetPack : 'true' +# ArtifactName: 'drop-linux' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/linux-x64 +# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 +# cd $(Build.BinariesDirectory) +# zip -r linux-x64.zip linux-x64 +# cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) +# ls -al $(Build.ArtifactStagingDirectory) # - template: ../../templates/linux-ci.yml # parameters: @@ -70,36 +70,50 @@ jobs: # cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) # ls -al $(Build.ArtifactStagingDirectory) -- template: ../../templates/mac-ci.yml - parameters: - AgentPool : $(AgentPoolMacOS) - JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' - DoNugetPack : 'true' - NuPackScript: | - set -e -x - mkdir $(Build.BinariesDirectory)/osx-x64 - find $(Build.BinariesDirectory) - cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ - dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM - strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib - find $(Build.BinariesDirectory)/osx-x64 -ls - cwd=`pwd` - cd $(Build.BinariesDirectory) - zip -r osx-x64.zip osx-x64 - cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) - cd $cwd +# - template: ../../templates/mac-ci.yml +# parameters: +# AgentPool : $(AgentPoolMacOS) +# JobName: 'MacOS_CI_Dev' +# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' +# DoNugetPack : 'true' +# NuPackScript: | +# set -e -x +# mkdir $(Build.BinariesDirectory)/osx-x64 +# find $(Build.BinariesDirectory) +# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ +# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM +# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib +# find $(Build.BinariesDirectory)/osx-x64 -ls +# cwd=`pwd` +# cd $(Build.BinariesDirectory) +# zip -r osx-x64.zip osx-x64 +# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) +# cd $cwd - job: NuGet_Packaging pool: $(AgentPoolWin) dependsOn: - - Windows_CI_Dev +# - Windows_CI_Dev # # - Windows_CI_Dev_x86 - - Linux_CI_Dev +# - Linux_CI_Dev # # - Linux_CI_Dev_x86 - - MacOS_CI_Dev +# - MacOS_CI_Dev condition: succeeded() steps: + - task: CmdLine@2 + displayName: 'check envinronment' + inputs: + script: | + echo check python + where python.exe + python --version + where conda + echo dump env vars + set + echo changing directory to artifact download path + + + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' inputs: From c391d761597eb7a61968efb2588024e30197913e Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 13:16:02 -0700 Subject: [PATCH 57/67] try installing python and conda --- .../azure-pipelines/nuget/templates/cpu.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 21aad48bc3b82..b3621b0a53d02 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -112,7 +112,25 @@ jobs: set echo changing directory to artifact download path + - task: PowerShell@2 + displayName: 'Download python' + inputs: + targetType: 'inline' + script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' + workingDirectory: '$(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Run python installer' + inputs: + filename: '$(Build.BinariesDirectory)\installer.exe' + arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' + timeoutInMinutes: 10 + - task: CmdLine@1 + displayName: 'Install conda modules' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' + arguments: 'install -q --insecure -y mysql-connector-python' + timeoutInMinutes: 10 - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' @@ -121,6 +139,16 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true + - task: CmdLine@2 + displayName: 'Post binary sizes to the dashboard database using command line' + inputs: + script: | + echo changing directory to artifact download path + echo os,arch,build_config,size > binary_size_data.txt + echo win,x64,openmp,3276987 >> binary_size_data.txt + echo calling python script to post to database + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Win-x86' # inputs: From 7d318a98eb16cbb1e2547df167f329c18676cef2 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 14:39:55 -0700 Subject: [PATCH 58/67] try installing python and conda --- .../azure-pipelines/nuget/templates/cpu.yml | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index b3621b0a53d02..290ef1a0852c4 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -132,13 +132,6 @@ jobs: arguments: 'install -q --insecure -y mysql-connector-python' timeoutInMinutes: 10 - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet' - inputs: - artifactName: 'drop-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - - task: CmdLine@2 displayName: 'Post binary sizes to the dashboard database using command line' inputs: @@ -148,6 +141,16 @@ jobs: echo win,x64,openmp,3276987 >> binary_size_data.txt echo calling python script to post to database $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - NuGet' + # inputs: + # artifactName: 'drop-nuget' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true + # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Win-x86' @@ -156,12 +159,12 @@ jobs: # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' # continueOnError: true - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Linux' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - Linux' + # inputs: + # artifactName: 'drop-linux' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - Linux-x86' @@ -170,30 +173,30 @@ jobs: # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' # continueOnError: true - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - MacOS' - inputs: - artifactName: 'drop-osx' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true + # - task: DownloadPipelineArtifact@0 + # displayName: 'Download Pipeline Artifact - MacOS' + # inputs: + # artifactName: 'drop-osx' + # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + # continueOnError: true - - template: bundle_dlls.yml + # - template: bundle_dlls.yml - - template: ../../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' - DoEsrp: ${{ parameters.DoEsrp }} + # - template: ../../templates/esrp_nuget.yml + # parameters: + # DisplayName: 'ESRP - sign NuGet package' + # FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + # DoEsrp: ${{ parameters.DoEsrp }} - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget' - targetPath: '$(Build.ArtifactStagingDirectory)' + # - task: PublishPipelineArtifact@0 + # displayName: 'Publish Pipeline NuGet Artifact' + # inputs: + # artifactName: 'drop-signed-nuget' + # targetPath: '$(Build.ArtifactStagingDirectory)' -- template: test_all_os.yml +# - template: test_all_os.yml - job: Publish_NuGet_Package_And_Report pool: From 3848abad28d19cb812c3315963e057e2e2200de3 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 14:43:57 -0700 Subject: [PATCH 59/67] cleanup --- .../azure-pipelines/nuget/templates/cpu.yml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 290ef1a0852c4..d7de185838c52 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -198,26 +198,26 @@ jobs: # - template: test_all_os.yml -- job: Publish_NuGet_Package_And_Report - pool: - name: Hosted Windows 2019 with VS2019 - # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, - # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. - # So, all the copy/publish jobs are being run on hosted agent - # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy - demands: azureps - dependsOn: - - NuGet_Test_Win - - NuGet_Test_Linux - - NuGet_Test_MacOS - steps: - - - template: ../../templates/set-version-number-variables-step.yml - - template: upload-binary-sizes-from-nuget-package.yml - parameters: - nugetArtifactName: 'drop-signed-nuget' - downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - gitCommitHash: $(OnnxRuntimeGitCommitHashShort) +# - job: Publish_NuGet_Package_And_Report +# pool: +# name: Hosted Windows 2019 with VS2019 +# # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, +# # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. +# # So, all the copy/publish jobs are being run on hosted agent +# # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy +# demands: azureps +# dependsOn: +# - NuGet_Test_Win +# - NuGet_Test_Linux +# - NuGet_Test_MacOS +# steps: + +# - template: ../../templates/set-version-number-variables-step.yml +# - template: upload-binary-sizes-from-nuget-package.yml +# parameters: +# nugetArtifactName: 'drop-signed-nuget' +# downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' +# gitCommitHash: $(OnnxRuntimeGitCommitHashShort) # - task: AzureFileCopy@3 # displayName: 'Copy Signed NuGet Package to Blob Store' From f6fe97d9d8449a205828422b57c1a04257f7e930 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 16:36:57 -0700 Subject: [PATCH 60/67] check env --- .../azure-pipelines/nuget/templates/cpu.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index d7de185838c52..6d3076fc524d8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -100,17 +100,6 @@ jobs: # - MacOS_CI_Dev condition: succeeded() steps: - - task: CmdLine@2 - displayName: 'check envinronment' - inputs: - script: | - echo check python - where python.exe - python --version - where conda - echo dump env vars - set - echo changing directory to artifact download path - task: PowerShell@2 displayName: 'Download python' @@ -136,6 +125,13 @@ jobs: displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | + echo checking env var for MYSQL password + set DASHBOARD_MYSQL_ORT_PASSWORD + echo %DASHBOARD_MYSQL_ORT_PASSWORD% + + echo checking all env + set + echo changing directory to artifact download path echo os,arch,build_config,size > binary_size_data.txt echo win,x64,openmp,3276987 >> binary_size_data.txt @@ -144,6 +140,7 @@ jobs: env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + - template: ../../templates/clean-agent-build-directory-step.yml # - task: DownloadPipelineArtifact@0 # displayName: 'Download Pipeline Artifact - NuGet' # inputs: From 7244f1117f6fbfc0b3c7d7aa1e86a3651f6d6927 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 17:03:15 -0700 Subject: [PATCH 61/67] echo --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 6d3076fc524d8..e259a513a7c74 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -128,6 +128,7 @@ jobs: echo checking env var for MYSQL password set DASHBOARD_MYSQL_ORT_PASSWORD echo %DASHBOARD_MYSQL_ORT_PASSWORD% + echo variable[dashboard-mysql-ort-password] = $(dashboard-mysql-ort-password) echo checking all env set From 24cb52cdc79a93efae98a93d7d73eceeb5833810 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 17:14:10 -0700 Subject: [PATCH 62/67] add the variable group --- tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index e259a513a7c74..821619dd67f64 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -92,6 +92,8 @@ jobs: - job: NuGet_Packaging pool: $(AgentPoolWin) + variables: + - group: Dashboard_MySQL_Secret dependsOn: # - Windows_CI_Dev # # - Windows_CI_Dev_x86 From 8a771268c2ba21b4642ed2d62b6648b9748645fb Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 17:27:50 -0700 Subject: [PATCH 63/67] cleaned up --- .../azure-pipelines/nuget/templates/cpu.yml | 376 +++++++++--------- 1 file changed, 188 insertions(+), 188 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 821619dd67f64..6977cedfc8dcb 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -9,215 +9,215 @@ parameters: DoCompliance: 'false' jobs: -# - template: ../../templates/win-ci.yml -# parameters: -# AgentPool : $(AgentPoolWin) -# JobName: 'Windows_CI_Dev' -# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' -# DoDebugBuild: 'false' -# DoNugetPack : 'true' -# DoCompliance: ${{ parameters.DoCompliance }} -# DoEsrp: ${{ parameters.DoEsrp }} -# NuPackScript: | -# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage -# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - - -# - template: ../../templates/win-x86-ci.yml -# parameters: -# AgentPool : $(AgentPoolWin) -# JobName: 'Windows_CI_Dev_x86' -# BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' -# DoDebugBuild: 'false' -# DoNugetPack : 'true' -# DoCompliance: 'false' -# DoEsrp: ${{ parameters.DoEsrp }} -# NuPackScript: | -# msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage -# cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ -# ren *.nupkg win-x86.zip -# copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/linux-ci.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# JobName: 'Linux_CI_Dev' -# BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' -# DoNugetPack : 'true' -# ArtifactName: 'drop-linux' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/linux-x64 -# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 -# cd $(Build.BinariesDirectory) -# zip -r linux-x64.zip linux-x64 -# cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) -# ls -al $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/linux-ci.yml -# parameters: -# AgentPool : $(AgentPoolLinux) -# JobName: 'Linux_CI_Dev_x86' -# BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' -# DoNugetPack : 'true' -# ArtifactName: 'drop-linux-x86' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/linux-x86 -# cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 -# cd $(Build.BinariesDirectory) -# zip -r linux-x86.zip linux-x86 -# cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) -# ls -al $(Build.ArtifactStagingDirectory) - -# - template: ../../templates/mac-ci.yml -# parameters: -# AgentPool : $(AgentPoolMacOS) -# JobName: 'MacOS_CI_Dev' -# BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' -# DoNugetPack : 'true' -# NuPackScript: | -# set -e -x -# mkdir $(Build.BinariesDirectory)/osx-x64 -# find $(Build.BinariesDirectory) -# cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ -# dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM -# strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -# find $(Build.BinariesDirectory)/osx-x64 -ls -# cwd=`pwd` -# cd $(Build.BinariesDirectory) -# zip -r osx-x64.zip osx-x64 -# cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) -# cd $cwd +- template: ../../templates/win-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_Dev' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + + +- template: ../../templates/win-x86-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_Dev_x86' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage + cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ + ren *.nupkg win-x86.zip + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) + +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' + DoNugetPack : 'true' + ArtifactName: 'drop-linux' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x64 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 + cd $(Build.BinariesDirectory) + zip -r linux-x64.zip linux-x64 + cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) + +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_Dev_x86' + BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86"' + DoNugetPack : 'true' + ArtifactName: 'drop-linux-x86' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x86 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86 + cd $(Build.BinariesDirectory) + zip -r linux-x86.zip linux-x86 + cp $(Build.BinariesDirectory)/linux*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) + +- template: ../../templates/mac-ci.yml + parameters: + AgentPool : $(AgentPoolMacOS) + JobName: 'MacOS_CI_Dev' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' + DoNugetPack : 'true' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/osx-x64 + find $(Build.BinariesDirectory) + cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ + dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM + strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib + find $(Build.BinariesDirectory)/osx-x64 -ls + cwd=`pwd` + cd $(Build.BinariesDirectory) + zip -r osx-x64.zip osx-x64 + cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) + cd $cwd - job: NuGet_Packaging pool: $(AgentPoolWin) - variables: - - group: Dashboard_MySQL_Secret dependsOn: -# - Windows_CI_Dev -# # - Windows_CI_Dev_x86 -# - Linux_CI_Dev -# # - Linux_CI_Dev_x86 -# - MacOS_CI_Dev + - Windows_CI_Dev + - Windows_CI_Dev_x86 + - Linux_CI_Dev + - Linux_CI_Dev_x86 + - MacOS_CI_Dev condition: succeeded() steps: - - task: PowerShell@2 - displayName: 'Download python' - inputs: - targetType: 'inline' - script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' - workingDirectory: '$(Build.BinariesDirectory)' + # - task: PowerShell@2 + # displayName: 'Download python' + # inputs: + # targetType: 'inline' + # script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' + # workingDirectory: '$(Build.BinariesDirectory)' - - task: CmdLine@1 - displayName: 'Run python installer' + # - task: CmdLine@1 + # displayName: 'Run python installer' + # inputs: + # filename: '$(Build.BinariesDirectory)\installer.exe' + # arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' + # timeoutInMinutes: 10 + # - task: CmdLine@1 + # displayName: 'Install conda modules' + # inputs: + # filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' + # arguments: 'install -q --insecure -y mysql-connector-python' + # timeoutInMinutes: 10 + + # - task: CmdLine@2 + # displayName: 'Post binary sizes to the dashboard database using command line' + # inputs: + # script: | + # echo checking env var for MYSQL password + # set DASHBOARD_MYSQL_ORT_PASSWORD + # echo %DASHBOARD_MYSQL_ORT_PASSWORD% + # echo variable[dashboard-mysql-ort-password] = $(dashboard-mysql-ort-password) + + # echo checking all env + # set + + # echo changing directory to artifact download path + # echo os,arch,build_config,size > binary_size_data.txt + # echo win,x64,openmp,3276987 >> binary_size_data.txt + # echo calling python script to post to database + # $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + # env: + # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + + # - template: ../../templates/clean-agent-build-directory-step.yml + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' inputs: - filename: '$(Build.BinariesDirectory)\installer.exe' - arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' - timeoutInMinutes: 10 - - task: CmdLine@1 - displayName: 'Install conda modules' + artifactName: 'drop-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true + + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Win-x86' inputs: - filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' - arguments: 'install -q --insecure -y mysql-connector-python' - timeoutInMinutes: 10 + artifactName: 'drop-win-x86-zip' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true - - task: CmdLine@2 - displayName: 'Post binary sizes to the dashboard database using command line' + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' inputs: - script: | - echo checking env var for MYSQL password - set DASHBOARD_MYSQL_ORT_PASSWORD - echo %DASHBOARD_MYSQL_ORT_PASSWORD% - echo variable[dashboard-mysql-ort-password] = $(dashboard-mysql-ort-password) - - echo checking all env - set - - echo changing directory to artifact download path - echo os,arch,build_config,size > binary_size_data.txt - echo win,x64,openmp,3276987 >> binary_size_data.txt - echo calling python script to post to database - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) - env: - DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) - - - template: ../../templates/clean-agent-build-directory-step.yml - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - NuGet' - # inputs: - # artifactName: 'drop-nuget' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux-x86' + inputs: + artifactName: 'drop-linux-x86' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Win-x86' - # inputs: - # artifactName: 'drop-win-x86-zip' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - MacOS' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Linux' - # inputs: - # artifactName: 'drop-linux' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - template: bundle_dlls.yml - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - Linux-x86' - # inputs: - # artifactName: 'drop-linux-x86' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} - # - task: DownloadPipelineArtifact@0 - # displayName: 'Download Pipeline Artifact - MacOS' - # inputs: - # artifactName: 'drop-osx' - # targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # continueOnError: true - # - template: bundle_dlls.yml + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' - # - template: ../../templates/esrp_nuget.yml - # parameters: - # DisplayName: 'ESRP - sign NuGet package' - # FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' - # DoEsrp: ${{ parameters.DoEsrp }} +- template: test_all_os.yml - # - task: PublishPipelineArtifact@0 - # displayName: 'Publish Pipeline NuGet Artifact' - # inputs: - # artifactName: 'drop-signed-nuget' - # targetPath: '$(Build.ArtifactStagingDirectory)' - - -# - template: test_all_os.yml - -# - job: Publish_NuGet_Package_And_Report -# pool: -# name: Hosted Windows 2019 with VS2019 -# # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, -# # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. -# # So, all the copy/publish jobs are being run on hosted agent -# # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy -# demands: azureps -# dependsOn: -# - NuGet_Test_Win -# - NuGet_Test_Linux -# - NuGet_Test_MacOS -# steps: - -# - template: ../../templates/set-version-number-variables-step.yml -# - template: upload-binary-sizes-from-nuget-package.yml -# parameters: -# nugetArtifactName: 'drop-signed-nuget' -# downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' -# gitCommitHash: $(OnnxRuntimeGitCommitHashShort) +- job: Publish_NuGet_Package_And_Report + variables: + - group: Dashboard_MySQL_Secret + pool: + name: Hosted Windows 2019 with VS2019 + # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, + # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. + # So, all the copy/publish jobs are being run on hosted agent + # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy + demands: azureps + dependsOn: + - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS + steps: + + - template: ../../templates/set-version-number-variables-step.yml + - template: upload-binary-sizes-from-nuget-package.yml + parameters: + nugetArtifactName: 'drop-signed-nuget' + downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + gitCommitHash: $(OnnxRuntimeGitCommitHashShort) # - task: AzureFileCopy@3 # displayName: 'Copy Signed NuGet Package to Blob Store' From 632dc11c256d3da7e5f460be4c0b3f678ee56715 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sat, 27 Jul 2019 19:35:25 -0700 Subject: [PATCH 64/67] fix the script to get the bin size --- .../azure-pipelines/nuget/templates/cpu.yml | 18 +++++++++--------- .../upload-binary-sizes-from-nuget-package.yml | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 6977cedfc8dcb..c8af048b6eace 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -219,13 +219,13 @@ jobs: downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' gitCommitHash: $(OnnxRuntimeGitCommitHashShort) - # - task: AzureFileCopy@3 - # displayName: 'Copy Signed NuGet Package to Blob Store' - # condition: ${{ parameters.DoEsrp }} - # inputs: - # SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - # azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' - # Destination: AzureBlob - # storage: ortpackages - # ContainerName: ortpackages + - task: AzureFileCopy@3 + displayName: 'Copy Signed NuGet Package to Blob Store' + condition: ${{ parameters.DoEsrp }} + inputs: + SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' + Destination: AzureBlob + storage: ortpackages + ContainerName: ortpackages diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index fa5e36ddf9deb..0dece267c306b 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -37,12 +37,12 @@ steps: echo copied to zip echo listing lib files in the zip REM use a single .csv file to put the data - echo "os,arch,build_config,size" > binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "linux,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "linux,x86,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "osx,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "win,x64,openmp,%%a" >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo "win,x86,openmp,%%a" >> binary_size_data.txt + echo os,arch,build_config,size > binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,openmp,%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x86\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x86,openmp,%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,openmp,%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,openmp,%%a >> binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,openmp,%%a >> binary_size_data.txt echo calling python script to post to database %CONDA%\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) ) From 2a8ea31346e071216bede4614b0b34db39face27 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sun, 28 Jul 2019 22:12:07 -0700 Subject: [PATCH 65/67] cleanup --- .../azure-pipelines/nuget/templates/cpu.yml | 57 +------------------ ...upload-binary-sizes-from-nuget-package.yml | 3 - .../windows/post_binary_sizes_to_dashboard.py | 10 +--- 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index c8af048b6eace..dca49ee42d915 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -101,47 +101,6 @@ jobs: condition: succeeded() steps: - # - task: PowerShell@2 - # displayName: 'Download python' - # inputs: - # targetType: 'inline' - # script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe' - # workingDirectory: '$(Build.BinariesDirectory)' - - # - task: CmdLine@1 - # displayName: 'Run python installer' - # inputs: - # filename: '$(Build.BinariesDirectory)\installer.exe' - # arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' - # timeoutInMinutes: 10 - # - task: CmdLine@1 - # displayName: 'Install conda modules' - # inputs: - # filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' - # arguments: 'install -q --insecure -y mysql-connector-python' - # timeoutInMinutes: 10 - - # - task: CmdLine@2 - # displayName: 'Post binary sizes to the dashboard database using command line' - # inputs: - # script: | - # echo checking env var for MYSQL password - # set DASHBOARD_MYSQL_ORT_PASSWORD - # echo %DASHBOARD_MYSQL_ORT_PASSWORD% - # echo variable[dashboard-mysql-ort-password] = $(dashboard-mysql-ort-password) - - # echo checking all env - # set - - # echo changing directory to artifact download path - # echo os,arch,build_config,size > binary_size_data.txt - # echo win,x64,openmp,3276987 >> binary_size_data.txt - # echo calling python script to post to database - # $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) - # env: - # DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) - - # - template: ../../templates/clean-agent-build-directory-step.yml - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' inputs: @@ -199,12 +158,9 @@ jobs: - job: Publish_NuGet_Package_And_Report variables: - group: Dashboard_MySQL_Secret + condition: ${{ parameters.DoEsrp }} pool: name: Hosted Windows 2019 with VS2019 - # AzureFileCopy@3 task has some bug that it depends on a particular version of azure power shell, - # which is not available in OnnxRuntime build VMs, but available in the latest hosted agents. - # So, all the copy/publish jobs are being run on hosted agent - # TODO: install the desired azureps on our VMs or use later bugfixed version of AzureFileCopy demands: azureps dependsOn: - NuGet_Test_Win @@ -215,17 +171,6 @@ jobs: - template: ../../templates/set-version-number-variables-step.yml - template: upload-binary-sizes-from-nuget-package.yml parameters: - nugetArtifactName: 'drop-signed-nuget' - downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' gitCommitHash: $(OnnxRuntimeGitCommitHashShort) - - task: AzureFileCopy@3 - displayName: 'Copy Signed NuGet Package to Blob Store' - condition: ${{ parameters.DoEsrp }} - inputs: - SourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' - azureSubscription: 'AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)' - Destination: AzureBlob - storage: ortpackages - ContainerName: ortpackages diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 0dece267c306b..55e1813951721 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -1,6 +1,4 @@ parameters: - nugetArtifactName: 'drop-signed-nuget' - downloadPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' gitCommitHash: '' steps: @@ -9,7 +7,6 @@ steps: inputs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - #{{ parameters.downloadPath }} - task: UsePythonVersion@0 inputs: diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index dbd697dc91f4c..7161b6f897457 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -39,7 +39,6 @@ def get_binary_sizes(size_data_file): else: tablerow[headers[i]] = linedata[i] binary_size.append(tablerow) - print(binary_size) return binary_size @@ -64,7 +63,6 @@ def write_to_db(binary_size_data, args): #insert current records for row in binary_size_data: - print (row) insert_query = ('INSERT INTO onnxruntime.binary_size ' '(build_time, build_project, build_id, commit_id, os, arch, build_config, size) ' 'VALUES (Now(), "%s", "%s", "%s", "%s", "%s", "%s", %d) ' @@ -83,15 +81,14 @@ def write_to_db(binary_size_data, args): args.build_id, row['size'] ) - print (insert_query) cursor.execute(insert_query) cnx.commit() # # Use below for debugging: - cursor.execute('select * from onnxruntime.binary_size') - for r in cursor: - print(r) + # cursor.execute('select * from onnxruntime.binary_size') + # for r in cursor: + # print(r) cursor.close() cnx.close() @@ -103,7 +100,6 @@ def write_to_db(binary_size_data, args): if __name__ == "__main__": try: args = parse_arguments() - print(args) binary_size_data = get_binary_sizes(args.size_data_file) write_to_db(binary_size_data, args) except BaseException as e: From 21d2b27527ec383ee6f260afabc9845e6f914e73 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sun, 28 Jul 2019 22:17:42 -0700 Subject: [PATCH 66/67] revert changes in the test scripts --- .../nuget/templates/test_linux.yml | 25 ++++++------------- .../nuget/templates/test_macos.yml | 3 +-- .../nuget/templates/test_win.yml | 3 +-- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index e5e34d11d0276..1eabf3fdf8408 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,10 +1,8 @@ parameters: AgentPool: 'Linux-CPU' - JobName: 'NuGet_Test_Linux' - TestGPU: 'false' jobs: -- job: ${{ parameters.JobName }} +- job: NuGet_Test_Linux pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging @@ -20,21 +18,12 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - ${{ if eq(parameters['TestGPU'], 'false') }}: - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact - displayName: 'Run Package Test (x64)' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - - - ${{ if eq(parameters['TestGPU'], 'true') }}: - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact - displayName: 'Run Package Test GPU (x64)' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + - script: | + set -e -x + $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact + displayName: 'Run Package Test (x64)' + env: + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 6db2d125cab99..03bb2cf4198b2 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -1,9 +1,8 @@ parameters: AgentPool : 'Hosted macOS High Sierra' - JobName: 'NuGet_Test_MacOS' jobs: -- job: ${{ parameters.JobName }} +- job: NuGet_Test_MacOS pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index d27c4e2515db9..ce445baa6977e 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -1,9 +1,8 @@ parameters: AgentPool : 'Win-CPU' - JobName: 'NuGet_Test_Win' jobs: -- job: ${{ parameters.JobName }} +- job: NuGet_Test_Win pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging From 19502a79efb3652a169f0271ac1eff5dc950fc54 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Sun, 28 Jul 2019 22:20:59 -0700 Subject: [PATCH 67/67] checkout test-linux from master --- .../nuget/templates/test_linux.yml | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 1eabf3fdf8408..2cb8763745653 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,5 +1,6 @@ parameters: AgentPool: 'Linux-CPU' + TestGPU: 'false' jobs: - job: NuGet_Test_Linux @@ -18,12 +19,21 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact - displayName: 'Run Package Test (x64)' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + - ${{ if eq(parameters['TestGPU'], 'false') }}: + - script: | + set -e -x + $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact + displayName: 'Run Package Test (x64)' + env: + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + + - ${{ if eq(parameters['TestGPU'], 'true') }}: + - script: | + set -e -x + $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact + displayName: 'Run Package Test GPU (x64)' + env: + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection'