Fix ARM64 build + Add NuGet pipeline including ARM binaries

cmake/CMakeLists.txt

@@ -132,8 +132,8 @@ add_definitions(-DEIGEN_MPL2_ONLY)
 
 if(onnxruntime_CROSS_COMPILING)
   set(CMAKE_CROSSCOMPILING ON)
-  string(APPEND CMAKE_CXX_FLAGS " -Wno-error=attributes")
-  string(APPEND CMAKE_C_FLAGS " -Wno-error=attributes")
+  #string(APPEND CMAKE_CXX_FLAGS " -Wno-error=attributes")
+  #string(APPEND CMAKE_C_FLAGS " -Wno-error=attributes")
 endif()
 
 #must after OpenMP settings

tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml

@@ -0,0 +1,22 @@
+# Defined as pipeline variables
+# variables:
+#   AgentPoolWin : 'Win-CPU'
+#   AgentPoolLinux : 'Linux-CPU'
+#   AgentPoolMacOS : 'macOS-10.13'
+
+schedules:
+- cron: "0 14 * * *"
+  displayName: Daily Build
+  branches:
+    include:
+    - master
+  always: true
+
+variables:
+  DisableContribOps: 'ON'
+
+jobs:
+- template: templates/cpu-nocontribops-arm64.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    DoEsrp: 'true'

tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml

@@ -0,0 +1,14 @@
+# Defined as pipeline variables
+# variables:
+#   AgentPoolWin : 'Win-CPU'
+#   AgentPoolLinux : 'Linux-CPU'
+#   AgentPoolMacOS : 'macOS-10.13'
+
+variables:
+  DisableContribOps: 'ON'
+
+jobs:
+- template: templates/cpu-nocontribops-arm64.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    DoEsrp: 'false'

tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml

@@ -0,0 +1,219 @@
+# Defined as pipeline variables
+# variables:
+#   AgentPoolWin : 'Win-CPU'
+#   AgentPoolLinux : 'Linux-CPU'
+#   AgentPoolMacOS : 'macOS-10.13'
+
+parameters:
+  DoEsrp: 'false'
+
+jobs:
+- template: ../../templates/win-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    JobName: 'Windows_CI_Dev'
+    BuildCommand:  '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
+    DoDebugBuild: 'false'
+    DoNugetPack : 'true'
+    DoCompliance: 'false'
+    DoEsrp: ${{ parameters.DoEsrp }}
+    NuPackScript: |
+     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage
+     copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
+
+
+- template: ../../templates/win-x86-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    JobName: 'Windows_CI_Dev_x86'
+    BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib  --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86'
+    DoDebugBuild: 'false'
+    DoNugetPack : 'true'
+    DoCompliance: 'false'
+    DoEsrp: ${{ parameters.DoEsrp }}
+    NuPackScript: |
+     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage
+     cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\
+     ren *.nupkg win-x86.zip
+     copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory)
+
+- template: ../../templates/win-ci-arm.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    JobName: 'Windows_Arm64_Dev'
+    BuildCommand:  '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64 --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
+    DoDebugBuild: 'false'
+    DoNugetPack : 'true'
+    DoCompliance: 'false'
+    MsbuildArguments: '/m /p:platform=arm64'
+    DoEsrp: ${{ parameters.DoEsrp }}
+    ArtifactName: 'drop-nuget-arm64'
+    NuPackScript: |
+     mkdir $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native
+     cd $(Build.BinariesDirectory)\arm64
+     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime.dll $(Build.BinariesDirectory)\arm64\runtimes\win10-arm\native
+     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
+     zip -r win10-arm.zip runtimes
+     copy *.zip $(Build.ArtifactStagingDirectory)
+
+- template: ../../templates/linux-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolLinux)
+    JobName: 'Linux_CI_Dev'
+    BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--disable_contrib_ops"'
+    DoNugetPack : 'true'
+    ArtifactName: 'drop-linux'
+    NuPackScript: |
+     set -e -x
+     mkdir $(Build.BinariesDirectory)/linux-x64
+     cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64
+     cd $(Build.BinariesDirectory)
+     zip -r linux-x64.zip linux-x64
+     cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory)
+     ls -al $(Build.ArtifactStagingDirectory)
+
+- template: ../../templates/linux-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolLinux)
+    JobName: 'Linux_CI_Dev_x86'
+    BuildCommand : 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -a x86 -x "--x86 --disable_contrib_ops"'
+    DoNugetPack : 'true'
+    ArtifactName: 'drop-linux-x86'
+    NuPackScript: |
+     set -e -x
+     mkdir $(Build.BinariesDirectory)/linux-x86
+     cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x86
+     cd $(Build.BinariesDirectory)
+     zip -r linux-x86.zip linux-x86
+     cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory)
+     ls -al $(Build.ArtifactStagingDirectory)
+
+- template: ../../templates/mac-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolMacOS)
+    JobName: 'MacOS_CI_Dev'
+    BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --use_openmp --enable_onnx_tests --config RelWithDebInfo'
+    DoNugetPack : 'true'
+    NuPackScript: |
+     set -e -x
+     mkdir $(Build.BinariesDirectory)/osx-x64
+     find $(Build.BinariesDirectory)
+     cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/
+     dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM
+     strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib
+     find $(Build.BinariesDirectory)/osx-x64
+     cwd=`pwd`
+     cd $(Build.BinariesDirectory)
+     zip -r osx-x64.zip osx-x64
+     cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory)
+     cd $cwd
+
+- job: NuGet_Packaging
+  pool: $(AgentPoolWin)
+  dependsOn:
+  - Windows_CI_Dev
+  - Windows_CI_Dev_x86
+  - Windows_Arm64_Dev
+  - Linux_CI_Dev
+  - Linux_CI_Dev_x86
+  - MacOS_CI_Dev
+  condition: succeeded()
+  steps:
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - NuGet'
+    inputs:
+      artifactName: 'drop-nuget'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - Win-x86'
+    inputs:
+      artifactName: 'drop-win-x86-zip'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - Arm64'
+    inputs:
+      artifactName: 'drop-nuget-arm64'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - Linux'
+    inputs:
+      artifactName: 'drop-linux'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - Linux-x86'
+    inputs:
+      artifactName: 'drop-linux-x86'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - MacOS'
+    inputs:
+      artifactName: 'drop-osx'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
+  - script: |
+     pushd $(Build.BinariesDirectory)\nuget-artifact
+     dir
+     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
+     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
+     set PATH=%CD%;%PATH%
+     FOR /R %%i IN (*.nupkg) do (
+         rename %%~ni.nupkg %%~ni.zip
+         unzip %%~ni.zip -d %%~ni
+         del /Q %%~ni.zip
+         unzip win-x86.zip -d win-x86
+         unzip win10-arm.zip -d win10-arm
+         unzip linux-x64.zip -d linux-x64
+         unzip linux-x86.zip -d linux-x86
+         mkdir %%~ni\runtimes\win-x86
+         mkdir %%~ni\runtimes\win10-arm
+         mkdir %%~ni\runtimes\linux-x64
+         mkdir %%~ni\runtimes\linux-x86
+         mkdir %%~ni\runtimes\win-x86\native
+         mkdir %%~ni\runtimes\win10-arm\native
+         mkdir %%~ni\runtimes\linux-x64\native
+         mkdir %%~ni\runtimes\linux-x86\native
+         move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll
+         move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib
+         move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb
+         move win10-arm\runtimes\win-x64\native\onnxruntime.dll %%~ni\runtimes\win10-arm\native\onnxruntime.dll
+         move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so
+         move linux-x86\linux-x86\libonnxruntime.so %%~ni\runtimes\linux-x86\native\libonnxruntime.so
+         unzip osx-x64.zip -d osx-x64
+         dir osx-x64 /s
+         mkdir %%~ni\runtimes\osx-x64
+         mkdir %%~ni\runtimes\osx-x64\native
+         move osx-x64\osx-x64\libonnxruntime.dylib %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib
+         move osx-x64\osx-x64\libonnxruntime.dylib.dSYM %%~ni\runtimes\osx-x64\native\libonnxruntime.dylib.dSYM
+         pushd %%~ni
+         zip -r ..\%%~ni.zip .
+         popd
+         move %%~ni.zip %%~ni.nupkg
+     )
+     popd
+     copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory)
+    displayName: 'Bundle NuGet and other binaries'
+
+  - template: ../../templates/esrp_nuget.yml
+    parameters:
+      DisplayName: 'ESRP - sign NuGet package'
+      FolderPath: '$(Build.BinariesDirectory)/nuget-artifact'
+      DoEsrp: ${{ parameters.DoEsrp }}
+
+  - task: PublishPipelineArtifact@0
+    displayName: 'Publish Pipeline NuGet Artifact'
+    inputs:
+      artifactName: 'drop-signed-nuget'
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+
+- template: test_all_os.yml

tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml

@@ -0,0 +1,103 @@
+parameters:
+  AgentPool : 'Win-CPU'
+  DoDebugBuild: 'true'
+  BuildCommand: ''
+  JobName: 'Windows_CI_Dev'
+  DoNugetPack:  'false'
+  NuPackScript : ''
+  ArtifactName: 'drop-nuget'
+  DoEsrp: 'false'
+  BuildArch: 'x64'
+  SetVcvars: 'false'
+  MsbuildArguments: '/m'
+  EnvSetupScript: 'setup_env.bat'
+  CudaVersion: ''
+
+jobs:
+- job: ${{ parameters.JobName }}
+  timeoutInMinutes: 120
+  pool: ${{ parameters.AgentPool }}
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+    BuildCommand: ${{ parameters.BuildCommand }}
+    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
+    DotNetExe: 'dotnet.exe'
+    CUDA_VERSION: ${{ parameters.CudaVersion }}
+
+  steps:
+    - template: set-test-data-variables-step.yml
+    - template: windows-build-tools-setup-steps.yml
+      parameters:
+        EnvSetupScript: ${{ parameters.EnvSetupScript }}
+        buildArch: ${{ parameters.BuildArch }}
+        setVcvars: ${{ parameters.SetVcvars }}
+
+    - task: CmdLine@1
+      displayName: 'Download test data and generate cmake config'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(BuildCommand) --update --config Debug RelWithDebInfo'
+        workingDirectory: "$(Build.BinariesDirectory)"
+
+    # Build Debug Mode
+    - ${{ if eq(parameters['DoDebugBuild'], 'true') }}:
+      - task: VSBuild@1
+        displayName: 'Build Debug'
+        inputs:
+          solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln'
+          platform: 'arm64'
+          configuration: 'Debug'
+          msbuildArguments: ${{ parameters.MsbuildArguments }}
+          msbuildArchitecture: 'x64'
+          logProjectEvents: true
+          workingFolder: '$(Build.BinariesDirectory)\Debug'
+          createLogFile: true
+
+    # Build RelWithDebInfo -- this variable required to build C#
+    - script: |
+       @echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo
+
+    - task: VSBuild@1
+      displayName: 'Build RelWithDebInfo'
+      inputs:
+        solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
+        platform: 'arm64'
+        configuration: 'RelWithDebInfo'
+        msbuildArguments: ${{ parameters.MsbuildArguments }}
+        msbuildArchitecture: 'x64'
+        logProjectEvents: true
+        workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
+
+    # Nuget packaging if needed
+    - ${{ if eq(parameters['DoNugetPack'], 'true') }}:
+      - task: BatchScript@1
+        displayName: 'Setup VS2017 env vars'
+        inputs:
+          filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat'
+          arguments: 'x64 -vcvars_ver=14.11'
+          modifyEnvironment: true
+
+      # Esrp signing
+      - template: esrp_dll.yml
+        parameters:
+          FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo'
+          DisplayName: 'ESRP - Sign Native dlls'
+          DoEsrp: ${{ parameters.DoEsrp }}
+
+      - script: |
+         ${{ parameters.NuPackScript }}
+        workingDirectory: '$(Build.SourcesDirectory)\csharp'
+        displayName: 'Create NuGet Package'
+
+      - task: PublishPipelineArtifact@0
+        displayName: 'Publish Pipeline Artifact: drop-nuget'
+        inputs:
+          artifactName: ${{ parameters.ArtifactName }}
+          targetPath: '$(Build.ArtifactStagingDirectory)'
+
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
+      condition: succeeded()
+
+    - template: clean-agent-build-directory-step.yml
+

tools/ci_build/github/azure-pipelines/templates/win-ci.yml

@@ -6,6 +6,7 @@ parameters:
   JobName: 'Windows_CI_Dev'
   DoNugetPack:  'false'
   NuPackScript : ''
+  ArtifactName: 'drop-nuget'
   DoEsrp: 'false'
   DoTestCoverage: 'false'
   BuildArch: 'x64'
@@ -163,7 +164,7 @@ jobs:
       - task: PublishPipelineArtifact@0
         displayName: 'Publish Pipeline Artifact: drop-nuget'
         inputs:
-          artifactName: 'drop-nuget'
+          artifactName: ${{ parameters.ArtifactName }}
           targetPath: '$(Build.ArtifactStagingDirectory)'
 
     # Compliance tasks require logs from Debug Build