diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml index ddf5d8103cac75..7ea5e7af112cf9 100644 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml @@ -13,6 +13,21 @@ parameters: pool: '' jobs: +- template: /eng/pipelines/coreclr/templates/build-corelib-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + container: ${{ parameters.container }} + testGroup: ${{ parameters.testGroup }} + crossrootfsDir: ${{ parameters.crossrootfsDir }} + timeoutInminutes: ${{ parameters.timeoutInMinutes }} + signBinaries: ${{ parameters.signBinaries }} + stagedBuild: ${{ parameters.stagedBuild }} + variables: ${{ parameters.variables }} + pool: ${{ parameters.pool }} + - template: /eng/pipelines/coreclr/templates/build-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 0249dc33eba230..0848c4cc3ea8cd 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -31,6 +31,17 @@ jobs: # - template: /eng/pipelines/common/checkout-job.yml +# +# Checked Corelib builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: checked + platformGroup: all + jobParameters: + testGroup: outerloop + # # Debug builds # diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml new file mode 100644 index 00000000000000..bfb91cb4d9095d --- /dev/null +++ b/eng/pipelines/coreclr/templates/build-corelib-job.yml @@ -0,0 +1,109 @@ +parameters: + archType: '' + buildConfig: '' + condition: true + container: '' + crossrootfsDir: '' + isOfficialBuild: false + osGroup: '' + osSubgroup: '' + platform: '' + pool: '' + signBinaries: false + stagedBuild: false + testGroup: '' + timeoutInMinutes: '' + variables: {} + +### Product build +jobs: +- template: xplat-pipeline-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + _BuildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + testGroup: ${{ parameters.testGroup }} + helixType: 'build/product/' + enableMicrobuild: true + stagedBuild: ${{ parameters.stagedBuild }} + pool: ${{ parameters.pool }} + condition: ${{ parameters.condition }} + + name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('CoreCLR CoreLib Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + # Run all steps in the container. + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + + gatherAssetManifests: true + variables: + - name: osGroup + value: ${{ parameters.osGroup }} + - name: osSubgroup + value: ${{ parameters.osSubgroup }} + - name: compilerArg + value: '' + - name: publishLogsArtifactPrefix + value: 'BuildLogs_CoreLib' + - name: officialBuildIdArg + value: '' + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - name: officialBuildIdArg + value: '/p:OfficialBuildId=$(Build.BuildNumber)' + + - ${{ parameters.variables }} + + steps: + + # Install native dependencies + # Linux builds use docker images with dependencies preinstalled, + # and FreeBSD builds use a build agent with dependencies + # preinstalled, so we only need this step for OSX and Windows. + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - script: sh $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) + displayName: Install native dependencies + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + # Necessary to install python + - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + displayName: Install native dependencies + + # # Install internal tools on official builds + # # Since our internal tools are behind an authenticated feed, + # # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. + # # We can't do this from within the build, so we need to do this as a separate step. + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/restore-internal-tools.yml + parameters: + installDotnet: true + + # Build Private CoreLib + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subsetCategory coreclr -subset corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci + displayName: Build System.Private.CoreLib + + + # Publish corelib output directory for consumption by libraries. + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(buildProductRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(corelibProductArtifactName) + displayName: Upload System.Private.CoreLib + + # Publish Logs + - task: PublishPipelineArtifact@1 + displayName: Publish Logs + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + continueOnError: true + condition: always() diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index ae418ed1533e71..aad6f7bf69fa40 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -75,6 +75,9 @@ jobs: - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' + - name: corelibProductArtifactName + value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + - name: managedTestArtifactName value: 'CoreCLRManagedTestArtifacts_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)' diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index c39cdd40427375..deee63825eff1e 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -95,7 +95,12 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + # Download corelib dependencies for coreclr libraries and not test + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: + - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + # Download full product dependencies for mono or test + - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: + - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 4f95464e683df5..14057daa4ff907 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -37,6 +37,7 @@ jobs: isOfficialBuild: ${{ parameters.isOfficialBuild }} isOfficialAllConfigurations: ${{ parameters.isOfficialAllConfigurations }} liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} @@ -49,7 +50,11 @@ jobs: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: dependsOn: - ${{ if ne(parameters.osGroup, 'WebAssembly') }}: - - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + # Use corelib dependencies for coreclr and non test builds + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: + - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, 'linux', parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index 15a05408761501..b821c45cdda4fc 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -6,6 +6,7 @@ parameters: framework: '' isOfficialBuild: false liveRuntimeBuildConfig: '' + runtimeFlavor: 'coreclr' timeoutInMinutes: 150 container: '' publishTestArtifacs: true @@ -38,6 +39,9 @@ jobs: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if in(parameters.framework, 'allConfigurations', 'net472') }}: - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # Libraries Test also depends on Product, now that the libraries build only depends on corelib + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} variables: - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index dd7fb83ae9a730..eb3b2e9493ddf5 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -14,6 +14,29 @@ variables: jobs: - template: /eng/pipelines/common/checkout-job.yml + # + # CoreLib Build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: + - Windows_NT_x64 + - Windows_NT_x86 + - ${{ if eq(variables['includeLinuxOuterloop'], true) }}: + - Linux_x64 + - Linux_musl_x64 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Linux_arm + - Linux_arm64 + - Linux_musl_arm64 + - ${{ if eq(variables['includeOsxOuterloop'], true) }}: + - OSX_x64 + jobParameters: + testGroup: innerloop + # # CoreCLR Build # diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 8b5d6ed0aad192..6035ec9730c7bf 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -31,6 +31,7 @@ jobs: container: '' # we just send to helix, no need to use a container. condition: ${{ parameters.condition }} pool: ${{ parameters.pool }} + testScope: ${{ parameters.testScope }} ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }} name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }} @@ -78,9 +79,19 @@ jobs: /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:Configuration=${{ parameters.buildConfig }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideCoreclrFromLiveDrop.binlog + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog displayName: Override $(runtimeFlavorName) from live drop + - script: $(_buildScript) -projects $(Build.SourcesDirectory)/src/libraries/pretest.proj + $(_runtimeArtifactsPathArg) + $(_runtimeOSArg) + /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} + /p:TargetArchitecture=${{ parameters.archType }} + /p:Configuration=${{ parameters.buildConfig }} + /p:SkipBuildProjects=true + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/pretest.binlog + displayName: Build PreTest to Generate Deps Json File + - template: /eng/pipelines/libraries/helix.yml parameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index a49a4b1bbda73e..45c5740583268f 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -39,6 +39,27 @@ stages: # - template: /eng/pipelines/common/checkout-job.yml + # + # Build CoreLib + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: ${{ variables.isOfficialBuild }} + # # Build CoreCLR # diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 8f909825a4311b..5edbdb1d88a357 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -118,6 +118,27 @@ jobs: - eng/pipelines/mono/* - eng/pipelines/libraries/* +# +# Build CoreLib release +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml + buildConfig: release + platforms: + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - Windows_NT_x64 + - Windows_NT_x86 + - Windows_NT_arm + - Windows_NT_arm64 + jobParameters: + testGroup: innerloop + # # Build CoreCLR checked # Only when CoreCLR is changed diff --git a/src/libraries/pretest.proj b/src/libraries/pretest.proj index 317bf140a39af2..10cd2407793876 100644 --- a/src/libraries/pretest.proj +++ b/src/libraries/pretest.proj @@ -78,7 +78,8 @@ Projects="@(Project)" Properties="TargetFramework=$(BuildTargetFramework);%(Project.AdditionalProperties)" BuildInParallel="true" - ContinueOnError="ErrorAndStop" /> + ContinueOnError="ErrorAndStop" + Condition="'$(SkipBuildProjects)' != 'true'"/> diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index d47a26ac972b8b..975b8c7763d460 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -41,8 +41,19 @@ + + + + + + +