Add MSVC-internal testing scripts to public GitHub.

scripts/azure-pipelines/test-modified-ports.ps1

@@ -9,18 +9,20 @@ Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms.
 .PARAMETER Triplet
 The triplet to test.
 
-.PARAMETER ArchivesRoot
-The location where the binary caching archives are stored. Shared across runs of this script.
-
 .PARAMETER WorkingRoot
 The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.
 
 .PARAMETER ArtifactStagingDirectory
 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
 
+.PARAMETER ArchivesRoot
+The location where the binary caching archives are stored. Shared across runs of this script. If
+this parameter is not set, binary caching will not be used.
+
 .PARAMETER BuildReason
-The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not
-supplied, binary caching will be used.
+The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).
+If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set and this is not,
+binary caching will default to read-write mode.
 #>
 
 [CmdletBinding()]
@@ -30,12 +32,10 @@ Param(
     [string]$Triplet,
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
-    $ArchivesRoot,
-    [Parameter(Mandatory = $true)]
-    [ValidateNotNullOrEmpty()]
     $WorkingRoot,
     [ValidateNotNullOrEmpty()]
     $ArtifactStagingDirectory = '.',
+    $ArchivesRoot = $null,
     $BuildReason = $null
 )
 
@@ -48,29 +48,39 @@ $env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'
 $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'packages'
-$commonArgs = @(
-    '--binarycaching',
+
+$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot))
+$commonArgs = @()
+if ($usingBinaryCaching) {
+    $commonArgs += @('--binarycaching')
+} else {
+    $commonArgs += @('--no-binarycaching')
+}
+
+$commonArgs += @(
     "--x-buildtrees-root=$buildtreesRoot",
     "--x-install-root=$installRoot",
     "--x-packages-root=$packagesRoot",
     "--overlay-ports=scripts/test_ports"
 )
 
-$binaryCachingMode = 'readwrite'
 $skipFailures = $false
-if ([string]::IsNullOrWhiteSpace($BuildReason)) {
-    Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
-}
-elseif ($BuildReason -eq 'PullRequest') {
-    Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'
-    $skipFailures = $true
+if ($usingBinaryCaching) {
+    $binaryCachingMode = 'readwrite'
+    if ([string]::IsNullOrWhiteSpace($BuildReason)) {
+        Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
+    }
+    elseif ($BuildReason -eq 'PullRequest') {
+        Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'
+        $skipFailures = $true
+    }
+    else {
+        Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
+        $binaryCachingMode = 'write'
+    }
+
+    $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
 }
-else {
-    Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
-    $binaryCachingMode = 'write'
-}
-
-$commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
 
 if ($Triplet -eq 'x64-linux') {
     $env:HOME = '/home/agent'

scripts/azure-pipelines/windows-unstable/README.md

@@ -0,0 +1,4 @@
+The "unstable" build is used internally by Microsoft to test prerelease versions
+of our C++ compiler; not seeing results from these build definitions in the 
+GitHub portal is normal as these builds depend on acquisition of private
+compiler bits that aren't yet shipping.

scripts/azure-pipelines/windows-unstable/azure-pipelines.yml

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+variables:
+  unstable-pool: 'VcpkgUnstable-2020-09-01'
+
+jobs:
+- template: job.yml
+  parameters:
+    triplet: x64-windows
+    jobName: x64_windows

scripts/azure-pipelines/windows-unstable/job.yml

@@ -0,0 +1,90 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+jobs:
+- job: ${{ parameters.jobName }}
+  pool:
+    name: $(unstable-pool)
+  workspace:
+    clean: resources
+  timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: D:\
+  - name: VCPKG_DOWNLOADS
+    value: D:\downloads
+
+  steps:
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download DropBuildNumber if not specified'
+    inputs:
+      buildType: specific
+      project: '0bdbc590-a062-4c3f-b0f6-9383f67865ee'
+      pipeline: 8136
+      buildVersionToDownload: latestFromBranch
+      branchName: 'refs/heads/$(MSVCBranchName)'
+      artifactName: BuildNumber
+      downloadPath: 'D:\msvc-drops'
+    condition: eq(variables['DropBuildNumber'], '')
+  - task: PowerShell@2
+    displayName: 'Set DropBuildNumber if not specified'
+    inputs:
+      targetType: inline
+      script: |
+        $DropBuildNumber = Get-Content -Path D:\msvc-drops\BuildNumber\Build.BuildNumber.txt
+        Write-Host "##vso[task.setvariable variable=DropBuildNumber]$DropBuildNumber"
+        Write-Host "Build Number set to: $DropBuildNumber"
+      pwsh: true
+    condition: eq(variables['DropBuildNumber'], '')
+  - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
+    displayName: 'Download msvc x86 ret'
+    inputs:
+      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
+      buildNumber: 'msvc/builds/$(DropBuildNumber)/x86ret'
+      destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.x86ret'
+  - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
+    displayName: 'Download msvc amd64 ret'
+    inputs:
+      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
+      buildNumber: 'msvc/builds/$(DropBuildNumber)/amd64ret'
+      destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.amd64ret'
+  - task: PowerShell@2
+    displayName: 'Rearrange MSVC Drop Layout'
+    inputs:
+      targetType: filePath
+      filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
+      arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Initialize Environment'
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Report on Disk Space'
+    condition: always()
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
+      pwsh: true
+  - script: .\bootstrap-vcpkg.bat
+    displayName: 'Build vcpkg'
+  - task: PowerShell@2
+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
+    inputs:
+      failOnStderr: true
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Report on Disk Space After Build'
+    condition: always()
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
+      pwsh: true
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs'
+      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
+    condition: failed()

scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1

@@ -0,0 +1,75 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+<#
+.SYNOPSIS
+Moves files from an MSVC compiler drop to the locations where they are installed in a Visual Studio installation.
+
+.PARAMETER DropRoot
+The location where the MSVC compiler drop has been downloaded.
+
+.PARAMETER BuildType
+The MSVC drop build type set with /p:_BuildType when MSVC was built. Defaults to 'ret'.
+
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)][string]$DropRoot,
+    [Parameter(Mandatory = $false)][ValidateSet('ret', 'chk')][string]$BuildType = 'ret'
+)
+
+Set-StrictMode -Version Latest
+
+$MSVCRoot = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC"
+
+$ErrorActionPreference = "Stop"
+
+$tempRoot = "$DropRoot\readytodeploy"
+
+New-Item -ItemType Directory -Path $tempRoot | Out-Null
+
+Write-Host "Rearranging x86$BuildType"
+New-Item -ItemType Directory -Path "$tempRoot\bin\HostX86" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\i386" "$tempRoot\bin\HostX86\x86"
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_amd64" "$tempRoot\bin\HostX86\x64"
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_arm" "$tempRoot\bin\HostX86\arm"
+
+Write-Host "Rearranging amd64$BuildType"
+New-Item -ItemType Directory -Path "$tempRoot\bin\HostX64" | Out-Null
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64" "$tempRoot\bin\HostX64\x64"
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_x86" "$tempRoot\bin\HostX64\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_arm" "$tempRoot\bin\HostX64\arm"
+
+# Only copy files and directories that already exist in the VS installation.
+Write-Host "Rearranging inc, lib"
+New-Item -ItemType Directory -Path "$tempRoot\lib" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\inc" "$tempRoot\include"
+Move-Item "$DropRoot\binaries.x86$BuildType\lib\i386" "$tempRoot\lib\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\lib\amd64" "$tempRoot\lib\x64"
+
+Write-Host "Rearranging atlmfc"
+New-Item -ItemType Directory -Path "$tempRoot\atlmfc" | Out-Null
+New-Item -ItemType Directory -Path "$tempRoot\atlmfc\lib" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\include" "$tempRoot\atlmfc\include"
+Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\lib\i386" "$tempRoot\atlmfc\lib\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\atlmfc\lib\amd64" "$tempRoot\atlmfc\lib\x64"
+
+$toolsets = Get-ChildItem -Path $MSVCRoot -Directory | Sort-Object -Descending
+if ($toolsets.Length -eq 0) {
+    throw "Could not find Visual Studio toolset!"
+}
+
+Write-Host "Found toolsets:`n$($toolsets -join `"`n`")`n"
+$selectedToolset = $toolsets[0]
+Write-Host "Using toolset: $selectedToolset"
+for ($idx = 1; $idx -lt $toolsets.Length; $idx++) {
+    $badToolset = $toolsets[$idx]
+    Write-Host "Deleting toolset: $badToolset"
+    Remove-Item $badToolset -Recurse -Force
+}
+
+Write-Host "Deploying $tempRoot => $selectedToolset"
+Copy-Item "$tempRoot\*" $selectedToolset -Recurse -Force
+Write-Host "Deleting $DropRoot..."
+Remove-Item $DropRoot -Recurse -Force
+Write-Host "Done!"