-
Notifications
You must be signed in to change notification settings - Fork 536
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ci] Add nightly build and test pipeline (#5549)
Context: #5098 A new `build-tools/automation/azure-pipelines-nightly.yaml` build pipeline file has been added to run our longer running build and test tasks that we don't want to run against every commit or PR build. This new pipeline is scheduled to run every day at 3:00am UTC, and will only run if there have been source code changes since the last successful scheduled run. For more information see the [scheduled triggers documentation][1]. This new nightly pipeline starts with a commercial build on macOS, which provides an opportunity to run time consuming Roslyn analyzers; see e249d77: > By default, *no* analyzers are run, as we found that enabling them > added ~90 minutes to our build times. Now we can run them! After the build completes, the artifacts that are produced will be used by new supplemental test jobs. Five new emulator test environments have been added to this nightly pipeline which will allow us to run tests on a broader set of Android versions. The following emulators are now available for testing: * Android 5.0 (API 21) - x86 * Android 6.0 (API 23) - x86 * Android 7.0 (API 24) - x86 * Android 8.0 (API 26) - x86_64 * Android 9.0 (API 28) - x86_64 These new test jobs currently only run the default configuration of the `tests/Mono.Android-Tests` suite against our legacy installers. It should be easy to extend these jobs to run additional tests as needed, e.g. using "checked sanitizer" binaries from commit aaa37c3. Some clean-up has also been applied to the main `azure-pipelines.yaml` file. Usage of `XA.Commercial.Build` has been removed, as any builds using this file will always be "commercial". Additionally, the test environment update from commit 75317db allows us to remove a supplemental mono installation step as a newer mono is available on these machines. Aside: support has been added for installing and launching emulators with different API levels and architectures, by setting the following MSBuild properties: * `$(SdkManagerImageName)`: The "name" of the emulator image to create/install. This is the same value that the Android SDK `sdkmanager` utility. * `$(TestAvdAbi)`: The ABI of the emulator to create, e.g. `x86`. * `$(TestAvdApiLevel)`: The API level of the emulator to create. For example, to create an API-21 emulator: msbuild /t:InstallAvdImage;AcquireAndroidTarget tests\Mono.Android-Tests\Mono.Android-Tests.csproj /p:TestAvdApiLevel=21 [1]: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml
- Loading branch information
Showing
9 changed files
with
392 additions
and
152 deletions.
There are no files selected for viewing
46 changes: 46 additions & 0 deletions
46
...droid.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/WaitForAndroidEmulator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Threading; | ||
|
|
||
| namespace Xamarin.Android.Tools.BootstrapTasks | ||
| { | ||
| public class WaitForAndroidEmulator : Adb | ||
| { | ||
| int bootCompleted = -1; | ||
|
|
||
| public override bool Execute () | ||
| { | ||
| var endTime = DateTime.UtcNow.AddMilliseconds (Timeout); | ||
| while (DateTime.UtcNow < endTime && bootCompleted != 1) { | ||
| base.Execute (); | ||
| Thread.Sleep (3000); | ||
| } | ||
|
|
||
| if (bootCompleted != 1) { | ||
| Log.LogError ($"Emulator '{AdbTarget}' did not finish launching in {Timeout} ms."); | ||
| } | ||
|
|
||
| return !Log.HasLoggedErrors; | ||
| } | ||
|
|
||
| protected override List <CommandInfo> GenerateCommandArguments () | ||
| { | ||
| return new List <CommandInfo> { | ||
| new CommandInfo { | ||
| ArgumentsString = $"{AdbTarget} shell getprop sys.boot_completed", | ||
| }, | ||
| }; | ||
| } | ||
|
|
||
| protected override void ProcessStdout (string line) | ||
| { | ||
| if (string.IsNullOrEmpty (line)) | ||
| return; | ||
|
|
||
| if (!int.TryParse (line, out int bootCompletedPropValue)) | ||
| return; | ||
|
|
||
| bootCompleted = bootCompletedPropValue; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| # Xamarin.Android Nightly Pipeline | ||
|
|
||
| name: $(Build.SourceBranchName)-$(Build.SourceVersion)-$(Rev:r) | ||
|
|
||
| trigger: | ||
| - none | ||
|
|
||
| pr: | ||
| - none | ||
|
|
||
| schedules: | ||
| - cron: "0 3 * * *" | ||
| displayName: Run daily at 3:00 UTC | ||
| branches: | ||
| include: | ||
| - master | ||
| - d16-9 | ||
|
|
||
| # External sources, scripts, tests, and yaml template files. | ||
| resources: | ||
| repositories: | ||
| - repository: yaml | ||
| type: github | ||
| name: xamarin/yaml-templates | ||
| ref: refs/heads/main | ||
| endpoint: xamarin | ||
| - repository: monodroid | ||
| type: github | ||
| name: xamarin/monodroid | ||
| endpoint: xamarin | ||
|
|
||
| # Global variables | ||
| variables: | ||
| RunningOnCI: true | ||
| XA.Build.Configuration: Release | ||
| XA.Jdk8.Folder: jdk-1.8 | ||
| NuGetArtifactName: nupkgs | ||
| InstallerArtifactName: installers | ||
| TestAssembliesArtifactName: test-assemblies | ||
| DotNetCoreVersion: 3.1.201 | ||
| DotNet5Version: 5.0.100 | ||
| MacMojaveBuildPool: VSEng-Xamarin-RedmondMac-Android-Untrusted | ||
| HostedMacImage: macOS-10.15 | ||
| VSEngWinVS2019: Xamarin-Android-Win2019-1120 | ||
| GitHub.Token: $(github--pat--vs-mobiletools-engineering-service2) | ||
|
|
||
| stages: | ||
| - stage: mac_build | ||
| displayName: Build | ||
| dependsOn: [] | ||
| jobs: | ||
| - job: mac_build_create_installers | ||
| displayName: macOS | ||
| pool: $(MacMojaveBuildPool) | ||
| timeoutInMinutes: 240 | ||
| cancelTimeoutInMinutes: 5 | ||
| workspace: | ||
| clean: all | ||
| variables: | ||
| - group: Xamarin-Secrets | ||
| - group: Xamarin Signing | ||
| - group: xamops-azdev-secrets | ||
| steps: | ||
| - checkout: self | ||
| submodules: recursive | ||
|
|
||
| - template: yaml-templates/commercial-build.yaml | ||
| parameters: | ||
| makeMSBuildArgs: /p:EnableRoslynAnalyzers=true | ||
|
|
||
| - stage: test | ||
| displayName: Test | ||
| dependsOn: mac_build | ||
| jobs: | ||
| - job: emulator_tests | ||
| displayName: Emulator | ||
| timeoutInMinutes: 180 | ||
| cancelTimeoutInMinutes: 2 | ||
| strategy: | ||
| matrix: | ||
| Android21-x86: | ||
| avdApiLevel: 21 | ||
| avdAbi: x86 | ||
| Android23-x86: | ||
| avdApiLevel: 23 | ||
| avdAbi: x86 | ||
| Android24-x86: | ||
| avdApiLevel: 24 | ||
| avdAbi: x86 | ||
| Android26-x86_64: | ||
| avdApiLevel: 26 | ||
| avdAbi: x86_64 | ||
| Android28-x86_64: | ||
| avdApiLevel: 28 | ||
| avdAbi: x86_64 | ||
| pool: | ||
| vmImage: $(HostedMacImage) | ||
| workspace: | ||
| clean: all | ||
| variables: | ||
| - group: Xamarin-Secrets | ||
| - group: xamops-azdev-secrets | ||
| steps: | ||
| - template: yaml-templates/setup-test-environment.yaml | ||
| parameters: | ||
| configuration: $(XA.Build.Configuration) | ||
|
|
||
| - script: > | ||
| mono $(System.DefaultWorkingDirectory)/build-tools/xaprepare/xaprepare/bin/$(XA.Build.Configuration)/xaprepare.exe | ||
| --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI | ||
| displayName: install required brew tools and prepare java.interop | ||
| - script: > | ||
| mono $(System.DefaultWorkingDirectory)/build-tools/xaprepare/xaprepare/bin/$(XA.Build.Configuration)/xaprepare.exe | ||
| --s=EmulatorTestDependencies --no-emoji --run-mode=CI | ||
| displayName: install emulator | ||
| - script: echo "##vso[task.setvariable variable=Java8SdkDirectory]$HOME/Library/Android/$(XA.Jdk8.Folder)" | ||
| displayName: set Java8SdkDirectory | ||
|
|
||
| - task: MSBuild@1 | ||
| displayName: install and launch emulator | ||
| inputs: | ||
| solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj | ||
| configuration: $(XA.Build.Configuration) | ||
| msbuildArguments: /t:InstallAvdImage;AcquireAndroidTarget /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-$(avdApiLevel).binlog | ||
|
|
||
| - template: yaml-templates/apk-instrumentation.yaml | ||
| parameters: | ||
| configuration: $(XA.Build.Configuration) | ||
| testName: Mono.Android_Tests-$(avdApiLevel)-$(avdAbi) | ||
| project: tests/Mono.Android-Tests/Mono.Android-Tests.csproj | ||
| testResultsFiles: TestResult-Mono.Android_Tests-$(XA.Build.Configuration).xml | ||
| extraBuildArgs: /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) | ||
| artifactName: Mono.Android_Tests-Signed.apk | ||
| artifactFolder: Default | ||
|
|
||
| - task: MSBuild@1 | ||
| displayName: shut down emulator | ||
| inputs: | ||
| solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj | ||
| configuration: $(XA.Build.Configuration) | ||
| msbuildArguments: /t:AcquireAndroidTarget,ReleaseAndroidTarget /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog | ||
| condition: always() | ||
|
|
||
| - template: yaml-templates/upload-results.yaml | ||
| parameters: | ||
| configuration: $(XA.Build.Configuration) | ||
| artifactName: Test Results - Emulator $(avdApiLevel)-$(avdAbi) - macOS | ||
|
|
||
| - template: yaml-templates/fail-on-issue.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.