[Release/7.0] Add Maui Mobile Android Benchmarks

azure-pipelines.yml

@@ -247,6 +247,19 @@ jobs:
         projectFile: maui_scenarios.proj
         channels:
           - release/7.0
+
+  # Maui Android scenario benchmarks
+  - template: /eng/performance/build_machine_matrix.yml
+    parameters:
+      jobTemplate: /eng/performance/scenarios.yml
+      buildMachines:
+        - win-x64-android-arm64
+      isPublic: false
+      jobParameters:
+        kind: maui_scenarios_android
+        projectFile: maui_scenarios_android.proj
+        channels:
+          - release/7.0
 
 ################################################
 # Scheduled Private jobs

eng/performance/build_machine_matrix.yml

@@ -91,4 +91,16 @@ jobs:
         vmImage: windows-2019
       machinePool: Tiger
       queue: Windows.10.Arm64.Perf.Surf
-      ${{ insert }}: ${{ parameters.jobParameters }}
+      ${{ insert }}: ${{ parameters.jobParameters }}
+
+- ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently
+  - template: ${{ parameters.jobTemplate }}
+    parameters:
+      osName: windows
+      architecture: x64
+      osVersion: 19H1
+      pool:
+        vmImage: 'windows-2022'
+      queue: Windows.10.Amd64.Pixel.Perf
+      machinePool: Pixel
+      ${{ insert }}: ${{ parameters.jobParameters }}

eng/performance/maui_scenarios_android.proj

@@ -0,0 +1,82 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <Import Project="Scenarios.Common.props" />
+
+  <PropertyGroup>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+    <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21566.2</MicrosoftDotNetXHarnessCLIVersion>
+    <XharnessPath>%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</XharnessPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
+    <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+  </PropertyGroup>
+
+  <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
+    <Message Text="Removing Dotnet from Correlation Staging" Importance="high" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
+  </Target>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>00:30</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <MAUIAndroidScenario Include="Maui Android Default">
+      <ScenarioDirectoryName>mauiandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Android Podcast">
+      <ScenarioDirectoryName>mauiandroidpodcast</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.Microsoft.NetConf2021.Maui-Signed</ApkName>
+      <PackageName>com.Microsoft.NetConf2021.Maui</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Blazor Android Default">
+      <ScenarioDirectoryName>mauiblazorandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiblazorandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiblazorandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+  </ItemGroup>
+
+
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+
+<!-- We only run the android tests from Windows machines (at least for now) -->
+  <ItemGroup>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) APK Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) Extracted Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "&amp; {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity)')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity) NoAnimation')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot; --disable-animations</Command>
+    </HelixWorkItem>
+  </ItemGroup>
+
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+</Project>

scripts/dotnet.py

@@ -5,13 +5,14 @@
 """
 
 import ssl
+import datetime
 from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError
 from collections import namedtuple
 from glob import iglob
 from json import loads
 from logging import getLogger
 from os import chmod, environ, listdir, makedirs, path, pathsep, system
-from re import search
+from re import search, match, MULTILINE
 from shutil import rmtree
 from stat import S_IRWXU
 from subprocess import CalledProcessError, check_output
@@ -611,27 +612,30 @@ def get_commit_date(
     if not commit_sha:
         raise ValueError('.NET Commit sha was not defined.')
 
+    # Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch
     url = None
-    urlformat = 'https://api.github.com/repos/%s/%s/commits/%s'
     if repository is None:
         # The origin of the repo where the commit belongs to has changed
         # between release. Here we attempt to naively guess the repo.
         core_sdk_frameworks = ChannelMap.get_supported_frameworks()
         repo = 'core-sdk' if framework  in core_sdk_frameworks else 'cli'
-        url = urlformat % ('dotnet', repo, commit_sha)
+        url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch'
     else:
         owner, repo = get_repository(repository)
-        url = urlformat % (owner, repo, commit_sha)
+        url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch'
 
     build_timestamp = None
-    sleep_time = 10 # Start with 10 second sleep timer
+    sleep_time = 10 # Start with 10 second sleep timer        
     for retrycount in range(5):
         try:
             with urlopen(url) as response:
                 getLogger().info("Commit: %s", url)
-                item = loads(response.read().decode('utf-8'))
-                build_timestamp = item['commit']['committer']['date']
-                break
+                patch = response.read().decode('utf-8')
+                dateMatch = search(r'^Date: (.+)$', patch, MULTILINE)
+                if dateMatch:
+                    build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+                    getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}")
+                    break
         except URLError as error:
             getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}")
             sleep(sleep_time)

src/scenarios/mauiandroid/pre.py

@@ -2,48 +2,37 @@
 pre-command
 '''
 import sys
-import os
-from zipfile import ZipFile
 from performance.logger import setup_loggers, getLogger
-from shutil import copyfile
+from shared import const
+from shared.mauisharedpython import remove_aab_files, install_versioned_maui
 from shared.precommands import PreCommands
-from shared.const import PUBDIR
-from argparse import ArgumentParser
+from shared.versionmanager import versions_write_json, get_version_from_dll_powershell
+from test import EXENAME
 
 setup_loggers(True)
 
-parser = ArgumentParser()
-parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False)
-parser.add_argument(
-        '--apk-name',
-        dest='apk',
-        required=True,
-        type=str,
-        help='Name of the APK to setup')
-args = parser.parse_args()
-
-if not os.path.exists(PUBDIR):
-    os.mkdir(PUBDIR)
-apkname = args.apk
-apknamezip = '%s.zip' % (apkname)
-if not os.path.exists(apkname):
-    getLogger().error('Cannot find %s' % (apkname))
-    exit(-1)
-if args.unzip:
-    if not os.path.exists(apknamezip):
-        copyfile(apkname, apknamezip)
-
-    with ZipFile(apknamezip) as zip:
-        zip.extractall(os.path.join('.', PUBDIR))
-
-    assets_dir = os.path.join(PUBDIR, 'assets')
-    assets_zip = os.path.join(assets_dir, 'assets.zip')
-    with ZipFile(assets_zip) as zip:
-        zip.extractall(assets_dir)
-
-    os.remove(assets_zip)
-else:
-    copyfile(apkname, os.path.join(PUBDIR, apkname))
-
-
-
+precommands = PreCommands()
+install_versioned_maui(precommands)
+
+# Setup the Maui folder
+precommands.new(template='maui',
+                output_dir=const.APPDIR,
+                bin_dir=const.BINDIR,
+                exename=EXENAME,
+                working_directory=sys.path[0],
+                no_restore=False)
+
+# Build the APK
+precommands.execute(['--no-restore', '--source', 'MauiNuGet.config'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+remove_aab_files(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versions_write_json(version_dict, rf"{output_dir}\versions.json")
+print(f"Versions: {version_dict}")

src/scenarios/mauiandroid/test.py

@@ -1,11 +1,15 @@
 '''
-C# Console app
+Mobile Maui App
 '''
+from shared.const import PUBDIR
 from shared.runner import TestTraits, Runner
+from shared.versionmanager import versions_read_json_file_save_env
 
 EXENAME = 'MauiAndroidDefault'
 
-if __name__ == "__main__":
+if __name__ == "__main__":    
+    versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json")
+
     traits = TestTraits(exename=EXENAME, 
                         guiapp='false',
                         )

src/scenarios/mauiandroidpodcast/post.py

@@ -0,0 +1,9 @@
+'''
+post cleanup script
+'''
+
+from shared.postcommands import clean_directories
+from performance.common import remove_directory
+
+remove_directory("dotnet-podcasts")
+clean_directories()

src/scenarios/mauiandroidpodcast/pre.py

@@ -0,0 +1,35 @@
+'''
+pre-command
+'''
+import subprocess
+from performance.logger import setup_loggers, getLogger
+from shared.precommands import PreCommands
+from shared.mauisharedpython import remove_aab_files, install_versioned_maui
+from shared.versionmanager import versions_write_json, get_version_from_dll_powershell
+from shared import const
+
+setup_loggers(True)
+precommands = PreCommands()
+install_versioned_maui(precommands)
+
+branch = f'{precommands.framework[:6]}'
+subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1'])
+subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately
+
+precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj')
+
+# Build the APK
+precommands._restore()
+precommands.execute(['--no-restore'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+remove_aab_files(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versions_write_json(version_dict, rf"{output_dir}\versions.json")
+print(f"Versions: {version_dict}")

src/scenarios/mauiandroidpodcast/test.py

@@ -0,0 +1,16 @@
+'''
+Mobile Maui App
+'''
+from shared.const import PUBDIR
+from shared.runner import TestTraits, Runner
+from shared.versionmanager import versions_read_json_file_save_env
+
+EXENAME = 'MauiAndroidPodcast'
+
+if __name__ == "__main__":
+    versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json")
+
+    traits = TestTraits(exename=EXENAME, 
+                        guiapp='false',
+                        )
+    Runner(traits).run()