diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3f85b7f5d94..0f7802ab2d3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -260,6 +260,19 @@ jobs: projectFile: maui_scenarios_android.proj channels: - release/7.0 + + # Maui iOS scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_ios + projectFile: maui_scenarios_ios.proj + channels: + - release/7.0 ################################################ # Scheduled Private jobs diff --git a/eng/Versions.props b/eng/Versions.props index c79ca93c9c4..ed74d47dbec 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,6 +8,7 @@ 5.0.104-servicing.21213.7 + 1.0.0-prerelease.22411.1 5.0.4 diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml index 95e18ac7850..eee6d7676d3 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/build_machine_matrix.yml @@ -104,3 +104,15 @@ jobs: queue: Windows.10.Amd64.Pixel.Perf machinePool: Pixel ${{ insert }}: ${{ parameters.jobParameters }} + +- ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 12mini only used in private builds currently + - template: ${{ parameters.jobTemplate }} + parameters: + osName: osx + architecture: x64 + osVersion: 12 + pool: + vmImage: 'macos-12' + queue: OSX.1015.Amd64.Iphone.Perf + machinePool: iPhoneMini12 + ${{ insert }}: ${{ parameters.jobParameters }} diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj new file mode 100644 index 00000000000..393529f1020 --- /dev/null +++ b/eng/performance/maui_scenarios_ios.proj @@ -0,0 +1,161 @@ + + + + + true + + + + $(Python) post.py + scenarios_out + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + + + + + + + 00:30 + + + + + + mauiios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiiOSDefault + net.dot.mauitesting + + + + + mauiblazorios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiBlazoriOSDefault + net.dot.mauiblazortesting + + + + + + $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub + $(Python) test.py sod --scenario-name "%(Identity)" + + + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip + $(Python) test.py sod --scenario-name "%(Identity)" + + + $(ScenariosDir)mauiios.zip + 00:15:00 + ios-device + + + + + + $(ScenariosDir)mauiblazorios.zip + 00:15:00 + ios-device + + + + + + + + + + + + export PYTHONPATH=$ORIGPYPATH;$(HelixPostCommands) + + + + + diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 1509785405f..69d08d9a65f 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -128,7 +128,30 @@ jobs: displayName: Delete old dotnet - script: xcopy tools\dotnet\arm64\* $(CorrelationStaging)dotnet\/E /I /Y displayName: Copy Arm64 Dotnet to Correlation Payload - - ${{ if ne(parameters.osName, 'windows') }}: + - ${{ if eq(parameters.osName, 'osx') }}: + - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps + displayName: Copy python libraries and NuGet.config + - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(PERFLAB_Framework) -r osx-${{parameters.architecture}} --self-contained $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true + displayName: Build startup tool + env: + PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework) + - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)SOD -f $(PERFLAB_Framework) -r osx-${{parameters.architecture}} --self-contained $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true + displayName: Build SOD tool + env: + PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework) + - script: | + $(Python) -m pip install --user --upgrade pip + $(Python) -m pip install --user requests + . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + displayName: Prepare scenarios + env: + CorrelationPayloadDirectory: $(CorrelationStaging) + Architecture: ${{ parameters.architecture }} + TargetsWindows: 'false' + WorkItemDirectory: $(Build.SourcesDirectory) + HelixTargetQueues: ${{ parameters.queue }} + - ${{ if and(ne(parameters.osName, 'windows'), ne(parameters.osName, 'osx')) }}: - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps displayName: Copy python libraries and NuGet.config - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(PERFLAB_Framework) -r linux-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true diff --git a/src/scenarios/mauiblazorios/pre.py b/src/scenarios/mauiblazorios/pre.py index bf9b8a17304..38ba650897a 100644 --- a/src/scenarios/mauiblazorios/pre.py +++ b/src/scenarios/mauiblazorios/pre.py @@ -1,42 +1,79 @@ ''' pre-command ''' +import shutil import sys -import os -from zipfile import ZipFile from performance.logger import setup_loggers, getLogger -from shutil import copyfile, copytree, move -from shared.const import PUBDIR -from argparse import ArgumentParser +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Maui folder +precommands.new(template='maui-blazor', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Add the index.razor.cs file +with open(f"{const.APPDIR}/Pages/Index.razor.cs", "w") as indexCSFile: + indexCSFile.write(''' + using Microsoft.AspNetCore.Components; + #if ANDROID + using Android.App; + #endif\n\n''' + + f" namespace {EXENAME}.Pages" + +''' + { + public partial class Index + { + protected override void OnAfterRender(bool firstRender) + { + if (firstRender) + { + #if ANDROID + var activity = MainActivity.Context as Activity; + activity.ReportFullyDrawn(); + #else + System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\"); + #endif + } + } + } + } +''') + +# Replace line in the Android MainActivity.cs file +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "r") as mainActivityFile: + mainActivityFileLines = mainActivityFile.readlines() + +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "w") as mainActivityFile: + for line in mainActivityFileLines: + if line.startswith("{"): + mainActivityFile.write("{\npublic static Android.Content.Context Context { get; private set; }\npublic MainActivity() { Context = this; }") + else: + mainActivityFile.write(line) + +# Build the APK +# NuGet.config file cannot be in the build directory currently due to https://github.com/dotnet/aspnetcore/issues/41397 +# shutil.copy('./MauiNuGet.config', './app/Nuget.config') +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauiblazortesting']) + +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") -parser = ArgumentParser() -parser.add_argument('--unzip', help='Unzip ipa file and report extracted tree', action='store_true', default=False) -parser.add_argument( - '--name', - dest='name', - required=True, - type=str, - help='Name of the file/folder to setup (with .app or .ipa)') -args = parser.parse_args() - -name = args.name -namezip = '%s.zip' % (name) -if not os.path.exists(PUBDIR): - os.mkdir(PUBDIR) -if not os.path.exists(name): - getLogger().error('Cannot find %s' % (name)) - exit(-1) -if args.unzip: - if not os.path.exists(namezip): - copyfile(name, namezip) - - with ZipFile(namezip) as zip: - zip.extractall(os.path.join('.', PUBDIR)) - -else: - if(os.path.isdir(name)): - copytree(name, PUBDIR, dirs_exist_ok=True) - else: - copyfile(name, os.path.join(PUBDIR, name)) diff --git a/src/scenarios/mauiblazorios/test.py b/src/scenarios/mauiblazorios/test.py index 55fa0d77bc0..0a9067fb615 100644 --- a/src/scenarios/mauiblazorios/test.py +++ b/src/scenarios/mauiblazorios/test.py @@ -1,11 +1,15 @@ ''' -C# Console app +Mobile Maui App ''' +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiBlazoriOSDefault' if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiios/pre.py b/src/scenarios/mauiios/pre.py index bf9b8a17304..fce66fe9bb1 100644 --- a/src/scenarios/mauiios/pre.py +++ b/src/scenarios/mauiios/pre.py @@ -1,42 +1,41 @@ ''' pre-command ''' +import shutil import sys -import os -from zipfile import ZipFile +import subprocess from performance.logger import setup_loggers, getLogger -from shutil import copyfile, copytree, move -from shared.const import PUBDIR -from argparse import ArgumentParser +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME setup_loggers(True) -parser = ArgumentParser() -parser.add_argument('--unzip', help='Unzip ipa file and report extracted tree', action='store_true', default=False) -parser.add_argument( - '--name', - dest='name', - required=True, - type=str, - help='Name of the file/folder to setup (with .app or .ipa)') -args = parser.parse_args() +precommands = PreCommands() +install_versioned_maui(precommands) -name = args.name -namezip = '%s.zip' % (name) -if not os.path.exists(PUBDIR): - os.mkdir(PUBDIR) -if not os.path.exists(name): - getLogger().error('Cannot find %s' % (name)) - exit(-1) -if args.unzip: - if not os.path.exists(namezip): - copyfile(name, namezip) +# Setup the Maui folder +precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) - with ZipFile(namezip) as zip: - zip.extractall(os.path.join('.', PUBDIR)) - -else: - if(os.path.isdir(name)): - copytree(name, PUBDIR, dirs_exist_ok=True) - else: - copyfile(name, os.path.join(PUBDIR, name)) +# Build the APK +shutil.copy('./MauiNuGet.config', './app/Nuget.config') +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauitesting']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") diff --git a/src/scenarios/mauiios/test.py b/src/scenarios/mauiios/test.py index 0aca010a88e..471a4aec5bd 100644 --- a/src/scenarios/mauiios/test.py +++ b/src/scenarios/mauiios/test.py @@ -1,11 +1,15 @@ ''' -C# Console app +Mobile Maui App ''' +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiiOSDefault' -if __name__ == "__main__": +if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiiospodcast/post.py b/src/scenarios/mauiiospodcast/post.py new file mode 100644 index 00000000000..abdb7d8db9d --- /dev/null +++ b/src/scenarios/mauiiospodcast/post.py @@ -0,0 +1,9 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories +from performance.common import remove_directory + +remove_directory("dotnet-podcasts") +clean_directories() diff --git a/src/scenarios/mauiiospodcast/pre.py b/src/scenarios/mauiiospodcast/pre.py new file mode 100644 index 00000000000..7a7b87e1946 --- /dev/null +++ b/src/scenarios/mauiiospodcast/pre.py @@ -0,0 +1,34 @@ +''' +pre-command +''' +import subprocess +from performance.logger import setup_loggers, getLogger +from shared.precommands import PreCommands +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell +from shared import const + +setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +branch = f'{precommands.framework[:6]}' +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1']) +subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately + +precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') + +# Build the APK +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.netconf2021.maui']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +# maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\android-arm64\linked\Microsoft.Maui.dll") +# version_dict = { "mauiVersion": maui_version } +# versions_write_json(version_dict, rf"{output_dir}\versions.json") +# print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiiospodcast/test.py b/src/scenarios/mauiiospodcast/test.py new file mode 100644 index 00000000000..642bc4b3abc --- /dev/null +++ b/src/scenarios/mauiiospodcast/test.py @@ -0,0 +1,16 @@ +''' +Mobile Maui App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'MauiiOSPodcast' + +if __name__ == "__main__": + #versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() diff --git a/src/scenarios/mauishared/mauisharedpython.py b/src/scenarios/mauishared/mauisharedpython.py deleted file mode 100644 index 127c712bfee..00000000000 --- a/src/scenarios/mauishared/mauisharedpython.py +++ /dev/null @@ -1,9 +0,0 @@ -import subprocess -import os - -# Remove the aab files as we don't need them, this saves space in the correlation payload -def RemoveAABFiles(output_dir="."): - file_list = os.listdir(output_dir) - for file in file_list: - if file.endswith(".aab"): - os.remove(os.path.join(output_dir, file)) diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py index a61be2cfa65..889235dd150 100644 --- a/src/scenarios/shared/mauisharedpython.py +++ b/src/scenarios/shared/mauisharedpython.py @@ -17,8 +17,8 @@ def install_versioned_maui(precommands): with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) - workload_install_args = ['--configfile', 'MauiNuGet.config'] + workload_install_args = ['--configfile', 'MauiNuGet.config', '--skip-sign-check'] if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 - workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] + workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] # Skip sign check per https://github.com/dotnet/maui/issues/9573 precommands.install_workload('maui', workload_install_args) \ No newline at end of file diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py index 3eeaf795a5a..170c0230e72 100644 --- a/src/scenarios/shared/versionmanager.py +++ b/src/scenarios/shared/versionmanager.py @@ -28,4 +28,8 @@ def versions_read_json_file_save_env(inputfile = 'versions.json'): def get_version_from_dll_powershell(dll_path: str): result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + return result.stdout.decode('utf-8').strip() + +def get_version_from_dll_powershell_ios(dll_path: str): + result = subprocess.run(['pwsh', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) return result.stdout.decode('utf-8').strip() \ No newline at end of file