dotnet · LoopedBard3 · Apr 19, 2023 · Apr 4, 2023 · Apr 5, 2023 · Apr 5, 2023
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -260,6 +260,19 @@ jobs:
         projectFile: maui_scenarios_android.proj
         channels:
           - release/7.0
+
+  # Maui iOS scenario benchmarks
+  - template: /eng/performance/build_machine_matrix.yml
+    parameters:
+      jobTemplate: /eng/performance/scenarios.yml
+      buildMachines:
+        - osx-x64-ios-arm64
+      isPublic: false
+      jobParameters:
+        kind: maui_scenarios_ios
+        projectFile: maui_scenarios_ios.proj
+        channels:
+          - release/7.0
 
 ################################################
 # Scheduled Private jobs

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -8,6 +8,7 @@
   <!--Package versions-->
   <PropertyGroup>
     <MicrosoftDotnetSdkInternalPackageVersion>5.0.104-servicing.21213.7</MicrosoftDotnetSdkInternalPackageVersion>
+    <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.22411.1</MicrosoftDotNetXHarnessCLIVersion>
     <MicrosoftNETCoreAppRuntimewinx64PackageVersion>5.0.4</MicrosoftNETCoreAppRuntimewinx64PackageVersion>
     <MicrosoftNETSdkInternalPackageVersion>
     </MicrosoftNETSdkInternalPackageVersion>

diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml
@@ -104,3 +104,15 @@ jobs:
       queue: Windows.10.Amd64.Pixel.Perf
       machinePool: Pixel
       ${{ insert }}: ${{ parameters.jobParameters }}
+
+- ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 12mini only used in private builds currently
+  - template: ${{ parameters.jobTemplate }}
+    parameters:
+      osName: osx
+      architecture: x64
+      osVersion: 12
+      pool:
+        vmImage: 'macos-12'
+      queue: OSX.1015.Amd64.Iphone.Perf
+      machinePool: iPhoneMini12
+      ${{ insert }}: ${{ parameters.jobParameters }}
diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj
@@ -0,0 +1,161 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <Import Project="Scenarios.Common.props" />
+
+  <PropertyGroup>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
+    <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+  </PropertyGroup>
+
+  <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
+    <Message Text="Removing Dotnet Packs from Correlation Staging" Importance="high" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
+  </Target>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>00:30</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <MAUIiOSScenario Include="Maui iOS Default">
+      <ScenarioDirectoryName>mauiios</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <IPAName>MauiiOSDefault</IPAName>
+      <PackageName>net.dot.mauitesting</PackageName>
+    </MAUIiOSScenario>
+    <!-- MessagingCenter was removed: https://github.com/dotnet/maui/pull/12582, needs to be replaced likely with something like: https://github.com/dotnet/maui/commit/762e07e04d1fdf8fca3edd948bab294c8762cd2c -->
+    <!-- <MAUIiOSScenario Include="Maui iOS Podcast">
+      <ScenarioDirectoryName>mauiiospodcast</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <IPAName>MauiiOSPodcast</IPAName>
+      <PackageName>net.dot.net.dot.netconf2021.maui</PackageName>
+    </MAUIiOSScenario> -->
+    <MAUIiOSScenario Include="Maui Blazor iOS Default">
+      <ScenarioDirectoryName>mauiblazorios</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <IPAName>MauiBlazoriOSDefault</IPAName>
+      <PackageName>net.dot.mauiblazortesting</PackageName>
+    </MAUIiOSScenario>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(MAUIiOSScenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command> 
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="@(MAUIiOSScenario -> 'SOD - %(Identity) IPA Size')">
+      <PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIiOSScenario -> 'SOD - %(Identity) Unzipped')">
+      <PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default">
+      <AppBundlePath>$(ScenariosDir)mauiios.zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
+            # PreCommands
+            export XHARNESSPATH=$XHARNESS_CLI_PATH
+
+            cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/mauiios/MauiiOSDefault.ipa $HELIX_WORKITEM_ROOT/mauiios/MauiiOSDefault.zip
+            mkdir $HELIX_WORKITEM_ROOT/mauiios/pub
+            cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/mauiios/versions.json $HELIX_WORKITEM_ROOT/mauiios/pub/versions.json
+            unzip -d $HELIX_WORKITEM_ROOT/mauiios $HELIX_WORKITEM_ROOT/mauiios/MauiiOSDefault.zip
+            mv $HELIX_WORKITEM_ROOT/mauiios/Payload/MauiiOSDefault.app $HELIX_WORKITEM_ROOT/mauiios/MauiiOSDefault.app
+            cp -f embedded.mobileprovision $HELIX_WORKITEM_ROOT/mauiios/MauiiOSDefault.app
+            cd $HELIX_WORKITEM_ROOT/mauiios
+            sign MauiiOSDefault.app
+
+            # Testing commands
+            $(Python) test.py devicestartup --device-type ios --package-path MauiiOSDefault.app --package-name net.dot.mauitesting --scenario-name "%(Identity)"
+            ((result=$?))
+
+            # Post commands
+            $(Python) post.py
+            exit $result
+          ]]>
+        </CustomCommands>
+    </XHarnessAppBundleToTest>
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default">
+      <AppBundlePath>$(ScenariosDir)mauiblazorios.zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
+          # PreCommands
+          export XHARNESSPATH=$XHARNESS_CLI_PATH
+
+          cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/mauiblazorios/MauiBlazoriOSDefault.ipa $HELIX_WORKITEM_ROOT/mauiblazorios/MauiBlazoriOSDefault.zip
+          mkdir $HELIX_WORKITEM_ROOT/mauiblazorios/pub
+          cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/mauiblazorios/versions.json $HELIX_WORKITEM_ROOT/mauiblazorios/pub/versions.json
+          unzip -d $HELIX_WORKITEM_ROOT/mauiblazorios $HELIX_WORKITEM_ROOT/mauiblazorios/MauiBlazoriOSDefault.zip
+          mv $HELIX_WORKITEM_ROOT/mauiblazorios/Payload/MauiBlazoriOSDefault.app $HELIX_WORKITEM_ROOT/mauiblazorios/MauiBlazoriOSDefault.app
+          cp -f embedded.mobileprovision $HELIX_WORKITEM_ROOT/mauiblazorios/MauiBlazoriOSDefault.app
+          cd $HELIX_WORKITEM_ROOT/mauiblazorios
+          sign MauiBlazoriOSDefault.app
+
+          # Testing commands
+          $(Python) test.py devicestartup --device-type ios --package-path MauiBlazoriOSDefault.app --package-name net.dot.mauiblazortesting --scenario-name "%(Identity)" --use-fully-drawn-time --fully-drawn-magic-string __MAUI_Blazor_WebView_OnAfterRender__ --startup-iterations 5
+          ((result=$?))
+
+          # Post commands
+          $(Python) post.py
+          exit $result
+        ]]>
+      </CustomCommands>
+    </XHarnessAppBundleToTest>
+    <!-- <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Podcast" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
+          # PreCommands
+          export XHARNESSPATH=$XHARNESS_CLI_PATH
+
+          cp -r $HELIX_CORRELATION_PAYLOAD/Microsoft.NetConf2021.Maui.app $(ScenarioDirectory)mauiios/Microsoft.NetConf2021.Maui.app
+          cp -f embedded.mobileprovision $(ScenarioDirectory)mauiios/Microsoft.NetConf2021.Maui.app
+          cd $(ScenarioDirectory)mauiios
+          sign Microsoft.NetConf2021.Maui.app
+
+          $(Python) pre.py -1-name Microsoft.NetConf2021.Maui.app
+
+          # Testing commands
+          $(Python) test.py devicestartup -1-device-type ios -1-package-path Microsoft.NetConf2021.Maui.app -1-package-name net.dot.netconf2021.maui -1-scenario-name "%(Identity)"
+          ((result=$?))
+
+          # Post commands
+          $(Python) post.py
+          exit $result
+        ]]>
+      </CustomCommands>
+    </XHarnessAppBundleToTest> -->
+  </ItemGroup>
+
+
+  <!--
+    This target is to work around the XHarness command that depend on scripts in ORIGPYPATH
+    being run before we get to run our normal Post commands. AddXHarnessCLI is the XHarness
+    Target so we just make sure we add this after that.
+  -->
+  <Target Name="ResetPYTHONPATHBeforeXHarnessCommand" AfterTargets="AddXHarnessCLI">
+    <PropertyGroup>
+      <HelixPostCommands>export PYTHONPATH=$ORIGPYPATH;$(HelixPostCommands)</HelixPostCommands>
+    </PropertyGroup>
+  </Target>
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+</Project>
diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml
@@ -128,7 +128,30 @@ jobs:
               displayName: Delete old dotnet
             - script: xcopy tools\dotnet\arm64\* $(CorrelationStaging)dotnet\/E /I /Y
               displayName: Copy Arm64 Dotnet to Correlation Payload
-        - ${{ if ne(parameters.osName, 'windows') }}:
+        - ${{ if eq(parameters.osName, 'osx') }}:
+          - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps
+            displayName: Copy python libraries and NuGet.config
+          - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(PERFLAB_Framework) -r osx-${{parameters.architecture}} --self-contained $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true
+            displayName: Build startup tool
+            env:
+              PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework)
+          - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)SOD -f $(PERFLAB_Framework) -r osx-${{parameters.architecture}} --self-contained $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true
+            displayName: Build SOD tool
+            env:
+              PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework)
+          - script: |
+              $(Python) -m pip install --user --upgrade pip
+              $(Python) -m pip install --user requests
+              . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
+              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
+            displayName: Prepare scenarios
+            env:
+              CorrelationPayloadDirectory: $(CorrelationStaging)
+              Architecture: ${{ parameters.architecture }}
+              TargetsWindows: 'false'
+              WorkItemDirectory: $(Build.SourcesDirectory)
+              HelixTargetQueues: ${{ parameters.queue }}
+        - ${{ if and(ne(parameters.osName, 'windows'), ne(parameters.osName, 'osx')) }}:
           - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps
             displayName: Copy python libraries and NuGet.config
           - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(PERFLAB_Framework) -r linux-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true

diff --git a/src/scenarios/mauiblazorios/pre.py b/src/scenarios/mauiblazorios/pre.py
@@ -1,42 +1,79 @@
 '''
 pre-command
 '''
+import shutil
 import sys
-import os
-from zipfile import ZipFile
 from performance.logger import setup_loggers, getLogger
-from shutil import copyfile, copytree, move
-from shared.const import PUBDIR
-from argparse import ArgumentParser
+from shared import const
+from shared.mauisharedpython import remove_aab_files, install_versioned_maui
+from shared.precommands import PreCommands
+from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios
+from test import EXENAME
 
 setup_loggers(True)
+precommands = PreCommands()
+install_versioned_maui(precommands)
+
+# Setup the Maui folder
+precommands.new(template='maui-blazor',
+                output_dir=const.APPDIR,
+                bin_dir=const.BINDIR,
+                exename=EXENAME,
+                working_directory=sys.path[0],
+                no_restore=False)
+
+# Add the index.razor.cs file
+with open(f"{const.APPDIR}/Pages/Index.razor.cs", "w") as indexCSFile:
+    indexCSFile.write('''
+    using Microsoft.AspNetCore.Components;
+    #if ANDROID
+        using Android.App;
+    #endif\n\n''' 
+    + f"    namespace {EXENAME}.Pages" + 
+'''
+    {
+        public partial class Index
+        {
+            protected override void OnAfterRender(bool firstRender)
+            {
+                if (firstRender)
+                {
+                    #if ANDROID
+                        var activity = MainActivity.Context as Activity;
+                        activity.ReportFullyDrawn();
+                    #else
+                        System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\");
+                    #endif
+                }
+            }
+        }
+    }
+''')
+
+# Replace line in the Android MainActivity.cs file
+with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "r") as mainActivityFile:
+    mainActivityFileLines = mainActivityFile.readlines()
+
+with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "w") as mainActivityFile:
+    for line in mainActivityFileLines:
+        if line.startswith("{"):
+            mainActivityFile.write("{\npublic static Android.Content.Context Context { get; private set; }\npublic MainActivity() { Context = this; }")
+        else:
+            mainActivityFile.write(line)
+
+# Build the APK
+# NuGet.config file cannot be in the build directory currently due to https://github.com/dotnet/aspnetcore/issues/41397
+# shutil.copy('./MauiNuGet.config', './app/Nuget.config')
+precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauiblazortesting'])
+
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+remove_aab_files(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versions_write_json(version_dict, rf"{output_dir}/versions.json")
+print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
 
-parser = ArgumentParser()
-parser.add_argument('--unzip', help='Unzip ipa file and report extracted tree', action='store_true', default=False)
-parser.add_argument(
-        '--name',
-        dest='name',
-        required=True,
-        type=str,
-        help='Name of the file/folder to setup (with .app or .ipa)')
-args = parser.parse_args()
-
-name = args.name
-namezip = '%s.zip' % (name)
-if not os.path.exists(PUBDIR):
-    os.mkdir(PUBDIR)
-if not os.path.exists(name):
-    getLogger().error('Cannot find %s' % (name))
-    exit(-1)
-if args.unzip:
-    if not os.path.exists(namezip):
-        copyfile(name, namezip)
-
-    with ZipFile(namezip) as zip:
-        zip.extractall(os.path.join('.', PUBDIR))
-
-else:
-    if(os.path.isdir(name)):
-        copytree(name, PUBDIR, dirs_exist_ok=True)
-    else:
-        copyfile(name, os.path.join(PUBDIR, name))
diff --git a/src/scenarios/mauiblazorios/test.py b/src/scenarios/mauiblazorios/test.py
@@ -1,11 +1,15 @@
 '''
-C# Console app
+Mobile Maui App
 '''
+from shared.const import PUBDIR
 from shared.runner import TestTraits, Runner
+from shared.versionmanager import versions_read_json_file_save_env
 
 EXENAME = 'MauiBlazoriOSDefault'
 
 if __name__ == "__main__":
+    versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json")
+
     traits = TestTraits(exename=EXENAME, 
                         guiapp='false',
                         )