From f0b1e8a53054c5620b256b7f5613ef5b3610146c Mon Sep 17 00:00:00 2001 From: Radek Doulik Date: Fri, 2 Aug 2019 11:45:16 -0700 Subject: [PATCH] [Xamarin.Android.Build.Tasks] Bundle Profiled AOT-required NDK tools (#3317) Fixes: https://github.com/xamarin/xamarin-android/issues/3299 Profiled AOT (6e88ffa6) requires a handful of things to work: 1. The mono cross-compilers, and 2. Various Android NDK tools such as `as` (assembler), `ld` (linker), and `strip`. As of decfbccf, Xamarin.Android *already* redistributes `as`. Update `make prepare` and installer creation so that Xamarin.AndroidAotMode also redistributes `ld` and `strip` for all supported ABIs. This has two benefits: 1. Profiled AOT and "normal" AOT use will not require that the full Android NDK be installed. 2. Redistribution helps "firewall" the binaries for macOS Catalina. The full Android NDK is a ~700-800MB download (r19c is 770MB, r20 is 804MB) and ~3GB extracted/installed, while adding `ld` and `strip` to our installation only increases the `.pkg` size by ~8MB and installation size by ~20MB. (`as` is already installed.) Redistribution of `ld` and `strip` thus makes it much easier for customers to use AOT. **Note**: AOT+LLVM (`$(AndroidAotMode)` != None *and* `$(EnableLLVM)`=True) still requires a full NDK. Finally, macOS Catalina is [removing support for 32-bit apps][0]. The Android NDK r8d and earlier provide 32-bit binaries for some (all?) of these utilities, and thus they won't work on the forthcoming macOS 10.15 Catalina release. The [NDK r8e][1] release notes state: > Added 64-bit host toolchain set (package name suffix *-x86_64.*). > For more information, see `CHANGES.HTML` and `NDK-BUILD.html`. Including these utilities thus removes a partial dependency on the installed Android NDK, which should increase the likelihood of successful Profiled AOT use on macOS Catalina. [0]: https://support.apple.com/en-us/HT208436 [1]: https://developer.android.com/ndk/downloads/revision_history --- Documentation/guides/BuildProcess.md | 2 ++ .../installers/create-installers.targets | 16 ++++++++++++++++ .../Scenarios/Scenario_Standard.Unix.cs | 2 +- ...ix.cs => Step_Get_Windows_Binutils.Unix.cs} | 12 ++++++++++-- .../xaprepare/Steps/Step_PrepareBundle.Unix.cs | 2 +- .../xaprepare/xaprepare/xaprepare.csproj | 2 +- src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs | 18 +++++++++++++----- .../Tasks/MakeBundleNativeCodeExternalTests.cs | 1 + .../Xamarin.Android.Build.Tasks.targets | 18 +++++++++++++----- .../Xamarin.Android.Common.targets | 1 + 10 files changed, 59 insertions(+), 15 deletions(-) rename build-tools/xaprepare/xaprepare/Steps/{Step_Get_Windows_GAS.Unix.cs => Step_Get_Windows_Binutils.Unix.cs} (95%) diff --git a/Documentation/guides/BuildProcess.md b/Documentation/guides/BuildProcess.md index 0b7c277a7b5..7fdebc0d194 100644 --- a/Documentation/guides/BuildProcess.md +++ b/Documentation/guides/BuildProcess.md @@ -735,6 +735,8 @@ when packaging Release applications. or not LLVM will be used when Ahead-of-Time compiling assemblies into native code. + Enabling this propery requires Android NDK to be installed. + Support for this property was added in Xamarin.Android 5.1. This property is `False` by default. diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index ccf789236e7..1cff48c557d 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -137,9 +137,17 @@ <_MSBuildFilesWin Include="$(MSBuildSrcDir)\aapt2.exe" /> <_MSBuildFilesWin Include="$(MSBuildSrcDir)\libwinpthread-1.dll" /> <_MSBuildFilesWin Include="$(MSBuildSrcDir)\aarch64-linux-android-as.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\aarch64-linux-android-ld.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\aarch64-linux-android-strip.exe" Condition=" '$(HostOS)' != 'Windows' "/> <_MSBuildFilesWin Include="$(MSBuildSrcDir)\arm-linux-androideabi-as.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\arm-linux-androideabi-ld.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\arm-linux-androideabi-strip.exe" Condition=" '$(HostOS)' != 'Windows' "/> <_MSBuildFilesWin Include="$(MSBuildSrcDir)\i686-linux-android-as.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\i686-linux-android-ld.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\i686-linux-android-strip.exe" Condition=" '$(HostOS)' != 'Windows' "/> <_MSBuildFilesWin Include="$(MSBuildSrcDir)\x86_64-linux-android-as.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\x86_64-linux-android-ld.exe" Condition=" '$(HostOS)' != 'Windows' "/> + <_MSBuildFilesWin Include="$(MSBuildSrcDir)\x86_64-linux-android-strip.exe" Condition=" '$(HostOS)' != 'Windows' "/> <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libmono-android.debug.dll" Condition=" '$(HostOS)' != 'Windows' " /> <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libmono-android.release.dll" Condition=" '$(HostOS)' != 'Windows' " /> <_MSBuildLibHostFilesWin Include="$(MSBuildSrcDir)\lib\host-mxe-Win64\libMonoPosixHelper.dll" /> @@ -148,9 +156,17 @@ <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\aarch64-linux-android-as" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\aarch64-linux-android-ld" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\aarch64-linux-android-strip" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\arm-linux-androideabi-as" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\arm-linux-androideabi-ld" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\arm-linux-androideabi-strip" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\i686-linux-android-as" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\i686-linux-android-ld" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\i686-linux-android-strip" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\x86_64-linux-android-as" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\x86_64-linux-android-ld" /> + <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\x86_64-linux-android-strip" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\illinkanalyzer" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\jit-times" /> <_MSBuildFilesUnix Include="$(MSBuildSrcDir)\$(HostOS)\mono" /> diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.Unix.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.Unix.cs index 5d9fe386c42..fcdb2a0feaf 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.Unix.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.Unix.cs @@ -13,7 +13,7 @@ partial void AddRequiredOSSpecificSteps (bool beforeBundle) // (because they're not useful for every day work with XA) so they must be downloaded after the bundle // is unpacked. Log.DebugLine ("Adding Windows GAS download step (AFTER bundle)"); - Steps.Add (new Step_Get_Windows_GAS ()); + Steps.Add (new Step_Get_Windows_Binutils ()); return; } diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_GAS.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_Binutils.Unix.cs similarity index 95% rename from build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_GAS.Unix.cs rename to build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_Binutils.Unix.cs index c0e294d15a3..f5e3f55f384 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_GAS.Unix.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Get_Windows_Binutils.Unix.cs @@ -9,7 +9,7 @@ namespace Xamarin.Android.Prepare { - class Step_Get_Windows_GAS : Step + class Step_Get_Windows_Binutils : Step { const int EndFileChunkSize = 65535 + 22; // Maximum comment size + EOCD size const uint EOCDSignature = 0x06054b50; @@ -69,7 +69,7 @@ class LFHeader public byte[] ExtraField; }; - public Step_Get_Windows_GAS () + public Step_Get_Windows_Binutils () : base ("Downloading NDK tools for Windows") {} @@ -79,9 +79,17 @@ protected override async Task Execute (Context context) var neededFiles = new HashSet (StringComparer.OrdinalIgnoreCase) { $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/i686-linux-android-as.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/i686-linux-android-ld.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/i686-linux-android-strip.exe", $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/arm-linux-androideabi-as.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/arm-linux-androideabi-ld.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/arm-linux-androideabi-strip.exe", $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/x86_64-linux-android-as.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/x86_64-linux-android-ld.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/x86_64-linux-android-strip.exe", $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/aarch64-linux-android-as.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/aarch64-linux-android-ld.exe", + $"android-ndk-r{ndkVersion}/toolchains/llvm/prebuilt/windows-x86_64/bin/aarch64-linux-android-strip.exe", }; string destinationDirectory = Path.Combine (Configurables.Paths.InstallMSBuildDir); diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareBundle.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareBundle.Unix.cs index 34a31f3355e..0d778cb6aac 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareBundle.Unix.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareBundle.Unix.cs @@ -16,7 +16,7 @@ void InitOS () // We need it here (even though Scenario_Standard runs the step, because if we failed to download the // bundle, the Step_BuildMonoRuntimes above will clean the destination directory and the Windows GAS // executables with it. - AddFailureStep (new Step_Get_Windows_GAS ()); + AddFailureStep (new Step_Get_Windows_Binutils ()); AddFailureStep (new Step_CreateBundle ()); } diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj index de16e1e6e31..f6d3f997eab 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.csproj +++ b/build-tools/xaprepare/xaprepare/xaprepare.csproj @@ -196,7 +196,7 @@ - + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs index c990d31633e..7556fef0590 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs @@ -35,7 +35,6 @@ public class Aot : AsyncTask [Required] public string AndroidAotMode { get; set; } - [Required] public string AndroidNdkDirectory { get; set; } [Required] @@ -71,6 +70,8 @@ public class Aot : AsyncTask public ITaskItem [] Profiles { get; set; } + public string ToolsDirectory { get; set; } + [Output] public string[] NativeLibrariesReferences { get; set; } @@ -83,7 +84,7 @@ public Aot () public override bool Execute () { - if (!NdkUtil.Init (Log, AndroidNdkDirectory)) + if (EnableLLVM && !NdkUtil.Init (Log, AndroidNdkDirectory)) return false; try { @@ -346,7 +347,7 @@ IEnumerable GetAotConfigs () throw new Exception ("Unsupported Android target architecture ABI: " + abi); } - if (!NdkUtil.ValidateNdkPlatform (Log, AndroidNdkDirectory, arch, enableLLVM:EnableLLVM)) { + if (EnableLLVM && !NdkUtil.ValidateNdkPlatform (Log, AndroidNdkDirectory, arch, enableLLVM:EnableLLVM)) { yield return Config.Invalid; yield break; } @@ -360,11 +361,18 @@ IEnumerable GetAotConfigs () if (!Directory.Exists (outdir)) Directory.CreateDirectory (outdir); - int level = GetNdkApiLevel (AndroidNdkDirectory, AndroidApiLevel, arch); - string toolPrefix = NdkUtil.GetNdkToolPrefix (AndroidNdkDirectory, arch, level); + int level = 0; + string toolPrefix = EnableLLVM + ? NdkUtil.GetNdkToolPrefix (AndroidNdkDirectory, arch, level = GetNdkApiLevel (AndroidNdkDirectory, AndroidApiLevel, arch)) + : $"{ToolsDirectory}{NdkUtil.GetArchDirName (arch)}-"; var toolchainPath = toolPrefix.Substring(0, toolPrefix.LastIndexOf(Path.DirectorySeparatorChar)); var ldFlags = string.Empty; if (EnableLLVM) { + if (string.IsNullOrEmpty (AndroidNdkDirectory)) { + yield return Config.Invalid; + yield break; + } + string androidLibPath = string.Empty; try { androidLibPath = NdkUtil.GetNdkPlatformLibPath(AndroidNdkDirectory, arch, level); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/MakeBundleNativeCodeExternalTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/MakeBundleNativeCodeExternalTests.cs index 538ef487bc8..242097e6f04 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/MakeBundleNativeCodeExternalTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/MakeBundleNativeCodeExternalTests.cs @@ -53,6 +53,7 @@ public void XA5101AndroidNdkNotFound (string androidNdkDirectory) AndroidNdkDirectory = androidNdkDirectory, AndroidAotMode = "normal", AndroidApiLevel = "28", + EnableLLVM = true, ResolvedAssemblies = new ITaskItem [0], SupportedAbis = new [] { "armeabi-v7a" }, AotOutputDirectory = path, diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index 9f7e3693811..3974810ad1a 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -151,14 +151,22 @@ <_ToolchainPrefix Include="x86_64"> x86_64-linux-android + <_ToolName Include="as"/> + <_ToolName Include="ld"/> + <_ToolName Include="strip"/> + + + + - + @@ -167,9 +175,9 @@ - <_NDKToolSource Include="$(_NDKToolLocation%(_ToolchainPrefix.Identity))" /> - <_NDKToolDestination Condition=" '$(HostOS)' != 'Windows' " Include="$(_NDKToolDestinationBase)\$(HostOS)\$([System.IO.Path]::GetFileName ('$(_NDKToolLocation%(_ToolchainPrefix.Identity))'))" /> - <_NDKToolDestination Condition=" '$(HostOS)' == 'Windows' " Include="$(_NDKToolDestinationBase)\$([System.IO.Path]::GetFileName ('$(_NDKToolLocation%(_ToolchainPrefix.Identity))'))" /> + <_NDKToolSource Include="$(_NDKToolLocation%(_ToolchainTool.Identity)-%(_ToolchainTool.Tool))" /> + <_NDKToolDestination Condition=" '$(HostOS)' != 'Windows' " Include="$(_NDKToolDestinationBase)\$(HostOS)\$([System.IO.Path]::GetFileName ('$(_NDKToolLocation%(_ToolchainTool.Identity)-%(_ToolchainTool.Tool))'))" /> + <_NDKToolDestination Condition=" '$(HostOS)' == 'Windows' " Include="$(_NDKToolDestinationBase)\$([System.IO.Path]::GetFileName ('$(_NDKToolLocation%(_ToolchainTool.Identity)-%(_ToolchainTool.Tool))'))" /> diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index e4b9fe1ccd8..56644c86804 100755 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2986,6 +2986,7 @@ because xbuild doesn't support framework reference assemblies. Condition="'$(AotAssemblies)' == 'True'" AndroidAotMode="$(AndroidAotMode)" AndroidNdkDirectory="$(_AndroidNdkDirectory)" + ToolsDirectory="$(MonoAndroidBinDirectory)" AndroidApiLevel="$(_AndroidApiLevel)" ManifestFile="$(IntermediateOutputPath)android\AndroidManifest.xml" SupportedAbis="@(_BuildTargetAbis)"