From e13e42ea16ab78a93894108322a417377e1e5f40 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Sep 2021 10:34:52 +0200 Subject: [PATCH] [WIP] More test fixes --- .../Xamarin.Android.Build.Tests/AotTests.cs | 58 ++++++++----- .../Tasks/LinkerTests.cs | 15 ++-- .../Utilities/ArchiveAssemblyHelper.cs | 84 ++++++++++++++++++- 3 files changed, 123 insertions(+), 34 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index e45814b8bb3..84ad4a6e4df 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -115,50 +115,64 @@ public void BuildBasicApplicationReleaseProfiledAotWithoutDefaultProfile () /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* supportedAbis */ "armeabi-v7a", + /* enableLLVM */ false, + /* expectedResult */ true, + /* usesAssemblyBlobs */ true, }, new object[] { /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, }; [Test] [TestCaseSource (nameof (AotChecks))] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = false, @@ -168,6 +182,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); if (checkMinLlvmPath) { // Set //uses-sdk/@android:minSdkVersion so that LLVM uses the right libc.so @@ -211,13 +226,13 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -234,9 +249,9 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL [TestCaseSource (nameof (AotChecks))] [Category ("Minor"), Category ("MkBundle")] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = true, @@ -246,6 +261,7 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { if (!b.CrossCompilerAvailable (supportedAbis)) Assert.Ignore ("Cross compiler was not available"); @@ -264,13 +280,12 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsFalse (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -381,6 +396,8 @@ public static void Foo () { [Category ("HybridAOT")] public void HybridAOT ([Values ("armeabi-v7a;arm64-v8a", "armeabi-v7a", "arm64-v8a")] string abis) { + // There's no point in testing all of the ABIs with and without assembly blobs, let's test just one of them this way + bool usesAssemblyBlobs = String.Compare ("arm64-v8a", abis, StringComparison.Ordinal) == 0; var proj = new XamarinAndroidApplicationProject () { IsRelease = true, AotAssemblies = true, @@ -388,6 +405,7 @@ public static void Foo () { proj.SetProperty ("AndroidAotMode", "Hybrid"); // So we can use Mono.Cecil to open assemblies directly proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); + proj.SetProperty ("AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ()); proj.SetAndroidSupportedAbis (abis); using (var b = CreateApkBuilder ()) { @@ -412,17 +430,15 @@ public static void Foo () { var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{proj.ProjectName}.dll"); - Assert.IsNotNull (entry, $"{proj.ProjectName}.dll should exist in apk!"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); - var method = type.Methods.First (m => m.Name == "OnCreate"); - Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); - } + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ($"assemblies/{proj.ProjectName}.dll"), $"{proj.ProjectName}.dll should exist in apk!"); + + using (var stream = helper.ReadEntry ($"assemblies/{proj.ProjectName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); + var method = type.Methods.First (m => m.Name == "OnCreate"); + Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index ceb4948aacb..5106be3e92b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -208,16 +208,11 @@ public void RemoveDesigner ([Values (true, false)] bool usesAssemblyBlobs) FileAssert.Exists (apk); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); - // TODO: helper must be able to extract assembly from the blob - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{assemblyName}.dll"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); - Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); - } + using (var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); + Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 5bc9e160ac8..3e5de3a0744 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; @@ -11,7 +12,8 @@ namespace Xamarin.Android.Build.Tests { public class ArchiveAssemblyHelper { - public const string DefaultBlobEntryPrefix = "{blob}"; + public const string DefaultBlobEntryPrefix = "{blobReader}"; + const int BlobReadBufferSize = 8192; static readonly HashSet SpecialExtensions = new HashSet (StringComparer.OrdinalIgnoreCase) { ".dll", @@ -27,6 +29,8 @@ public class ArchiveAssemblyHelper {"arm64_v8a", "arm64-v8a"}, }; + static readonly ArrayPool buffers = ArrayPool.Create (); + readonly string archivePath; readonly string assembliesRootDir; bool useAssemblyBlobs; @@ -55,6 +59,80 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs) } } + public Stream ReadEntry (string path) + { + if (useAssemblyBlobs) { + return ReadBlobEntry (path); + } + + return ReadZipEntry (path); + } + + Stream ReadZipEntry (string path) + { + using (var zip = ZipHelper.OpenZip (archivePath)) { + ZipEntry entry = zip.ReadEntry (path); + var ret = new MemoryStream (); + entry.Extract (ret); + ret.Flush (); + return ret; + } + } + + Stream ReadBlobEntry (string path) + { + BlobReader blobReader = null; + BlobAssembly assembly = null; + string name = Path.GetFileNameWithoutExtension (path); + var explorer = new BlobExplorer (archivePath); + + foreach (var asm in explorer.Assemblies) { + if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { + continue; + } + assembly = asm; + blobReader = asm.Blob; + break; + } + + if (blobReader == null) { + Console.WriteLine ($"Blob for entry {path} not found, will try a standard Zip read"); + return ReadZipEntry (path); + } + + string blobEntryName; + if (String.IsNullOrEmpty (blobReader.Arch)) { + blobEntryName = $"{assembliesRootDir}assemblies.blob"; + } else { + blobEntryName = $"{assembliesRootDir}assemblies_{blobReader.Arch}.blob"; + } + + Stream blob = ReadZipEntry (blobEntryName); + if (blob == null) { + Console.WriteLine ($"Blob zip entry {blobEntryName} does not exist"); + return null; + } + + blob.Seek (assembly.DataOffset, SeekOrigin.Begin); + var ret = new MemoryStream (); + byte[] buffer = buffers.Rent (BlobReadBufferSize); + int toRead = (int)assembly.DataSize; + while (toRead > 0) { + int nread = blob.Read (buffer, 0, BlobReadBufferSize); + if (nread <= 0) { + break; + } + + ret.Write (buffer, 0, nread); + toRead -= nread; + } + ret.Flush (); + blob.Dispose (); + buffers.Return (buffer); + + return ret; + } + public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix, bool forceRefresh = false) { if (!forceRefresh && archiveContents != null) { @@ -97,7 +175,7 @@ public List ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt } } - Console.WriteLine ("Archive entries with synthetised assembly blob entries:"); + Console.WriteLine ("Archive entries with synthetised assembly blobReader entries:"); foreach (string e in entries) { Console.WriteLine ($" {e}"); } @@ -170,7 +248,7 @@ void BlobContains (string[] fileNames, out List existingFiles, out List< if (explorer.AssembliesByName.Count != 0) { existingFiles.AddRange (blobAssemblies); - // We need to fake config and debug files since they have no named entries in the blob + // We need to fake config and debug files since they have no named entries in the blobReader foreach (string file in configFiles) { BlobAssembly asm = GetBlobAssembly (file); if (asm == null) {