implement temporary 5.4 fix for extra starting null bytes; override c…

…ustom version container if versioned asset

UAssetAPI.Tests/AssetUnitTests.cs

@@ -71,8 +71,8 @@ public bool CheckAllExportsParsedCorrectly(UAsset tester)
         /// <returns>An array of paths to assets that should be tested.</returns>
         public string[] GetAllTestAssets(string folder)
         {
-            List<string> allFilesToTest = Directory.GetFiles(folder, "*.uasset").ToList();
-            allFilesToTest.AddRange(Directory.GetFiles(folder, "*.umap"));
+            List<string> allFilesToTest = Directory.GetFiles(folder, "*.uasset", SearchOption.AllDirectories).ToList();
+            allFilesToTest.AddRange(Directory.GetFiles(folder, "*.umap", SearchOption.AllDirectories));
             return allFilesToTest.ToArray();
         }
 
@@ -303,6 +303,19 @@ private void TestUE5_3Subsection(string game, EngineVersion version, Usmap mappi
             }
         }
 
+        private void TestUE5_4Subsection(string game, EngineVersion version, Usmap mappings = null)
+        {
+            string[] allTestingAssets = GetAllTestAssets(Path.Combine("TestAssets", "TestUE5_4", game));
+            foreach (string assetPath in allTestingAssets)
+            {
+                Console.WriteLine(assetPath);
+                var tester = new UAsset(assetPath, version, mappings);
+                Assert.IsTrue(tester.VerifyBinaryEquality());
+                Assert.IsTrue(CheckAllExportsParsedCorrectly(tester));
+                Console.WriteLine(tester.GetEngineVersion());
+            }
+        }
+
         /// <summary>
         /// Tests the GUID/string conversion operations to ensure that they match the Unreal implementation.
         /// </summary>
@@ -552,6 +565,18 @@ public void TestTraditionalUE5_3()
             TestUE5_3Subsection("RON", EngineVersion.VER_UE5_3, new Usmap(Path.Combine("TestAssets", "TestUE5_3", "RON", "ReadyOrNot.usmap")));
         }
 
+        /// <summary>
+        /// In this test, we test several traditional assets specifically from Unreal Engine 5.3 games.
+        /// Binary equality is expected.
+        /// </summary>
+        [TestMethod]
+        public void TestTraditionalUE5_4()
+        {
+            TestUE5_4Subsection("BlankGame", EngineVersion.VER_UE5_4, new Usmap(Path.Combine("TestAssets", "TestUE5_4", "BlankGame", "BlankGame_Dumper-7.usmap")));
+            TestUE5_4Subsection("Bellwright", EngineVersion.VER_UE5_4, new Usmap(Path.Combine("TestAssets", "TestUE5_4", "Bellwright", "Bellwright.usmap")));
+            TestUE5_4Subsection("TheForeverWinter", EngineVersion.VER_UE5_4, new Usmap(Path.Combine("TestAssets", "TestUE5_4", "TheForeverWinter", "TheForeverWinter.usmap")));
+        }
+
         /// <summary>
         /// In this test, we test the Clone function, along with indexers for assets and exports.
         /// </summary>

UAssetAPI.Tests/UAssetAPI.Tests.csproj

@@ -284,6 +284,198 @@
     <None Update="TestAssets\TestUE5_3\RON\ReadyOrNot.usmap">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Bellwright.usmap">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeBlacksmithNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeBlacksmithNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeCarpenterNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeCarpenterNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeEngineerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeEngineerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeFarmerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeFarmerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeHealerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeHealerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeInkeeperNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeInkeeperNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeLabourerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeLabourerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeWeaverNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeWeaverNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeWoodsmanNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\ApprenticeWoodsmanNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\HighNoTraitsVillagerIdleNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\HighNoTraitsVillagerIdleNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\HighVillagerIdleNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\HighSkill\HighVillagerIdleNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowBeggarNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowBeggarNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowVillagerIdleNoTraitsNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowVillagerIdleNoTraitsNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowVillagerIdleNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\LowSkill\LowVillagerIdleNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\MediumNoTraitsVillagerIdleNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\MediumNoTraitsVillagerIdleNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\MediumVillagerIdleNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\MediumVillagerIdleNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceBlacksmithNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceBlacksmithNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceCarpenterNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceCarpenterNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceEngineerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceEngineerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceFarmerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceFarmerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceHealerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceHealerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceInkeeperNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceInkeeperNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceLabourerNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceLabourerNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceWeaverNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceWeaverNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceWoodsmanNPCTemplate.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\Content\Mist\Characters\NPC\Villager\MediumSkill\NoviceWoodsmanNPCTemplate.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\Bellwright\UMG_TreeViewDemo.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\BP_CubePawn.combined">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\BP_CubePawn.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\BP_CubePawn.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\MI_BlueCube.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\MI_BlueCube.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\M_Cube.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\M_Cube.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\SM_BlueCube.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\BlankGame\SM_BlueCube.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\TheForeverWinter\DA_DailyGiftList.uasset">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\TheForeverWinter\DA_DailyGiftList.uexp">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\TestUE5_4\TheForeverWinter\TheForeverWinter.usmap">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="TestAssets\TestUnknownProperties\BP_DetPack_Charge.uasset">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
@@ -447,4 +639,7 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="TestAssets\TestUE5_4\Bellwright\" />
+  </ItemGroup>
 </Project>

UAssetAPI/ExportTypes/NormalExport.cs

@@ -121,6 +121,13 @@ public NormalExport()
 
         public override void Read(AssetBinaryReader reader, int nextStarting = 0)
         {
+            // 5.4-specific problem; unclear why this occurs
+            if (reader.Asset.ObjectVersionUE5 >= ObjectVersionUE5.PROPERTY_TAG_COMPLETE_TYPE_NAME && !ObjectFlags.HasFlag(EObjectFlags.RF_ClassDefaultObject))
+            {
+                int dummy = reader.ReadInt32();
+                if (dummy != 0) throw new FormatException("Expected 4 null bytes at start of NormalExport; got " + dummy);
+            }
+
             Data = new List<PropertyData>();
             PropertyData bit;
 
@@ -158,6 +165,12 @@ public override void ResolveAncestries(UAsset asset, AncestryInfo ancestrySoFar)
 
         public override void Write(AssetBinaryWriter writer)
         {
+            // 5.4-specific problem; unclear why this occurs
+            if (writer.Asset.ObjectVersionUE5 >= ObjectVersionUE5.PROPERTY_TAG_COMPLETE_TYPE_NAME && !ObjectFlags.HasFlag(EObjectFlags.RF_ClassDefaultObject))
+            {
+                writer.Write((int)0); // "false" bool?
+            }
+
             FName parentName = GetClassTypeForAncestry(writer.Asset, out FName parentModulePath);
 
             MainSerializer.GenerateUnversionedHeader(ref Data, parentName, parentModulePath, writer.Asset)?.Write(writer);

UAssetAPI/UAsset.cs

@@ -1539,6 +1539,12 @@ private void ReadHeader(AssetBinaryReader reader)
             }
             if (ObjectVersionUE5 == ObjectVersionUE5.UNKNOWN && Mappings != null && Mappings.FileVersionUE5 > 0) ObjectVersionUE5 = Mappings.FileVersionUE5;
 
+            // if wasn't unversioned, we'll ignore the current custom version container and just read it from disk
+            if (!IsUnversioned)
+            {
+                CustomVersionContainer = null;
+            }
+
             FileVersionLicenseeUE = reader.ReadInt32();
 
             // Custom versions container