Skip to content

Commit

Permalink
zen no longer planned feature; remove incomplete zen/io store impleme…
Browse files Browse the repository at this point in the history 
…ntation to simplify architecture, remove cityhash
  • Loading branch information
@atenfyr
atenfyr committed Jan 11, 2025
1 parent 3d259eb commit f54fdc4
Show file tree
Hide file tree
Showing 212 changed files with 2,420 additions and 9,319 deletions.
24 changes: 0 additions & 24 deletions NOTICE.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,30 +78,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
```

## cityhash
https://github.com/google/cityhash
```
// Copyright (c) 2011 Google, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
```

## ZstdNet
https://github.com/skbkontur/ZstdNet
```
Expand Down
12 changes: 0 additions & 12 deletions UAssetAPI.Benchmark/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.Reflection;
using UAssetAPI.CustomVersions;
using UAssetAPI.ExportTypes;
using UAssetAPI.IO;
using UAssetAPI.UnrealTypes;
using UAssetAPI.Unversioned;

Expand Down Expand Up @@ -347,17 +346,6 @@ public static void Run(string[] args)
//Usmap with_skip = new Usmap(@"C:\Dumper-7\with_skip.usmap");
Usmap no_skip = new Usmap(@"C:\Users\Alexandros\AppData\Local\UAssetGUI\Mappings\ReadyOrNot-D7-PPTH.usmap");
break;
case "zen":
IOStoreContainer test1 = new IOStoreContainer(@"C:\Program Files (x86)\Steam\steamapps\common\Garten Of Banban\Clay\Content\Paks\global.utoc");
ZenAsset test = new ZenAsset(EngineVersion.VER_UE5_1, new Usmap(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "UAssetGUI", "Mappings", "Clay.usmap")));
test.GlobalData = new IOGlobalData(test1, EngineVersion.VER_UE5_1);
test.Read(test.PathToReader(Path.Combine("TestAssets", "B_Gamemode.uasset")));
Console.WriteLine(test.Name);

MemoryStream testStrm = new MemoryStream();
new AssetBinaryWriter(testStrm, test).WriteNameBatch(test.HashVersion, (IList<FString>)test.GetNameMapIndexList());
Console.WriteLine(BitConverter.ToString(testStrm.ToArray()));
break;
}
}

Expand Down
130 changes: 24 additions & 106 deletions UAssetAPI/AssetBinaryReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.IO;
using System.Text;
using UAssetAPI.CustomVersions;
using UAssetAPI.IO;
using UAssetAPI.Kismet.Bytecode;
using UAssetAPI.UnrealTypes;
using UAssetAPI.Unversioned;
Expand Down Expand Up @@ -82,47 +81,31 @@ public override string ReadString()
return ReadFString()?.Value;
}

public virtual FString ReadFString(FSerializedNameHeader nameHeader = null)
public virtual FString ReadFString()
{
if (nameHeader == null)
int length = this.ReadInt32();
switch (length)
{
int length = this.ReadInt32();
switch (length)
{
case 0:
return null;
default:
if (length < 0)
{
byte[] data = this.ReadBytes(-length * 2);
return new FString(Encoding.Unicode.GetString(data, 0, data.Length - 2), Encoding.Unicode);
}
else
{
byte[] data = this.ReadBytes(length);
return new FString(Encoding.ASCII.GetString(data, 0, data.Length - 1), Encoding.ASCII);
}
}
}
else
{
if (nameHeader.bIsWide)
{
byte[] data = this.ReadBytes(nameHeader.Len * 2); // TODO: are we actually supposed to divide by two?
return new FString(Encoding.Unicode.GetString(data, 0, data.Length), Encoding.Unicode);
}
else
{
byte[] data = this.ReadBytes(nameHeader.Len);
return new FString(Encoding.ASCII.GetString(data, 0, data.Length), Encoding.ASCII);
}
case 0:
return null;
default:
if (length < 0)
{
byte[] data = this.ReadBytes(-length * 2);
return new FString(Encoding.Unicode.GetString(data, 0, data.Length - 2), Encoding.Unicode);
}
else
{
byte[] data = this.ReadBytes(length);
return new FString(Encoding.ASCII.GetString(data, 0, data.Length - 1), Encoding.ASCII);
}
}
}

public virtual FString ReadNameMapString(FSerializedNameHeader nameHeader, out uint hashes)
public virtual FString ReadNameMapString(out uint hashes)
{
hashes = 0;
FString str = this.ReadFString(nameHeader);
FString str = this.ReadFString();
if (this is AssetBinaryReader abr)
{
if (abr.Asset is UAsset abrUa && abrUa.WillSerializeNameHashes != false && !string.IsNullOrEmpty(str.Value))
Expand All @@ -143,59 +126,6 @@ public virtual FString ReadNameMapString(FSerializedNameHeader nameHeader, out u
return str;
}

internal const ulong CityHash64 = 0x00000000C1640000;
public void ReadNameBatch(bool VerifyHashes, out ulong HashVersion, out List<FString> nameMap)
{
// TODO: implement pre-ue5 serialization

HashVersion = 0;
nameMap = new List<FString>();

int numStrings = ReadInt32();
if (numStrings == 0) return;
ReadInt32(); // length of strings in bytes

// read hashes
HashVersion = ReadUInt64();
ulong[] hashes = new ulong[numStrings];
switch (HashVersion)
{
case UnrealBinaryReader.CityHash64:
for (int i = 0; i < numStrings; i++) hashes[i] = ReadUInt64(); // CityHash64 of str.ToLowerCase();
break;
default:
throw new InvalidOperationException("Unknown algorithm ID " + HashVersion);
}

// read headers
FSerializedNameHeader[] nameHeaders = new FSerializedNameHeader[numStrings];
for (int i = 0; i < numStrings; i++) nameHeaders[i] = FSerializedNameHeader.Read(this);

// read strings
for (int i = 0; i < numStrings; i++)
{
FString newStr = ReadNameMapString(nameHeaders[i], out _);
nameMap.Add(newStr);
}

// verify hashes if requested
if (VerifyHashes)
{
for (int i = 0; i < nameMap.Count; i++)
{
switch (HashVersion)
{
case UnrealBinaryReader.CityHash64:
ulong expectedHash = CRCGenerator.CityHash64WithLower(nameMap[i]);
if (expectedHash != hashes[i]) throw new IOException("Expected hash \"" + expectedHash + "\", received \"" + hashes[i] + "\" for string " + nameMap[i].Value + " in name map; corrupt data?");
break;
default:
throw new InvalidOperationException("Unknown algorithm ID " + HashVersion);
}
}
}
}

public List<CustomVersion> ReadCustomVersionContainer(ECustomVersionSerializationFormat format, List<CustomVersion> oldCustomVersionContainer = null, Usmap Mappings = null)
{
var newCustomVersionContainer = new List<CustomVersion>();
Expand Down Expand Up @@ -252,15 +182,15 @@ public List<CustomVersion> ReadCustomVersionContainer(ECustomVersionSerializatio
/// </summary>
public class AssetBinaryReader : UnrealBinaryReader
{
public UnrealPackage Asset;
public UAsset Asset;
public bool LoadUexp = true;

public AssetBinaryReader(Stream stream, UnrealPackage asset = null) : base(stream)
public AssetBinaryReader(Stream stream, UAsset asset = null) : base(stream)
{
Asset = asset;
}

public AssetBinaryReader(Stream stream, bool inLoadUexp, UnrealPackage asset = null) : base(stream)
public AssetBinaryReader(Stream stream, bool inLoadUexp, UAsset asset = null) : base(stream)
{
Asset = asset;
LoadUexp = inLoadUexp;
Expand All @@ -279,21 +209,9 @@ public AssetBinaryReader(Stream stream, bool inLoadUexp, UnrealPackage asset = n

public virtual FName ReadFName()
{
if (Asset is ZenAsset)
{
uint Index = this.ReadUInt32();
uint Number = this.ReadUInt32();

var res = new FName(Asset, (int)(Index & FName.IndexMask), (int)Number);
res.Type = (EMappedNameType)((Index & FName.TypeMask) >> FName.TypeShift);
return res;
}
else
{
int nameMapPointer = this.ReadInt32();
int number = this.ReadInt32();
return new FName(Asset, nameMapPointer, number);
}
int nameMapPointer = this.ReadInt32();
int number = this.ReadInt32();
return new FName(Asset, nameMapPointer, number);
}

public FObjectThumbnail ReadObjectThumbnail()
Expand Down
64 changes: 7 additions & 57 deletions UAssetAPI/AssetBinaryWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.IO;
using System.Text;
using UAssetAPI.CustomVersions;
using UAssetAPI.IO;
using UAssetAPI.Kismet.Bytecode;
using UAssetAPI.UnrealTypes;
using UAssetAPI.Unversioned;
Expand Down Expand Up @@ -102,47 +101,6 @@ public virtual int Write(FString value)
}
}

public void WriteNameBatch(ulong HashVersion, IList<FString> nameMap)
{
Write(nameMap.Count);
if (nameMap.Count == 0) return;
long numBytesOfStringsPos = this.BaseStream.Position;
Write((int)0);

// write hashes
Write(HashVersion);
switch (HashVersion)
{
case UnrealBinaryReader.CityHash64:
for (int i = 0; i < nameMap.Count; i++)
{
Write(CRCGenerator.CityHash64(CRCGenerator.ToLower(nameMap[i].Value), nameMap[i].Encoding));
}
break;
default:
throw new InvalidOperationException("Unknown algorithm ID " + HashVersion);
}

// write headers
for (int i = 0; i < nameMap.Count; i++)
{
FSerializedNameHeader.Write(this, nameMap[i].Encoding is UnicodeEncoding, nameMap[i].Value.Length);
}

// write strings
long stringsStartPos = BaseStream.Position;
for (int i = 0; i < nameMap.Count; i++)
{
Write(nameMap[i].Encoding.GetBytes(nameMap[i].Value));
}
long stringsEndPos = BaseStream.Position;

// fix length
Seek((int)numBytesOfStringsPos, SeekOrigin.Begin);
Write((int)(stringsEndPos - stringsStartPos));
Seek((int)stringsEndPos, SeekOrigin.Begin);
}

public void WriteCustomVersionContainer(ECustomVersionSerializationFormat format, List<CustomVersion> CustomVersionContainer)
{
// TODO: support for enum-based custom versions
Expand Down Expand Up @@ -198,41 +156,33 @@ public void WriteCustomVersionContainer(ECustomVersionSerializationFormat format
/// </summary>
public class AssetBinaryWriter : UnrealBinaryWriter
{
public UnrealPackage Asset;
public UAsset Asset;

public AssetBinaryWriter(UnrealPackage asset) : base()
public AssetBinaryWriter(UAsset asset) : base()
{
Asset = asset;
}

public AssetBinaryWriter(Stream stream, UnrealPackage asset) : base(stream)
public AssetBinaryWriter(Stream stream, UAsset asset) : base(stream)
{
Asset = asset;
}

public AssetBinaryWriter(Stream stream, Encoding encoding, UnrealPackage asset) : base(stream, encoding)
public AssetBinaryWriter(Stream stream, Encoding encoding, UAsset asset) : base(stream, encoding)
{
Asset = asset;
}

public AssetBinaryWriter(Stream stream, Encoding encoding, bool leaveOpen, UnrealPackage asset) : base(stream, encoding, leaveOpen)
public AssetBinaryWriter(Stream stream, Encoding encoding, bool leaveOpen, UAsset asset) : base(stream, encoding, leaveOpen)
{
Asset = asset;
}

public virtual void Write(FName name)
{
if (name == null) name = new FName(Asset, 0, 0);
if (Asset is ZenAsset)
{
this.Write(((uint)name.Type << FName.TypeShift) | (uint)name.Index);
this.Write(name.Number);
}
else
{
this.Write(name.Index);
this.Write(name.Number);
}
this.Write(name.Index);
this.Write(name.Number);
}

public virtual void WritePropertyGuid(Guid? guid)
Expand Down
38 changes: 0 additions & 38 deletions UAssetAPI/CRCGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,44 +6,6 @@ namespace UAssetAPI;

public static class CRCGenerator
{
public static unsafe ulong GenerateImportHashFromObjectPath(FString text)
{
return GenerateImportHashFromObjectPath(text?.Value);
}

public static unsafe ulong GenerateImportHashFromObjectPath(string text)
{
return CityHash64(Encoding.Unicode.GetBytes(ToLower(text, true))) & FPackageObjectIndex.IndexMask;
}

public static unsafe ulong CityHash64WithLower(FString text)
{
return CityHash64WithLower(text?.Value, text?.Encoding);
}

public static unsafe ulong CityHash64WithLower(string text, Encoding encoding)
{
return CityHash64(encoding.GetBytes(ToLower(text)));
}

public static unsafe ulong CityHash64(FString text)
{
return CityHash64(text?.Value, text?.Encoding);
}

public static unsafe ulong CityHash64(string text, Encoding encoding)
{
return CityHash64(encoding.GetBytes(text));
}

public static unsafe ulong CityHash64(byte[] data)
{
fixed (byte* arr = data)
{
return CityHash.CityHash64(arr, (uint)data.Length);
}
}

public static uint GenerateHash(FString text, bool disableCasePreservingHash, bool version420 = false)
{
return GenerateHash(text?.Value, text?.Encoding, disableCasePreservingHash, version420);
Expand Down
Loading

0 comments on commit f54fdc4

Please sign in to comment.