Skip to content

Commit

Permalink
usmap thread safety fixes, remove usmap CityHash64Map, add SkipPreloa…
Browse files Browse the repository at this point in the history 
…dDependencyLoading flag
  • Loading branch information
@atenfyr
atenfyr committed Oct 26, 2024
1 parent 43aecfe commit ff22022
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 44 deletions.
2 changes: 1 addition & 1 deletion UAssetAPI/IO/ZenAsset.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ private void AddCityHash64MapEntryRaw(string val)
public string GetStringFromCityHash64(ulong val)
{
if (CityHash64Map.ContainsKey(val)) return CityHash64Map[val];
if (Mappings.CityHash64Map.ContainsKey(val)) return Mappings.CityHash64Map[val];
//if (Mappings.CityHash64Map.ContainsKey(val)) return Mappings.CityHash64Map[val];
return null;
}

Expand Down
13 changes: 10 additions & 3 deletions UAssetAPI/UAsset.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,12 @@ public enum CustomSerializationFlags : int
/// <summary>
/// Skip Kismet bytecode serialization.
/// </summary>
SkipParsingBytecode = 2
SkipParsingBytecode = 2,

/// <summary>
/// Skip loading other assets referenced in preload dependencies. You may wish to set this flag when possible in multi-threading applications, since preload dependency loading could lead to file handle race conditions.
/// </summary>
SkipPreloadDependencyLoading = 4
}


Expand Down Expand Up @@ -279,6 +284,8 @@ public int SearchForImport(FName objectName)

public override bool PullSchemasFromAnotherAsset(FName path, FName desiredObject = null)
{
if (CustomSerializationFlags.HasFlag(CustomSerializationFlags.SkipPreloadDependencyLoading)) return false;

if (Mappings?.Schemas == null) return false;
if (path?.Value?.Value == null) return false;
if (!path.Value.Value.StartsWith("/") || path.Value.Value.StartsWith("/Script")) return false;
Expand All @@ -291,15 +298,15 @@ public override bool PullSchemasFromAnotherAsset(FName path, FName desiredObject
}

// basic circular referencing guard
if (Mappings.PathsAlreadyProcessedForSchemas.Contains(assetPath))
if (Mappings.PathsAlreadyProcessedForSchemas.ContainsKey(assetPath))
{
return false;
}

bool success = false;
try
{
Mappings.PathsAlreadyProcessedForSchemas.Add(assetPath);
Mappings.PathsAlreadyProcessedForSchemas[assetPath] = 1;
UAsset otherAsset = new UAsset(this.ObjectVersion, this.ObjectVersionUE5, this.CustomVersionContainer.Select(item => (CustomVersion)item.Clone()).ToList(), this.Mappings);
otherAsset.InternalAssetPath = assetPath;
otherAsset.FilePath = pathOnDisk;
Expand Down
2 changes: 1 addition & 1 deletion UAssetAPI/UnrealPackage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -829,7 +829,7 @@ protected void ConvertExportToChildExportAndRead(AssetBinaryReader reader, int i
string enumName = fetchedEnumExp.ObjectName?.ToString();
if (Mappings?.EnumMap != null && enumName != null)
{
var newEnum = new UsmapEnum(enumName, new Dictionary<long, string>());
var newEnum = new UsmapEnum(enumName, new ConcurrentDictionary<long, string>());
foreach (Tuple<FName, long> entry in fetchedEnumExp.Enum.Names)
{
newEnum.Values[entry.Item2] = entry.Item1.ToString();
Expand Down
57 changes: 18 additions & 39 deletions UAssetAPI/Unversioned/Usmap.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
Expand Down Expand Up @@ -248,8 +249,8 @@ public class UsmapSchema

public IReadOnlyDictionary<int, UsmapProperty> Properties => properties;

private Dictionary<int, UsmapProperty> properties;
private Dictionary<Tuple<string, int>, UsmapProperty> propertiesMap;
private ConcurrentDictionary<int, UsmapProperty> properties;
private ConcurrentDictionary<Tuple<string, int>, UsmapProperty> propertiesMap;

public UsmapStructKind StructKind;
public int StructOrClassFlags;
Expand All @@ -262,14 +263,14 @@ public UsmapProperty GetProperty(string key, int dupIndex)

public void ConstructPropertiesMap(bool isCaseInsensitive)
{
propertiesMap = new Dictionary<Tuple<string, int>, UsmapProperty>(new PropertyMapComparer { Comparer = isCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture });
propertiesMap = new ConcurrentDictionary<Tuple<string, int>, UsmapProperty>(new PropertyMapComparer { Comparer = isCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture });
foreach (KeyValuePair<int, UsmapProperty> entry in properties)
{
propertiesMap[new Tuple<string, int>(entry.Value.Name, entry.Value.ArrayIndex)] = entry.Value;
}
}

public UsmapSchema(string name, string superType, ushort propCount, Dictionary<int, UsmapProperty> props, bool isCaseInsensitive, bool fromAsset = false)
public UsmapSchema(string name, string superType, ushort propCount, ConcurrentDictionary<int, UsmapProperty> props, bool isCaseInsensitive, bool fromAsset = false)
{
Name = name;
SuperType = superType;
Expand All @@ -291,9 +292,9 @@ public class UsmapEnum
public string Name;
public string ModulePath;
public int EnumFlags;
public Dictionary<long, string> Values;
public ConcurrentDictionary<long, string> Values;

public UsmapEnum(string name, Dictionary<long, string> values)
public UsmapEnum(string name, ConcurrentDictionary<long, string> values)
{
Name = name;
Values = values;
Expand Down Expand Up @@ -389,30 +390,11 @@ public bool AreFNamesCaseInsensitive
/// </summary>
public IDictionary<string, UsmapSchema> Schemas;

/// <summary>
/// Pre-computed CityHash64 map for all relevant strings
/// </summary>
public IDictionary<ulong, string> CityHash64Map;

/// <summary>
/// List of extensions that failed to parse.
/// </summary>
public List<string> FailedExtensions;

private void AddCityHash64MapEntry(string val)
{
// for now, we don't actually use this
return;

/*ulong hsh = CRCGenerator.GenerateImportHashFromObjectPath(val);
if (CityHash64Map.ContainsKey(hsh))
{
if (CRCGenerator.ToLower(CityHash64Map[hsh]) == CRCGenerator.ToLower(val)) return;
throw new FormatException("CityHash64 hash collision between \"" + CityHash64Map[hsh] + "\" and \"" + val + "\"");
}
CityHash64Map.Add(hsh, val);*/
}

private static UsmapPropertyData ConvertFPropertyToUsmapPropertyData(StructExport exp, FProperty entry)
{
var typ = entry.GetUsmapPropertyType();
Expand Down Expand Up @@ -621,14 +603,14 @@ public static UsmapSchema GetSchemaFromStructExport(string exportName, UnrealPac

public static UsmapSchema GetSchemaFromStructExport(StructExport exp, bool isCaseInsensitive)
{
var res = new Dictionary<int, UsmapProperty>();
var res = new ConcurrentDictionary<int, UsmapProperty>();
int idx = 0;
if (exp.Asset.GetCustomVersion<FCoreObjectVersion>() >= FCoreObjectVersion.FProperties)
{
foreach (FProperty entry in exp.LoadedProperties)
{
UsmapProperty converted = new UsmapProperty(entry.Name.ToString(), (ushort)idx, 0, 1, ConvertFPropertyToUsmapPropertyData(exp, entry));
res.Add(idx, converted);
res[idx] = converted;
idx++;
}
}
Expand Down Expand Up @@ -656,7 +638,7 @@ public static UsmapSchema GetSchemaFromStructExport(StructExport exp, bool isCas
if (entry.ToExport(exp.Asset) is not PropertyExport field) continue;

UsmapProperty converted = new UsmapProperty(field.ObjectName.ToString(), (ushort)idx, 0, 1, ConvertUPropertyToUsmapPropertyData(field));
res.Add(idx, converted);
res[idx] = converted;
idx++;
}
}
Expand Down Expand Up @@ -724,7 +706,8 @@ public string GetAllPropertiesAnnotated(string schemaName, UnrealPackage asset,
return string.Join("\n", res.ToArray());
}

public ISet<string> PathsAlreadyProcessedForSchemas = new HashSet<string>();
// not a set to ensure thread safety
public ConcurrentDictionary<string, byte> PathsAlreadyProcessedForSchemas = new ConcurrentDictionary<string, byte>();
public UsmapSchema GetSchemaFromName(string nm, UnrealPackage asset = null, string modulePath = null, bool throwExceptions = true)
{
if (string.IsNullOrEmpty(nm)) return null;
Expand Down Expand Up @@ -960,7 +943,6 @@ private UsmapPropertyData DeserializePropData(UsmapBinaryReader reader)
public void Read(UsmapBinaryReader compressedReader)
{
var reader = ReadHeader(compressedReader);
CityHash64Map = new Dictionary<ulong, string>();

// part 1: names
//Console.WriteLine(reader.BaseStream.Position);
Expand All @@ -975,18 +957,18 @@ public void Read(UsmapBinaryReader compressedReader)

// part 2: enums
//Console.WriteLine(reader.BaseStream.Position);
EnumMap = new Dictionary<string, UsmapEnum>(AreFNamesCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture);
EnumMap = new ConcurrentDictionary<string, UsmapEnum>(AreFNamesCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture);
int numEnums = reader.ReadInt32();
UsmapEnum[] enumIndexMap = new UsmapEnum[numEnums];
for (int i = 0; i < numEnums; i++)
{
string enumName = reader.ReadName();

var newEnum = new UsmapEnum(enumName, new Dictionary<long, string>());
var newEnum = new UsmapEnum(enumName, new ConcurrentDictionary<long, string>());
int numEnumEntries = Version >= UsmapVersion.LargeEnums ? (int)reader.ReadInt16() : (int)reader.ReadByte();
for (int j = 0; j < numEnumEntries; j++)
{
newEnum.Values.Add(j, reader.ReadName());
newEnum.Values[j] = reader.ReadName();
}

if (!EnumMap.ContainsKey(enumName))
Expand All @@ -998,7 +980,7 @@ public void Read(UsmapBinaryReader compressedReader)

// part 3: schema
//Console.WriteLine(reader.BaseStream.Position);
Schemas = new Dictionary<string, UsmapSchema>(AreFNamesCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture);
Schemas = new ConcurrentDictionary<string, UsmapSchema>(AreFNamesCaseInsensitive ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture);
int numSchema = reader.ReadInt32();
UsmapSchema[] schemaIndexMap = new UsmapSchema[numSchema];
for (int i = 0; i < numSchema; i++)
Expand All @@ -1007,7 +989,7 @@ public void Read(UsmapBinaryReader compressedReader)
string schemaSuperName = reader.ReadName();
ushort numProps = reader.ReadUInt16();
ushort serializablePropCount = reader.ReadUInt16();
Dictionary<int, UsmapProperty> props = new Dictionary<int, UsmapProperty>();
ConcurrentDictionary<int, UsmapProperty> props = new ConcurrentDictionary<int, UsmapProperty>();
for (int j = 0; j < serializablePropCount; j++)
{
ushort SchemaIdx = reader.ReadUInt16();
Expand All @@ -1021,7 +1003,7 @@ public void Read(UsmapBinaryReader compressedReader)
var cln = (UsmapProperty)currProp.Clone();
cln.SchemaIndex = (ushort)(SchemaIdx + k);
cln.ArrayIndex = (ushort)k;
props.Add(SchemaIdx + k, cln);
props[SchemaIdx + k] = cln;
}
}

Expand Down Expand Up @@ -1050,14 +1032,12 @@ void ReadExtension(string extId, uint extLeng)
for (int i = 0; i < ppthNumEnums; i++)
{
enumIndexMap[i].ModulePath = reader.ReadName();
AddCityHash64MapEntry(enumIndexMap[i].ModulePath + "." + enumIndexMap[i].Name);
}
int ppthNumSchemas = reader.ReadInt32();
for (int i = 0; i < ppthNumSchemas; i++)
{
schemaIndexMap[i].ModulePath = reader.ReadName();
Schemas[schemaIndexMap[i].ModulePath + "." + schemaIndexMap[i].Name] = schemaIndexMap[i];
AddCityHash64MapEntry(schemaIndexMap[i].ModulePath + "." + schemaIndexMap[i].Name);
}

if (reader.BaseStream.Position != endPos) throw new FormatException("Failed to parse extension " + extId + ": ended at " + reader.BaseStream.Position + ", expected " + endPos);
Expand Down Expand Up @@ -1112,7 +1092,6 @@ void ReadExtension(string extId, uint extLeng)
for (int i = 0; i < schemaIndexMap.Length; i++)
{
schemaIndexMap[i].ModulePath = modulePaths[numModulePaths > byte.MaxValue ? reader.ReadUInt16() : reader.ReadByte()];
AddCityHash64MapEntry(schemaIndexMap[i].ModulePath + "." + schemaIndexMap[i].Name);
}

if (reader.BaseStream.Position != endPos) throw new FormatException("Failed to parse extension " + extId + ": ended at " + reader.BaseStream.Position + ", expected " + endPos);
Expand Down

0 comments on commit ff22022

Please sign in to comment.