From cecd85d1f444de02f885ec1cb7104aec0dadd2db Mon Sep 17 00:00:00 2001 From: Ada Date: Fri, 9 Feb 2024 09:31:41 +0000 Subject: [PATCH 1/5] Chunked patch readers for large files --- CUE4Parse/Globals.cs | 3 +- CUE4Parse/UE4/IO/IoStoreReader.cs | 2 +- CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs | 2 +- CUE4Parse/UE4/Pak/Objects/FPakEntry.cs | 2 +- CUE4Parse/UE4/Pak/PakFileReader.cs | 93 ++++++++++++++--------- CUE4Parse/UE4/Readers/FChunkedArchive.cs | 82 ++++++++++++++++++++ CUE4Parse/UE4/Readers/FIoChunkArchive.cs | 26 +++++++ CUE4Parse/UE4/Readers/FPakChunkArchive.cs | 26 +++++++ 8 files changed, 198 insertions(+), 38 deletions(-) create mode 100644 CUE4Parse/UE4/Readers/FChunkedArchive.cs create mode 100644 CUE4Parse/UE4/Readers/FIoChunkArchive.cs create mode 100644 CUE4Parse/UE4/Readers/FPakChunkArchive.cs diff --git a/CUE4Parse/Globals.cs b/CUE4Parse/Globals.cs index 34b5acb2d..152e283dc 100644 --- a/CUE4Parse/Globals.cs +++ b/CUE4Parse/Globals.cs @@ -9,5 +9,6 @@ public static class Globals public static bool LogVfsMounts = true; public static bool FatalObjectSerializationErrors = false; public static bool WarnMissingImportPackage = true; + public static bool AlwaysUseChunkedReader = false; } -} \ No newline at end of file +} diff --git a/CUE4Parse/UE4/IO/IoStoreReader.cs b/CUE4Parse/UE4/IO/IoStoreReader.cs index 2036ffdd9..a09e4e375 100644 --- a/CUE4Parse/UE4/IO/IoStoreReader.cs +++ b/CUE4Parse/UE4/IO/IoStoreReader.cs @@ -199,7 +199,7 @@ public virtual byte[] Read(FIoChunkId chunkId) throw new KeyNotFoundException($"Couldn't find chunk {chunkId} in IoStore {Name}"); } - private byte[] Read(long offset, long length) + public byte[] Read(long offset, long length) { var compressionBlockSize = TocResource.Header.CompressionBlockSize; var dst = new byte[length]; diff --git a/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs b/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs index ae597223b..87dc2c460 100644 --- a/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs +++ b/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs @@ -39,6 +39,6 @@ public IoStoreReader IoStoreReader public override byte[] Read() => Vfs.Extract(this); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override FArchive CreateReader() => new FByteArchive(Path, Read(), Vfs.Versions); + public override FArchive CreateReader() => Size > int.MaxValue || Globals.AlwaysUseChunkedReader ? new FIoChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); } } diff --git a/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs b/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs index 52c7cfaea..3493022da 100644 --- a/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs +++ b/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs @@ -328,6 +328,6 @@ public PakFileReader PakFileReader public override byte[] Read() => Vfs.Extract(this); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override FArchive CreateReader() => new FByteArchive(Path, Read(), Vfs.Versions); + public override FArchive CreateReader() => Size > int.MaxValue || Globals.AlwaysUseChunkedReader ? new FPakChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); } } diff --git a/CUE4Parse/UE4/Pak/PakFileReader.cs b/CUE4Parse/UE4/Pak/PakFileReader.cs index 12b0d977b..bc6542b49 100644 --- a/CUE4Parse/UE4/Pak/PakFileReader.cs +++ b/CUE4Parse/UE4/Pak/PakFileReader.cs @@ -55,40 +55,7 @@ public PakFileReader(string filePath, Stream stream, VersionContainer? versions public override byte[] Extract(VfsEntry entry) { if (entry is not FPakEntry pakEntry || entry.Vfs != this) throw new ArgumentException($"Wrong pak file reader, required {entry.Vfs.Name}, this is {Name}"); - // If this reader is used as a concurrent reader create a clone of the main reader to provide thread safety - var reader = IsConcurrent ? (FArchive) Ar.Clone() : Ar; - if (pakEntry.IsCompressed) - { -#if DEBUG - Log.Debug($"{pakEntry.Name} is compressed with {pakEntry.CompressionMethod}"); -#endif - var uncompressed = new byte[(int) pakEntry.UncompressedSize]; - var uncompressedOff = 0; - foreach (var block in pakEntry.CompressionBlocks) - { - reader.Position = block.CompressedStart; - var blockSize = (int) block.Size; - var srcSize = blockSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); - // Read the compressed block - byte[] compressed = ReadAndDecrypt(srcSize, reader, pakEntry.IsEncrypted); - // Calculate the uncompressed size, - // its either just the compression block size - // or if its the last block its the remaining data size - var uncompressedSize = (int) Math.Min(pakEntry.CompressionBlockSize, pakEntry.UncompressedSize - uncompressedOff); - Decompress(compressed, 0, blockSize, uncompressed, uncompressedOff, uncompressedSize, pakEntry.CompressionMethod); - uncompressedOff += (int) pakEntry.CompressionBlockSize; - } - - return uncompressed; - } - - // Pak Entry is written before the file data, - // but its the same as the one from the index, just without a name - // We don't need to serialize that again so + file.StructSize - reader.Position = pakEntry.Offset + pakEntry.StructSize; // Doesn't seem to be the case with older pak versions - var size = (int) pakEntry.UncompressedSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); - var data = ReadAndDecrypt(size, reader, pakEntry.IsEncrypted); - return size != pakEntry.UncompressedSize ? data.SubByteArray((int) pakEntry.UncompressedSize) : data; + return Read(pakEntry, 0, pakEntry.UncompressedSize).ToArray(); } public override IReadOnlyDictionary Mount(bool caseInsensitive = false) @@ -304,5 +271,63 @@ public override void Dispose() { Ar.Dispose(); } + + public Span Read(FPakEntry pakEntry, long offset, long size) { + // If this reader is used as a concurrent reader create a clone of the main reader to provide thread safety + var reader = IsConcurrent ? (FArchive) Ar.Clone() : Ar; + if (pakEntry.IsCompressed) + { +#if DEBUG + Log.Debug($"{pakEntry.Name} is compressed with {pakEntry.CompressionMethod}"); +#endif + var uncompressed = new byte[(int) size + pakEntry.CompressionBlockSize]; + var uncompressedOff = 0; + var shift = 0; + foreach (var block in pakEntry.CompressionBlocks) + { + if (offset >= block.Size) { + offset -= block.Size; + continue; + } + + if (offset != 0) { + shift = (int) offset; + } + + offset = 0; + reader.Position = block.CompressedStart; + var blockSize = (int) block.Size; + var srcSize = blockSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); + // Read the compressed block + byte[] compressed = ReadAndDecrypt(srcSize, reader, pakEntry.IsEncrypted); + // Calculate the uncompressed size, + // its either just the compression block size + // or if its the last block its the remaining data size + var uncompressedSize = (int) Math.Min(pakEntry.CompressionBlockSize, size - uncompressedOff); + Decompress(compressed, 0, blockSize, uncompressed, uncompressedOff, uncompressedSize, pakEntry.CompressionMethod); + uncompressedOff += (int) pakEntry.CompressionBlockSize; + + if (uncompressedOff >= uncompressed.Length) { + break; + } + } + + return uncompressed.AsSpan(shift, (int) size); + } + + // Pak Entry is written before the file data, + // but its the same as the one from the index, just without a name + // We don't need to serialize that again so + file.StructSize + var readSize = (int) size.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); + var readShift = 0; + if (pakEntry.IsEncrypted && offset.Align(Aes.ALIGN) != offset) { + var tmp = offset.Align(Aes.ALIGN) - Aes.ALIGN; + readShift = (int)(offset - tmp); + offset = tmp; + readSize += Aes.ALIGN; + } + reader.Position = pakEntry.Offset + offset + pakEntry.StructSize; // Doesn't seem to be the case with older pak versions + return ReadAndDecrypt(readSize, reader, pakEntry.IsEncrypted).AsSpan(readShift, (int) size); + } } } diff --git a/CUE4Parse/UE4/Readers/FChunkedArchive.cs b/CUE4Parse/UE4/Readers/FChunkedArchive.cs new file mode 100644 index 000000000..d09bdb6a1 --- /dev/null +++ b/CUE4Parse/UE4/Readers/FChunkedArchive.cs @@ -0,0 +1,82 @@ +using System; +using System.Buffers; +using System.IO; +using System.Runtime.CompilerServices; +using CUE4Parse.UE4.Versions; +using CUE4Parse.Utils; + +namespace CUE4Parse.UE4.Readers; + +public abstract class FChunkedArchive : FArchive { + private const int BUFFER_SIZE = 1024 * 1024 * 128; // 128 MiB. + + protected FChunkedArchive(string path, VersionContainer versions) : base(versions) { + Name = path; + Buffer = ArrayPool.Shared.Rent(BUFFER_SIZE); + } + + protected abstract Span ReadChunks(long offset, long size); + + public override int Read(byte[] buffer, int offset, int count) { + if (Position < 0) { + return 0; + } + + var n = (int) (Length - Position); + if (n > count) n = count; + if (n <= 0) + return 0; + + Span data; + if (n < BUFFER_SIZE) { + var bufferRangeStart = BufferOffset; + var bufferRangeEnd = BufferOffset + BUFFER_SIZE; + if (!(bufferRangeStart <= Position && Position <= bufferRangeEnd)) { + BufferOffset = Position; + var blockSize = Math.Min(BUFFER_SIZE, Length - Position).Align(BUFFER_SIZE); + if (Position.Align(BUFFER_SIZE) != Position) { + BufferOffset = Position.Align(BUFFER_SIZE) - BUFFER_SIZE; + } + + ReadChunks(BufferOffset, blockSize).CopyTo(Buffer); + } + + data = Buffer.AsSpan().Slice((int)(Position - BufferOffset), n); + } else { + data = ReadChunks(Position, n); + } + + data.CopyTo(buffer.AsSpan(offset)); + Position += n; + return n; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override long Seek(long offset, SeekOrigin origin) + { + Position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => Position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException() + }; + return Position; + } + + public override bool CanSeek => true; + public override long Position { get; set; } + public override string Name { get; } + private byte[] Buffer { get; } + private long BufferOffset { get; set; } = -BUFFER_SIZE - 1; + + protected void ImportBuffer(FChunkedArchive other) { + BufferOffset = other.BufferOffset; + other.Buffer.AsSpan().CopyTo(Buffer.AsSpan()); + } + + public override void Close() { + base.Close(); + ArrayPool.Shared.Return(Buffer); + } +} diff --git a/CUE4Parse/UE4/Readers/FIoChunkArchive.cs b/CUE4Parse/UE4/Readers/FIoChunkArchive.cs new file mode 100644 index 000000000..5bfafa908 --- /dev/null +++ b/CUE4Parse/UE4/Readers/FIoChunkArchive.cs @@ -0,0 +1,26 @@ +using System; +using CUE4Parse.UE4.IO.Objects; +using CUE4Parse.UE4.Versions; + +namespace CUE4Parse.UE4.Readers; + +public class FIoChunkArchive : FChunkedArchive { + public FIoChunkArchive(string path, FIoStoreEntry entry, VersionContainer versions) : base(path, versions) { + Entry = entry; + } + + public override long Length => Entry.Size; + public FIoStoreEntry Entry { get; } + public override object Clone() { + var instance = new FIoChunkArchive(Name, Entry, Versions) { + Position = Position, + }; + instance.ImportBuffer(this); + return instance; + } + + protected override Span ReadChunks(long offset, long size) { + var remaining = Math.Min(size, Entry.Size - offset); + return remaining <= 0 ? Span.Empty : Entry.IoStoreReader.Read(offset + Entry.Offset, remaining); + } +} diff --git a/CUE4Parse/UE4/Readers/FPakChunkArchive.cs b/CUE4Parse/UE4/Readers/FPakChunkArchive.cs new file mode 100644 index 000000000..4a5a44f90 --- /dev/null +++ b/CUE4Parse/UE4/Readers/FPakChunkArchive.cs @@ -0,0 +1,26 @@ +using System; +using CUE4Parse.UE4.Pak.Objects; +using CUE4Parse.UE4.Versions; + +namespace CUE4Parse.UE4.Readers; + +public class FPakChunkArchive : FChunkedArchive { + public FPakChunkArchive(string path, FPakEntry entry, VersionContainer versions) : base(path, versions) { + Entry = entry; + } + + public override long Length => Entry.UncompressedSize; + public FPakEntry Entry { get; } + public override object Clone() { + var instance = new FPakChunkArchive(Name, Entry, Versions) { + Position = Position, + }; + instance.ImportBuffer(this); + return instance; + } + + protected override Span ReadChunks(long offset, long size) { + var remaining = Math.Min(size, Entry.UncompressedSize - offset); + return remaining <= 0 ? Span.Empty : Entry.PakFileReader.Read(Entry, offset, remaining); + } +} From 00c37083e505d5daf2cb136a803b610c48d40fbb Mon Sep 17 00:00:00 2001 From: Ada Date: Fri, 9 Feb 2024 09:41:13 +0000 Subject: [PATCH 2/5] Fix formatting --- CUE4Parse/UE4/Pak/PakFileReader.cs | 34 +++++++++-------- CUE4Parse/UE4/Readers/FChunkedArchive.cs | 45 +++++++++++++---------- CUE4Parse/UE4/Readers/FIoChunkArchive.cs | 16 +++++--- CUE4Parse/UE4/Readers/FPakChunkArchive.cs | 16 +++++--- 4 files changed, 66 insertions(+), 45 deletions(-) diff --git a/CUE4Parse/UE4/Pak/PakFileReader.cs b/CUE4Parse/UE4/Pak/PakFileReader.cs index bc6542b49..adbf55271 100644 --- a/CUE4Parse/UE4/Pak/PakFileReader.cs +++ b/CUE4Parse/UE4/Pak/PakFileReader.cs @@ -272,62 +272,64 @@ public override void Dispose() Ar.Dispose(); } - public Span Read(FPakEntry pakEntry, long offset, long size) { + public Span Read(FPakEntry pakEntry, long offset, long size) + { // If this reader is used as a concurrent reader create a clone of the main reader to provide thread safety - var reader = IsConcurrent ? (FArchive) Ar.Clone() : Ar; + var reader = IsConcurrent ? (FArchive)Ar.Clone() : Ar; if (pakEntry.IsCompressed) { #if DEBUG Log.Debug($"{pakEntry.Name} is compressed with {pakEntry.CompressionMethod}"); #endif - var uncompressed = new byte[(int) size + pakEntry.CompressionBlockSize]; + var uncompressed = new byte[(int)size + pakEntry.CompressionBlockSize]; var uncompressedOff = 0; var shift = 0; foreach (var block in pakEntry.CompressionBlocks) { - if (offset >= block.Size) { + if (offset >= block.Size) + { offset -= block.Size; continue; } - if (offset != 0) { - shift = (int) offset; - } + if (offset != 0) + shift = (int)offset; offset = 0; reader.Position = block.CompressedStart; - var blockSize = (int) block.Size; + var blockSize = (int)block.Size; var srcSize = blockSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); // Read the compressed block byte[] compressed = ReadAndDecrypt(srcSize, reader, pakEntry.IsEncrypted); // Calculate the uncompressed size, // its either just the compression block size // or if its the last block its the remaining data size - var uncompressedSize = (int) Math.Min(pakEntry.CompressionBlockSize, size - uncompressedOff); + var uncompressedSize = (int)Math.Min(pakEntry.CompressionBlockSize, size - uncompressedOff); Decompress(compressed, 0, blockSize, uncompressed, uncompressedOff, uncompressedSize, pakEntry.CompressionMethod); - uncompressedOff += (int) pakEntry.CompressionBlockSize; + uncompressedOff += (int)pakEntry.CompressionBlockSize; - if (uncompressedOff >= uncompressed.Length) { + if (uncompressedOff >= uncompressed.Length) break; - } } - return uncompressed.AsSpan(shift, (int) size); + return uncompressed.AsSpan(shift, (int)size); } // Pak Entry is written before the file data, // but its the same as the one from the index, just without a name // We don't need to serialize that again so + file.StructSize - var readSize = (int) size.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); + var readSize = (int)size.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1); var readShift = 0; - if (pakEntry.IsEncrypted && offset.Align(Aes.ALIGN) != offset) { + if (pakEntry.IsEncrypted && offset.Align(Aes.ALIGN) != offset) + { var tmp = offset.Align(Aes.ALIGN) - Aes.ALIGN; readShift = (int)(offset - tmp); offset = tmp; readSize += Aes.ALIGN; } + reader.Position = pakEntry.Offset + offset + pakEntry.StructSize; // Doesn't seem to be the case with older pak versions - return ReadAndDecrypt(readSize, reader, pakEntry.IsEncrypted).AsSpan(readShift, (int) size); + return ReadAndDecrypt(readSize, reader, pakEntry.IsEncrypted).AsSpan(readShift, (int)size); } } } diff --git a/CUE4Parse/UE4/Readers/FChunkedArchive.cs b/CUE4Parse/UE4/Readers/FChunkedArchive.cs index d09bdb6a1..b8281f2f5 100644 --- a/CUE4Parse/UE4/Readers/FChunkedArchive.cs +++ b/CUE4Parse/UE4/Readers/FChunkedArchive.cs @@ -7,42 +7,47 @@ namespace CUE4Parse.UE4.Readers; -public abstract class FChunkedArchive : FArchive { +public abstract class FChunkedArchive : FArchive +{ private const int BUFFER_SIZE = 1024 * 1024 * 128; // 128 MiB. - protected FChunkedArchive(string path, VersionContainer versions) : base(versions) { + protected FChunkedArchive(string path, VersionContainer versions) : base(versions) + { Name = path; Buffer = ArrayPool.Shared.Rent(BUFFER_SIZE); } protected abstract Span ReadChunks(long offset, long size); - public override int Read(byte[] buffer, int offset, int count) { - if (Position < 0) { + public override int Read(byte[] buffer, int offset, int count) + { + if (Position < 0) return 0; - } - var n = (int) (Length - Position); + var n = (int)(Length - Position); if (n > count) n = count; if (n <= 0) return 0; Span data; - if (n < BUFFER_SIZE) { + if (n < BUFFER_SIZE) + { var bufferRangeStart = BufferOffset; var bufferRangeEnd = BufferOffset + BUFFER_SIZE; - if (!(bufferRangeStart <= Position && Position <= bufferRangeEnd)) { + if (!(bufferRangeStart <= Position && Position <= bufferRangeEnd)) + { BufferOffset = Position; var blockSize = Math.Min(BUFFER_SIZE, Length - Position).Align(BUFFER_SIZE); - if (Position.Align(BUFFER_SIZE) != Position) { + if (Position.Align(BUFFER_SIZE) != Position) BufferOffset = Position.Align(BUFFER_SIZE) - BUFFER_SIZE; - } ReadChunks(BufferOffset, blockSize).CopyTo(Buffer); } data = Buffer.AsSpan().Slice((int)(Position - BufferOffset), n); - } else { + } + else + { data = ReadChunks(Position, n); } @@ -55,12 +60,12 @@ public override int Read(byte[] buffer, int offset, int count) { public override long Seek(long offset, SeekOrigin origin) { Position = origin switch - { - SeekOrigin.Begin => offset, - SeekOrigin.Current => Position + offset, - SeekOrigin.End => Length + offset, - _ => throw new ArgumentOutOfRangeException() - }; + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => Position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException() + }; return Position; } @@ -70,12 +75,14 @@ public override long Seek(long offset, SeekOrigin origin) private byte[] Buffer { get; } private long BufferOffset { get; set; } = -BUFFER_SIZE - 1; - protected void ImportBuffer(FChunkedArchive other) { + protected void ImportBuffer(FChunkedArchive other) + { BufferOffset = other.BufferOffset; other.Buffer.AsSpan().CopyTo(Buffer.AsSpan()); } - public override void Close() { + public override void Close() + { base.Close(); ArrayPool.Shared.Return(Buffer); } diff --git a/CUE4Parse/UE4/Readers/FIoChunkArchive.cs b/CUE4Parse/UE4/Readers/FIoChunkArchive.cs index 5bfafa908..63552f6a9 100644 --- a/CUE4Parse/UE4/Readers/FIoChunkArchive.cs +++ b/CUE4Parse/UE4/Readers/FIoChunkArchive.cs @@ -4,22 +4,28 @@ namespace CUE4Parse.UE4.Readers; -public class FIoChunkArchive : FChunkedArchive { - public FIoChunkArchive(string path, FIoStoreEntry entry, VersionContainer versions) : base(path, versions) { +public class FIoChunkArchive : FChunkedArchive +{ + public FIoChunkArchive(string path, FIoStoreEntry entry, VersionContainer versions) : base(path, versions) + { Entry = entry; } public override long Length => Entry.Size; public FIoStoreEntry Entry { get; } - public override object Clone() { - var instance = new FIoChunkArchive(Name, Entry, Versions) { + + public override object Clone() + { + var instance = new FIoChunkArchive(Name, Entry, Versions) + { Position = Position, }; instance.ImportBuffer(this); return instance; } - protected override Span ReadChunks(long offset, long size) { + protected override Span ReadChunks(long offset, long size) + { var remaining = Math.Min(size, Entry.Size - offset); return remaining <= 0 ? Span.Empty : Entry.IoStoreReader.Read(offset + Entry.Offset, remaining); } diff --git a/CUE4Parse/UE4/Readers/FPakChunkArchive.cs b/CUE4Parse/UE4/Readers/FPakChunkArchive.cs index 4a5a44f90..f3a136e71 100644 --- a/CUE4Parse/UE4/Readers/FPakChunkArchive.cs +++ b/CUE4Parse/UE4/Readers/FPakChunkArchive.cs @@ -4,22 +4,28 @@ namespace CUE4Parse.UE4.Readers; -public class FPakChunkArchive : FChunkedArchive { - public FPakChunkArchive(string path, FPakEntry entry, VersionContainer versions) : base(path, versions) { +public class FPakChunkArchive : FChunkedArchive +{ + public FPakChunkArchive(string path, FPakEntry entry, VersionContainer versions) : base(path, versions) + { Entry = entry; } public override long Length => Entry.UncompressedSize; public FPakEntry Entry { get; } - public override object Clone() { - var instance = new FPakChunkArchive(Name, Entry, Versions) { + + public override object Clone() + { + var instance = new FPakChunkArchive(Name, Entry, Versions) + { Position = Position, }; instance.ImportBuffer(this); return instance; } - protected override Span ReadChunks(long offset, long size) { + protected override Span ReadChunks(long offset, long size) + { var remaining = Math.Min(size, Entry.UncompressedSize - offset); return remaining <= 0 ? Span.Empty : Entry.PakFileReader.Read(Entry, offset, remaining); } From fcf259f20d12128e5aa90aa9a82582b3c599d075 Mon Sep 17 00:00:00 2001 From: Ada Date: Fri, 9 Feb 2024 10:49:08 +0000 Subject: [PATCH 3/5] Fix boundary issues --- CUE4Parse/UE4/Pak/PakFileReader.cs | 4 ++-- CUE4Parse/UE4/Readers/FChunkedArchive.cs | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CUE4Parse/UE4/Pak/PakFileReader.cs b/CUE4Parse/UE4/Pak/PakFileReader.cs index adbf55271..4f2e5e729 100644 --- a/CUE4Parse/UE4/Pak/PakFileReader.cs +++ b/CUE4Parse/UE4/Pak/PakFileReader.cs @@ -304,11 +304,11 @@ public Span Read(FPakEntry pakEntry, long offset, long size) // Calculate the uncompressed size, // its either just the compression block size // or if its the last block its the remaining data size - var uncompressedSize = (int)Math.Min(pakEntry.CompressionBlockSize, size - uncompressedOff); + var uncompressedSize = (int)Math.Min(pakEntry.CompressionBlockSize, pakEntry.UncompressedSize - uncompressedOff); Decompress(compressed, 0, blockSize, uncompressed, uncompressedOff, uncompressedSize, pakEntry.CompressionMethod); uncompressedOff += (int)pakEntry.CompressionBlockSize; - if (uncompressedOff >= uncompressed.Length) + if (uncompressedOff >= size) break; } diff --git a/CUE4Parse/UE4/Readers/FChunkedArchive.cs b/CUE4Parse/UE4/Readers/FChunkedArchive.cs index b8281f2f5..2540a123f 100644 --- a/CUE4Parse/UE4/Readers/FChunkedArchive.cs +++ b/CUE4Parse/UE4/Readers/FChunkedArchive.cs @@ -41,10 +41,16 @@ public override int Read(byte[] buffer, int offset, int count) if (Position.Align(BUFFER_SIZE) != Position) BufferOffset = Position.Align(BUFFER_SIZE) - BUFFER_SIZE; - ReadChunks(BufferOffset, blockSize).CopyTo(Buffer); + if ((int) (Position - BufferOffset) + n <= BUFFER_SIZE) //overflow check + ReadChunks(BufferOffset, blockSize).CopyTo(Buffer); + else + BufferOffset = -1; // reset buffer position because we didn't actually read } - data = Buffer.AsSpan().Slice((int)(Position - BufferOffset), n); + if (BufferOffset == -1 || (int) (Position - BufferOffset) + n > BUFFER_SIZE) + data = ReadChunks(Position, n); + else + data = Buffer.AsSpan().Slice((int) (Position - BufferOffset), n); } else { From 7bfb62aa7fe7ebfb0c8fb8c8e580792b1cf5e9af Mon Sep 17 00:00:00 2001 From: Ada Date: Sat, 10 Feb 2024 09:12:56 +0000 Subject: [PATCH 4/5] Add Globals.AllowLargeFiles and Globals.LargeFileLimit --- CUE4Parse/FileProvider/AbstractFileProvider.cs | 16 ++++++++++++++++ CUE4Parse/Globals.cs | 2 ++ CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs | 5 +++-- CUE4Parse/UE4/Pak/Objects/FPakEntry.cs | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CUE4Parse/FileProvider/AbstractFileProvider.cs b/CUE4Parse/FileProvider/AbstractFileProvider.cs index 558dc490e..f04bb6c77 100644 --- a/CUE4Parse/FileProvider/AbstractFileProvider.cs +++ b/CUE4Parse/FileProvider/AbstractFileProvider.cs @@ -549,6 +549,10 @@ public virtual async Task LoadPackageAsync(GameFile file) Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); + if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + ubulkFile = null; + if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + uptnlFile = null; var uassetTask = file.CreateReaderAsync(); var uexpTask = uexpFile?.CreateReaderAsync(); var ubulkTask = ubulkFile?.CreateReaderAsync(); @@ -589,6 +593,10 @@ public virtual async Task LoadPackageAsync(GameFile file) Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); + if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + ubulkFile = null; + if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + uptnlFile = null; var uassetTask = file.TryCreateReaderAsync().ConfigureAwait(false); var uexpTask = uexpFile?.TryCreateReaderAsync().ConfigureAwait(false); var lazyUbulk = ubulkFile != null ? new Lazy(() => ubulkFile.TryCreateReader(out var reader) ? reader : null) : null; @@ -653,6 +661,10 @@ public virtual async Task> SavePackageAsync( Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); + if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + ubulkFile = null; + if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + uptnlFile = null; var uassetTask = file.ReadAsync(); var uexpTask = uexpFile?.ReadAsync(); var ubulkTask = ubulkFile?.ReadAsync(); @@ -688,6 +700,10 @@ public virtual async Task> SavePackageAsync( Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); + if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + ubulkFile = null; + if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + uptnlFile = null; var uassetTask = file.TryReadAsync().ConfigureAwait(false); var uexpTask = uexpFile?.TryReadAsync().ConfigureAwait(false); var ubulkTask = ubulkFile?.TryReadAsync().ConfigureAwait(false); diff --git a/CUE4Parse/Globals.cs b/CUE4Parse/Globals.cs index 152e283dc..762f85abc 100644 --- a/CUE4Parse/Globals.cs +++ b/CUE4Parse/Globals.cs @@ -10,5 +10,7 @@ public static class Globals public static bool FatalObjectSerializationErrors = false; public static bool WarnMissingImportPackage = true; public static bool AlwaysUseChunkedReader = false; + public static bool AllowLargeFiles = false; + public static long LargeFileLimit = int.MaxValue; } } diff --git a/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs b/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs index 87dc2c460..98dcff6c2 100644 --- a/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs +++ b/CUE4Parse/UE4/IO/Objects/FIoStoreEntry.cs @@ -1,4 +1,5 @@ -using System.Runtime.CompilerServices; +using System; +using System.Runtime.CompilerServices; using CUE4Parse.Compression; using CUE4Parse.UE4.Readers; using CUE4Parse.UE4.VirtualFileSystem; @@ -39,6 +40,6 @@ public IoStoreReader IoStoreReader public override byte[] Read() => Vfs.Extract(this); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override FArchive CreateReader() => Size > int.MaxValue || Globals.AlwaysUseChunkedReader ? new FIoChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); + public override FArchive CreateReader() => Globals.AlwaysUseChunkedReader || Size > Math.Min(Globals.LargeFileLimit, int.MaxValue) ? new FIoChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); } } diff --git a/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs b/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs index 3493022da..041a3c509 100644 --- a/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs +++ b/CUE4Parse/UE4/Pak/Objects/FPakEntry.cs @@ -328,6 +328,6 @@ public PakFileReader PakFileReader public override byte[] Read() => Vfs.Extract(this); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override FArchive CreateReader() => Size > int.MaxValue || Globals.AlwaysUseChunkedReader ? new FPakChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); + public override FArchive CreateReader() => Globals.AlwaysUseChunkedReader || Size > Math.Min(Globals.LargeFileLimit, int.MaxValue) ? new FPakChunkArchive(Path, this, Vfs.Versions) : new FByteArchive(Path, Read(), Vfs.Versions); } } From 6dfedac4c83198d510ef03f8c64a09c7429b8a11 Mon Sep 17 00:00:00 2001 From: Ada Date: Sun, 11 Feb 2024 00:04:41 +0000 Subject: [PATCH 5/5] Add GameFile.HasValidSize --- CUE4Parse/FileProvider/AbstractFileProvider.cs | 18 +++++++++--------- CUE4Parse/FileProvider/Objects/GameFile.cs | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CUE4Parse/FileProvider/AbstractFileProvider.cs b/CUE4Parse/FileProvider/AbstractFileProvider.cs index f04bb6c77..4d2b0b817 100644 --- a/CUE4Parse/FileProvider/AbstractFileProvider.cs +++ b/CUE4Parse/FileProvider/AbstractFileProvider.cs @@ -549,9 +549,9 @@ public virtual async Task LoadPackageAsync(GameFile file) Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); - if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + if (ubulkFile?.HasValidSize() == false) ubulkFile = null; - if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + if (uptnlFile?.HasValidSize() == false) uptnlFile = null; var uassetTask = file.CreateReaderAsync(); var uexpTask = uexpFile?.CreateReaderAsync(); @@ -593,9 +593,9 @@ public virtual async Task LoadPackageAsync(GameFile file) Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); - if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) - ubulkFile = null; - if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + if (ubulkFile?.HasValidSize() == false) + ubulkFile = null; + if (uptnlFile?.HasValidSize() == false) uptnlFile = null; var uassetTask = file.TryCreateReaderAsync().ConfigureAwait(false); var uexpTask = uexpFile?.TryCreateReaderAsync().ConfigureAwait(false); @@ -661,9 +661,9 @@ public virtual async Task> SavePackageAsync( Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); - if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + if (ubulkFile?.HasValidSize() == false) ubulkFile = null; - if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + if (uptnlFile?.HasValidSize() == false) uptnlFile = null; var uassetTask = file.ReadAsync(); var uexpTask = uexpFile?.ReadAsync(); @@ -700,9 +700,9 @@ public virtual async Task> SavePackageAsync( Files.TryGetValue(file.PathWithoutExtension + ".uexp", out var uexpFile); Files.TryGetValue(file.PathWithoutExtension + ".ubulk", out var ubulkFile); Files.TryGetValue(file.PathWithoutExtension + ".uptnl", out var uptnlFile); - if (ubulkFile != null && !Globals.AllowLargeFiles && ubulkFile.Size >= Globals.LargeFileLimit) + if (ubulkFile?.HasValidSize() == false) ubulkFile = null; - if (uptnlFile != null && !Globals.AllowLargeFiles && uptnlFile.Size >= Globals.LargeFileLimit) + if (uptnlFile?.HasValidSize() == false) uptnlFile = null; var uassetTask = file.TryReadAsync().ConfigureAwait(false); var uexpTask = uexpFile?.TryReadAsync().ConfigureAwait(false); diff --git a/CUE4Parse/FileProvider/Objects/GameFile.cs b/CUE4Parse/FileProvider/Objects/GameFile.cs index c230357dc..ffa00b4bc 100644 --- a/CUE4Parse/FileProvider/Objects/GameFile.cs +++ b/CUE4Parse/FileProvider/Objects/GameFile.cs @@ -88,5 +88,8 @@ public virtual bool TryCreateReader(out FArchive reader) }).ConfigureAwait(false); public override string ToString() => Path; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool HasValidSize() => Size > 0 && (Globals.AllowLargeFiles || Size < Globals.LargeFileLimit); } }