Fix large file corruption (#245)

* test: adds Platform{Read,Write}LargeFile
* test: adds jc_test_print_value overload for uint64_t
* longtail_platform: fix >4GB file read/writes on win32
* longtail_platform: fix >4GB file read/writes on linux
* longtail_platform: set MaxChunkSize to 16 MB
* test: disable Platform{Read,Write}LargeFile by default

lib/longtail_platform.c

@@ -5,6 +5,7 @@
 
 static const uint32_t HostnamePrime = 0x01000193;
 static const uint32_t HostnameSeed  = 0x811C9DC5;
+static const uint32_t MaxChunkSize  = 16u * 1024u * 1024u;
 
 static uint32_t HostnameFNV1A(const void* data, uint32_t numBytes)
 {
@@ -1073,12 +1074,21 @@ int Longtail_Read(HLongtail_OpenFile handle, uint64_t offset, uint64_t length, v
     OVERLAPPED ReadOp;
     memset(&ReadOp, 0, sizeof(ReadOp));
 
-    ReadOp.Offset  = (DWORD)(offset & 0xffffffff);
-    ReadOp.OffsetHigh = (DWORD)(offset >> 32);
-
-    if (FALSE == ReadFile(h, output, (DWORD)length, 0, &ReadOp))
+    char* cur = (char*)output;
+    const char* end = cur + length;
+    while (cur < end)
     {
-        return Win32ErrorToErrno(GetLastError());
+        ReadOp.Offset  = (DWORD)(offset & 0xffffffff);
+        ReadOp.OffsetHigh = (DWORD)(offset >> 32);
+
+        const DWORD chunk_size = ((end - cur) > MaxChunkSize) ? (DWORD)MaxChunkSize : (DWORD)(end - cur);
+        DWORD bytes_read = 0;
+        if (FALSE == ReadFile(h, cur, chunk_size, &bytes_read, &ReadOp))
+        {
+            return Win32ErrorToErrno(GetLastError());
+        }
+        cur += bytes_read;
+        offset += bytes_read;
     }
 
     return 0;
@@ -1091,13 +1101,23 @@ int Longtail_Write(HLongtail_OpenFile handle, uint64_t offset, uint64_t length,
     OVERLAPPED WriteOp;
     memset(&WriteOp, 0, sizeof(WriteOp));
 
-    WriteOp.Offset  = (DWORD)(offset & 0xffffffff);
-    WriteOp.OffsetHigh = (DWORD)(offset >> 32);
-
-    if (FALSE == WriteFile(h, input, (DWORD)length, 0, &WriteOp))
+    const char* cur = (const char*)input;
+    const char* end = cur + length;
+    while (cur < end)
     {
-        return Win32ErrorToErrno(GetLastError());
+        WriteOp.Offset  = (DWORD)(offset & 0xffffffff);
+        WriteOp.OffsetHigh = (DWORD)(offset >> 32);
+
+        const DWORD chunk_size = ((end - cur) > MaxChunkSize) ? (DWORD)MaxChunkSize : (DWORD)(end - cur);
+        DWORD bytes_written = 0;
+        if (FALSE == WriteFile(h, cur, chunk_size, &bytes_written, &WriteOp))
+        {
+            return Win32ErrorToErrno(GetLastError());
+        }
+        cur += bytes_written;
+        offset += bytes_written;
     }
+
     return 0;
 }
 
@@ -2202,10 +2222,18 @@ int Longtail_Read(HLongtail_OpenFile handle, uint64_t offset, uint64_t length, v
 {
     FILE* f = (FILE*)handle;
     int fd = fileno(f);
-    ssize_t length_read = pread(fd, output, (off_t)length, (off_t)offset);
-    if (length_read == -1)
+    char* cur = (char*)output;
+    const char* end = cur + length;
+    while (cur < end)
     {
-        return errno;
+        const size_t chunk_size = ((end - cur) > MaxChunkSize) ? MaxChunkSize : (end - cur);
+        ssize_t length_read = pread(fd, cur, chunk_size, (off_t)offset);
+        if (length_read == -1)
+        {
+            return errno;
+        }
+        cur += length_read;
+        offset += length_read;
     }
     return 0;
 }
@@ -2215,10 +2243,18 @@ int Longtail_Write(HLongtail_OpenFile handle, uint64_t offset, uint64_t length,
     FILE* f = (FILE*)handle;
 
     int fd = fileno(f);
-    ssize_t length_written = pwrite(fd, input, length, offset);
-    if (length_written == -1)
+    const char* cur = (const char*)input;
+    const char* end = cur + length;
+    while (cur < end)
     {
-        return errno;
+        const size_t chunk_size = ((end - cur) > MaxChunkSize) ? MaxChunkSize : (end - cur);
+        ssize_t length_written = pwrite(fd, cur, chunk_size, offset);
+        if (length_written == -1)
+        {
+            return errno;
+        }
+        cur += length_written;
+        offset += length_written;
     }
     return 0;
 }

test/test.cpp

@@ -37,6 +37,13 @@
 #define TEST_LOG(fmt, ...) \
     fprintf(stderr, "--- ");fprintf(stderr, fmt, __VA_ARGS__);
 
+
+template<>
+char* jc_test_print_value(char* buffer, size_t buffer_len, uint64_t value)
+{
+    return buffer + JC_TEST_SNPRINTF(buffer, buffer_len, "%" PRId64, value);
+}
+
 static int CreateParentPath(struct Longtail_StorageAPI* storage_api, const char* path)
 {
     char* dir_path = Longtail_Strdup(path);
@@ -8194,4 +8201,65 @@ TEST(Longtail, Longtail_CaseSensitivePaths)
     SAFE_DISPOSE_API(compression_registry);
     SAFE_DISPOSE_API(target_storage);
     SAFE_DISPOSE_API(source_storage);
-}
+}
+
+#if 0
+
+TEST(Longtail, PlatformWriteLargeFile)
+{
+    static const uint64_t expected_size = 4ULL * 1024ULL * 1024ULL * 1024ULL;
+
+    HLongtail_OpenFile large_file;
+    ASSERT_EQ(0, Longtail_OpenWriteFile("testdata/write_large_file.bin", 0, &large_file));
+    ASSERT_NE((HLongtail_OpenFile)0, large_file);
+
+    uint8_t* buf = (uint8_t*)malloc(expected_size);
+    ASSERT_NE(nullptr, buf);
+    ASSERT_EQ(0, Longtail_Write(large_file, 0, expected_size, buf));
+    free(buf);
+
+    uint64_t actual_size;
+    ASSERT_EQ(0, Longtail_GetFileSize(large_file, &actual_size));
+    ASSERT_EQ(actual_size, expected_size);
+
+    Longtail_CloseFile(large_file);
+}
+
+TEST(Longtail, PlatformReadLargeFile)
+{
+    static const uint64_t expected_size = 4ULL * 1024ULL * 1024ULL * 1024ULL;
+
+    uint8_t magic[256];
+    GenerateRandomData(magic, sizeof(magic));
+
+    // Generate large file
+    {
+        HLongtail_OpenFile large_file;
+        ASSERT_EQ(0, Longtail_OpenWriteFile("testdata/read_large_file.bin", expected_size, &large_file));
+        ASSERT_NE((HLongtail_OpenFile)0, large_file);
+
+        uint64_t actual_size;
+        ASSERT_EQ(0, Longtail_GetFileSize(large_file, &actual_size));
+        ASSERT_EQ(actual_size, expected_size);
+
+        ASSERT_EQ(0, Longtail_Write(large_file, expected_size - sizeof(magic), sizeof(magic), magic));
+
+        Longtail_CloseFile(large_file);
+    }
+
+    // Read large file
+    HLongtail_OpenFile large_file;
+    ASSERT_EQ(0, Longtail_OpenReadFile("testdata/read_large_file.bin", &large_file));
+    ASSERT_NE((HLongtail_OpenFile)0, large_file);
+
+    uint8_t* buf = (uint8_t*)malloc(expected_size);
+    ASSERT_NE(nullptr, buf);
+    ASSERT_EQ(0, Longtail_Read(large_file, 0, expected_size, buf));
+    ASSERT_EQ(0, memcmp(buf + expected_size - sizeof(magic), magic, sizeof(magic)));
+    free(buf);
+
+    Longtail_CloseFile(large_file);
+}
+
+#endif
+