Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mmap for windows #3557

Merged
merged 6 commits into from
Mar 28, 2023
Merged

mmap for windows #3557

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
48766b2
mmap for windows
daniellerozenblit Mar 17, 2023
3966167
remove enabling mmap for testing
daniellerozenblit Mar 17, 2023
ffc7bcf
rename FIO dictionary initialization methods + un-const dictionary ob…
daniellerozenblit Mar 27, 2023
104f7a5
remove enabling mmap for testing
daniellerozenblit Mar 27, 2023
5e79902
initDict returns void, underlying setDictBuffer methods return the si…
daniellerozenblit Mar 28, 2023
854f25f
fix comment
daniellerozenblit Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 89 additions & 35 deletions programs/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,16 +687,17 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
}
}

/* FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`,
/* FIO_setDictBufferMalloc() :
* allocates a buffer, pointed by `dict->dictBuffer`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;

assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
Expand Down Expand Up @@ -733,20 +734,17 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p

#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static void* FIO_mmap(size_t fileSize, int fileHandle)
static void FIO_munmap(FIO_Dict_t* dict)
{
return mmap
(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
munmap(dict->dictBuffer, dict->dictBufferSize);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static int FIO_munmap(void* buffer, size_t bufferSize)
{
return munmap(buffer, bufferSize);
}
/* We might want to also do mapping for windows */
static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
int fileHandle;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;

assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
Expand All @@ -770,27 +768,90 @@ static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, F
}
}

*bufferPtr = FIO_mmap((size_t)fileSize, fileHandle);
*bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
daniellerozenblit marked this conversation as resolved.
Show resolved Hide resolved
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));

close(fileHandle);
return (size_t)fileSize;
}
#elif defined(_MSC_VER) || defined(_WIN32)
#include <windows.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
UnmapViewOfFile(dict->dictBuffer);
CloseHandle(dict->dictHandle);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
HANDLE fileHandle, mapping;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;

assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;

DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);

fileHandle = CreateFileA(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);

if (fileHandle == INVALID_HANDLE_VALUE) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}

fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
}

mapping = CreateFileMapping(fileHandle, NULL, PAGE_READONLY, 0, 0, NULL);
if (mapping == NULL) {
EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno));
}

*bufferPtr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, (DWORD)fileSize); /* we can only cast to DWORD here because dictSize <= 2GB */
if (*bufferPtr==NULL) EXM_THROW(36, "%s", strerror(errno));

dict->dictHandle = fileHandle;
return (size_t)fileSize;
}
#else
static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
return FIO_createDictBuffer(bufferPtr, fileName, prefs, dictFileStat);
return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
}
static void FIO_munmap(void* buffer, size_t bufferSize) {
(void)bufferSize;
free(buffer);
static void FIO_munmap(FIO_Dict_t* dict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
#endif

static void FIO_freeDict(const FIO_Dict_t* dict) {
static void FIO_freeDict(FIO_Dict_t* dict) {
if (dict->dictBufferType == FIO_mallocDict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
} else if (dict->dictBufferType == FIO_mmapDict) {
FIO_munmap(dict->dictBuffer, dict->dictBufferSize);
FIO_munmap(dict);
} else {
assert(0); /* Should not reach this case */
}
}

static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
dict->dictBufferType = dictBufferType;
if (dict->dictBufferType == FIO_mallocDict) {
dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
} else if (dict->dictBufferType == FIO_mmapDict) {
dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat);
} else {
assert(0); /* Should not reach this case */
}
Expand Down Expand Up @@ -1045,6 +1106,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
int cLevel, ZSTD_compressionParameters comprParams) {
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
FIO_dictBufferType_t dictBufferType;
cRess_t ress;
memset(&ress, 0, sizeof(ress));

Expand All @@ -1065,13 +1127,8 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}

ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;

if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */
} else {
ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat);
}
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */

ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
Expand Down Expand Up @@ -1135,7 +1192,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
return ress;
}

static void FIO_freeCResources(const cRess_t* const ress)
static void FIO_freeCResources(cRess_t* const ress)
{
FIO_freeDict(&(ress->dict));
AIO_WritePool_free(ress->writeCtx);
Expand Down Expand Up @@ -1976,7 +2033,7 @@ int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);

#define DISPLAY_LEVEL_DEFAULT 2
Expand Down Expand Up @@ -2169,19 +2226,16 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
}

/* Allocation */
ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));

/* dictionary */
{ if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf);
} else {
ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf);
}
{
FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);

CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );

Expand Down
3 changes: 3 additions & 0 deletions programs/fileio_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ typedef struct {
void* dictBuffer;
size_t dictBufferSize;
FIO_dictBufferType_t dictBufferType;
#if defined(_MSC_VER) || defined(_WIN32)
HANDLE dictHandle;
#endif
} FIO_Dict_t;

#endif /* FILEIO_TYPES_HEADER */