Skip to content

Commit

Permalink
rename FIO dictionary initialization methods + un-const dictionary ob…
Browse files Browse the repository at this point in the history 
…jects in free functions
  • Loading branch information
@daniellerozenblit
daniellerozenblit committed Mar 27, 2023
1 parent 3966167 commit 7467668
Showing 1 changed file with 43 additions and 28 deletions.
71 changes: 43 additions & 28 deletions programs/fileio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -687,13 +687,13 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
}
}

/* FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`,
/* FIO_initDictMalloc() :
* allocates a buffer, pointed by `dict->dictBuffer`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBuffer(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_initDictMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
Expand Down Expand Up @@ -734,11 +734,13 @@ static size_t FIO_createDictBuffer(FIO_Dict_t* dict, const char* fileName, FIO_p

#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static int FIO_munmap(const FIO_Dict_t* dict)
static void FIO_munmap(FIO_Dict_t* dict)
{
return munmap(dict->dictBuffer, dict->dictBufferSize);
munmap(dict->dictBuffer, dict->dictBufferSize);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static size_t FIO_createDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_initDictMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
int fileHandle;
U64 fileSize;
Expand Down Expand Up @@ -773,12 +775,14 @@ static size_t FIO_createDictBufferMMap(FIO_Dict_t* dict, const char* fileName, F
}
#elif defined(_MSC_VER) || defined(_WIN32)
#include <windows.h>
static int FIO_munmap(const FIO_Dict_t* dict)
static void FIO_munmap(FIO_Dict_t* dict)
{
UnmapViewOfFile(dict->dictBuffer);
return CloseHandle(dict->dictHandle);
CloseHandle(dict->dictHandle);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static size_t FIO_createDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_initDictMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
HANDLE fileHandle, mapping;
U64 fileSize;
Expand Down Expand Up @@ -819,25 +823,41 @@ static size_t FIO_createDictBufferMMap(FIO_Dict_t* dict, const char* fileName, F
return (size_t)fileSize;
}
#else
static size_t FIO_createDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
static size_t FIO_initDictMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
return FIO_createDictBuffer(dict, fileName, prefs, dictFileStat);
return FIO_initDictMalloc(dict, fileName, prefs, dictFileStat);
}
static void FIO_munmap(const FIO_Dict_t* dict) {
static void FIO_munmap(FIO_Dict_t* dict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
#endif

static void FIO_freeDict(const FIO_Dict_t* dict) {
static void FIO_freeDict(FIO_Dict_t* dict) {
if (dict->dictBufferType == FIO_mallocDict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
} else if (dict->dictBufferType == FIO_mmapDict) {
FIO_munmap(dict);
} else {
assert(0); /* Should not reach this case */
}
}

static size_t FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
dict->dictBufferType = dictBufferType;
if (dict->dictBufferType == FIO_mallocDict) {
return FIO_initDictMalloc(dict, fileName, prefs, dictFileStat);
} else if (dict->dictBufferType == FIO_mmapDict) {
return FIO_initDictMMap(dict, fileName, prefs, dictFileStat);
} else {
assert(0); /* Should not reach this case */
return 0;
}
}


/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
Expand Down Expand Up @@ -1086,7 +1106,9 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
useMMap = 1;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
FIO_dictBufferType_t dictBufferType;
cRess_t ress;
memset(&ress, 0, sizeof(ress));

Expand All @@ -1107,13 +1129,8 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}

ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;

if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */
} else {
ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict, dictFileName, prefs, &ress.dictFileStat);
}
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
ress.dict.dictBufferSize = FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */

ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
Expand Down Expand Up @@ -1177,7 +1194,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
return ress;
}

static void FIO_freeCResources(const cRess_t* const ress)
static void FIO_freeCResources(cRess_t* const ress)
{
FIO_freeDict(&(ress->dict));
AIO_WritePool_free(ress->writeCtx);
Expand Down Expand Up @@ -2018,7 +2035,7 @@ int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);

#define DISPLAY_LEVEL_DEFAULT 2
Expand Down Expand Up @@ -2197,6 +2214,7 @@ typedef struct {
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
useMMap = 1;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
stat_t statbuf;
dRess_t ress;
Expand All @@ -2211,19 +2229,16 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
}

/* Allocation */
ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));

/* dictionary */
{ if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict, dictFileName, prefs, &statbuf);
} else {
ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict, dictFileName, prefs, &statbuf);
}
{
FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
ress.dict.dictBufferSize = FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);

CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );

Expand Down

0 comments on commit 7467668

Please sign in to comment.