add manual flag to mmap dictionary

facebook · Feb 14, 2023 · 2d8afd9 · Cyan4973 · Apr 4, 2023 · 2d8afd9
1 parent 8a189b1
commit 2d8afd9
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 5 deletions.
diff --git a/programs/fileio.c b/programs/fileio.c
@@ -485,6 +485,11 @@ void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
     prefs->passThrough = (value != 0);
 }
 
+void FIO_setMMapDict(FIO_prefs_t* const prefs, int value)
+{
+    prefs->mmapDict = value;
+}
+
 /* FIO_ctx_t functions */
 
 void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@@ -1028,7 +1033,7 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
 static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
                                     const char* dictFileName, unsigned long long const maxSrcFileSize,
                                     int cLevel, ZSTD_compressionParameters comprParams) {
-    int mmapDict = 0;
+    int mmapDict = prefs->mmapDict;
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
@@ -1045,7 +1050,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
     if (prefs->patchFromMode) {
         U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
         unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
-        mmapDict = dictSize > prefs->memLimit;
+        mmapDict |= dictSize > prefs->memLimit;
         FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
     }
 
@@ -2136,7 +2141,7 @@ typedef struct {
 
 static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
 {
-    int mmapDict = 0;
+    int mmapDict = prefs->mmapDict;
     stat_t statbuf;
     dRess_t ress;
     memset(&ress, 0, sizeof(ress));
@@ -2145,7 +2150,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
 
     if (prefs->patchFromMode){
         U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
-        mmapDict = dictSize > prefs->memLimit;
+        mmapDict |= dictSize > prefs->memLimit;
         FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
     }
 

diff --git a/programs/fileio.h b/programs/fileio.h
@@ -106,6 +106,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
 void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
 void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
 void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
+void FIO_setMMapDict(FIO_prefs_t* const prefs, int value);
 
 /* FIO_ctx_t functions */
 void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

diff --git a/programs/fileio_types.h b/programs/fileio_types.h
@@ -69,6 +69,7 @@ typedef struct FIO_prefs_s {
     int contentSize;
     int allowBlockDevices;
     int passThrough;
+    int mmapDict;
 } FIO_prefs_t;
 
 #endif /* FILEIO_TYPES_HEADER */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
@@ -254,6 +254,7 @@ static void usage_advanced(const char* programName)
 
     DISPLAYOUT("\n");
     DISPLAYOUT("  --format=zstd                 Compress files to the `.zst` format. [Default]\n");
+    DISPLAYOUT("  --mmap-dict                   Memory-map dictionary file rather than mallocing and loading all at once");
 #ifdef ZSTD_GZCOMPRESS
     DISPLAYOUT("  --format=gzip                 Compress files to the `.gz` format.\n");
 #endif
@@ -850,7 +851,8 @@ int main(int argCount, const char* argv[])
         showDefaultCParams = 0,
         ultra=0,
         contentSize=1,
-        removeSrcFile=0;
+        removeSrcFile=0,
+        mmapDict=0;
     ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
     FIO_compressionType_t cType = FIO_zstdCompression;
     unsigned nbWorkers = 0;
@@ -984,6 +986,7 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
                 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
                 if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
+                if (!strcmp(argument, "--mmap-dict")) { mmapDict = 1; continue; }
 #ifdef ZSTD_GZCOMPRESS
                 if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
                 if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
@@ -1526,6 +1529,7 @@ int main(int argCount, const char* argv[])
     FIO_setNotificationLevel(g_displayLevel);
     FIO_setAllowBlockDevices(prefs, allowBlockDevices);
     FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
+    FIO_setMMapDict(prefs, mmapDict);
     if (memLimit == 0) {
         if (compressionParams.windowLog == 0) {
             memLimit = (U32)1 << g_defaultMaxWindowLog;