reference dictionaries rather than copying

facebook · Feb 10, 2023 · bc0ead9 · bc0ead9
1 parent 18147e6
commit bc0ead9
Showing 1 changed file with 19 additions and 16 deletions.
diff --git a/programs/fileio.c b/programs/fileio.c
@@ -751,7 +751,6 @@ static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, F
     }
 
     fileSize = UTIL_getFileSizeStat(dictFileStat);
-
     {
         size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
         if (fileSize >  dictSizeMax) {
@@ -1044,7 +1043,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
     if (prefs->patchFromMode) {
         U64 const dictSize = UTIL_getFileSize(dictFileName);
         unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
-        mmapDict = prefs->patchFromMode && dictSize > prefs->memLimit;
+        mmapDict = dictSize > prefs->memLimit;
         FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
     }
 
@@ -1113,7 +1112,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
     if (prefs->patchFromMode) {
         CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
     } else {
-        CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
+        CHECK( ZSTD_CCtx_loadDictionary_byReference(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
     }
 
     return ress;
@@ -2126,9 +2125,12 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
  *  Decompression
  ***************************************************************************/
 typedef struct {
+    void* dictBuffer;
+    size_t dictBufferSize;
     ZSTD_DStream* dctx;
     WritePoolCtx_t *writeCtx;
     ReadPoolCtx_t *readCtx;
+    int mmapDict;
 } dRess_t;
 
 static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
@@ -2139,37 +2141,33 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
 
     if (prefs->patchFromMode){
         U64 const dictSize = UTIL_getFileSize(dictFileName);
-        mmapDict = prefs->patchFromMode && dictSize > prefs->memLimit;
+        mmapDict = dictSize > prefs->memLimit;
         FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
     }
 
     mmapDict = 1;
-
     /* Allocation */
+    ress.mmapDict = mmapDict;
     ress.dctx = ZSTD_createDStream();
     if (ress.dctx==NULL)
         EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
     CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
     CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
 
     /* dictionary */
-    {   void* dictBuffer;
-        stat_t statbuf;
-        size_t dictBufferSize;
-
+    {   stat_t statbuf;
         if (!mmapDict) {
-            dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs, &statbuf);
+            ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &statbuf);
         } else {
-            dictBufferSize = FIO_createDictBufferMMap(&dictBuffer, dictFileName, prefs, &statbuf);
+            ress.dictBufferSize = FIO_createDictBufferMMap(&ress.dictBuffer, dictFileName, prefs, &statbuf);
         }
 
-        CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
-        CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
+        CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
 
-        if (!mmapDict) {
-            free(dictBuffer);
+        if (prefs->patchFromMode){
+            CHECK(ZSTD_DCtx_refPrefix(ress.dctx, ress.dictBuffer, ress.dictBufferSize));
         } else {
-            FIO_freeDictBufferMMap(dictBuffer, dictBufferSize);
+            CHECK(ZSTD_DCtx_loadDictionary_byReference(ress.dctx, ress.dictBuffer, ress.dictBufferSize));
         }
     }
 
@@ -2181,6 +2179,11 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
 
 static void FIO_freeDResources(dRess_t ress)
 {
+    if (!ress.mmapDict) {
+        free(ress.dictBuffer);
+    } else {
+        FIO_freeDictBufferMMap(ress.dictBuffer, ress.dictBufferSize);
+    }
     CHECK( ZSTD_freeDStream(ress.dctx) );
     AIO_WritePool_free(ress.writeCtx);
     AIO_ReadPool_free(ress.readCtx);