Skip to content

Commit

Permalink
Merge pull request #3526 from facebook/bench_zstd_api
Browse files Browse the repository at this point in the history 
Simplify benchmark unit invocation API from CLI
  • Loading branch information
@Cyan4973
Cyan4973 authored Mar 9, 2023
2 parents 6bedef8 + 1e38e07 commit e769da1
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 65 deletions.
49 changes: 31 additions & 18 deletions programs/benchzstd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,22 +697,26 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
displayLevel, displayName, adv);
}

BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
{
char name[20] = {0};
size_t const benchedSize = 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;

if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 15;
}

/* Memory allocation */
srcBuffer = malloc(benchedSize);
if (!srcBuffer) RETURN_ERROR(21, BMK_benchOutcome_t, "not enough memory");
if (!srcBuffer) {
DISPLAYLEVEL(1, "allocation error : not enough memory");
return 16;
}

/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
Expand All @@ -728,7 +732,7 @@ BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
/* clean up */
free(srcBuffer);

return res;
return !BMK_isSuccessful_benchOutcome(res);
}


Expand Down Expand Up @@ -790,7 +794,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
return 0;
}

BMK_benchOutcome_t BMK_benchFilesAdvanced(
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel,
const ZSTD_compressionParameters* compressionParams,
Expand All @@ -805,38 +809,47 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);

if (!nbFiles) {
RETURN_ERROR(14, BMK_benchOutcome_t, "No Files to Benchmark");
DISPLAYLEVEL(1, "No Files to Benchmark");
return 13;
}

if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 14;
}

if (totalSizeToLoad == UTIL_FILESIZE_UNKNOWN) {
RETURN_ERROR(9, BMK_benchOutcome_t, "Error loading files");
DISPLAYLEVEL(1, "Error loading files");
return 15;
}

fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
if (!fileSizes) RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory for fileSizes");
if (!fileSizes) {
DISPLAYLEVEL(1, "not enough memory for fileSizes");
return 16;
}

/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
free(fileSizes);
RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
DISPLAYLEVEL(1, "benchmark aborted");
return 17;
}
if (dictFileSize > 64 MB) {
free(fileSizes);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
DISPLAYLEVEL(1, "dictionary file %s too large", dictFileName);
return 18;
}
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) {
free(fileSizes);
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
DISPLAYLEVEL(1, "not enough memory for dictionary (%u bytes)",
(unsigned)dictBufferSize);
return 19;
}

{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
Expand All @@ -858,7 +871,8 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (!srcBuffer) {
free(dictBuffer);
free(fileSizes);
RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory");
DISPLAYLEVEL(1, "not enough memory for srcBuffer");
return 20;
}

/* Load input buffer */
Expand Down Expand Up @@ -886,12 +900,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
return res;
return !BMK_isSuccessful_benchOutcome(res);
}


BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
int BMK_benchFiles(const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel)
Expand Down
51 changes: 17 additions & 34 deletions programs/benchzstd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,13 @@ BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
* 2 : + result + interaction + warnings;
* 3 : + information;
* 4 : + debug
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
int BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);


typedef enum {
Expand Down Expand Up @@ -126,11 +118,11 @@ BMK_advancedParams_t BMK_initAdvancedParams(void);
/*! BMK_benchFilesAdvanced():
* Same as BMK_benchFiles(),
* with more controls, provided through advancedParams_t structure */
BMK_benchOutcome_t BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);

/*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using compressibility argument */
Expand All @@ -139,20 +131,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);



Expand Down Expand Up @@ -190,8 +173,8 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
int displayLevel, const char* displayName);


/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
/* BMK_benchMemAdvanced() : used by Paramgrill
* same as BMK_benchMem() with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
* adv = see advancedParams_t
Expand Down
11 changes: 4 additions & 7 deletions programs/zstdcli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "benchzstd.h" /* BMK_benchFiles */
# include "benchzstd.h" /* BMK_benchFilesAdvanced */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
Expand Down Expand Up @@ -1395,18 +1395,15 @@ int main(int argCount, const char* argv[])
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
} }

#else
Expand Down
12 changes: 6 additions & 6 deletions zlibWrapper/examples/zwrapbench.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"

#include "zstd_zlibwrapper.h"
#include "../zstd_zlibwrapper.h"



Expand Down Expand Up @@ -109,17 +109,17 @@ static unsigned g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;

void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }

void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
static void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }

void BMK_SetNbIterations(unsigned nbLoops)
static void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
}

void BMK_SetBlockSize(size_t blockSize)
static void BMK_SetBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10));
Expand Down Expand Up @@ -798,7 +798,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}


int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
static int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
{
double const compressibility = (double)g_compressibilityDefault / 100;
Expand Down

0 comments on commit e769da1

Please sign in to comment.