Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify benchmark unit invocation API from CLI #3526

Merged
merged 3 commits into from
Mar 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 31 additions & 18 deletions programs/benchzstd.c
Original file line number Diff line number Diff line change
Expand Up @@ -697,22 +697,26 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
displayLevel, displayName, adv);
}

BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
{
char name[20] = {0};
size_t const benchedSize = 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;

if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 15;
}

/* Memory allocation */
srcBuffer = malloc(benchedSize);
if (!srcBuffer) RETURN_ERROR(21, BMK_benchOutcome_t, "not enough memory");
if (!srcBuffer) {
DISPLAYLEVEL(1, "allocation error : not enough memory");
return 16;
}

/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
Expand All @@ -728,7 +732,7 @@ BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
/* clean up */
free(srcBuffer);

return res;
return !BMK_isSuccessful_benchOutcome(res);
}


Expand Down Expand Up @@ -790,7 +794,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
return 0;
}

BMK_benchOutcome_t BMK_benchFilesAdvanced(
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel,
const ZSTD_compressionParameters* compressionParams,
Expand All @@ -805,38 +809,47 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);

if (!nbFiles) {
RETURN_ERROR(14, BMK_benchOutcome_t, "No Files to Benchmark");
DISPLAYLEVEL(1, "No Files to Benchmark");
return 13;
}

if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 14;
}

if (totalSizeToLoad == UTIL_FILESIZE_UNKNOWN) {
RETURN_ERROR(9, BMK_benchOutcome_t, "Error loading files");
DISPLAYLEVEL(1, "Error loading files");
return 15;
}

fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
if (!fileSizes) RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory for fileSizes");
if (!fileSizes) {
DISPLAYLEVEL(1, "not enough memory for fileSizes");
return 16;
}

/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
free(fileSizes);
RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
DISPLAYLEVEL(1, "benchmark aborted");
return 17;
}
if (dictFileSize > 64 MB) {
free(fileSizes);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
DISPLAYLEVEL(1, "dictionary file %s too large", dictFileName);
return 18;
}
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) {
free(fileSizes);
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
DISPLAYLEVEL(1, "not enough memory for dictionary (%u bytes)",
(unsigned)dictBufferSize);
return 19;
}

{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
Expand All @@ -858,7 +871,8 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (!srcBuffer) {
free(dictBuffer);
free(fileSizes);
RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory");
DISPLAYLEVEL(1, "not enough memory for srcBuffer");
return 20;
}

/* Load input buffer */
Expand Down Expand Up @@ -886,12 +900,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
return res;
return !BMK_isSuccessful_benchOutcome(res);
}


BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
int BMK_benchFiles(const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel)
Expand Down
51 changes: 17 additions & 34 deletions programs/benchzstd.h
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,13 @@ BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
* 2 : + result + interaction + warnings;
* 3 : + information;
* 4 : + debug
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
int BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);


typedef enum {
Expand Down Expand Up @@ -126,11 +118,11 @@ BMK_advancedParams_t BMK_initAdvancedParams(void);
/*! BMK_benchFilesAdvanced():
* Same as BMK_benchFiles(),
* with more controls, provided through advancedParams_t structure */
BMK_benchOutcome_t BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);

/*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using compressibility argument */
Expand All @@ -139,20 +131,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);



Expand Down Expand Up @@ -190,8 +173,8 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
int displayLevel, const char* displayName);


/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
/* BMK_benchMemAdvanced() : used by Paramgrill
* same as BMK_benchMem() with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
* adv = see advancedParams_t
Expand Down
11 changes: 4 additions & 7 deletions programs/zstdcli.c
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "benchzstd.h" /* BMK_benchFiles */
# include "benchzstd.h" /* BMK_benchFilesAdvanced */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
Expand Down Expand Up @@ -1391,18 +1391,15 @@ int main(int argCount, const char* argv[])
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
} }

#else
Expand Down
12 changes: 6 additions & 6 deletions zlibWrapper/examples/zwrapbench.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"

#include "zstd_zlibwrapper.h"
#include "../zstd_zlibwrapper.h"



Expand Down Expand Up @@ -109,17 +109,17 @@ static unsigned g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;

void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }

void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
static void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }

void BMK_SetNbIterations(unsigned nbLoops)
static void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
}

void BMK_SetBlockSize(size_t blockSize)
static void BMK_SetBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10));
Expand Down Expand Up @@ -798,7 +798,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}


int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
static int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
{
double const compressibility = (double)g_compressibilityDefault / 100;
Expand Down