Skip to content

Commit

Permalink
misc(fuzzer): Log fuzzer stats to stderr
Browse files Browse the repository at this point in the history 
Summary:
Fuzzer jobs in GitHub Actions currently only print logs to stderr to screen and save logs to 
stdout to the artifact. Because the fuzzer stats provide useful signals about the test 
coverage, we'd like to print them on the screen so that developers do not have to download 
artifacts to see them.

Differential Revision: D69624523
  • Loading branch information
@kagamiori @facebook-github-bot
kagamiori authored and facebook-github-bot committed Feb 13, 2025
1 parent 0d0a815 commit c19aaef
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 57 deletions.
24 changes: 12 additions & 12 deletions velox/exec/fuzzer/AggregationFuzzer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1037,18 +1037,18 @@ void AggregationFuzzer::verifyAggregation(
}

void AggregationFuzzer::Stats::print(size_t numIterations) const {
LOG(INFO) << "Total masked aggregations: "
<< printPercentageStat(numMask, numIterations);
LOG(INFO) << "Total global aggregations: "
<< printPercentageStat(numGlobal, numIterations);
LOG(INFO) << "Total group-by aggregations: "
<< printPercentageStat(numGroupBy, numIterations);
LOG(INFO) << "Total distinct aggregations: "
<< printPercentageStat(numDistinct, numIterations);
LOG(INFO) << "Total aggregations over distinct inputs: "
<< printPercentageStat(numDistinctInputs, numIterations);
LOG(INFO) << "Total window expressions: "
<< printPercentageStat(numWindow, numIterations);
LOG(ERROR) << "Total masked aggregations: "
<< printPercentageStat(numMask, numIterations);
LOG(ERROR) << "Total global aggregations: "
<< printPercentageStat(numGlobal, numIterations);
LOG(ERROR) << "Total group-by aggregations: "
<< printPercentageStat(numGroupBy, numIterations);
LOG(ERROR) << "Total distinct aggregations: "
<< printPercentageStat(numDistinct, numIterations);
LOG(ERROR) << "Total aggregations over distinct inputs: "
<< printPercentageStat(numDistinctInputs, numIterations);
LOG(ERROR) << "Total window expressions: "
<< printPercentageStat(numWindow, numIterations);
AggregationFuzzerBase::Stats::print(numIterations);
}

Expand Down
42 changes: 21 additions & 21 deletions velox/exec/fuzzer/AggregationFuzzerBase.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -670,47 +670,47 @@ void AggregationFuzzerBase::Stats::updateReferenceQueryStats(
}

void AggregationFuzzerBase::Stats::print(size_t numIterations) const {
LOG(INFO) << "Total functions tested: " << functionNames.size();
LOG(INFO) << "Total iterations requiring sorted inputs: "
<< printPercentageStat(numSortedInputs, numIterations);
LOG(INFO) << "Total iterations verified against reference DB: "
<< printPercentageStat(numVerified, numIterations);
LOG(INFO)
LOG(ERROR) << "Total functions tested: " << functionNames.size();
LOG(ERROR) << "Total iterations requiring sorted inputs: "
<< printPercentageStat(numSortedInputs, numIterations);
LOG(ERROR) << "Total iterations verified against reference DB: "
<< printPercentageStat(numVerified, numIterations);
LOG(ERROR)
<< "Total functions not verified (verification skipped / not supported by reference DB / reference DB failed): "
<< printPercentageStat(numVerificationSkipped, numIterations) << " / "
<< printPercentageStat(numReferenceQueryNotSupported, numIterations)
<< " / " << printPercentageStat(numReferenceQueryFailed, numIterations);
LOG(INFO) << "Total failed functions: "
<< printPercentageStat(numFailed, numIterations);
LOG(ERROR) << "Total failed functions: "
<< printPercentageStat(numFailed, numIterations);
}

std::string printPercentageStat(size_t n, size_t total) {
return fmt::format("{} ({:.2f}%)", n, (double)n / total * 100);
}

void printStats(const AggregationFuzzerBase::FunctionsStats& stats) {
LOG(INFO) << fmt::format(
LOG(ERROR) << fmt::format(
"Total functions: {} ({} signatures)",
stats.numFunctions,
stats.numSignatures);
LOG(INFO) << "Functions with at least one supported signature: "
<< printPercentageStat(
stats.numSupportedFunctions, stats.numFunctions);
LOG(ERROR) << "Functions with at least one supported signature: "
<< printPercentageStat(
stats.numSupportedFunctions, stats.numFunctions);

size_t numNotSupportedFunctions =
stats.numFunctions - stats.numSupportedFunctions;
LOG(INFO) << "Functions with no supported signature: "
<< printPercentageStat(
numNotSupportedFunctions, stats.numFunctions);
LOG(INFO) << "Supported function signatures: "
<< printPercentageStat(
stats.numSupportedSignatures, stats.numSignatures);
LOG(ERROR) << "Functions with no supported signature: "
<< printPercentageStat(
numNotSupportedFunctions, stats.numFunctions);
LOG(ERROR) << "Supported function signatures: "
<< printPercentageStat(
stats.numSupportedSignatures, stats.numSignatures);

size_t numNotSupportedSignatures =
stats.numSignatures - stats.numSupportedSignatures;
LOG(INFO) << "Unsupported function signatures: "
<< printPercentageStat(
numNotSupportedSignatures, stats.numSignatures);
LOG(ERROR) << "Unsupported function signatures: "
<< printPercentageStat(
numNotSupportedSignatures, stats.numSignatures);
}

std::string makeFunctionCall(
Expand Down
4 changes: 2 additions & 2 deletions velox/exec/fuzzer/WindowFuzzer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -891,8 +891,8 @@ void windowFuzzer(

void WindowFuzzer::Stats::print(size_t numIterations) const {
AggregationFuzzerBase::Stats::print(numIterations);
LOG(INFO) << "Total functions verified in reference DB: "
<< verifiedFunctionNames.size();
LOG(ERROR) << "Total functions verified in reference DB: "
<< verifiedFunctionNames.size();
}

} // namespace facebook::velox::exec::test
45 changes: 23 additions & 22 deletions velox/expression/fuzzer/ExpressionFuzzerVerifier.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,48 +215,49 @@ void ExpressionFuzzerVerifier::logStats() {
return left.second.numProcessedRows > right.second.numProcessedRows;
});
int maxEntriesLimit = std::min<size_t>(10, entries.size());
LOG(INFO) << "==============================> Top " << maxEntriesLimit
<< " by number of rows processed";
LOG(INFO)
LOG(ERROR) << "==============================> Top " << maxEntriesLimit
<< " by number of rows processed";
LOG(ERROR)
<< "Format: functionName numTimesSelected proportionOfTimesSelected "
"numProcessedRows";
for (int i = 0; i < maxEntriesLimit; i++) {
LOG(INFO) << entries[i].first << " " << entries[i].second.numTimesSelected
<< " " << std::fixed << std::setprecision(2)
<< (entries[i].second.numTimesSelected * 100.00) / totalSelections
<< "% " << entries[i].second.numProcessedRows;
LOG(ERROR) << entries[i].first << " " << entries[i].second.numTimesSelected
<< " " << std::fixed << std::setprecision(2)
<< (entries[i].second.numTimesSelected * 100.00) /
totalSelections
<< "% " << entries[i].second.numProcessedRows;
}

LOG(INFO) << "==============================> Bottom " << maxEntriesLimit
<< " by number of rows processed";
LOG(INFO)
LOG(ERROR) << "==============================> Bottom " << maxEntriesLimit
<< " by number of rows processed";
LOG(ERROR)
<< "Format: functionName numTimesSelected proportionOfTimesSelected "
"numProcessedRows";
for (int i = 0; i < maxEntriesLimit; i++) {
int idx = entries.size() - 1 - i;
LOG(INFO) << entries[idx].first << " "
<< entries[idx].second.numTimesSelected << " " << std::fixed
<< std::setprecision(2)
<< (entries[idx].second.numTimesSelected * 100.00) /
LOG(ERROR) << entries[idx].first << " "
<< entries[idx].second.numTimesSelected << " " << std::fixed
<< std::setprecision(2)
<< (entries[idx].second.numTimesSelected * 100.00) /
totalSelections
<< "% " << entries[idx].second.numProcessedRows;
<< "% " << entries[idx].second.numProcessedRows;
}

// sort by numTimesSelected
std::sort(entries.begin(), entries.end(), [](auto& left, auto& right) {
return left.second.numTimesSelected > right.second.numTimesSelected;
});

LOG(INFO) << "==============================> All stats sorted by number "
"of times the function was chosen";
LOG(INFO)
LOG(ERROR) << "==============================> All stats sorted by number "
"of times the function was chosen";
LOG(ERROR)
<< "Format: functionName numTimesSelected proportionOfTimesSelected "
"numProcessedRows";
for (auto& elem : entries) {
LOG(INFO) << elem.first << " " << elem.second.numTimesSelected << " "
<< std::fixed << std::setprecision(2)
<< (elem.second.numTimesSelected * 100.00) / totalSelections
<< "% " << elem.second.numProcessedRows;
LOG(ERROR) << elem.first << " " << elem.second.numTimesSelected << " "
<< std::fixed << std::setprecision(2)
<< (elem.second.numTimesSelected * 100.00) / totalSelections
<< "% " << elem.second.numProcessedRows;
}
}

Expand Down

0 comments on commit c19aaef

Please sign in to comment.