Skip to content

Commit

Permalink
Add physicalWrittenBytes to PlanNodeStats (#8213)
Browse files Browse the repository at this point in the history
Summary:
The physicalWrittenBytes metric is available in the OperatorStats class, but not
in the PlanNodeStats. When accessing metrics from Task::taskStats(), we are
unable to retrieve the physicalWrittenBytes' value.

Ad physicalWrittenBytes metric to PlanNodeStats.

Pull Request resolved: #8213

Reviewed By: xiaoxmeng

Differential Revision: D52554834

Pulled By: mbasmanova

fbshipit-source-id: 7edb3b468cc8c32c5861a0bd3309a4473b837079
  • Loading branch information
JkSelf authored and facebook-github-bot committed Jan 5, 2024
1 parent 7376fb2 commit e5355f3
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 11 deletions.
3 changes: 3 additions & 0 deletions velox/exec/PlanNodeStats.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ void PlanNodeStats::addTotals(const OperatorStats& stats) {
peakMemoryBytes += stats.memoryStats.peakTotalMemoryReservation;
numMemoryAllocations += stats.memoryStats.numMemoryAllocations;

physicalWrittenBytes += stats.physicalWrittenBytes;

for (const auto& [name, runtimeStats] : stats.runtimeStats) {
if (UNLIKELY(customStats.count(name) == 0)) {
customStats.insert(std::make_pair(name, runtimeStats));
Expand Down Expand Up @@ -154,6 +156,7 @@ folly::dynamic toPlanStatsJson(const facebook::velox::exec::TaskStats& stats) {
stat["blockedWallNanos"] = operatorStat.second->blockedWallNanos;
stat["peakMemoryBytes"] = operatorStat.second->peakMemoryBytes;
stat["numMemoryAllocations"] = operatorStat.second->numMemoryAllocations;
stat["physicalWrittenBytes"] = operatorStat.second->physicalWrittenBytes;
stat["numDrivers"] = operatorStat.second->numDrivers;
stat["numSplits"] = operatorStat.second->numSplits;
stat["spilledInputBytes"] = operatorStat.second->spilledInputBytes;
Expand Down
2 changes: 2 additions & 0 deletions velox/exec/PlanNodeStats.h
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ struct PlanNodeStats {

uint64_t numMemoryAllocations{0};

uint64_t physicalWrittenBytes{0};

/// Operator-specific counters.
std::unordered_map<std::string, RuntimeMetric> customStats;

Expand Down
23 changes: 12 additions & 11 deletions velox/exec/tests/TableWriteTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2972,17 +2972,18 @@ TEST_P(AllTableWriterTest, tableWriterStats) {
const int32_t ORC_HEADER_LEN{3};
const auto fixedWrittenBytes =
numWrittenFiles * (fileFormat_ == FileFormat::DWRF ? ORC_HEADER_LEN : 0);
for (int i = 0; i < task->taskStats().pipelineStats.size(); ++i) {
auto operatorStats = task->taskStats().pipelineStats.at(i).operatorStats;
for (int j = 0; j < operatorStats.size(); ++j) {
if (operatorStats.at(j).operatorType == "TableWrite") {
ASSERT_GT(operatorStats.at(j).physicalWrittenBytes, fixedWrittenBytes);
ASSERT_EQ(
operatorStats.at(j).runtimeStats.at("numWrittenFiles").sum,
numWrittenFiles);
}
}
}

auto planStats = exec::toPlanStats(task->taskStats());
auto& stats = planStats.at(tableWriteNodeId_);
ASSERT_GT(stats.physicalWrittenBytes, fixedWrittenBytes);
ASSERT_GT(
stats.operatorStats.at("TableWrite")->physicalWrittenBytes,
fixedWrittenBytes);
ASSERT_EQ(
stats.operatorStats.at("TableWrite")
->customStats.at("numWrittenFiles")
.sum,
numWrittenFiles);
}

DEBUG_ONLY_TEST_P(
Expand Down

0 comments on commit e5355f3

Please sign in to comment.