app_stats.proto

// Copyright 2017 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

syntax = "proto2";
package cockroach.sql;
option go_package = "roachpb";

import "gogoproto/gogo.proto";
import "google/protobuf/timestamp.proto";

// N.B. When fields are added to this struct, make sure to update
// (*StatementStatistics).Add and (*StatementStatistics).AlmostEqual
// in app_stats.go.
message StatementStatistics {
  // Count is the total number of times this statement was executed
  // since the begin of the reporting period.
  // When transmitted to the reporting server, this value gets
  // quantized into buckets (few <10, dozens 10+, 100 or more).
  optional int64 count = 1 [(gogoproto.nullable) = false];

  // FirstAttemptCount collects the total number of times a first
  // attempt was executed (either the one time in explicitly committed
  // statements, or the first time in implicitly committed statements
  // with implicit retries).
  // The proportion of statements that could be executed without retry
  // can be computed as FirstAttemptCount / Count.
  // The cumulative number of retries can be computed with
  // Count - FirstAttemptCount.
  //
  // When transmitted to the reporting server, this value gets
  // simplified so that the proportion of statements that could be
  // executed without retry remains as FirstAttemptCount / Count.
  optional int64 first_attempt_count = 2 [(gogoproto.nullable) = false];

  // MaxRetries collects the maximum observed number of automatic
  // retries in the reporting period.
  // When transmitted to the reporting server, this value gets
  // quantized into buckets (few <10, dozens 10+, 100 or more).
  optional int64 max_retries = 3 [(gogoproto.nullable) = false];

  // DEPRECATED: LastErr collects the last error encountered.
  // Use sensitive_info.last_err instead.
  optional string legacy_last_err = 4 [(gogoproto.nullable) = false];

  // DEPRECATED: LastErrRedacted collects the last error, redacted for reporting.
  optional string legacy_last_err_redacted = 11 [(gogoproto.nullable) = false];

  // NumRows collects the number of rows returned or observed.
  optional NumericStat num_rows = 5 [(gogoproto.nullable) = false];

  // Phase latencies:

  // ParseLat is the time to transform the SQL string into an AST.
  optional NumericStat parse_lat = 6 [(gogoproto.nullable) = false];

  // PlanLat is the time to transform the AST into a logical query plan.
  optional NumericStat plan_lat = 7 [(gogoproto.nullable) = false];

  // RunLat is the time to run the query and fetch/compute the result rows.
  optional NumericStat run_lat = 8 [(gogoproto.nullable) = false];

  // ServiceLat is the time to service the query, from start of parse to end of execute.
  optional NumericStat service_lat = 9 [(gogoproto.nullable) = false];

  // OverheadLat is the difference between ServiceLat and the sum of parse+plan+run latencies.
  // We store it separately (as opposed to computing it post-hoc) because the combined
  // variance for the overhead cannot be derived from the variance of the separate latencies.
  optional NumericStat overhead_lat = 10 [(gogoproto.nullable) = false];

  // SensitiveInfo is info that needs to be scrubbed or redacted before being
  // sent to the reg cluster.
  optional SensitiveInfo sensitive_info = 12 [(gogoproto.nullable) = false];

  reserved 13, 14;

  // BytesRead collects the number of bytes read from disk.
  optional NumericStat bytes_read = 15 [(gogoproto.nullable) = false];

  // RowsRead collects the number of rows read from disk.
  optional NumericStat rows_read = 16 [(gogoproto.nullable) = false];

  // Note: be sure to update `sql/app_stats.go` when adding/removing fields here!
}

message TransactionStatistics {
  // StatementIds is the IDs of the statements which this transaction comprises,
  // in order.
  repeated string statement_ids = 1;

  // Count is the total number of times this transaction was executed
  // since the begin of the reporting period.
  optional int64 count = 2 [(gogoproto.nullable) = false];

  // MaxRetries collects the maximum observed number of automatic
  // retries in the reporting period.
  optional int64 max_retries = 3 [(gogoproto.nullable) = false];

  // NumRows collects the total number of rows returned or observed across all
  // statements.
  optional NumericStat num_rows = 4 [(gogoproto.nullable) = false];

  // ServiceLat is the time to service the transaction, from start of parse to
  // end of execute.
  optional NumericStat service_lat = 5 [(gogoproto.nullable) = false];
}


message SensitiveInfo {
  option (gogoproto.equal) = true;
  // LastErr collects the last error encountered.
  // It is only reported once it's been redacted.
  optional string last_err = 1 [(gogoproto.nullable) = false];

  // MostRecentPlanDescription is a serialized representation of the logical plan most recently captured for this query.
  optional ExplainTreePlanNode most_recent_plan_description = 2 [(gogoproto.nullable) = false];

  // Timestamp is the time at which the logical plan was last sampled.
  optional google.protobuf.Timestamp most_recent_plan_timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true];
}

// N.B. When this changes, make sure to update (*NumericStat).AlmostEqual
// in app_stats.go.
message NumericStat {
  // NumericStat keeps track of two running values --- the running mean and
  // the running sum of squared differences from the mean. Using this along
  // with the total count of values, we can compute variance using Welford's
  // method. This is more reliable than keeping track of the sum of
  // squared values, which is liable to overflow. See
  // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
  optional double mean = 1 [(gogoproto.nullable) = false];
  optional double squared_diffs = 2 [(gogoproto.nullable) = false];
}

message StatementStatisticsKey {
  optional string query = 1 [(gogoproto.nullable) = false];
  optional string app = 2 [(gogoproto.nullable) = false];
  optional bool distSQL = 3 [(gogoproto.nullable) = false];
  optional bool failed = 4 [(gogoproto.nullable) = false];
  optional bool opt = 5 [(gogoproto.nullable) = false];
  optional bool implicit_txn = 6 [(gogoproto.nullable) = false];
  optional bool vec = 7 [(gogoproto.nullable) = false];

  // Id is a hash of the query fingerprint which can be used to identify the
  // statement, for instance in transaction statistics.
  optional string id = 8 [(gogoproto.nullable) = false];
}

// CollectedStats wraps collected timings and metadata for some query's execution.
message CollectedStatementStatistics {
  optional StatementStatisticsKey key = 1 [(gogoproto.nullable) = false];
  optional StatementStatistics stats = 2 [(gogoproto.nullable) = false];
}

// ExplainTreePlanNode is a serialized representation of an EXPLAIN tree for a logical plan.
message ExplainTreePlanNode {
  option (gogoproto.equal) = true;
  // Name is the type of node this is, e.g. "scan" or "index-join".
  optional string name = 1 [(gogoproto.nullable) = false];

  message Attr {
    option (gogoproto.equal) = true;
    optional string key = 1 [(gogoproto.nullable) = false];
    optional string value = 2 [(gogoproto.nullable) = false];
  }

  // Attrs are attributes of this plan node.
  // Often there are many attributes with the same key, e.g. "render".
  repeated Attr attrs = 2;

  // Children are the nodes that feed into this one, e.g. two scans for a join.
  repeated ExplainTreePlanNode children = 3;
}

// TxnStats contains statistics about transactions of one application.
// N.B. When field are added to this struct, make sure to updated
// (*TxnStats).Add in app_stats.go.
message TxnStats {
  optional int64 txn_count = 1 [(gogoproto.nullable) = false];

  optional NumericStat txn_time_sec = 2 [(gogoproto.nullable) = false];

  optional int64 committed_count = 3 [(gogoproto.nullable) = false];

  optional int64 implicit_count = 4 [(gogoproto.nullable) = false];

  // Note: be sure to update `sql/app_stats.go` when adding/removing fields here!
}