Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

softnet: additionals metrics from softnet_data, #2592

Merged
merged 18 commits into from
May 24, 2023
Merged

softnet: additionals metrics from softnet_data, #2592

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6c4b29f
softnet: additionals metrics from softnet_data, https://github.com/pr…
remijouannet Feb 6, 2023
a3c7730
fix fixtures
remijouannet Feb 6, 2023
1d38a8f
Merge branch 'prometheus:master' into softnet_additional_metrics
remijouannet Mar 7, 2023
fa101c6
Merge branch 'prometheus:master' into softnet_additional_metrics
remijouannet Mar 20, 2023
e124db7
fix e2e test
remijouannet Mar 20, 2023
b483720
fix tests e2e-64k-page-output
remijouannet Mar 20, 2023
7698933
rollback the typo fix
remijouannet May 11, 2023
83d9cc8
fix cpuStats.Index
remijouannet May 11, 2023
be8b2d4
fix cpuStats.Index
remijouannet May 11, 2023
2d9c38c
rollback e2e output
remijouannet May 11, 2023
3baee6b
rollback e2e output, rollback typo in e2e
remijouannet May 11, 2023
21d1434
rollback e2e output, rollback typo in e2e
remijouannet May 11, 2023
4498f88
remove width, softnet backlog must be a gauge not a counter
remijouannet May 11, 2023
b15361b
remove width, softnet backlog must be a gauge not a counter
remijouannet May 11, 2023
aa5fc6b
remove width, softnet backlog must be a gauge not a counter
remijouannet May 11, 2023
79530dc
comment to explain cpuStats.Index usage
remijouannet May 22, 2023
2c03cae
rollback to Index for procfs 0.10
remijouannet May 24, 2023
15b4afe
Merge branch 'master' into softnet_additional_metrics
remijouannet May 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions collector/fixtures/e2e-64k-page-output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3040,18 +3040,42 @@ node_softirqs_total{vector="rcu"} 508444
node_softirqs_total{vector="sched"} 622196
node_softirqs_total{vector="tasklet"} 1.783454e+06
node_softirqs_total{vector="timer"} 1.481983e+06
# HELP node_softnet_backlog_len Softnet backlog status
# TYPE node_softnet_backlog_len gauge
node_softnet_backlog_len{cpu="0"} 0
node_softnet_backlog_len{cpu="1"} 0
node_softnet_backlog_len{cpu="2"} 0
node_softnet_backlog_len{cpu="3"} 0
# HELP node_softnet_cpu_collision_total Number of collision occur while obtaining device lock while transmitting
# TYPE node_softnet_cpu_collision_total counter
node_softnet_cpu_collision_total{cpu="0"} 0
node_softnet_cpu_collision_total{cpu="1"} 0
node_softnet_cpu_collision_total{cpu="2"} 0
node_softnet_cpu_collision_total{cpu="3"} 0
# HELP node_softnet_dropped_total Number of dropped packets
# TYPE node_softnet_dropped_total counter
node_softnet_dropped_total{cpu="0"} 0
node_softnet_dropped_total{cpu="1"} 41
node_softnet_dropped_total{cpu="2"} 0
node_softnet_dropped_total{cpu="3"} 0
# HELP node_softnet_flow_limit_count_total Number of times flow limit has been reached
# TYPE node_softnet_flow_limit_count_total counter
node_softnet_flow_limit_count_total{cpu="0"} 0
node_softnet_flow_limit_count_total{cpu="1"} 0
node_softnet_flow_limit_count_total{cpu="2"} 0
node_softnet_flow_limit_count_total{cpu="3"} 0
# HELP node_softnet_processed_total Number of processed packets
# TYPE node_softnet_processed_total counter
node_softnet_processed_total{cpu="0"} 299641
node_softnet_processed_total{cpu="1"} 916354
node_softnet_processed_total{cpu="2"} 5.577791e+06
node_softnet_processed_total{cpu="3"} 3.113785e+06
# HELP node_softnet_received_rps_total Number of times cpu woken up received_rps
# TYPE node_softnet_received_rps_total counter
node_softnet_received_rps_total{cpu="0"} 0
node_softnet_received_rps_total{cpu="1"} 0
node_softnet_received_rps_total{cpu="2"} 0
node_softnet_received_rps_total{cpu="3"} 0
# HELP node_softnet_times_squeezed_total Number of times processing packets ran out of quota
# TYPE node_softnet_times_squeezed_total counter
node_softnet_times_squeezed_total{cpu="0"} 1
Expand Down
24 changes: 24 additions & 0 deletions collector/fixtures/e2e-output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3062,18 +3062,42 @@ node_softirqs_total{vector="rcu"} 508444
node_softirqs_total{vector="sched"} 622196
node_softirqs_total{vector="tasklet"} 1.783454e+06
node_softirqs_total{vector="timer"} 1.481983e+06
# HELP node_softnet_backlog_len Softnet backlog status
# TYPE node_softnet_backlog_len gauge
node_softnet_backlog_len{cpu="0"} 0
node_softnet_backlog_len{cpu="1"} 0
node_softnet_backlog_len{cpu="2"} 0
node_softnet_backlog_len{cpu="3"} 0
# HELP node_softnet_cpu_collision_total Number of collision occur while obtaining device lock while transmitting
# TYPE node_softnet_cpu_collision_total counter
node_softnet_cpu_collision_total{cpu="0"} 0
node_softnet_cpu_collision_total{cpu="1"} 0
node_softnet_cpu_collision_total{cpu="2"} 0
node_softnet_cpu_collision_total{cpu="3"} 0
# HELP node_softnet_dropped_total Number of dropped packets
# TYPE node_softnet_dropped_total counter
node_softnet_dropped_total{cpu="0"} 0
node_softnet_dropped_total{cpu="1"} 41
node_softnet_dropped_total{cpu="2"} 0
node_softnet_dropped_total{cpu="3"} 0
# HELP node_softnet_flow_limit_count_total Number of times flow limit has been reached
# TYPE node_softnet_flow_limit_count_total counter
node_softnet_flow_limit_count_total{cpu="0"} 0
node_softnet_flow_limit_count_total{cpu="1"} 0
node_softnet_flow_limit_count_total{cpu="2"} 0
node_softnet_flow_limit_count_total{cpu="3"} 0
# HELP node_softnet_processed_total Number of processed packets
# TYPE node_softnet_processed_total counter
node_softnet_processed_total{cpu="0"} 299641
node_softnet_processed_total{cpu="1"} 916354
node_softnet_processed_total{cpu="2"} 5.577791e+06
node_softnet_processed_total{cpu="3"} 3.113785e+06
# HELP node_softnet_received_rps_total Number of times cpu woken up received_rps
# TYPE node_softnet_received_rps_total counter
node_softnet_received_rps_total{cpu="0"} 0
node_softnet_received_rps_total{cpu="1"} 0
node_softnet_received_rps_total{cpu="2"} 0
node_softnet_received_rps_total{cpu="3"} 0
# HELP node_softnet_times_squeezed_total Number of times processing packets ran out of quota
# TYPE node_softnet_times_squeezed_total counter
node_softnet_times_squeezed_total{cpu="0"} 1
Expand Down
70 changes: 64 additions & 6 deletions collector/softnet_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,15 @@ import (
)

type softnetCollector struct {
fs procfs.FS
processed *prometheus.Desc
dropped *prometheus.Desc
timeSqueezed *prometheus.Desc
logger log.Logger
fs procfs.FS
processed *prometheus.Desc
dropped *prometheus.Desc
timeSqueezed *prometheus.Desc
cpuCollision *prometheus.Desc
receivedRps *prometheus.Desc
flowLimitCount *prometheus.Desc
softnetBacklogLen *prometheus.Desc
logger log.Logger
}

const (
Expand Down Expand Up @@ -65,19 +69,49 @@ func NewSoftnetCollector(logger log.Logger) (Collector, error) {
"Number of times processing packets ran out of quota",
[]string{"cpu"}, nil,
),
cpuCollision: prometheus.NewDesc(
prometheus.BuildFQName(namespace, softnetSubsystem, "cpu_collision_total"),
"Number of collision occur while obtaining device lock while transmitting",
[]string{"cpu"}, nil,
),
receivedRps: prometheus.NewDesc(
prometheus.BuildFQName(namespace, softnetSubsystem, "received_rps_total"),
"Number of times cpu woken up received_rps",
[]string{"cpu"}, nil,
),
flowLimitCount: prometheus.NewDesc(
prometheus.BuildFQName(namespace, softnetSubsystem, "flow_limit_count_total"),
"Number of times flow limit has been reached",
[]string{"cpu"}, nil,
),
softnetBacklogLen: prometheus.NewDesc(
prometheus.BuildFQName(namespace, softnetSubsystem, "backlog_len"),
"Softnet backlog status",
[]string{"cpu"}, nil,
),
logger: logger,
}, nil
}

// Update gets parsed softnet statistics using procfs.
func (c *softnetCollector) Update(ch chan<- prometheus.Metric) error {
var cpu string

stats, err := c.fs.NetSoftnetStat()
if err != nil {
return fmt.Errorf("could not get softnet statistics: %w", err)
}

for cpuNumber, cpuStats := range stats {
cpu := strconv.Itoa(cpuNumber)
// cpuStats.Index in /proc/net/softnet_stat was introduced in Linux 5.14
// for older kernel we still rely on the line index
// Linux 5.14 https://elixir.bootlin.com/linux/v5.14/source/net/core/net-procfs.c#L169
// https://github.com/prometheus/procfs/pull/517
if cpuStats.Width >= 13 {
cpu = strconv.FormatUint(uint64(cpuStats.Index), 10)
} else {
cpu = strconv.Itoa(cpuNumber)
}
remijouannet marked this conversation as resolved.
Show resolved Hide resolved

ch <- prometheus.MustNewConstMetric(
c.processed,
Expand All @@ -97,6 +131,30 @@ func (c *softnetCollector) Update(ch chan<- prometheus.Metric) error {
float64(cpuStats.TimeSqueezed),
cpu,
)
ch <- prometheus.MustNewConstMetric(
c.cpuCollision,
prometheus.CounterValue,
float64(cpuStats.CPUCollision),
cpu,
)
ch <- prometheus.MustNewConstMetric(
c.receivedRps,
prometheus.CounterValue,
float64(cpuStats.ReceivedRps),
cpu,
)
ch <- prometheus.MustNewConstMetric(
c.flowLimitCount,
prometheus.CounterValue,
float64(cpuStats.FlowLimitCount),
cpu,
)
ch <- prometheus.MustNewConstMetric(
c.softnetBacklogLen,
prometheus.GaugeValue,
float64(cpuStats.SoftnetBacklogLen),
cpu,
)
}

return nil
Expand Down