Skip to content

Commit

Permalink
NETOBSERV-1343: generate dashboards from metrics API (#609)
Browse files Browse the repository at this point in the history 
* NETOBSERV-1343: generate dashboards from metrics API

- Add dashboard config to metrics API
- Use this API internally for predefined dashboards
- Allow using SingleStats
- New dedicated buckets for latency histograms

* Document API, gen bundle, fix tests, update examples

* Manage multiple dashboards

- Allow users to reference/create new dashboards
  (this requires the monitoring controller to fetch all dashboards from
  the dashboards namespace)
  All dashboard names are prefixed "NetObserv / " ; so, rename the main
dashboard as "NetObserv / Main"
- Add/update tests

* update bundle

* enforce consistent charts order

* When too many panels are in top row, put them in

an Overview section

* Add configurable top

* fix test

* More examples

* Remove includeDuplicates, and some minor doc/samples changes

* Add FlowMetrics upstream doc

- generate markdown API reference
- document in Metrics.md

* Apply suggestions from code review

Co-authored-by: Mehul Modi <[email protected]>

* Add more info on cardinality

---------

Co-authored-by: Mehul Modi <[email protected]>
  • Loading branch information
@jotak @memodi
jotak and memodi authored May 7, 2024
1 parent 13b2a78 commit 33b5849
Show file tree
Hide file tree
Showing 40 changed files with 2,197 additions and 703 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ gencode: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and

doc: crdoc ## Generate markdown documentation
$(CRDOC) --resources config/crd/bases/flows.netobserv.io_flowcollectors.yaml --output docs/FlowCollector.md
$(CRDOC) --resources config/crd/bases/flows.netobserv.io_flowmetrics.yaml --output docs/FlowMetric.md

generate-go-conversions: $(CONVERSION_GEN) ## Run all generate-go-conversions
$(MAKE) clean-generated-conversions SRC_DIRS="./apis/flowcollector/v1beta1"
Expand Down
86 changes: 78 additions & 8 deletions apis/flowmetrics/v1alpha1/flowmetric_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ type MetricFilter struct {
// +required
Field string `json:"field"`

// Value to filter on
// Value to filter on. When `matchType` is `Equal` or `NotEqual`, you can use field injection with `$(SomeField)` to refer to any other field of the flow.
// +optional
Value string `json:"value"`

Expand All @@ -61,7 +61,7 @@ type MetricFilter struct {
// usage of Prometheus workloads as this could potentially have a high impact. Cf https://rhobs-handbook.netlify.app/products/openshiftmonitoring/telemetry.md/#what-is-the-cardinality-of-a-metric<br>
// To check the cardinality of all NetObserv metrics, run as `promql`: `count({__name__=~"netobserv.*"}) by (__name__)`.
type FlowMetricSpec struct {
// Name of the metric in Prometheus. It will be automatically prefixed with "netobserv_".
// Name of the metric. In Prometheus, it is automatically prefixed with "netobserv_".
// +required
MetricName string `json:"metricName"`

Expand Down Expand Up @@ -93,11 +93,6 @@ type FlowMetricSpec struct {
// +optional
Labels []string `json:"labels"`

// When set to `true`, flows duplicated across several interfaces will add up in the generated metrics.
// When set to `false` (default), it is equivalent to adding the exact filter on `Duplicate` != `true`.
// +optional
IncludeDuplicates bool `json:"includeDuplicates,omitempty"`

// Filter for ingress, egress or any direction flows.
// When set to `Ingress`, it is equivalent to adding the regex filter on `FlowDirection`: `0|2`.
// When set to `Egress`, it is equivalent to adding the regex filter on `FlowDirection`: `1|2`.
Expand All @@ -106,9 +101,84 @@ type FlowMetricSpec struct {
// +optional
Direction FlowDirection `json:"direction,omitempty"`

// A list of buckets to use when `type` is "Histogram". The list must be parseable as floats. Prometheus default buckets will be used if unset.
// A list of buckets to use when `type` is "Histogram". The list must be parseable as floats. When not set, Prometheus default buckets are used.
// +optional
Buckets []string `json:"buckets,omitempty"`

// When non-zero, scale factor (divider) of the value. Metric value = Flow value / Divider.
// +optional
Divider string `json:"divider"`

// Charts configuration, for the OpenShift Console in the administrator view, Dashboards menu.
// +optional
Charts []Chart `json:"charts,omitempty"`
}

type Unit string
type ChartType string

const (
UnitBytes Unit = "bytes"
UnitSeconds Unit = "seconds"
UnitBPS Unit = "Bps"
UnitPPS Unit = "pps"
UnitPercent Unit = "percent"
ChartTypeSingleStat ChartType = "SingleStat"
ChartTypeLine ChartType = "Line"
ChartTypeStackArea ChartType = "StackArea"
)

// Configures charts / dashboard generation associated to a metric
type Chart struct {
// Name of the containing dashboard. If this name does not refer to an existing dashboard, a new dashboard is created.
// +kubebuilder:default:="Main"
DashboardName string `json:"dashboardName"`

// Name of the containing dashboard section. If this name does not refer to an existing section, a new section is created.
// If `sectionName` is omitted or empty, the chart is placed in the global top section.
// +optional
SectionName string `json:"sectionName,omitempty"`

// Title of the chart.
// +required
Title string `json:"title"`

// Unit of this chart. Only a few units are currently supported. Leave empty to use generic number.
// +kubebuilder:validation:Enum:="bytes";"seconds";"Bps";"pps";"percent"
// +optional
Unit Unit `json:"unit,omitempty"`

// Type of the chart.
// +kubebuilder:validation:Enum:="SingleStat";"Line";"StackArea"
// +required
Type ChartType `json:"type"`

// List of queries to be displayed on this chart. If `type` is `SingleStat` and multiple queries are provided,
// this chart is automatically expanded in several panels (one per query).
// +required
Queries []Query `json:"queries"`
}

// Configures PromQL queries
type Query struct {
// The `promQL` query to be run against Prometheus. If the chart `type` is `SingleStat`, this query should only return
// a single timeseries. For other types, a top 7 is displayed.
// You can use `$METRIC` to refer to the metric defined in this resource. For example: `sum(rate($METRIC[2m]))`.
// To learn more about `promQL`, refer to the Prometheus documentation: https://prometheus.io/docs/prometheus/latest/querying/basics/
// +required
PromQL string `json:"promQL"`

// The query legend that applies to each timeseries represented in this chart. When multiple timeseries are displayed, you should set a legend
// that distinguishes each of them. It can be done with the following format: `{{ Label }}`. For example, if the `promQL` groups timeseries per
// label such as: `sum(rate($METRIC[2m])) by (Label1, Label2)`, you may write as the legend: `Label1={{ Label1 }}, Label2={{ Label2 }}`.
// +required
Legend string `json:"legend"`

// Top N series to display per timestamp. Does not apply to `SingleStat` chart type.
// +kubebuilder:default:=7
// +kubebuilder:validation:Minimum=1
// +required
Top int `json:"top"`
}

// FlowMetricStatus defines the observed state of FlowMetric
Expand Down
42 changes: 42 additions & 0 deletions apis/flowmetrics/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

96 changes: 87 additions & 9 deletions bundle/manifests/flows.netobserv.io_flowmetrics.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,88 @@ spec:
properties:
buckets:
description: A list of buckets to use when `type` is "Histogram".
The list must be parseable as floats. Prometheus default buckets
will be used if unset.
The list must be parseable as floats. When not set, Prometheus default
buckets are used.
items:
type: string
type: array
charts:
description: Charts configuration, for the OpenShift Console in the
administrator view, Dashboards menu.
items:
description: Configures charts / dashboard generation associated
to a metric
properties:
dashboardName:
default: Main
description: Name of the containing dashboard. If this name
does not refer to an existing dashboard, a new dashboard is
created.
type: string
queries:
description: |-
List of queries to be displayed on this chart. If `type` is `SingleStat` and multiple queries are provided,
this chart is automatically expanded in several panels (one per query).
items:
description: Configures PromQL queries
properties:
legend:
description: |-
The query legend that applies to each timeseries represented in this chart. When multiple timeseries are displayed, you should set a legend
that distinguishes each of them. It can be done with the following format: `{{ Label }}`. For example, if the `promQL` groups timeseries per
label such as: `sum(rate($METRIC[2m])) by (Label1, Label2)`, you may write as the legend: `Label1={{ Label1 }}, Label2={{ Label2 }}`.
type: string
promQL:
description: |-
The `promQL` query to be run against Prometheus. If the chart `type` is `SingleStat`, this query should only return
a single timeseries. For other types, a top 7 is displayed.
You can use `$METRIC` to refer to the metric defined in this resource. For example: `sum(rate($METRIC[2m]))`.
To learn more about `promQL`, refer to the Prometheus documentation: https://prometheus.io/docs/prometheus/latest/querying/basics/
type: string
top:
default: 7
description: Top N series to display per timestamp. Does
not apply to `SingleStat` chart type.
minimum: 1
type: integer
required:
- legend
- promQL
- top
type: object
type: array
sectionName:
description: |-
Name of the containing dashboard section. If this name does not refer to an existing section, a new section is created.
If `sectionName` is omitted or empty, the chart is placed in the global top section.
type: string
title:
description: Title of the chart.
type: string
type:
description: Type of the chart.
enum:
- SingleStat
- Line
- StackArea
type: string
unit:
description: Unit of this chart. Only a few units are currently
supported. Leave empty to use generic number.
enum:
- bytes
- seconds
- Bps
- pps
- percent
type: string
required:
- dashboardName
- queries
- title
- type
type: object
type: array
direction:
default: Any
description: |-
Expand All @@ -72,6 +149,10 @@ spec:
- Egress
- Ingress
type: string
divider:
description: When non-zero, scale factor (divider) of the value. Metric
value = Flow value / Divider.
type: string
filters:
description: |-
`filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must
Expand All @@ -94,18 +175,15 @@ spec:
- NotMatchRegex
type: string
value:
description: Value to filter on
description: Value to filter on. When `matchType` is `Equal`
or `NotEqual`, you can use field injection with `$(SomeField)`
to refer to any other field of the flow.
type: string
required:
- field
- matchType
type: object
type: array
includeDuplicates:
description: |-
When set to `true`, flows duplicated across several interfaces will add up in the generated metrics.
When set to `false` (default), it is equivalent to adding the exact filter on `Duplicate` != `true`.
type: boolean
labels:
description: |-
`labels` is a list of fields that should be used as Prometheus labels, also known as dimensions.
Expand All @@ -118,7 +196,7 @@ spec:
type: string
type: array
metricName:
description: Name of the metric in Prometheus. It will be automatically
description: Name of the metric. In Prometheus, it is automatically
prefixed with "netobserv_".
type: string
type:
Expand Down
49 changes: 35 additions & 14 deletions bundle/manifests/netobserv-operator.clusterserviceversion.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,49 @@ metadata:
"name": "flowmetric-sample"
},
"spec": {
"filters": [
"charts": [
{
"field": "DstPort",
"matchType": "Regex",
"value": "^\\d\\d?\\d?\\d?$"
"dashboardName": "Main",
"queries": [
{
"legend": "",
"promQL": "sum(rate($METRIC[2m]))"
}
],
"title": "External ingress traffic",
"type": "SingleStat",
"unit": "Bps"
},
{
"field": "Duplicate",
"matchType": "NotEqual",
"value": "true"
},
"dashboardName": "Main",
"queries": [
{
"legend": "{{DstK8S_Namespace}} / {{DstK8S_OwnerName}}",
"promQL": "sum(rate($METRIC{DstK8S_Namespace!=\"\"}[2m])) by (DstK8S_Namespace, DstK8S_OwnerName)"
}
],
"sectionName": "External",
"title": "Top external ingress traffic per workload",
"type": "StackArea",
"unit": "Bps"
}
],
"direction": "Ingress",
"filters": [
{
"field": "FlowDirection",
"matchType": "Regex",
"value": "1|2"
"field": "SrcSubnetLabel",
"matchType": "Absence"
}
],
"labels": [
"DstPort"
"DstK8S_HostName",
"DstK8S_Namespace",
"DstK8S_OwnerName",
"DstK8S_OwnerType"
],
"metricName": "service_ports_total",
"type": "Counter"
"metricName": "cluster_external_ingress_bytes_total",
"type": "Counter",
"valueField": "Bytes"
}
},
{
Expand Down
Loading

0 comments on commit 33b5849

Please sign in to comment.