Skip to content

Commit

Permalink
Added field to disable alerts creation
Browse files Browse the repository at this point in the history
  • Loading branch information
OlivierCazade committed Mar 13, 2023
1 parent 8a4db3c commit 5d23deb
Show file tree
Hide file tree
Showing 9 changed files with 86 additions and 37 deletions.
11 changes: 11 additions & 0 deletions api/v1alpha1/flowcollector_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {
dst.Spec.Processor.ConnectionEndTimeout = restored.Spec.Processor.ConnectionEndTimeout
}

if restored.Spec.Processor.Metrics.DisableAlerts != nil {
dst.Spec.Processor.Metrics.DisableAlerts = restored.Spec.Processor.Metrics.DisableAlerts
}

return nil
}

Expand Down Expand Up @@ -89,3 +93,10 @@ func (r *FlowCollectorList) ConvertFrom(srcRaw conversion.Hub) error {
func Convert_v1beta1_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in *v1beta1.FlowCollectorFLP, out *FlowCollectorFLP, s apiconversion.Scope) error {
return autoConvert_v1beta1_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in, out, s)
}

// This function need to be manually created because conversion-gen not able to create it intentionally because
// we have new defined fields in v1beta1 not in v1alpha1
// nolint:golint,stylecheck,revive
func Convert_v1beta1_FLPMetrics_To_v1alpha1_FLPMetrics(in *v1beta1.FLPMetrics, out *FLPMetrics, s apiconversion.Scope) error {
return autoConvert_v1beta1_FLPMetrics_To_v1alpha1_FLPMetrics(in, out, s)
}
16 changes: 6 additions & 10 deletions api/v1alpha1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions api/v1beta1/flowcollector_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ type FLPMetrics struct {
// ignoreTags is a list of tags to specify which metrics to ignore
//+kubebuilder:default:={"egress","packets"}
IgnoreTags []string `json:"ignoreTags,omitempty"`

// disableAlerts is a list of alerts related to FLP that should not be created
// +optional
DisableAlerts []string `json:"disableAlerts,omitempty"`
}

const (
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions config/crd/bases/flows.netobserv.io_flowcollectors.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4069,6 +4069,12 @@ spec:
description: Metrics define the processor configuration regarding
metrics
properties:
disableAlerts:
description: disableAlerts is a list of alerts related to
FLP that should not be created
items:
type: string
type: array
ignoreTags:
default:
- egress
Expand Down
1 change: 1 addition & 0 deletions config/samples/flows_v1beta1_flowcollector.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ spec:
ignoreTags:
- egress
- packets
disableAlerts: []
dropUnusedFields: true
resources:
requests:
Expand Down
1 change: 1 addition & 0 deletions config/samples/flows_v1beta1_flowcollector_versioned.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ spec:
ignoreTags:
- egress
- packets
disableAlerts: []
dropUnusedFields: true
resources:
requests:
Expand Down
72 changes: 45 additions & 27 deletions controllers/flowlogspipeline/flp_common_objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,50 @@ func (b *builder) serviceMonitor() *monitoringv1.ServiceMonitor {
return &flpServiceMonitorObject
}

func shouldAddAlert(name string, disabledList []string) bool {
for _, disabledAlert := range disabledList {
if name == disabledAlert {
return false
}
}
return true
}

func (b *builder) prometheusRule() *monitoringv1.PrometheusRule {
rules := []monitoringv1.Rule{}

// Not receiving flows
if shouldAddAlert("NetObservNoFlows", b.desired.Processor.Metrics.DisableAlerts) {
rules = append(rules, monitoringv1.Rule{
Alert: "NetObservNoFlows",
Annotations: map[string]string{
"description": "NetObserv flowlogs-pipeline is not receiving any flow, this is either a connection issue with the agent, or an agent issue",
"summary": "NetObserv flowlogs-pipeline is not receiving any flow",
},
Expr: intstr.FromString("sum(rate(netobserv_ingest_flows_processed[5m])) == 0"),
For: "10m",
Labels: map[string]string{
"severity": "warning",
},
})
}

// Flows getting dropped by loki library
if shouldAddAlert("NetObservLokiError", b.desired.Processor.Metrics.DisableAlerts) {
rules = append(rules, monitoringv1.Rule{
Alert: "NetObservLokiError",
Annotations: map[string]string{
"description": "NetObserv flowlogs-pipeline is dropping flows because of loki errors, loki may be down or having issues ingesting every flows. Please check loki and flowlogs-pipeline logs.",
"summary": "NetObserv flowlogs-pipeline is dropping flows because of loki errors",
},
Expr: intstr.FromString("sum(rate(netobserv_loki_dropped_entries_total[5m])) > 0"),
For: "10m",
Labels: map[string]string{
"severity": "warning",
},
})
}

flpPrometheusRuleObject := monitoringv1.PrometheusRule{
ObjectMeta: metav1.ObjectMeta{
Name: b.prometheusRuleName(),
Expand All @@ -722,33 +765,8 @@ func (b *builder) prometheusRule() *monitoringv1.PrometheusRule {
Spec: monitoringv1.PrometheusRuleSpec{
Groups: []monitoringv1.RuleGroup{
{
Name: "NetobservFlowLogsPipeline",
Rules: []monitoringv1.Rule{
{
Alert: "NetObservNoFlows",
Annotations: map[string]string{
"description": "NetObserv flowlogs-pipeline is not receiving any flow, this is either a connection issue with the agent, or an agent issue",
"summary": "NetObserv flowlogs-pipeline is not receiving any flow",
},
Expr: intstr.FromString("sum(rate(netobserv_ingest_flows_processed[5m])) == 0"),
For: "10m",
Labels: map[string]string{
"severity": "warning",
},
},
{
Alert: "NetObservLokiError",
Annotations: map[string]string{
"description": "NetObserv flowlogs-pipeline is dropping flows because of loki errors, loki may be down or having issues ingesting every flows. Please check loki and flowlogs-pipeline logs.",
"summary": "NetObserv flowlogs-pipeline is dropping flows because of loki errors",
},
Expr: intstr.FromString("sum(rate(netobserv_loki_dropped_entries_total[5m])) > 0"),
For: "10m",
Labels: map[string]string{
"severity": "warning",
},
},
},
Name: "NetobservFlowLogsPipeline",
Rules: rules,
},
},
},
Expand Down
7 changes: 7 additions & 0 deletions docs/FlowCollector.md
Original file line number Diff line number Diff line change
Expand Up @@ -7293,6 +7293,13 @@ Metrics define the processor configuration regarding metrics
</tr>
</thead>
<tbody><tr>
<td><b>disableAlerts</b></td>
<td>[]string</td>
<td>
disableAlerts is a list of alerts related to FLP that should not be created<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>ignoreTags</b></td>
<td>[]string</td>
<td>
Expand Down

0 comments on commit 5d23deb

Please sign in to comment.