Implement regex exclusions for alerts

Signed-off-by: Stefan Prodan <[email protected]>

api/v1beta1/alert_types.go

@@ -42,6 +42,10 @@ type AlertSpec struct {
 	// +required
 	EventSources []CrossNamespaceObjectReference `json:"eventSources"`
 
+	// A list of golang regex expressions to be used for excluding meesages
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
+
 	// Short description of the impact and affected cluster.
 	// +optional
 	Summary string `json:"summary,omitempty"`

config/crd/bases/notification.toolkit.fluxcd.io_alerts.yaml

@@ -91,6 +91,12 @@ spec:
                   - name
                   type: object
                 type: array
+              exclusionList:
+                description: A list of golang regex expressions to be used for excluding
+                  meesages
+                items:
+                  type: string
+                type: array
               providerRef:
                 description: Send events using this provider
                 properties:

docs/api/notification.md

@@ -111,6 +111,18 @@ If set to &lsquo;info&rsquo; no events will be filtered.</p>
 </tr>
 <tr>
 <td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>A list of golang regex expressions to be used for excluding meesages</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>summary</code><br>
 <em>
 string
@@ -496,6 +508,18 @@ If set to &lsquo;info&rsquo; no events will be filtered.</p>
 </tr>
 <tr>
 <td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>A list of golang regex expressions to be used for excluding meesages</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>summary</code><br>
 <em>
 string

docs/spec/v1beta1/alert.md

@@ -21,6 +21,10 @@ type AlertSpec struct {
 	// +required
 	EventSources []CrossNamespaceObjectReference `json:"eventSources"`
 
+	// A list of golang regex expressions to be used for excluding meesages
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
+
 	// Short description of the impact and affected cluster.
 	// +optional
 	Summary string `json:"summary,omitempty"`
@@ -115,3 +119,28 @@ spec:
     - kind: HelmRelease
       name: nginx-ingress
 ```
+
+Skip alerting if the messages matches a golang regex from the exclusion list:
+
+```yaml
+apiVersion: notification.toolkit.fluxcd.io/v1beta1
+kind: Alert
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  providerRef: 
+    name: on-call-slack
+  eventSeverity: error
+  eventSources:
+    - kind: GitRepository
+      name: flux-system
+  exclusionList:
+    - "waiting.*socket"
+```
+
+The above definition will not send alerts for transient Git clone errors like:
+
+```
+unable to clone 'ssh://[email protected]/v3/...', error: SSH could not read data: Error waiting on socket
+```

internal/server/event_handlers.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
+	"regexp"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -72,6 +73,17 @@ func (s *EventServer) handleEvent() func(w http.ResponseWriter, r *http.Request)
 				continue
 			}
 
+			// skip alert if the message matches a regex from the exclusion list
+			if len(alert.Spec.ExclusionList) > 0 {
+				for _, exp := range alert.Spec.ExclusionList {
+					if r, err := regexp.Compile(exp); err == nil {
+						if r.Match([]byte(event.Message)) {
+							continue
+						}
+					}
+				}
+			}
+
 			// filter alerts by object and severity
 			for _, source := range alert.Spec.EventSources {
 				if source.Namespace == "" {