feat: notification fallback recipient

api/v1/notification_types.go

@@ -39,6 +39,13 @@ func (t *NotificationRecipientSpec) Empty() bool {
 	return t.Person == "" && t.Team == "" && t.Email == "" && t.Connection == "" && t.URL == "" && t.Playbook == nil
 }
 
+type NotificationFallback struct {
+	NotificationRecipientSpec `json:",inline" yaml:",inline"`
+
+	// wait this long before considering a send a failure
+	Delay string `json:"delay,omitempty" yaml:"delay,omitempty"`
+}
+
 // +kubebuilder:object:generate=true
 type NotificationSpec struct {
 	// List of events that can trigger this notification
@@ -56,13 +63,12 @@ type NotificationSpec struct {
 	// RepeatInterval is the waiting time to resend a notification after it has been succefully sent.
 	RepeatInterval string `json:"repeatInterval,omitempty" yaml:"repeatInterval,omitempty"`
 
-	// RepeatGroup allows notifications to be grouped by certain set of keys and only send
-	// one per group within the specified repeat interval.
-	// RepeatGroup []string `json:"repeatGroup,omitempty" yaml:"repeatGroup,omitempty"`
-
 	// Specify the recipient
 	To NotificationRecipientSpec `json:"to" yaml:"to"`
 
+	// In case of failure, send the notification to this recipient
+	Fallback *NotificationFallback `json:"fallback,omitempty" yaml:"fallback,omitempty"`
+
 	// WaitFor defines a duration to delay sending a health-based notification.
 	// After this period, the health status is reassessed to confirm it hasn't
 	// changed, helping prevent false alarms from transient issues.

config/crds/mission-control.flanksource.com_notifications.yaml

@@ -43,6 +43,41 @@ spec:
                 items:
                   type: string
                 type: array
+              fallback:
+                description: In case of failure, send the notification to this recipient
+                properties:
+                  connection:
+                    description: |-
+                      Specify connection string for an external service.
+                      Should be in the format of connection://<type>/name
+                      or the id of the connection.
+                    type: string
+                  delay:
+                    description: wait this long before considering a send a failure
+                    type: string
+                  email:
+                    description: Email of the recipient
+                    type: string
+                  person:
+                    description: ID or email of the person
+                    type: string
+                  playbook:
+                    description: |-
+                      Name or <namespace>/<name> of the playbook to run.
+                      When a playbook is set as the recipient, a run is triggered.
+                    type: string
+                  properties:
+                    additionalProperties:
+                      type: string
+                    description: Properties for Shoutrrr
+                    type: object
+                  team:
+                    description: name or ID of the recipient team
+                    type: string
+                  url:
+                    description: Specify shoutrrr URL
+                    type: string
+                type: object
               filter:
                 description: Cel-expression used to decide whether this notification
                   client should send the notification

config/schemas/notification.schema.json

@@ -56,6 +56,39 @@
       "additionalProperties": false,
       "type": "object"
     },
+    "NotificationFallback": {
+      "properties": {
+        "person": {
+          "type": "string"
+        },
+        "team": {
+          "type": "string"
+        },
+        "email": {
+          "type": "string"
+        },
+        "connection": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "properties": {
+          "additionalProperties": {
+            "type": "string"
+          },
+          "type": "object"
+        },
+        "playbook": {
+          "type": "string"
+        },
+        "delay": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
     "NotificationRecipientSpec": {
       "properties": {
         "person": {
@@ -109,6 +142,9 @@
         "to": {
           "$ref": "#/$defs/NotificationRecipientSpec"
         },
+        "fallback": {
+          "$ref": "#/$defs/NotificationFallback"
+        },
         "waitFor": {
           "type": "string"
         },

db/notifications.go

@@ -14,6 +14,7 @@ import (
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/query"
+	"github.com/flanksource/duty/types"
 	"github.com/flanksource/incident-commander/api"
 	v1 "github.com/flanksource/incident-commander/api/v1"
 	"github.com/google/uuid"
@@ -65,76 +66,108 @@ func PersistNotificationFromCRD(ctx context.Context, obj *v1.Notification) error
 	}
 
 	if len(obj.Spec.GroupBy) > 0 && obj.Spec.WaitFor != nil && *obj.Spec.WaitFor == "" {
-		return fmt.Errorf("groupBy provided with an empty waitFor. either remove the groupBy or set a waitFor period.")
+		return fmt.Errorf("groupBy provided with an empty waitFor. either remove the groupBy or set a waitFor period")
 	}
 
+	if recipient, err := resolveNotificationRecipient(ctx, obj.Spec.To); err != nil {
+		return fmt.Errorf("failed to resolve recipient: %w", err)
+	} else {
+		dbObj.PersonID = recipient.PersonID
+		dbObj.TeamID = recipient.TeamID
+		dbObj.PlaybookID = recipient.PlaybookID
+		dbObj.CustomServices = recipient.CustomServices
+	}
+
+	if obj.Spec.Fallback != nil {
+		if recipient, err := resolveNotificationRecipient(ctx, obj.Spec.Fallback.NotificationRecipientSpec); err != nil {
+			return fmt.Errorf("failed to resolve recipient: %w", err)
+		} else {
+			dbObj.FallbackPersonID = recipient.PersonID
+			dbObj.FallbackTeamID = recipient.TeamID
+			dbObj.FallbackPlaybookID = recipient.PlaybookID
+			dbObj.FallbackCustomServices = recipient.CustomServices
+		}
+	}
+
+	return ctx.DB().Save(&dbObj).Error
+}
+
+type notificationRecipient struct {
+	PersonID       *uuid.UUID
+	TeamID         *uuid.UUID
+	PlaybookID     *uuid.UUID
+	CustomServices types.JSON
+}
+
+func resolveNotificationRecipient(ctx context.Context, recipient v1.NotificationRecipientSpec) (*notificationRecipient, error) {
+	var result notificationRecipient
 	switch {
-	case obj.Spec.To.Person != "":
-		person, err := query.FindPerson(ctx, obj.Spec.To.Person)
+	case recipient.Person != "":
+		person, err := query.FindPerson(ctx, recipient.Person)
 		if err != nil {
-			return err
+			return nil, err
 		} else if person == nil {
-			return fmt.Errorf("person (%s) not found", obj.Spec.To.Person)
+			return nil, fmt.Errorf("person (%s) not found", recipient.Person)
 		}
 
-		dbObj.PersonID = &person.ID
+		result.PersonID = &person.ID
 
-	case obj.Spec.To.Team != "":
-		team, err := query.FindTeam(ctx, obj.Spec.To.Team)
+	case recipient.Team != "":
+		team, err := query.FindTeam(ctx, recipient.Team)
 		if err != nil {
-			return err
+			return nil, err
 		} else if team == nil {
-			return fmt.Errorf("team (%s) not found", obj.Spec.To.Team)
+			return nil, fmt.Errorf("team (%s) not found", recipient.Team)
 		}
 
-		dbObj.TeamID = &team.ID
+		result.TeamID = &team.ID
 
-	case lo.FromPtr(obj.Spec.To.Playbook) != "":
-		split := strings.Split(*obj.Spec.To.Playbook, "/")
+	case lo.FromPtr(recipient.Playbook) != "":
+		split := strings.Split(*recipient.Playbook, "/")
 		if len(split) == 1 {
 			name := split[0]
 			playbook, err := query.FindPlaybook(ctx, name)
 			if err != nil {
-				return err
+				return nil, err
 			} else if playbook == nil {
-				return fmt.Errorf("playbook (%s) not found", *obj.Spec.To.Playbook)
+				return nil, fmt.Errorf("playbook (%s) not found", *recipient.Playbook)
 			}
 
-			dbObj.PlaybookID = &playbook.ID
+			result.PlaybookID = &playbook.ID
 		} else if len(split) == 2 {
 			namespace := split[0]
 			name := split[1]
 
 			var playbook models.Playbook
 			if err := ctx.DB().Where("namespace = ?", namespace).Where("name = ?", name).Find(&playbook).Error; err != nil {
-				return err
+				return nil, err
 			} else if playbook.ID == uuid.Nil {
-				return fmt.Errorf("playbook %s not found", *obj.Spec.To.Playbook)
+				return nil, fmt.Errorf("playbook %s not found", *recipient.Playbook)
 			} else {
-				dbObj.PlaybookID = &playbook.ID
+				result.PlaybookID = &playbook.ID
 			}
 		}
 
 	default:
 		var customService api.NotificationConfig
 
-		if len(obj.Spec.To.Email) != 0 {
-			customService.URL = fmt.Sprintf("smtp://system/?To=%s", obj.Spec.To.Email)
-		} else if len(obj.Spec.To.Connection) != 0 {
-			customService.Connection = obj.Spec.To.Connection
-		} else if len(obj.Spec.To.URL) != 0 {
-			customService.URL = obj.Spec.To.URL
+		if len(recipient.Email) != 0 {
+			customService.URL = fmt.Sprintf("smtp://system/?To=%s", recipient.Email)
+		} else if len(recipient.Connection) != 0 {
+			customService.Connection = recipient.Connection
+		} else if len(recipient.URL) != 0 {
+			customService.URL = recipient.URL
 		}
 
 		customServices, err := json.Marshal([]api.NotificationConfig{customService})
 		if err != nil {
-			return err
+			return nil, err
 		}
 
-		dbObj.CustomServices = customServices
+		result.CustomServices = customServices
 	}
 
-	return ctx.DB().Save(&dbObj).Error
+	return &result, nil
 }
 
 func DeleteNotification(ctx context.Context, id string) error {

fixtures/notifications/health-playbook.yaml

@@ -9,4 +9,6 @@ spec:
     - config.unhealthy
     - config.warning
   to:
-    playbook: mc/echo-config
+    playbook: mc/diagnose-configs
+  fallback:
+    connection: connection://mc/slack

go.mod

@@ -368,7 +368,7 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
-// replace github.com/flanksource/duty => ../duty
+replace github.com/flanksource/duty => ../duty
 
 // replace github.com/flanksource/gomplate/v3 => ../gomplat3
 

go.sum

@@ -287,8 +287,6 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z
 github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
 github.com/flanksource/commons v1.36.1 h1:SDppXOYO6vk06d12SPOYVZJX+8gV72FmK1l9ar5fkjc=
 github.com/flanksource/commons v1.36.1/go.mod h1:nsYCn6JOhENXNLR5pz4zNco70DQV+bGObQ8NbOrUeuQ=
-github.com/flanksource/duty v1.0.862 h1:VsqkaEKjxezMordMu3YZNd64sWH0/GWvE4uD98ScPWY=
-github.com/flanksource/duty v1.0.862/go.mod h1:5NmEw4b/vz08vXNR4WPuHWNtD6YB2pGu2RrX1HrDT8E=
 github.com/flanksource/gomplate/v3 v3.24.55 h1:BW+KeMcggCkNawb4VTbY+Zn3s9eYIhwqb5/myiyJRb8=
 github.com/flanksource/gomplate/v3 v3.24.55/go.mod h1:hGEObNtnOQs8rNUX8sM8aJTAhnt4ehjyOw1MvDhl6AU=
 github.com/flanksource/is-healthy v1.0.62 h1:wa4Eq3YR1+Ku3UhKlVpos0CqieGeWWVF5gZeOsmDAIg=

jobs/jobs.go

@@ -10,6 +10,7 @@ import (
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/job"
 	"github.com/flanksource/duty/query"
+	"github.com/flanksource/duty/shutdown"
 	"github.com/flanksource/incident-commander/api"
 	"github.com/flanksource/incident-commander/incidents"
 	"github.com/flanksource/incident-commander/notification"
@@ -67,6 +68,10 @@ func Start(ctx context.Context) {
 		logger.Errorf("Failed to schedule job for cleaning up event queue table: %v", err)
 	}
 
+	if err := notification.ProcessFallbackCheckNotificationsJob(ctx).AddToScheduler(FuncScheduler); err != nil {
+		shutdown.ShutdownAndExit(1, fmt.Sprintf("failed to schedule job ProcessFallbackCheckNotificationsJob: %v", err))
+	}
+
 	if err := notification.ProcessPendingNotificationsJob(ctx).AddToScheduler(FuncScheduler); err != nil {
 		logger.Errorf("failed to schedule job: %v", err)
 	}