[ResponseOps][flapping] change action behavior when flapping

x-pack/plugins/alerting/common/alert_instance.ts

@@ -36,6 +36,7 @@ const metaSchema = t.partial({
   flappingHistory: t.array(t.boolean),
   // flapping flag that indicates whether the alert is flapping
   flapping: t.boolean,
+  pendingRecoveredCount: t.number,
 });
 export type AlertInstanceMeta = t.TypeOf<typeof metaSchema>;
 

x-pack/plugins/alerting/server/alert/alert.test.ts

@@ -414,11 +414,12 @@ describe('toJSON', () => {
           },
           flappingHistory: [false, true],
           flapping: false,
+          pendingRecoveredCount: 2,
         },
       }
     );
     expect(JSON.stringify(alertInstance)).toEqual(
-      '{"state":{"foo":true},"meta":{"lastScheduledActions":{"date":"1970-01-01T00:00:00.000Z","group":"default"},"flappingHistory":[false,true],"flapping":false}}'
+      '{"state":{"foo":true},"meta":{"lastScheduledActions":{"date":"1970-01-01T00:00:00.000Z","group":"default"},"flappingHistory":[false,true],"flapping":false,"pendingRecoveredCount":2}}'
     );
   });
 });
@@ -433,6 +434,7 @@ describe('toRaw', () => {
           group: 'default',
         },
         flappingHistory: [false, true, true],
+        pendingRecoveredCount: 2,
       },
     };
     const alertInstance = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>(
@@ -529,3 +531,43 @@ describe('getFlapping', () => {
     expect(alert.getFlapping()).toEqual(true);
   });
 });
+
+describe('incrementPendingRecoveredCount', () => {
+  test('correctly increments pendingRecoveredCount', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { pendingRecoveredCount: 3 },
+    });
+    alert.incrementPendingRecoveredCount();
+    expect(alert.getPendingRecoveredCount()).toEqual(4);
+  });
+
+  test('correctly increments pendingRecoveredCount when it is not already defined', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1');
+    alert.incrementPendingRecoveredCount();
+    expect(alert.getPendingRecoveredCount()).toEqual(1);
+  });
+});
+
+describe('getPendingRecoveredCount', () => {
+  test('returns pendingRecoveredCount', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { pendingRecoveredCount: 3 },
+    });
+    expect(alert.getPendingRecoveredCount()).toEqual(3);
+  });
+
+  test('defines and returns pendingRecoveredCount when it is not already defined', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1');
+    expect(alert.getPendingRecoveredCount()).toEqual(0);
+  });
+});
+
+describe('resetPendingRecoveredCount', () => {
+  test('resets pendingRecoveredCount to 0', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { pendingRecoveredCount: 3 },
+    });
+    alert.resetPendingRecoveredCount();
+    expect(alert.getPendingRecoveredCount()).toEqual(0);
+  });
+});

x-pack/plugins/alerting/server/alert/alert.ts

@@ -225,4 +225,19 @@ export class Alert<
   getFlapping() {
     return this.meta.flapping || false;
   }
+
+  incrementPendingRecoveredCount() {
+    if (!this.meta.pendingRecoveredCount) {
+      this.meta.pendingRecoveredCount = 0;
+    }
+    this.meta.pendingRecoveredCount++;
+  }
+
+  getPendingRecoveredCount() {
+    return this.meta.pendingRecoveredCount || 0;
+  }
+
+  resetPendingRecoveredCount() {
+    this.meta.pendingRecoveredCount = 0;
+  }
 }

x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts

@@ -17,6 +17,18 @@ jest.mock('../lib', () => ({
 
 let clock: sinon.SinonFakeTimers;
 const logger = loggingSystemMock.create().get();
+const alert1 = {
+  index: 1,
+  flappingHistory: [true, true, true, true],
+};
+const alert2 = {
+  index: 2,
+  flappingHistory: new Array(20).fill(false),
+};
+const alert3 = {
+  index: 3,
+  flappingHistory: [true, true],
+};
 
 describe('createAlertFactory()', () => {
   beforeAll(() => {
@@ -298,6 +310,34 @@ describe('createAlertFactory()', () => {
     alertFactory.alertLimit.setLimitReached(false);
     alertFactory.alertLimit.checkLimitUsage();
   });
+
+  test('trimRecovered should return longest recovered alerts', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 2,
+    });
+
+    const recoveredAlerts = [alert1, alert2, alert3];
+    const trimmedAlerts = alertFactory.alertLimit.trimRecovered(recoveredAlerts);
+    expect(trimmedAlerts).toEqual([alert2]);
+
+    expect(logger.warn).toBeCalledWith(
+      'Recovered alerts have exceeded the max alert limit: dropping 1 alert.'
+    );
+  });
+
+  test('trimRecovered should not return alerts if the num of recovered alerts is not at the limit', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 2,
+    });
+
+    const recoveredAlerts = [alert1, alert2];
+    const trimmedAlerts = alertFactory.alertLimit.trimRecovered(recoveredAlerts);
+    expect(trimmedAlerts).toEqual([]);
+  });
 });
 
 describe('getPublicAlertFactory', () => {
@@ -312,6 +352,7 @@ describe('getPublicAlertFactory', () => {
     expect(alertFactory.alertLimit.getValue).toBeDefined();
     expect(alertFactory.alertLimit.setLimitReached).toBeDefined();
     expect(alertFactory.alertLimit.checkLimitUsage).toBeDefined();
+    expect(alertFactory.alertLimit.trimRecovered).toBeDefined();
     expect(alertFactory.hasReachedAlertLimit).toBeDefined();
     expect(alertFactory.done).toBeDefined();
 
@@ -321,6 +362,7 @@ describe('getPublicAlertFactory', () => {
     expect(publicAlertFactory.done).toBeDefined();
     expect(publicAlertFactory.alertLimit.getValue).toBeDefined();
     expect(publicAlertFactory.alertLimit.setLimitReached).toBeDefined();
+    expect(publicAlertFactory.alertLimit.trimRecovered).toBeDefined();
 
     // @ts-expect-error
     expect(publicAlertFactory.alertLimit.checkLimitUsage).not.toBeDefined();

x-pack/plugins/alerting/server/alert/create_alert_factory.ts

@@ -11,6 +11,12 @@ import { AlertInstanceContext, AlertInstanceState } from '../types';
 import { Alert, PublicAlert } from './alert';
 import { processAlerts } from '../lib';
 
+export interface TrimRecoveredOpts {
+  index: number | string;
+  flappingHistory: boolean[];
+  trackedEvents?: boolean;
+}
+
 export interface AlertFactory<
   State extends AlertInstanceState,
   Context extends AlertInstanceContext,
@@ -21,6 +27,7 @@ export interface AlertFactory<
     getValue: () => number;
     setLimitReached: (reached: boolean) => void;
     checkLimitUsage: () => void;
+    trimRecovered: (recoveredAlerts: TrimRecoveredOpts[]) => TrimRecoveredOpts[];
   };
   hasReachedAlertLimit: () => boolean;
   done: () => AlertFactoryDoneUtils<State, Context, ActionGroupIds>;
@@ -33,7 +40,7 @@ export type PublicAlertFactory<
 > = Pick<AlertFactory<State, Context, ActionGroupIds>, 'create' | 'done'> & {
   alertLimit: Pick<
     AlertFactory<State, Context, ActionGroupIds>['alertLimit'],
-    'getValue' | 'setLimitReached'
+    'getValue' | 'setLimitReached' | 'trimRecovered'
   >;
 };
 
@@ -116,6 +123,22 @@ export function createAlertFactory<
           );
         }
       },
+      trimRecovered: (recoveredAlerts: TrimRecoveredOpts[]) => {
+        let earlyRecoveredAlerts: TrimRecoveredOpts[] = [];
+        if (recoveredAlerts.length > maxAlerts) {
+          recoveredAlerts.sort((a, b) => {
+            return a.flappingHistory.length - b.flappingHistory.length;
+          });
+
+          earlyRecoveredAlerts = recoveredAlerts.splice(maxAlerts * 1);
+          logger.warn(
+            `Recovered alerts have exceeded the max alert limit: dropping ${
+              earlyRecoveredAlerts.length
+            } ${earlyRecoveredAlerts.length > 1 ? 'alerts' : 'alert'}.`
+          );
+        }
+        return earlyRecoveredAlerts;
+      },
     },
     hasReachedAlertLimit: (): boolean => hasReachedAlertLimit,
     done: (): AlertFactoryDoneUtils<State, Context, ActionGroupIds> => {
@@ -164,6 +187,8 @@ export function getPublicAlertFactory<
     alertLimit: {
       getValue: (): number => alertFactory.alertLimit.getValue(),
       setLimitReached: (...args): void => alertFactory.alertLimit.setLimitReached(...args),
+      trimRecovered: (...args): TrimRecoveredOpts[] =>
+        alertFactory.alertLimit.trimRecovered(...args),
     },
     done: (): AlertFactoryDoneUtils<State, Context, ActionGroupIds> => alertFactory.done(),
   };

x-pack/plugins/alerting/server/lib/flapping_utils.ts

@@ -6,7 +6,7 @@
  */
 
 const MAX_CAPACITY = 20;
-const MAX_FLAP_COUNT = 4;
+export const MAX_FLAP_COUNT = 4;
 
 export function updateFlappingHistory(flappingHistory: boolean[], state: boolean) {
   const updatedFlappingHistory = flappingHistory.concat(state).slice(MAX_CAPACITY * -1);

x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getAlertsForNotification } from '.';
+import { Alert } from '../alert';
+
+describe('getAlertsForNotification', () => {
+  test('should set pendingRecoveredCount to zero for all active alerts', () => {
+    const alert1 = new Alert('1', { meta: { flapping: true, pendingRecoveredCount: 3 } });
+    const alert2 = new Alert('2', { meta: { flapping: false } });
+
+    const { newAlerts, activeAlerts } = getAlertsForNotification(
+      'default',
+      {
+        '1': alert1,
+      },
+      {
+        '1': alert1,
+        '2': alert2,
+      },
+      {},
+      {}
+    );
+    expect(newAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "flapping": true,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 0,
+          },
+          "state": Object {},
+        },
+      }
+    `);
+    expect(activeAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "flapping": true,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 0,
+          },
+          "state": Object {},
+        },
+        "2": Object {
+          "meta": Object {
+            "flapping": false,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 0,
+          },
+          "state": Object {},
+        },
+      }
+    `);
+  });
+
+  test('should return flapping pending recovered alerts as active alerts', () => {
+    const alert1 = new Alert('1', { meta: { flapping: true, pendingRecoveredCount: 3 } });
+    const alert2 = new Alert('2', { meta: { flapping: false } });
+    const alert3 = new Alert('3', { meta: { flapping: true } });
+
+    const { newAlerts, activeAlerts, recoveredAlerts, currentRecoveredAlerts } =
+      getAlertsForNotification(
+        'default',
+        {},
+        {},
+        {
+          '1': alert1,
+          '2': alert2,
+          '3': alert3,
+        },
+        {
+          '1': alert1,
+          '2': alert2,
+          '3': alert3,
+        }
+      );
+
+    expect(newAlerts).toMatchInlineSnapshot(`Object {}`);
+    expect(activeAlerts).toMatchInlineSnapshot(`
+      Object {
+        "3": Object {
+          "meta": Object {
+            "flapping": true,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 1,
+          },
+          "state": Object {},
+        },
+      }
+    `);
+    expect(Object.values(activeAlerts).map((a) => a.getScheduledActionOptions()))
+      .toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "actionGroup": "default",
+          "context": Object {},
+          "state": Object {},
+        },
+      ]
+    `);
+    expect(recoveredAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "flapping": true,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 0,
+          },
+          "state": Object {},
+        },
+        "2": Object {
+          "meta": Object {
+            "flapping": false,
+            "flappingHistory": Array [],
+          },
+          "state": Object {},
+        },
+      }
+    `);
+    expect(currentRecoveredAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "flapping": true,
+            "flappingHistory": Array [],
+            "pendingRecoveredCount": 0,
+          },
+          "state": Object {},
+        },
+        "2": Object {
+          "meta": Object {
+            "flapping": false,
+            "flappingHistory": Array [],
+          },
+          "state": Object {},
+        },
+      }
+    `);
+  });
+});