[Alerting] Show more user friendly ES error message when executor fails (#96254)

* WIP for ES error parser

* Fix tests

* Ensure the error shows up in the UI too

* wip

* Handle multiple types here

* Fix tests

* PR feedback

Co-authored-by: Kibana Machine <[email protected]>

Author: chrisronline

x-pack/plugins/alerting/server/index.ts

@@ -33,6 +33,7 @@ export { PluginSetupContract, PluginStartContract } from './plugin';
 export { FindResult } from './alerts_client';
 export { PublicAlertInstance as AlertInstance } from './alert_instance';
 export { parseDuration } from './lib';
+export { getEsErrorMessage } from './lib/errors';
 
 export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext);
 

x-pack/plugins/alerting/server/lib/alert_execution_status.ts

@@ -8,6 +8,7 @@
 import { Logger } from 'src/core/server';
 import { AlertTaskState, AlertExecutionStatus, RawAlertExecutionStatus } from '../types';
 import { getReasonFromError } from './error_with_reason';
+import { getEsErrorMessage } from './errors';
 
 export function executionStatusFromState(state: AlertTaskState): AlertExecutionStatus {
   const instanceIds = Object.keys(state.alertInstances ?? {});
@@ -23,7 +24,7 @@ export function executionStatusFromError(error: Error): AlertExecutionStatus {
     status: 'error',
     error: {
       reason: getReasonFromError(error),
-      message: error.message,
+      message: getEsErrorMessage(error),
     },
   };
 }

x-pack/plugins/alerting/server/lib/errors/es_error_parser.test.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEsErrorMessage } from './es_error_parser';
+
+describe('ES error parser', () => {
+  test('should return all the cause of the error', () => {
+    expect(
+      getEsErrorMessage({
+        name: '',
+        message: '',
+        error: {
+          meta: {
+            body: {
+              error: {
+                caused_by: {
+                  reason: 'reason1',
+                },
+              },
+            },
+          },
+        },
+      })
+    ).toStrictEqual(', caused by: "reason1"');
+
+    expect(
+      getEsErrorMessage({
+        name: '',
+        message: '',
+        meta: {
+          body: {
+            error: {
+              caused_by: {
+                reason: 'reason2',
+              },
+            },
+          },
+        },
+      })
+    ).toStrictEqual(', caused by: "reason2"');
+
+    expect(
+      getEsErrorMessage({
+        name: '',
+        message: '',
+        meta: {
+          body: {
+            error: {
+              caused_by: {
+                reason: 'reason3',
+                caused_by: {
+                  reason: 'reason4',
+                },
+              },
+            },
+          },
+        },
+      })
+    ).toStrictEqual(', caused by: "reason3,reason4"');
+
+    expect(
+      getEsErrorMessage({
+        name: '',
+        message: '',
+        meta: {
+          body: {
+            error: {
+              failed_shards: [
+                {
+                  reason: {
+                    caused_by: {
+                      reason: 'reason4',
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      })
+    ).toStrictEqual(', caused by: "reason4"');
+
+    expect(
+      getEsErrorMessage({
+        name: '',
+        message: '',
+        meta: {
+          body: {
+            error: {
+              failed_shards: [
+                {
+                  reason: {
+                    caused_by: {
+                      reason: 'reason5',
+                      caused_by: {
+                        reason: 'reason6',
+                      },
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      })
+    ).toStrictEqual(', caused by: "reason5,reason6"');
+  });
+});

x-pack/plugins/alerting/server/lib/errors/es_error_parser.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// import { ResponseError } from '@elastic/elasticsearch/lib/errors';
+import { ElasticsearchError, ElasticsearchErrorCausedByObject } from './types';
+
+const getEsCause = (
+  obj: ElasticsearchErrorCausedByObject = {},
+  causes: string[] = []
+): string[] => {
+  const updated = [...causes];
+
+  if (obj.caused_by) {
+    if (obj.caused_by?.reason) {
+      updated.push(obj.caused_by?.reason);
+    }
+
+    // Recursively find all the "caused by" reasons
+    return getEsCause(obj.caused_by, updated);
+  }
+
+  if (obj.failed_shards && obj.failed_shards.length) {
+    for (const failure of obj.failed_shards) {
+      if (failure && failure.reason) {
+        updated.push(...getEsCause(failure.reason));
+      }
+    }
+  }
+
+  return updated.filter(Boolean);
+};
+
+export const getEsErrorMessage = (error: ElasticsearchError) => {
+  let message = error?.message;
+  const apiError = error?.error?.meta?.body?.error ?? error?.meta?.body?.error;
+  if (apiError) {
+    message += `, caused by: "${getEsCause(apiError as ElasticsearchErrorCausedByObject)}"`;
+  }
+  return message;
+};

x-pack/plugins/alerting/server/lib/errors/index.ts

@@ -5,13 +5,14 @@
  * 2.0.
  */
 
-import { ErrorThatHandlesItsOwnResponse } from './types';
+import { ErrorThatHandlesItsOwnResponse, ElasticsearchError } from './types';
+import { getEsErrorMessage } from './es_error_parser';
 
 export function isErrorThatHandlesItsOwnResponse(
   e: ErrorThatHandlesItsOwnResponse
 ): e is ErrorThatHandlesItsOwnResponse {
   return typeof (e as ErrorThatHandlesItsOwnResponse).sendResponse === 'function';
 }
 
-export { ErrorThatHandlesItsOwnResponse };
+export { ErrorThatHandlesItsOwnResponse, ElasticsearchError, getEsErrorMessage };
 export { AlertTypeDisabledError, AlertTypeDisabledReason } from './alert_type_disabled';

x-pack/plugins/alerting/server/lib/errors/types.ts

@@ -4,9 +4,31 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 import { KibanaResponseFactory, IKibanaResponse } from '../../../../../../src/core/server';
 
 export interface ErrorThatHandlesItsOwnResponse extends Error {
   sendResponse(res: KibanaResponseFactory): IKibanaResponse;
 }
+
+export interface ElasticsearchErrorCausedByObject {
+  reason?: string;
+  caused_by?: ElasticsearchErrorCausedByObject;
+  failed_shards?: Array<{
+    reason?: {
+      caused_by?: ElasticsearchErrorCausedByObject;
+    };
+  }>;
+}
+
+interface ElasticsearchErrorMeta {
+  body?: {
+    error?: ElasticsearchErrorCausedByObject;
+  };
+}
+
+export interface ElasticsearchError extends Error {
+  error?: {
+    meta?: ElasticsearchErrorMeta;
+  };
+  meta?: ElasticsearchErrorMeta;
+}

x-pack/plugins/alerting/server/lib/index.ts

@@ -16,6 +16,7 @@ export {
   AlertTypeDisabledReason,
   ErrorThatHandlesItsOwnResponse,
   isErrorThatHandlesItsOwnResponse,
+  ElasticsearchError,
 } from './errors';
 export {
   executionStatusFromState,

x-pack/plugins/alerting/server/task_runner/task_runner.ts

@@ -20,6 +20,7 @@ import {
   executionStatusFromError,
   alertExecutionStatusToRaw,
   ErrorWithReason,
+  ElasticsearchError,
 } from '../lib';
 import {
   RawAlert,
@@ -49,6 +50,7 @@ import {
   WithoutReservedActionGroups,
 } from '../../common';
 import { NormalizedAlertType } from '../alert_type_registry';
+import { getEsErrorMessage } from '../lib/errors';
 
 const FALLBACK_RETRY_INTERVAL = '5m';
 
@@ -480,7 +482,7 @@ export class TaskRunner<
     const executionStatus: AlertExecutionStatus = map(
       state,
       (alertTaskState: AlertTaskState) => executionStatusFromState(alertTaskState),
-      (err: Error) => executionStatusFromError(err)
+      (err: ElasticsearchError) => executionStatusFromError(err)
     );
 
     // set the executionStatus date to same as event, if it's set
@@ -530,16 +532,18 @@ export class TaskRunner<
     }
 
     return {
-      state: map<AlertTaskState, Error, AlertTaskState>(
+      state: map<AlertTaskState, ElasticsearchError, AlertTaskState>(
         state,
         (stateUpdates: AlertTaskState) => {
           return {
             ...stateUpdates,
             previousStartedAt: startedAt,
           };
         },
-        (err: Error) => {
-          const message = `Executing Alert "${alertId}" has resulted in Error: ${err.message}`;
+        (err: ElasticsearchError) => {
+          const message = `Executing Alert "${alertId}" has resulted in Error: ${getEsErrorMessage(
+            err
+          )}`;
           if (isAlertSavedObjectNotFoundError(err, alertId)) {
             this.logger.debug(message);
           } else {

x-pack/plugins/triggers_actions_ui/server/data/lib/time_series_query.ts

@@ -7,6 +7,7 @@
 
 import type { estypes } from '@elastic/elasticsearch';
 import { Logger, ElasticsearchClient } from 'kibana/server';
+import { getEsErrorMessage } from '../../../../alerting/server';
 import { DEFAULT_GROUPS } from '../index';
 import { getDateRangeInfo } from './date_range_info';
 
@@ -137,7 +138,7 @@ export async function timeSeriesQuery(
     esResult = (await esClient.search(esQuery, { ignore: [404] })).body;
   } catch (err) {
     // console.log('time_series_query.ts error\n', JSON.stringify(err, null, 4));
-    logger.warn(`${logPrefix} error: ${err.message}`);
+    logger.warn(`${logPrefix} error: ${getEsErrorMessage(err)}`);
     return { results: [] };
   }