[APM] Syncs agent config settings to APM Fleet policies (#100744) (#1…

…01685) * [APM] Syncs agent config settings to APM Fleet policies (#95501) * fixes eslint issues * fixes malformed line comment * - consolidated logic that applies agent configurations to package policy objects - update package policy agent_configs to include etag, agent.name, and change settings -> config * Synchronizes agent configs whenever configuration is deleted. * PR feedback * nest agent_config within `apm-server` in the package policy input * nests agent_config under the requried 'value' property of config['apm-server'] in order to pass validation checks * - externalizes getApmPackagePolicies for reusability - parallelizes operations for improved performance Co-authored-by: Kibana Machine <[email protected]> Co-authored-by: Kibana Machine <[email protected]>
elastic · Jun 9, 2021 · 8a21bd5 · 8a21bd5
1 parent 7b6c599
commit 8a21bd5
Show file tree

Hide file tree

Showing 8 changed files with 286 additions and 17 deletions.
diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json
@@ -23,7 +23,8 @@
     "security",
     "ml",
     "home",
-    "maps"
+    "maps",
+    "fleet"
   ],
   "server": true,
   "ui": true,

diff --git a/x-pack/plugins/apm/server/lib/fleet/get_apm_package_policies.ts b/x-pack/plugins/apm/server/lib/fleet/get_apm_package_policies.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  CoreSetup,
+  CoreStart,
+  SavedObjectsClientContract,
+} from 'kibana/server';
+import { APMPluginStartDependencies } from '../../types';
+import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client';
+
+export async function getApmPackgePolicies({
+  core,
+  fleetPluginStart,
+}: {
+  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+}) {
+  // @ts-ignore
+  const savedObjectsClient: SavedObjectsClientContract = await getInternalSavedObjectsClient(
+    core.setup
+  );
+  return await fleetPluginStart.packagePolicyService.list(savedObjectsClient, {
+    kuery: 'ingest-package-policies.package.name:apm',
+  });
+}
diff --git a/x-pack/plugins/apm/server/lib/fleet/register_fleet_policy_callbacks.ts b/x-pack/plugins/apm/server/lib/fleet/register_fleet_policy_callbacks.ts
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { APMPlugin, APMRouteHandlerResources } from '../..';
+import { listConfigurations } from '../settings/agent_configuration/list_configurations';
+import { setupRequest } from '../helpers/setup_request';
+import { APMPluginStartDependencies } from '../../types';
+import { ExternalCallback } from '../../../../fleet/server';
+import { AGENT_NAME } from '../../../common/elasticsearch_fieldnames';
+import { AgentConfiguration } from '../../../common/agent_configuration/configuration_types';
+
+export async function registerFleetPolicyCallbacks({
+  plugins,
+  ruleDataClient,
+  config,
+  logger,
+}: {
+  plugins: APMRouteHandlerResources['plugins'];
+  ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
+  config: NonNullable<APMPlugin['currentConfig']>;
+  logger: NonNullable<APMPlugin['logger']>;
+}) {
+  if (!plugins.fleet) {
+    return;
+  }
+  const fleetPluginStart = await plugins.fleet.start();
+
+  // Registers a callback invoked when a policy is created to populate the APM
+  // integration policy with pre-existing agent configurations
+  registerAgentConfigExternalCallback({
+    fleetPluginStart,
+    callbackName: 'packagePolicyCreate',
+    plugins,
+    ruleDataClient,
+    config,
+    logger,
+  });
+
+  // Registers a callback invoked when a policy is updated to populate the APM
+  // integration policy with existing agent configurations
+  registerAgentConfigExternalCallback({
+    fleetPluginStart,
+    callbackName: 'packagePolicyUpdate',
+    plugins,
+    ruleDataClient,
+    config,
+    logger,
+  });
+}
+
+type ExternalCallbackParams = Parameters<ExternalCallback[1]>;
+type PackagePolicy = ExternalCallbackParams[0];
+type Context = ExternalCallbackParams[1];
+type Request = ExternalCallbackParams[2];
+
+function registerAgentConfigExternalCallback({
+  fleetPluginStart,
+  callbackName,
+  plugins,
+  ruleDataClient,
+  config,
+  logger,
+}: {
+  fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+  callbackName: ExternalCallback[0];
+  plugins: APMRouteHandlerResources['plugins'];
+  ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
+  config: NonNullable<APMPlugin['currentConfig']>;
+  logger: NonNullable<APMPlugin['logger']>;
+}) {
+  const callbackFn: ExternalCallback[1] = async (
+    packagePolicy: PackagePolicy,
+    context: Context,
+    request: Request
+  ) => {
+    if (packagePolicy.package?.name !== 'apm') {
+      return packagePolicy;
+    }
+    const setup = await setupRequest({
+      context: context as any,
+      params: { query: { _inspect: false } },
+      core: null as any,
+      plugins,
+      request,
+      config,
+      logger,
+      ruleDataClient,
+    });
+    const agentConfigurations = await listConfigurations({ setup });
+    return getPackagePolicyWithAgentConfigurations(
+      packagePolicy,
+      agentConfigurations
+    );
+  };
+
+  fleetPluginStart.registerExternalCallback(callbackName, callbackFn);
+}
+
+const APM_SERVER = 'apm-server';
+
+// Immutable function applies the given package policy with a set of agent configurations
+export function getPackagePolicyWithAgentConfigurations(
+  packagePolicy: PackagePolicy,
+  agentConfigurations: AgentConfiguration[]
+) {
+  const [firstInput, ...restInputs] = packagePolicy.inputs;
+  const apmServerValue = firstInput?.config?.[APM_SERVER].value;
+  return {
+    ...packagePolicy,
+    inputs: [
+      {
+        ...firstInput,
+        config: {
+          [APM_SERVER]: {
+            value: {
+              ...apmServerValue,
+              agent_config: agentConfigurations.map((configuration) => ({
+                service: configuration.service,
+                config: configuration.settings,
+                etag: configuration.etag,
+                [AGENT_NAME]: configuration.agent_name,
+              })),
+            },
+          },
+        },
+      },
+      ...restInputs,
+    ],
+  };
+}
diff --git a/x-pack/plugins/apm/server/lib/fleet/sync_agent_configs_to_apm_package_policies.ts b/x-pack/plugins/apm/server/lib/fleet/sync_agent_configs_to_apm_package_policies.ts
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  CoreSetup,
+  CoreStart,
+  SavedObjectsClientContract,
+} from 'kibana/server';
+import { APMPluginStartDependencies } from '../../types';
+import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client';
+import { Setup } from '../helpers/setup_request';
+import { listConfigurations } from '../settings/agent_configuration/list_configurations';
+import { getApmPackgePolicies } from './get_apm_package_policies';
+import { getPackagePolicyWithAgentConfigurations } from './register_fleet_policy_callbacks';
+
+export async function syncAgentConfigsToApmPackagePolicies({
+  core,
+  fleetPluginStart,
+  setup,
+}: {
+  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+  setup: Setup;
+}) {
+  const coreStart = await core.start();
+  const esClient = coreStart.elasticsearch.client.asInternalUser;
+  const [
+    savedObjectsClient,
+    agentConfigurations,
+    packagePolicies,
+  ] = await Promise.all([
+    getInternalSavedObjectsClient(core.setup),
+    listConfigurations({ setup }),
+    getApmPackgePolicies({
+      core,
+      fleetPluginStart,
+    }),
+  ]);
+
+  return Promise.all(
+    packagePolicies.items.map(async (item) => {
+      const { id, revision, updated_at, updated_by, ...packagePolicy } = item; // eslint-disable-line @typescript-eslint/naming-convention
+      const updatedPackagePolicy = getPackagePolicyWithAgentConfigurations(
+        packagePolicy,
+        agentConfigurations
+      );
+      return fleetPluginStart.packagePolicyService.update(
+        (savedObjectsClient as unknown) as SavedObjectsClientContract,
+        esClient,
+        id,
+        updatedPackagePolicy
+      );
+    })
+  );
+}
diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts
@@ -24,6 +24,7 @@ import { mergeConfigs } from './index';
 import { UI_SETTINGS } from '../../../../src/plugins/data/common';
 import { APM_FEATURE, registerFeaturesUsage } from './feature';
 import { registerApmAlerts } from './lib/alerts/register_apm_alerts';
+import { registerFleetPolicyCallbacks } from './lib/fleet/register_fleet_policy_callbacks';
 import { createApmTelemetry } from './lib/apm_telemetry';
 import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client';
 import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client';
@@ -186,6 +187,19 @@ export class APMPlugin
       ready,
     });
 
+    const resourcePlugins = mapValues(plugins, (value, key) => {
+      return {
+        setup: value,
+        start: () =>
+          core.getStartServices().then((services) => {
+            const [, pluginsStartContracts] = services;
+            return pluginsStartContracts[
+              key as keyof APMPluginStartDependencies
+            ];
+          }),
+      };
+    }) as APMRouteHandlerResources['plugins'];
+
     registerRoutes({
       core: {
         setup: core,
@@ -195,18 +209,7 @@ export class APMPlugin
       config: currentConfig,
       repository: getGlobalApmServerRouteRepository(),
       ruleDataClient,
-      plugins: mapValues(plugins, (value, key) => {
-        return {
-          setup: value,
-          start: () =>
-            core.getStartServices().then((services) => {
-              const [, pluginsStartContracts] = services;
-              return pluginsStartContracts[
-                key as keyof APMPluginStartDependencies
-              ];
-            }),
-        };
-      }) as APMRouteHandlerResources['plugins'],
+      plugins: resourcePlugins,
     });
 
     const boundGetApmIndices = async () =>
@@ -225,6 +228,13 @@ export class APMPlugin
       });
     }
 
+    registerFleetPolicyCallbacks({
+      plugins: resourcePlugins,
+      ruleDataClient,
+      config: this.currentConfig,
+      logger: this.logger,
+    });
+
     return {
       config$: mergedConfig$,
       getApmIndices: boundGetApmIndices,

diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration.ts
@@ -25,6 +25,7 @@ import {
 } from '../../../common/agent_configuration/runtime_types/agent_configuration_intake_rt';
 import { getSearchAggregatedTransactions } from '../../lib/helpers/aggregated_transactions';
 import { createApmServerRouteRepository } from '../create_apm_server_route_repository';
+import { syncAgentConfigsToApmPackagePolicies } from '../../lib/fleet/sync_agent_configs_to_apm_package_policies';
 
 // get list of configurations
 const agentConfigurationRoute = createApmServerRoute({
@@ -78,7 +79,7 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
   }),
   handler: async (resources) => {
     const setup = await setupRequest(resources);
-    const { params, logger } = resources;
+    const { params, logger, core } = resources;
 
     const { service } = params.body;
 
@@ -95,10 +96,23 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
       `Deleting config ${service.name}/${service.environment} (${config._id})`
     );
 
-    return await deleteConfiguration({
+    const deleteConfigurationResult = await deleteConfiguration({
       configurationId: config._id,
       setup,
     });
+
+    if (resources.plugins.fleet) {
+      await syncAgentConfigsToApmPackagePolicies({
+        core,
+        fleetPluginStart: await resources.plugins.fleet.start(),
+        setup,
+      });
+      logger.info(
+        `Updated Fleet integration policy for APM to remove the deleted agent configuration.`
+      );
+    }
+
+    return deleteConfigurationResult;
   },
 });
 
@@ -114,7 +128,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
   ]),
   handler: async (resources) => {
     const setup = await setupRequest(resources);
-    const { params, logger } = resources;
+    const { params, logger, core } = resources;
     const { body, query } = params;
 
     // if the config already exists, it is fetched and updated
@@ -142,6 +156,17 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
       configurationIntake: body,
       setup,
     });
+
+    if (resources.plugins.fleet) {
+      await syncAgentConfigsToApmPackagePolicies({
+        core,
+        fleetPluginStart: await resources.plugins.fleet.start(),
+        setup,
+      });
+      logger.info(
+        `Saved latest agent settings to Fleet integration policy for APM.`
+      );
+    }
   },
 });
 

diff --git a/x-pack/plugins/apm/server/types.ts b/x-pack/plugins/apm/server/types.ts
@@ -43,6 +43,10 @@ import {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '../../task_manager/server';
+import {
+  FleetSetupContract as FleetPluginSetup,
+  FleetStartContract as FleetPluginStart,
+} from '../../fleet/server';
 import { APMConfig } from '.';
 import { getApmIndices } from './lib/settings/apm_indices/get_apm_indices';
 import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client';
@@ -123,6 +127,10 @@ interface DependencyMap {
     setup: RuleRegistryPluginSetupContract;
     start: RuleRegistryPluginStartContract;
   };
+  fleet: {
+    setup: FleetPluginSetup;
+    start: FleetPluginStart;
+  };
 }
 
 const requiredDependencies = [
@@ -148,6 +156,7 @@ const optionalDependencies = [
   'ml',
   'home',
   'maps',
+  'fleet',
 ] as const;
 
 type RequiredDependencies = Pick<
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,7 +23,8 @@ @@
         "security",
         "ml",
         "home",
-        "maps"
+        "maps",
+        "fleet"
       ],
       "server": true,
       "ui": true,
@@ Expand Down @@