[Fleet] Support user overrides in composable templates

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { estypes } from '@elastic/elasticsearch';
 // Follow pattern from https://github.com/elastic/kibana/pull/52447
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import type { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
@@ -301,8 +302,8 @@ export interface RegistryDataStream {
 }
 
 export interface RegistryElasticsearch {
-  'index_template.settings'?: object;
-  'index_template.mappings'?: object;
+  'index_template.settings'?: estypes.IndicesIndexSettings;
+  'index_template.mappings'?: estypes.MappingTypeMapping;
 }
 
 export interface RegistryDataStreamPermissions {
@@ -431,7 +432,7 @@ export interface IndexTemplate {
   _meta: object;
 }
 
-export interface TemplateRef {
+export interface IndexTemplateEntry {
   templateName: string;
   indexTemplate: IndexTemplate;
 }

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -11,15 +11,15 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s
 import { ElasticsearchAssetType } from '../../../../types';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   RegistryElasticsearch,
   InstallablePackage,
 } from '../../../../types';
 import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
-import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install';
+import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install';
 
 import {
   generateMappings,
@@ -34,52 +34,44 @@ export const installTemplates = async (
   esClient: ElasticsearchClient,
   paths: string[],
   savedObjectsClient: SavedObjectsClientContract
-): Promise<TemplateRef[]> => {
+): Promise<IndexTemplateEntry[]> => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global index templates
   // Install component templates first, as they are used by the index templates
   await installPreBuiltComponentTemplates(paths, esClient);
   await installPreBuiltTemplates(paths, esClient);
 
   // remove package installation's references to index templates
-  await removeAssetsFromInstalledEsByType(
-    savedObjectsClient,
-    installablePackage.name,
-    ElasticsearchAssetType.indexTemplate
-  );
+  await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [
+    ElasticsearchAssetType.indexTemplate,
+    ElasticsearchAssetType.componentTemplate,
+  ]);
   // build templates per data stream from yml files
   const dataStreams = installablePackage.data_streams;
   if (!dataStreams) return [];
+
+  const installedTemplatesNested = await Promise.all(
+    dataStreams.map((dataStream) =>
+      installTemplateForDataStream({
+        pkg: installablePackage,
+        esClient,
+        dataStream,
+      })
+    )
+  );
+  const installedTemplates = installedTemplatesNested.flat();
+
   // get template refs to save
-  const installedTemplateRefs = dataStreams.map((dataStream) => ({
-    id: generateTemplateName(dataStream),
-    type: ElasticsearchAssetType.indexTemplate,
-  }));
+  const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates);
 
   // add package installation's references to index templates
-  await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs);
-
-  if (dataStreams) {
-    const installTemplatePromises = dataStreams.reduce<Array<Promise<TemplateRef>>>(
-      (acc, dataStream) => {
-        acc.push(
-          installTemplateForDataStream({
-            pkg: installablePackage,
-            esClient,
-            dataStream,
-          })
-        );
-        return acc;
-      },
-      []
-    );
-
-    const res = await Promise.all(installTemplatePromises);
-    const installedTemplates = res.flat();
+  await saveInstalledEsRefs(
+    savedObjectsClient,
+    installablePackage.name,
+    installedIndexTemplateRefs
+  );
 
-    return installedTemplates;
-  }
-  return [];
+  return installedTemplates;
 };
 
 const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => {
@@ -160,7 +152,7 @@ export async function installTemplateForDataStream({
   pkg: InstallablePackage;
   esClient: ElasticsearchClient;
   dataStream: RegistryDataStream;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const fields = await loadFieldsFromYaml(pkg, dataStream.path);
   return installTemplate({
     esClient,
@@ -171,84 +163,109 @@ export async function installTemplateForDataStream({
   });
 }
 
+interface TemplateMapEntry {
+  template:
+    | {
+        mappings: NonNullable<RegistryElasticsearch['index_template.mappings']>;
+      }
+    | {
+        settings: NonNullable<RegistryElasticsearch['index_template.settings']> | object;
+      };
+}
+type TemplateMap = Record<string, TemplateMapEntry>;
 function putComponentTemplate(
-  body: object | undefined,
-  name: string,
-  esClient: ElasticsearchClient
-): { clusterPromise: Promise<any>; name: string } | undefined {
-  if (body) {
-    const esClientParams = {
-      name,
-      body,
-    };
-
-    return {
-      // @ts-expect-error body expected to be ClusterPutComponentTemplateRequest
-      clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }),
-      name,
-    };
+  esClient: ElasticsearchClient,
+  params: {
+    body: TemplateMapEntry;
+    name: string;
+    create?: boolean;
   }
+): { clusterPromise: Promise<any>; name: string } {
+  const { name, body, create = false } = params;
+  return {
+    clusterPromise: esClient.cluster.putComponentTemplate(
+      // @ts-expect-error body is missing required key `settings`. TemplateMapEntry has settings *or* mappings
+      { name, body, create },
+      { ignore: [404] }
+    ),
+    name,
+  };
 }
 
-function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
-  let mappingsTemplate;
-  let settingsTemplate;
+const mappingsSuffix = '-mappings';
+const settingsSuffix = '-settings';
+const userSettingsSuffix = '-user_settings';
+type TemplateBaseName = string;
+type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`;
+
+const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
+  name.endsWith(userSettingsSuffix);
+
+function buildComponentTemplates(
+  templateName: string,
+  registryElasticsearch: RegistryElasticsearch | undefined
+) {
+  const mappingsTemplateName = `${templateName}${mappingsSuffix}`;
+  const settingsTemplateName = `${templateName}${settingsSuffix}`;
+  const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`;
+
+  const templatesMap: TemplateMap = {};
 
   if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
-    mappingsTemplate = {
+    templatesMap[mappingsTemplateName] = {
       template: {
-        mappings: {
-          ...registryElasticsearch['index_template.mappings'],
-        },
+        mappings: registryElasticsearch['index_template.mappings'],
       },
     };
   }
 
   if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
-    settingsTemplate = {
+    templatesMap[settingsTemplateName] = {
       template: {
         settings: registryElasticsearch['index_template.settings'],
       },
     };
+
+    // return empty/stub template
+    templatesMap[userSettingsTemplateName] = {
+      template: {
+        settings: {},
+      },
+    };
   }
-  return { settingsTemplate, mappingsTemplate };
+
+  return templatesMap;
 }
 
 async function installDataStreamComponentTemplates(
   templateName: string,
   registryElasticsearch: RegistryElasticsearch | undefined,
   esClient: ElasticsearchClient
 ) {
-  const templates: string[] = [];
-  const componentPromises: Array<Promise<any>> = [];
-
-  const compTemplates = buildComponentTemplates(registryElasticsearch);
-
-  const mappings = putComponentTemplate(
-    compTemplates.mappingsTemplate,
-    `${templateName}-mappings`,
-    esClient
-  );
+  const templates = buildComponentTemplates(templateName, registryElasticsearch);
+  const templateNames = Object.keys(templates);
+  const templateEntries = Object.entries(templates);
 
-  const settings = putComponentTemplate(
-    compTemplates.settingsTemplate,
-    `${templateName}-settings`,
-    esClient
+  // TODO: Check return values for errors
+  await Promise.all(
+    templateEntries.map(async ([name, body]) => {
+      if (isUserSettingsTemplate(name)) {
+        // look for existing user_settings template
+        const result = await esClient.cluster.getComponentTemplate({ name }, { ignore: [404] });
+        const hasUserSettingsTemplate = result.body.component_templates?.length === 1;
+        if (!hasUserSettingsTemplate) {
+          // only add if one isn't already present
+          const { clusterPromise } = putComponentTemplate(esClient, { body, name, create: true });
+          return clusterPromise;
+        }
+      } else {
+        const { clusterPromise } = putComponentTemplate(esClient, { body, name });
+        return clusterPromise;
+      }
+    })
   );
 
-  if (mappings) {
-    templates.push(mappings.name);
-    componentPromises.push(mappings.clusterPromise);
-  }
-
-  if (settings) {
-    templates.push(settings.name);
-    componentPromises.push(settings.clusterPromise);
-  }
-
-  // TODO: Check return values for errors
-  await Promise.all(componentPromises);
-  return templates;
+  return templateNames;
 }
 
 export async function installTemplate({
@@ -263,7 +280,7 @@ export async function installTemplate({
   dataStream: RegistryDataStream;
   packageVersion: string;
   packageName: string;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const validFields = processFields(fields);
   const mappings = generateMappings(validFields);
   const templateName = generateTemplateName(dataStream);
@@ -342,3 +359,21 @@ export async function installTemplate({
     indexTemplate: template,
   };
 }
+
+export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
+  return installedTemplates.flatMap((installedTemplate) => {
+    const indexTemplates = [
+      {
+        id: installedTemplate.templateName,
+        type: ElasticsearchAssetType.indexTemplate,
+      },
+    ];
+    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
+      (componentTemplateId) => ({
+        id: componentTemplateId,
+        type: ElasticsearchAssetType.componentTemplate,
+      })
+    );
+    return indexTemplates.concat(componentTemplates);
+  });
+}

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -10,7 +10,7 @@ import type { ElasticsearchClient } from 'kibana/server';
 import type { Field, Fields } from '../../fields/field';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   IndexTemplate,
   IndexTemplateMappings,
 } from '../../../../types';
@@ -456,7 +456,7 @@ function getBaseTemplate(
 
 export const updateCurrentWriteIndices = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<void> => {
   if (!templates.length) return;
 
@@ -471,7 +471,7 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur
 
 const queryDataStreamsFromTemplates = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<CurrentDataStream[]> => {
   const dataStreamPromises = templates.map((template) => {
     return getDataStreams(esClient, template);
@@ -482,7 +482,7 @@ const queryDataStreamsFromTemplates = async (
 
 const getDataStreams = async (
   esClient: ElasticsearchClient,
-  template: TemplateRef
+  template: IndexTemplateEntry
 ): Promise<CurrentDataStream[] | undefined> => {
   const { templateName, indexTemplate } = template;
   const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` });

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -10,10 +10,10 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } fro
 import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common';
 import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
-import { ElasticsearchAssetType } from '../../../types';
 import type { AssetReference, Installation, InstallType } from '../../../types';
 import { installTemplates } from '../elasticsearch/template/install';
 import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/';
+import { getAllTemplateRefs } from '../elasticsearch/template/install';
 import { installILMPolicy } from '../elasticsearch/ilm/install';
 import { installKibanaAssets, getKibanaAssets } from '../kibana/assets/install';
 import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
@@ -170,10 +170,7 @@ export async function _installPackage({
         installedPkg.attributes.install_version
       );
     }
-    const installedTemplateRefs = installedTemplates.map((template) => ({
-      id: template.templateName,
-      type: ElasticsearchAssetType.indexTemplate,
-    }));
+    const installedTemplateRefs = getAllTemplateRefs(installedTemplates);
 
     // make sure the assets are installed (or didn't error)
     if (installKibanaAssetsError) throw installKibanaAssetsError;