diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index dab8f3c0bf82e..527b93a66d130 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -223,7 +223,7 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX); -function buildComponentTemplates(params: { +export function buildComponentTemplates(params: { mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index ab8f60e172dcb..8c908ecc9ef87 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -7,65 +7,71 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; +import { safeLoad } from 'js-yaml'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { + PACKAGE_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, +} from '../../../../../common/constants'; +import { + buildComponentTemplates, + installComponentAndIndexTemplateForDataStream, +} from '../template/install'; +import { processFields } from '../../fields/field'; +import { generateMappings } from '../template/template'; +import { getESAssetMetadata } from '../meta'; import { updateEsAssetReferences } from '../../packages/install'; import { getPathParts } from '../../archive'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; -import type { EsAssetReference, InstallablePackage } from '../../../../../common/types/models'; +import type { + EsAssetReference, + InstallablePackage, + ESAssetMetadata, + IndexTemplate, + RegistryElasticsearch, +} from '../../../../../common/types/models'; import { getInstallation } from '../../packages'; - -import { getESAssetMetadata } from '../meta'; - import { retryTransientEsErrors } from '../retry'; import { deleteTransforms } from './remove'; import { getAsset } from './common'; -interface TransformInstallation { +const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250; +enum TRANSFORM_SPECS_TYPES { + MANIFEST = 'manifest', + FIELDS = 'fields', + TRANSFORM = 'transform', +} + +interface TransformModuleBase { + transformModuleId?: string; +} +interface DestinationIndexTemplateInstallation extends TransformModuleBase { + installationName: string; + _meta: ESAssetMetadata; + template: IndexTemplate['template']; +} +interface TransformInstallation extends TransformModuleBase { installationName: string; content: any; } -export const installTransform = async ( +const installLegacyTransformsAssets = async ( installablePackage: InstallablePackage, - paths: string[], + installNameSuffix: string, + transformPaths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, logger: Logger, - esReferences?: EsAssetReference[] + esReferences: EsAssetReference[] = [], + previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { - const installation = await getInstallation({ - savedObjectsClient, - pkgName: installablePackage.name, - }); - esReferences = esReferences ?? installation?.installed_es ?? []; - let previousInstalledTransformEsAssets: EsAssetReference[] = []; - if (installation) { - previousInstalledTransformEsAssets = installation.installed_es.filter( - ({ type, id }) => type === ElasticsearchAssetType.transform - ); - if (previousInstalledTransformEsAssets.length) { - logger.info( - `Found previous transform references:\n ${JSON.stringify( - previousInstalledTransformEsAssets - )}` - ); - } - } - - // delete all previous transform - await deleteTransforms( - esClient, - previousInstalledTransformEsAssets.map((asset) => asset.id) - ); - - const installNameSuffix = `${installablePackage.version}`; - const transformPaths = paths.filter((path) => isTransform(path)); let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const transformRefs = transformPaths.reduce((acc, path) => { acc.push({ - id: getTransformNameForInstallation(installablePackage, path, installNameSuffix), + id: getLegacyTransformNameForInstallation(installablePackage, path, installNameSuffix), type: ElasticsearchAssetType.transform, }); @@ -87,7 +93,7 @@ export const installTransform = async ( content._meta = getESAssetMetadata({ packageName: installablePackage.name }); return { - installationName: getTransformNameForInstallation( + installationName: getLegacyTransformNameForInstallation( installablePackage, path, installNameSuffix @@ -117,6 +123,289 @@ export const installTransform = async ( return { installedTransforms, esReferences }; }; +const processTransformAssetsPerModule = ( + installablePackage: InstallablePackage, + installNameSuffix: string, + transformPaths: string[] +) => { + const transformsSpecifications = new Map(); + const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = []; + const transforms: TransformInstallation[] = []; + + transformPaths.forEach((path: string) => { + const { transformModuleId, fileName } = getTransformFolderAndFileNames( + installablePackage, + path + ); + + // Since there can be multiple assets per transform definition + // We want to create a unique list of assets/specifications for each transform + if (transformsSpecifications.get(transformModuleId) === undefined) { + transformsSpecifications.set(transformModuleId, new Map()); + } + const packageAssets = transformsSpecifications.get(transformModuleId); + + const content = safeLoad(getAsset(path).toString('utf-8')); + + if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) { + const validFields = processFields(content); + const mappings = generateMappings(validFields); + packageAssets?.set('mappings', mappings); + } + + if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) { + transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest); + transformsSpecifications.get(transformModuleId)?.set('transform', content); + content._meta = getESAssetMetadata({ packageName: installablePackage.name }); + transforms.push({ + transformModuleId, + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + `default-${installNameSuffix}` + ), + content, + }); + } + + if (fileName === TRANSFORM_SPECS_TYPES.MANIFEST) { + if (isPopulatedObject(content, ['start']) && content.start === false) { + transformsSpecifications.get(transformModuleId)?.set('start', false); + } + // If manifest.yml contains destination_index_template + // Combine the mappings and other index template settings from manifest.yml into a single index template + // Create the index template and track the template in EsAssetReferences + if ( + isPopulatedObject(content, ['destination_index_template']) || + isPopulatedObject(packageAssets.get('mappings')) + ) { + const destinationIndexTemplate = + (content.destination_index_template as Record) ?? {}; + destinationIndexTemplates.push({ + transformModuleId, + _meta: getESAssetMetadata({ packageName: installablePackage.name }), + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ), + template: destinationIndexTemplate, + } as DestinationIndexTemplateInstallation); + packageAssets.set('destinationIndexTemplate', destinationIndexTemplate); + } + } + }); + + const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({ + id: template.installationName, + type: ElasticsearchAssetType.indexTemplate, + })); + const componentTemplatesRefs = [ + ...destinationIndexTemplates.map((template) => ({ + id: `${template.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`, + type: ElasticsearchAssetType.componentTemplate, + })), + ...destinationIndexTemplates.map((template) => ({ + id: `${template.installationName}${PACKAGE_TEMPLATE_SUFFIX}`, + type: ElasticsearchAssetType.componentTemplate, + })), + ]; + + const transformRefs = transforms.map((t) => ({ + id: t.installationName, + type: ElasticsearchAssetType.transform, + })); + + return { + indexTemplatesRefs, + componentTemplatesRefs, + transformRefs, + transforms, + destinationIndexTemplates, + transformsSpecifications, + }; +}; + +const installTransformsAssets = async ( + installablePackage: InstallablePackage, + installNameSuffix: string, + transformPaths: string[], + esClient: ElasticsearchClient, + savedObjectsClient: SavedObjectsClientContract, + logger: Logger, + esReferences: EsAssetReference[] = [], + previousInstalledTransformEsAssets: EsAssetReference[] = [] +) => { + let installedTransforms: EsAssetReference[] = []; + if (transformPaths.length > 0) { + const { + indexTemplatesRefs, + componentTemplatesRefs, + transformRefs, + transforms, + destinationIndexTemplates, + transformsSpecifications, + } = processTransformAssetsPerModule(installablePackage, installNameSuffix, transformPaths); + // get and save refs associated with the transforms before installing + esReferences = await updateEsAssetReferences( + savedObjectsClient, + installablePackage.name, + esReferences, + { + assetsToAdd: [...indexTemplatesRefs, ...componentTemplatesRefs, ...transformRefs], + assetsToRemove: previousInstalledTransformEsAssets, + } + ); + + // create index templates and component templates + await Promise.all( + destinationIndexTemplates + .map((destinationIndexTemplate) => { + const customMappings = transformsSpecifications + .get(destinationIndexTemplate.transformModuleId) + ?.get('mappings'); + const registryElasticsearch: RegistryElasticsearch = { + 'index_template.settings': destinationIndexTemplate.template.settings, + 'index_template.mappings': destinationIndexTemplate.template.mappings, + }; + + const componentTemplates = buildComponentTemplates({ + mappings: customMappings, + templateName: destinationIndexTemplate.installationName, + registryElasticsearch, + packageName: installablePackage.name, + defaultSettings: {}, + }); + + if (destinationIndexTemplate || customMappings) { + return installComponentAndIndexTemplateForDataStream({ + esClient, + logger, + componentTemplates, + indexTemplate: { + templateName: destinationIndexTemplate.installationName, + // @ts-expect-error We don't need to pass data_stream property here + // as this template is applied to only an index and not a data stream + indexTemplate: { + template: { settings: undefined, mappings: undefined }, + priority: DEFAULT_TRANSFORM_TEMPLATES_PRIORITY, + index_patterns: [ + transformsSpecifications + .get(destinationIndexTemplate.transformModuleId) + ?.get('destinationIndex').index, + ], + _meta: destinationIndexTemplate._meta, + composed_of: Object.keys(componentTemplates), + }, + }, + }); + } + }) + .filter((p) => p !== undefined) + ); + + // create destination indices + await Promise.all( + transforms.map(async (transform) => { + const index = transform.content.dest.index; + const pipelineId = transform.content.dest.pipeline; + + try { + await retryTransientEsErrors( + () => + esClient.indices.create( + { + index, + ...(pipelineId ? { settings: { default_pipeline: pipelineId } } : {}), + }, + { ignore: [400] } + ), + { logger } + ); + } catch (err) { + throw new Error(err.message); + } + }) + ); + + // create & optionally start transforms + const transformsPromises = transforms.map(async (transform) => { + return handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); + }); + + installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + } + + return { installedTransforms, esReferences }; +}; +export const installTransforms = async ( + installablePackage: InstallablePackage, + paths: string[], + esClient: ElasticsearchClient, + savedObjectsClient: SavedObjectsClientContract, + logger: Logger, + esReferences?: EsAssetReference[] +) => { + const transformPaths = paths.filter((path) => isTransform(path)); + + const installation = await getInstallation({ + savedObjectsClient, + pkgName: installablePackage.name, + }); + esReferences = esReferences ?? installation?.installed_es ?? []; + let previousInstalledTransformEsAssets: EsAssetReference[] = []; + if (installation) { + previousInstalledTransformEsAssets = installation.installed_es.filter( + ({ type, id }) => type === ElasticsearchAssetType.transform + ); + if (previousInstalledTransformEsAssets.length) { + logger.debug( + `Found previous transform references:\n ${JSON.stringify( + previousInstalledTransformEsAssets + )}` + ); + } + } + + // delete all previous transform + await deleteTransforms( + esClient, + previousInstalledTransformEsAssets.map((asset) => asset.id) + ); + + const installNameSuffix = `${installablePackage.version}`; + + // If package contains legacy transform specifications (i.e. with json instead of yml) + if (transformPaths.some((p) => p.endsWith('.json')) || transformPaths.length === 0) { + return await installLegacyTransformsAssets( + installablePackage, + installNameSuffix, + transformPaths, + esClient, + savedObjectsClient, + logger, + esReferences, + previousInstalledTransformEsAssets + ); + } + + return await installTransformsAssets( + installablePackage, + installNameSuffix, + transformPaths, + esClient, + savedObjectsClient, + logger, + esReferences, + previousInstalledTransformEsAssets + ); +}; + export const isTransform = (path: string) => { const pathParts = getPathParts(path); return !path.endsWith('/') && pathParts.type === ElasticsearchAssetType.transform; @@ -126,10 +415,12 @@ async function handleTransformInstall({ esClient, logger, transform, + startTransform, }: { esClient: ElasticsearchClient; logger: Logger; transform: TransformInstallation; + startTransform?: boolean; }): Promise { try { await retryTransientEsErrors( @@ -151,15 +442,21 @@ async function handleTransformInstall({ throw err; } } - await esClient.transform.startTransform( - { transform_id: transform.installationName }, - { ignore: [409] } - ); + + // start transform by default if not set in yml file + // else, respect the setting + if (startTransform === undefined || startTransform === true) { + await esClient.transform.startTransform( + { transform_id: transform.installationName }, + { ignore: [409] } + ); + logger.debug(`Started transform: ${transform.installationName}`); + } return { id: transform.installationName, type: ElasticsearchAssetType.transform }; } -const getTransformNameForInstallation = ( +const getLegacyTransformNameForInstallation = ( installablePackage: InstallablePackage, path: string, suffix: string @@ -169,3 +466,24 @@ const getTransformNameForInstallation = ( const folderName = pathPaths?.pop(); return `${installablePackage.name}.${folderName}-${filename}-${suffix}`; }; + +const getTransformAssetNameForInstallation = ( + installablePackage: InstallablePackage, + transformModuleId: string, + suffix?: string +) => { + return `logs-${installablePackage.name}.${transformModuleId}${suffix ? '-' + suffix : ''}`; +}; + +const getTransformFolderAndFileNames = (installablePackage: InstallablePackage, path: string) => { + const pathPaths = path.split('/'); + const fileName = pathPaths?.pop()?.split('.')[0]; + let transformModuleId = pathPaths?.pop(); + + // If fields.yml is located inside a directory called 'fields' (e.g. {exampleFolder}/fields/fields.yml) + // We need to go one level up to get the real folder name + if (transformModuleId === 'fields') { + transformModuleId = pathPaths?.pop(); + } + return { fileName: fileName ?? '', transformModuleId: transformModuleId ?? '' }; +}; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 1b8c347ad07d5..97fa1e94ca218 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -35,7 +35,7 @@ import { getESAssetMetadata } from '../meta'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; import { getAsset } from './common'; -import { installTransform } from './install'; +import { installTransforms } from './install'; describe('test transform install', () => { let esClient: ReturnType; @@ -122,7 +122,7 @@ describe('test transform install', () => { ], }); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -320,7 +320,7 @@ describe('test transform install', () => { } as unknown as SavedObject) ); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -422,7 +422,7 @@ describe('test transform install', () => { ], }); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -556,7 +556,7 @@ describe('test transform install', () => { ) ); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 1dc4039a7c2a7..4f98776f53d60 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -128,7 +128,7 @@ function getTest( test = { method: mocks.packageClient.reinstallEsAssets.bind(mocks.packageClient), args: [pkg, paths], - spy: jest.spyOn(epmTransformsInstall, 'installTransform'), + spy: jest.spyOn(epmTransformsInstall, 'installTransforms'), spyArgs: [pkg, paths, mocks.esClient, mocks.soClient, mocks.logger], spyResponse: { installedTransforms: [ diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index c43386acdbdf4..71ab87c7d92bd 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -24,7 +24,7 @@ import type { import { checkSuperuser } from '../../routes/security'; import { FleetUnauthorizedError } from '../../errors'; -import { installTransform, isTransform } from './elasticsearch/transform/install'; +import { installTransforms, isTransform } from './elasticsearch/transform/install'; import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry'; import { ensureInstalledPackage, getInstallation } from './packages'; @@ -151,7 +151,7 @@ class PackageClientImpl implements PackageClient { } async #reinstallTransforms(packageInfo: InstallablePackage, paths: string[]) { - const { installedTransforms } = await installTransform( + const { installedTransforms } = await installTransforms( packageInfo, paths, this.internalEsClient, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 8168bb05e53a4..4ecec17560731 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -42,7 +42,7 @@ import { import { installILMPolicy } from '../elasticsearch/ilm/install'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; -import { installTransform } from '../elasticsearch/transform/install'; +import { installTransforms } from '../elasticsearch/transform/install'; import { installMlModel } from '../elasticsearch/ml_model'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; import { saveArchiveEntries } from '../archive/storage'; @@ -219,7 +219,7 @@ export async function _installPackage({ ); ({ esReferences } = await withPackageSpan('Install transforms', () => - installTransform(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) + installTransforms(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) )); // If this is an update or retrying an update, delete the previous version's pipelines