Skip to content

Commit

Permalink
Separate out logic, add component template
Browse files Browse the repository at this point in the history
  • Loading branch information
@qn895
qn895 committed Sep 14, 2022
1 parent fa7e8e2 commit 8fa543f
Showing 1 changed file with 156 additions and 93 deletions.
249 changes: 156 additions & 93 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ import { errors } from '@elastic/elasticsearch';
import { safeLoad } from 'js-yaml';
import { isPopulatedObject } from '@kbn/ml-is-populated-object';

import {
PACKAGE_TEMPLATE_SUFFIX,
USER_SETTINGS_TEMPLATE_SUFFIX,
} from '../../../../../common/constants';

import { installComponentAndIndexTemplateForDataStream } from '../template/install';
import { processFields } from '../../fields/field';
import { generateMappings } from '../template/template';
Expand All @@ -26,10 +31,16 @@ import type {
import { getInstallation } from '../../packages';
import { retryTransientEsErrors } from '../retry';

import { mergeIndexTemplateWithMappings } from './merge_index_template_mappings';
import { deleteTransforms } from './remove';
import { getAsset } from './common';

const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250;
enum TRANSFORM_SPECS_TYPES {
MANIFEST = 'manifest',
FIELDS = 'fields',
TRANSFORM = 'transform',
}

interface TransformModuleBase {
transformModuleId?: string;
}
Expand All @@ -43,7 +54,7 @@ interface TransformInstallation extends TransformModuleBase {
content: any;
}

export const installLegacyTransformsAssets = async (
const installLegacyTransformsAssets = async (
installablePackage: InstallablePackage,
installNameSuffix: string,
transformPaths: string[],
Expand Down Expand Up @@ -109,146 +120,197 @@ export const installLegacyTransformsAssets = async (
return { installedTransforms, esReferences };
};

export const installTransformsAssets = async (
const processTransformAssetsPerModule = (
installablePackage: InstallablePackage,
installNameSuffix: string,
transformPaths: string[],
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract,
logger: Logger,
esReferences: EsAssetReference[] = [],
previousInstalledTransformEsAssets: EsAssetReference[] = []
transformPaths: string[]
) => {
let installedTransforms: EsAssetReference[] = [];
if (transformPaths.length > 0) {
const transformsSpecifications = new Map();
const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = [];
const indices = [];
const transforms: TransformInstallation[] = [];

transformPaths.forEach((path: string) => {
const { transformModuleId, fileName } = getTransformFolderAndFileNames(
installablePackage,
path
);
const transformsSpecifications = new Map();
const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = [];
const transforms: TransformInstallation[] = [];

// Since there can be multiple assets per transform definition
// We want to create a unique list of assets/specifications for each transform
if (transformsSpecifications.get(transformModuleId) === undefined) {
transformsSpecifications.set(transformModuleId, new Map());
}
const packageAssets = transformsSpecifications.get(transformModuleId);
transformPaths.forEach((path: string) => {
const { transformModuleId, fileName } = getTransformFolderAndFileNames(
installablePackage,
path
);

const content = safeLoad(getAsset(path).toString('utf-8'));
// Since there can be multiple assets per transform definition
// We want to create a unique list of assets/specifications for each transform
if (transformsSpecifications.get(transformModuleId) === undefined) {
transformsSpecifications.set(transformModuleId, new Map());
}
const packageAssets = transformsSpecifications.get(transformModuleId);

if (fileName === 'fields') {
const validFields = processFields(content);
const mappings = generateMappings(validFields);
packageAssets?.set('mappings', mappings);
}
const content = safeLoad(getAsset(path).toString('utf-8'));

if (fileName === 'transform') {
transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest);
indices.push(content.dest);
transformsSpecifications.get(transformModuleId)?.set('transform', content);
content._meta = getESAssetMetadata({ packageName: installablePackage.name });
transforms.push({
if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) {
const validFields = processFields(content);
const mappings = generateMappings(validFields);
packageAssets?.set('mappings', mappings);
}

if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) {
transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest);
transformsSpecifications.get(transformModuleId)?.set('transform', content);
content._meta = getESAssetMetadata({ packageName: installablePackage.name });
transforms.push({
transformModuleId,
installationName: getTransformNameForInstallation(
installablePackage,
transformModuleId,
installNameSuffix
),
content,
});
}

if (fileName === TRANSFORM_SPECS_TYPES.MANIFEST) {
if (isPopulatedObject(content, ['start']) && content.start === false) {
transformsSpecifications.get(transformModuleId)?.set('start', false);
}
// If manifest.yml contains destination_index_template
// Combine the mappings and other index template settings from manifest.yml into a single index template
// Create the index template and track the template in EsAssetReferences
if (
isPopulatedObject(content, ['destination_index_template']) ||
isPopulatedObject(packageAssets.get('mappings'))
) {
const destinationIndexTemplate =
(content.destination_index_template as Record<string, unknown>) ?? {};
destinationIndexTemplates.push({
transformModuleId,
_meta: getESAssetMetadata({ packageName: installablePackage.name }),
installationName: getTransformNameForInstallation(
installablePackage,
transformModuleId,
installNameSuffix
installNameSuffix,
'template'
),
content,
});
indices.push(content.dest);
template: destinationIndexTemplate,
} as DestinationIndexTemplateInstallation);
packageAssets.set('destinationIndexTemplate', destinationIndexTemplate);
}
}
});

if (fileName === 'manifest') {
if (isPopulatedObject(content, ['start']) && content.start === false) {
transformsSpecifications.get(transformModuleId)?.set('start', false);
}
// If manifest.yml contains destination_index_template
// Combine the mappings and other index template settings from manifest.yml into a single index template
// Create the index template and track the template in EsAssetReferences
if (
isPopulatedObject(content, ['destination_index_template']) ||
isPopulatedObject(packageAssets.get('mappings'))
) {
const destinationIndexTemplate =
(content.destination_index_template as Record<string, unknown>) ?? {};
destinationIndexTemplates.push({
transformModuleId,
_meta: getESAssetMetadata({ packageName: installablePackage.name }),
installationName: getTransformNameForInstallation(
installablePackage,
transformModuleId,
installNameSuffix,
'template'
),
template: destinationIndexTemplate,
} as DestinationIndexTemplateInstallation);
packageAssets.set('destinationIndexTemplate', destinationIndexTemplate);
}
}
});

const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({
id: template.installationName,
type: ElasticsearchAssetType.indexTemplate,
}));

const transformRefs = transforms.map((t) => ({
id: t.installationName,
type: ElasticsearchAssetType.transform,
}));
const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({
id: template.installationName,
type: ElasticsearchAssetType.indexTemplate,
}));
const componentTemplatesRefs = destinationIndexTemplates.map((template) => ({
id: `${template.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`,
type: ElasticsearchAssetType.componentTemplate,
}));

const transformRefs = transforms.map((t) => ({
id: t.installationName,
type: ElasticsearchAssetType.transform,
}));

return {
indexTemplatesRefs,
componentTemplatesRefs,
transformRefs,
transforms,
destinationIndexTemplates,
transformsSpecifications,
};
};

const installTransformsAssets = async (
installablePackage: InstallablePackage,
installNameSuffix: string,
transformPaths: string[],
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract,
logger: Logger,
esReferences: EsAssetReference[] = [],
previousInstalledTransformEsAssets: EsAssetReference[] = []
) => {
let installedTransforms: EsAssetReference[] = [];
if (transformPaths.length > 0) {
const {
indexTemplatesRefs,
componentTemplatesRefs,
transformRefs,
transforms,
destinationIndexTemplates,
transformsSpecifications,
} = processTransformAssetsPerModule(installablePackage, installNameSuffix, transformPaths);
// get and save refs associated with the transforms before installing
esReferences = await updateEsAssetReferences(
savedObjectsClient,
installablePackage.name,
esReferences,
{
assetsToAdd: [...indexTemplatesRefs, ...transformRefs],
assetsToAdd: [...indexTemplatesRefs, ...componentTemplatesRefs, ...transformRefs],
assetsToRemove: previousInstalledTransformEsAssets,
}
);

// create index templates and component templates
await Promise.all(
destinationIndexTemplates
.map((destinationIndexTemplate) => {
const mergedTemplate = mergeIndexTemplateWithMappings(
destinationIndexTemplate.template,
transformsSpecifications
.get(destinationIndexTemplate.transformModuleId)
?.get('mappings')
);
if (mergedTemplate !== undefined) {
const customMappings = transformsSpecifications
.get(destinationIndexTemplate.transformModuleId)
?.get('mappings');
const customTemplateName = `${destinationIndexTemplate.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`;
const packageTemplateName = `${destinationIndexTemplate.installationName}${PACKAGE_TEMPLATE_SUFFIX}`;
if (destinationIndexTemplate !== undefined || customMappings !== undefined) {
return installComponentAndIndexTemplateForDataStream({
esClient,
logger,
componentTemplates: {},
componentTemplates: {
...(customMappings
? {
[customTemplateName]: {
template: {
mappings: customMappings,
},
_meta: destinationIndexTemplate._meta,
},
}
: {}),
...(destinationIndexTemplate.template
? {
[packageTemplateName]: {
template: {
settings: destinationIndexTemplate.template.settings,
mappings: destinationIndexTemplate.template.mappings,
},
_meta: destinationIndexTemplate._meta,
},
}
: {}),
},
indexTemplate: {
templateName: destinationIndexTemplate.installationName,
// @ts-expect-error Index template here should not contain data_stream property
// as this template is applied to only an index and not a data stream
indexTemplate: {
template: mergedTemplate,
priority: 250,
template: { settings: undefined, mappings: undefined },
priority: DEFAULT_TRANSFORM_TEMPLATES_PRIORITY,
index_patterns: [
transformsSpecifications
.get(destinationIndexTemplate.transformModuleId)
?.get('destinationIndex').index,
],
_meta: destinationIndexTemplate._meta,
composed_of: [],
composed_of: [
...(customMappings ? [customTemplateName] : []),
...(destinationIndexTemplate.template ? [packageTemplateName] : []),
],
},
},
});
}
})
.filter((p) => p !== undefined)
);

// create destination indices
await Promise.all(
transforms.map(async (transform) => {
const index = transform.content.dest.index;
Expand All @@ -264,6 +326,7 @@ export const installTransformsAssets = async (
})
);

// create & optionally start transforms
const transformsPromises = transforms.map(async (transform) => {
return handleTransformInstall({
esClient,
Expand Down

0 comments on commit 8fa543f

Please sign in to comment.