diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts index 1e58319183c7d..83ad08d09de76 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import { Logger, SavedObjectsClientContract } from 'kibana/server'; import { saveInstalledEsRefs } from '../../packages/install'; import * as Registry from '../../registry'; @@ -33,89 +33,100 @@ export const installTransformForDataset = async ( registryPackage: RegistryPackage, paths: string[], callCluster: CallESAsCurrentUser, - savedObjectsClient: SavedObjectsClientContract + savedObjectsClient: SavedObjectsClientContract, + logger: Logger ) => { - const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name }); - let previousInstalledTransformEsAssets: EsAssetReference[] = []; - if (installation) { - previousInstalledTransformEsAssets = installation.installed_es.filter( - ({ type, id }) => type === ElasticsearchAssetType.transform + try { + const installation = await getInstallation({ + savedObjectsClient, + pkgName: registryPackage.name, + }); + let previousInstalledTransformEsAssets: EsAssetReference[] = []; + if (installation) { + previousInstalledTransformEsAssets = installation.installed_es.filter( + ({ type, id }) => type === ElasticsearchAssetType.transform + ); + } + + // delete all previous transform + await deleteTransforms( + callCluster, + previousInstalledTransformEsAssets.map((asset) => asset.id) ); - } - - // delete all previous transform - await deleteTransforms( - callCluster, - previousInstalledTransformEsAssets.map((asset) => asset.id) - ); - // install the latest dataset - const datasets = registryPackage.datasets; - if (!datasets?.length) return []; - const installNameSuffix = `${registryPackage.version}`; - - const transformPaths = paths.filter((path) => isTransform(path)); - let installedTransforms: EsAssetReference[] = []; - if (transformPaths.length > 0) { - const transformPathDatasets = datasets.reduce((acc, dataset) => { - transformPaths.forEach((path) => { - if (isDatasetTransform(path, dataset.path)) { - acc.push({ path, dataset }); - } - }); - return acc; - }, []); - - const transformRefs = transformPathDatasets.reduce( - (acc, transformPathDataset) => { - if (transformPathDataset) { - acc.push({ - id: getTransformNameForInstallation(transformPathDataset, installNameSuffix), - type: ElasticsearchAssetType.transform, - }); - } + // install the latest dataset + const datasets = registryPackage.datasets; + if (!datasets?.length) return []; + const installNameSuffix = `${registryPackage.version}`; + + const transformPaths = paths.filter((path) => isTransform(path)); + let installedTransforms: EsAssetReference[] = []; + if (transformPaths.length > 0) { + const transformPathDatasets = datasets.reduce((acc, dataset) => { + transformPaths.forEach((path) => { + if (isDatasetTransform(path, dataset.path)) { + acc.push({ path, dataset }); + } + }); return acc; - }, - [] - ); - - // get and save transform refs before installing transforms - await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, transformRefs); - - const transforms: TransformInstallation[] = transformPathDatasets.map( - (transformPathDataset: TransformPathDataset) => { - return { - installationName: getTransformNameForInstallation( - transformPathDataset, - installNameSuffix - ), - content: getAsset(transformPathDataset.path).toString('utf-8'), - }; - } - ); + }, []); + + const transformRefs = transformPathDatasets.reduce( + (acc, transformPathDataset) => { + if (transformPathDataset) { + acc.push({ + id: getTransformNameForInstallation(transformPathDataset, installNameSuffix), + type: ElasticsearchAssetType.transform, + }); + } + return acc; + }, + [] + ); + + // get and save transform refs before installing transforms + await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, transformRefs); + + const transforms: TransformInstallation[] = transformPathDatasets.map( + (transformPathDataset: TransformPathDataset) => { + return { + installationName: getTransformNameForInstallation( + transformPathDataset, + installNameSuffix + ), + content: getAsset(transformPathDataset.path).toString('utf-8'), + }; + } + ); - const installationPromises = transforms.map(async (transform) => { - return installTransform({ callCluster, transform }); - }); + const installationPromises = transforms.map(async (transform) => { + return installTransform({ callCluster, transform, logger }); + }); - installedTransforms = await Promise.all(installationPromises).then((results) => results.flat()); - } + installedTransforms = await Promise.all(installationPromises).then((results) => + results.flat() + ); + } - if (previousInstalledTransformEsAssets.length > 0) { - const currentInstallation = await getInstallation({ - savedObjectsClient, - pkgName: registryPackage.name, - }); + if (previousInstalledTransformEsAssets.length > 0) { + const currentInstallation = await getInstallation({ + savedObjectsClient, + pkgName: registryPackage.name, + }); - // remove the saved object reference - await deleteTransformRefs( - savedObjectsClient, - currentInstallation?.installed_es || [], - registryPackage.name, - previousInstalledTransformEsAssets.map((asset) => asset.id), - installedTransforms.map((installed) => installed.id) - ); + // remove the saved object reference + await deleteTransformRefs( + savedObjectsClient, + currentInstallation?.installed_es || [], + registryPackage.name, + previousInstalledTransformEsAssets.map((asset) => asset.id), + installedTransforms.map((installed) => installed.id) + ); + } + return installedTransforms; + } catch (err) { + logger.error(err); + throw err; } - return installedTransforms; }; const isTransform = (path: string) => { @@ -136,24 +147,31 @@ const isDatasetTransform = (path: string, datasetName: string) => { async function installTransform({ callCluster, transform, + logger, }: { callCluster: CallESAsCurrentUser; transform: TransformInstallation; + logger: Logger; }): Promise { - // defer validation on put if the source index is not available - await callCluster('transport.request', { - method: 'PUT', - path: `_transform/${transform.installationName}`, - query: 'defer_validation=true', - body: transform.content, - }); - - await callCluster('transport.request', { - method: 'POST', - path: `_transform/${transform.installationName}/_start`, - }); - - return { id: transform.installationName, type: ElasticsearchAssetType.transform }; + try { + // defer validation on put if the source index is not available + await callCluster('transport.request', { + method: 'PUT', + path: `/_transform/${transform.installationName}`, + query: 'defer_validation=true', + body: transform.content, + }); + + await callCluster('transport.request', { + method: 'POST', + path: `/_transform/${transform.installationName}/_start`, + }); + + return { id: transform.installationName, type: ElasticsearchAssetType.transform }; + } catch (err) { + logger.error(err); + throw err; + } } const getTransformNameForInstallation = ( diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts index 5c9d3e2846200..a527d05f1c49b 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts @@ -12,7 +12,7 @@ export const stopTransforms = async (transformIds: string[], callCluster: CallES for (const transformId of transformIds) { await callCluster('transport.request', { method: 'POST', - path: `_transform/${transformId}/_stop`, + path: `/_transform/${transformId}/_stop`, query: 'force=true', ignore: [404], }); @@ -29,7 +29,7 @@ export const deleteTransforms = async ( await callCluster('transport.request', { method: 'DELETE', query: 'force=true', - path: `_transform/${transformId}`, + path: `/_transform/${transformId}`, ignore: [404], }); }) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts index 0b66077b8699a..bb506ecad0ade 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts @@ -4,6 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { loggingSystemMock } from '../../../../../../../../src/core/server/logging/logging_system.mock'; + jest.mock('../../packages/get', () => { return { getInstallation: jest.fn(), getInstallationObject: jest.fn() }; }); @@ -15,7 +18,12 @@ jest.mock('./common', () => { }); import { installTransformForDataset } from './install'; -import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server'; +import { + ILegacyScopedClusterClient, + LoggerFactory, + SavedObject, + SavedObjectsClientContract, +} from 'kibana/server'; import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types'; import { getInstallation, getInstallationObject } from '../../packages'; import { getAsset } from './common'; @@ -25,6 +33,7 @@ import { savedObjectsClientMock } from '../../../../../../../../src/core/server/ describe('test transform install', () => { let legacyScopedClusterClient: jest.Mocked; let savedObjectsClient: jest.Mocked; + let logger: jest.Mocked; beforeEach(() => { legacyScopedClusterClient = { callAsInternalUser: jest.fn(), @@ -33,6 +42,7 @@ describe('test transform install', () => { (getInstallation as jest.MockedFunction).mockReset(); (getInstallationObject as jest.MockedFunction).mockReset(); savedObjectsClient = savedObjectsClientMock.create(); + logger = loggingSystemMock.create(); }); afterEach(() => { @@ -132,15 +142,15 @@ describe('test transform install', () => { 'endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json', ], legacyScopedClusterClient.callAsCurrentUser, - savedObjectsClient + savedObjectsClient, + logger.get('ingest') ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ [ 'transport.request', { method: 'POST', - path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0/_stop', + path: '/_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0/_stop', query: 'force=true', ignore: [404], }, @@ -150,7 +160,7 @@ describe('test transform install', () => { { method: 'DELETE', query: 'force=true', - path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0', + path: '/_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0', ignore: [404], }, ], @@ -158,7 +168,7 @@ describe('test transform install', () => { 'transport.request', { method: 'PUT', - path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0', + path: '/_transform/metrics-endpoint.metadata-default-0.16.0-dev.0', query: 'defer_validation=true', body: '{"content": "data"}', }, @@ -167,7 +177,7 @@ describe('test transform install', () => { 'transport.request', { method: 'PUT', - path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0', + path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0', query: 'defer_validation=true', body: '{"content": "data"}', }, @@ -176,14 +186,14 @@ describe('test transform install', () => { 'transport.request', { method: 'POST', - path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0/_start', + path: '/_transform/metrics-endpoint.metadata-default-0.16.0-dev.0/_start', }, ], [ 'transport.request', { method: 'POST', - path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start', + path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start', }, ], ]); @@ -287,7 +297,8 @@ describe('test transform install', () => { } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json'], legacyScopedClusterClient.callAsCurrentUser, - savedObjectsClient + savedObjectsClient, + logger.get('ingest') ); expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ @@ -295,7 +306,7 @@ describe('test transform install', () => { 'transport.request', { method: 'PUT', - path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0', + path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0', query: 'defer_validation=true', body: '{"content": "data"}', }, @@ -304,7 +315,7 @@ describe('test transform install', () => { 'transport.request', { method: 'POST', - path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start', + path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start', }, ], ]); @@ -384,26 +395,27 @@ describe('test transform install', () => { } as unknown) as RegistryPackage, [], legacyScopedClusterClient.callAsCurrentUser, - savedObjectsClient + savedObjectsClient, + logger.get('ingest') ); expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ [ 'transport.request', { - ignore: [404], method: 'POST', - path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0/_stop', + path: '/_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0/_stop', query: 'force=true', + ignore: [404], }, ], [ 'transport.request', { - ignore: [404], method: 'DELETE', - path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0', query: 'force=true', + path: '/_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0', + ignore: [404], }, ], ]); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 54b9c4d3fbb17..4179e82d6ad1d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -36,6 +36,7 @@ import { deleteKibanaSavedObjectsAssets } from './remove'; import { PackageOutdatedError } from '../../../errors'; import { getPackageSavedObjects } from './get'; import { installTransformForDataset } from '../elasticsearch/transform/install'; +import { appContextService } from '../../app_context'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; @@ -196,7 +197,8 @@ export async function installPackage({ registryPackageInfo, paths, callCluster, - savedObjectsClient + savedObjectsClient, + appContextService.getLogger() ); // if this is an update or retrying an update, delete the previous version's pipelines