From 58f2e4a6f3d4a3c376ed8ed5f18b9934d2f9c433 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 16:30:13 +0100 Subject: [PATCH 1/9] add build step to download gpg key --- src/dev/build/build_distributables.ts | 1 + .../tasks/fleet_download_elastic_gpg_key.ts | 38 +++++++++++++++++++ src/dev/build/tasks/index.ts | 5 ++- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 src/dev/build/tasks/fleet_download_elastic_gpg_key.ts diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 0a3db5dc36d07..876dbe220de6a 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -88,6 +88,7 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions await run(Tasks.CleanTypescript); await run(Tasks.CleanExtraFilesFromModules); await run(Tasks.CleanEmptyFolders); + await run(Tasks.FleetDownloadElasticGpgKey); await run(Tasks.BundleFleetPackages); } diff --git a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts new file mode 100644 index 0000000000000..d3d66521785dd --- /dev/null +++ b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Task, downloadToDisk } from '../lib'; + +const BUNDLED_KEYS_DIR = 'x-pack/plugins/fleet/target/keys'; +const ARTIFACTS_URL = 'https://artifacts.elastic.co/'; +const GPG_KEY_NAME = 'GPG-KEY-elasticsearch'; + +export const FleetDownloadElasticGpgKey: Task = { + description: 'Downloading Elastic GPG key for Fleet', + + async run(config, log, build) { + const gpgKeyUrl = ARTIFACTS_URL + GPG_KEY_NAME; + log.info(`Downloading Elastic GPG key from ${gpgKeyUrl}`); + const destination = build.resolvePath(BUNDLED_KEYS_DIR, GPG_KEY_NAME); + + try { + await downloadToDisk({ + log, + url: gpgKeyUrl, + destination, + shaChecksum: '', + shaAlgorithm: 'sha512', + skipChecksumCheck: true, + maxAttempts: 3, + }); + } catch (error) { + log.warning(`Failed to download Elastic GPG key`); + log.warning(error); + } + }, +}; diff --git a/src/dev/build/tasks/index.ts b/src/dev/build/tasks/index.ts index f158634829100..94e6d107ff8da 100644 --- a/src/dev/build/tasks/index.ts +++ b/src/dev/build/tasks/index.ts @@ -7,8 +7,8 @@ */ export * from './bin'; -export * from './build_kibana_platform_plugins'; export * from './build_kibana_example_plugins'; +export * from './build_kibana_platform_plugins'; export * from './build_packages_task'; export * from './bundle_fleet_packages'; export * from './clean_tasks'; @@ -18,6 +18,7 @@ export * from './create_archives_task'; export * from './create_empty_dirs_and_files_task'; export * from './create_readme_task'; export * from './download_cloud_dependencies'; +export * from './fleet_download_elastic_gpg_key'; export * from './generate_packages_optimized_assets'; export * from './install_dependencies_task'; export * from './license_file_task'; @@ -27,11 +28,11 @@ export * from './os_packages'; export * from './package_json'; export * from './patch_native_modules_task'; export * from './path_length_task'; +export * from './replace_favicon'; export * from './transpile_babel_task'; export * from './uuid_verification_task'; export * from './verify_env_task'; export * from './write_sha_sums_task'; -export * from './replace_favicon'; // @ts-expect-error this module can't be TS because it ends up pulling x-pack into Kibana export { InstallChromium } from './install_chromium'; From bd5729a260942cfc91958b46b0e07de2461a52c0 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 19:23:40 +0100 Subject: [PATCH 2/9] add gpg path to config --- x-pack/plugins/fleet/common/types/index.ts | 3 +++ x-pack/plugins/fleet/server/index.ts | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index ce1cb5a294f80..9f88d9b231854 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -33,6 +33,9 @@ export interface FleetConfigType { outputs?: PreconfiguredOutput[]; agentIdVerificationEnabled?: boolean; enableExperimental?: string[]; + packageVerification?: { + gpgKeyPath?: string; + }; developer?: { disableRegistryVersionCheck?: boolean; allowAgentUpgradeSourceUri?: boolean; diff --git a/x-pack/plugins/fleet/server/index.ts b/x-pack/plugins/fleet/server/index.ts index cd7c2e5ce8ff6..443bd00a3cf19 100644 --- a/x-pack/plugins/fleet/server/index.ts +++ b/x-pack/plugins/fleet/server/index.ts @@ -47,6 +47,7 @@ export type { export { AgentNotFoundError, FleetUnauthorizedError } from './errors'; const DEFAULT_BUNDLED_PACKAGE_LOCATION = path.join(__dirname, '../target/bundled_packages'); +const DEFAULT_GPG_KEY_PATH = path.join(__dirname, '../target/keys/GPG-KEY-elasticsearch'); export const config: PluginConfigDescriptor = { exposeToBrowser: { @@ -143,6 +144,9 @@ export const config: PluginConfigDescriptor = { allowAgentUpgradeSourceUri: schema.boolean({ defaultValue: false }), bundledPackageLocation: schema.string({ defaultValue: DEFAULT_BUNDLED_PACKAGE_LOCATION }), }), + packageVerification: schema.object({ + gpgKeyPath: schema.string({ defaultValue: DEFAULT_GPG_KEY_PATH }), + }), /** * For internal use. A list of string values (comma delimited) that will enable experimental * type of functionality that is not yet released. From b90347a957e387df95f7739bd9b884be68c527bd Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 19:24:07 +0100 Subject: [PATCH 3/9] add getGpgKey method --- .../epm/packages/package_verification.test.ts | 28 ++++++++++++++++++ .../epm/packages/package_verification.ts | 29 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts new file mode 100644 index 0000000000000..efe3b56270048 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { readFile } from 'fs/promises'; + +import { getGpgKey } from './package_verification'; +jest.mock('fs/promises', () => ({ + readFile: jest.fn(), +})); + +const mockedReadFile = readFile as jest.MockedFunction; + +describe('getGpgKey', () => { + it('should cache the gpg key after reading file once', async () => { + const config = { gpgKeyPath: 'somePath' }; + const keyContent = 'this is the gpg key'; + mockedReadFile.mockResolvedValue(Buffer.from(keyContent)); + + expect(await getGpgKey(config)).toEqual(keyContent); + expect(await getGpgKey(config)).toEqual(keyContent); + expect(mockedReadFile).toHaveBeenCalledWith('somePath'); + expect(mockedReadFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts new file mode 100644 index 0000000000000..82ef4db4893da --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { readFile } from 'fs/promises'; + +import type { FleetConfigType } from '../../../../common/types'; + +let cachedKey: string = ''; + +export async function getGpgKey(config: FleetConfigType['packageVerification']): Promise { + if (cachedKey) return cachedKey; + + const gpgKeyPath = config?.gpgKeyPath; + + if (!gpgKeyPath) { + throw new Error('No gpg key path specified, unable to get GPG key'); + } + + const buffer = await readFile(gpgKeyPath); + + const key = buffer.toString(); + + cachedKey = key; + return key; +} From 78ac737b8c30dc2083ba9d886f820a965be924a1 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 19:34:44 +0100 Subject: [PATCH 4/9] getGpgKey reads config directly --- .../epm/packages/package_verification.test.ts | 12 +++++++++--- .../services/epm/packages/package_verification.ts | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts index efe3b56270048..74d623dc6d945 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts @@ -8,6 +8,13 @@ import { readFile } from 'fs/promises'; import { getGpgKey } from './package_verification'; + +jest.mock('../../app_context', () => ({ + appContextService: { + getConfig: () => ({ packageVerification: { gpgKeyPath: 'somePath' } }), + }, +})); + jest.mock('fs/promises', () => ({ readFile: jest.fn(), })); @@ -16,12 +23,11 @@ const mockedReadFile = readFile as jest.MockedFunction; describe('getGpgKey', () => { it('should cache the gpg key after reading file once', async () => { - const config = { gpgKeyPath: 'somePath' }; const keyContent = 'this is the gpg key'; mockedReadFile.mockResolvedValue(Buffer.from(keyContent)); - expect(await getGpgKey(config)).toEqual(keyContent); - expect(await getGpgKey(config)).toEqual(keyContent); + expect(await getGpgKey()).toEqual(keyContent); + expect(await getGpgKey()).toEqual(keyContent); expect(mockedReadFile).toHaveBeenCalledWith('somePath'); expect(mockedReadFile).toHaveBeenCalledTimes(1); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts index 82ef4db4893da..cf4e89368a1b8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts @@ -7,17 +7,17 @@ import { readFile } from 'fs/promises'; -import type { FleetConfigType } from '../../../../common/types'; +import { appContextService } from '../../app_context'; let cachedKey: string = ''; -export async function getGpgKey(config: FleetConfigType['packageVerification']): Promise { +export async function getGpgKey(): Promise { if (cachedKey) return cachedKey; - const gpgKeyPath = config?.gpgKeyPath; + const gpgKeyPath = appContextService.getConfig()?.packageVerification?.gpgKeyPath; if (!gpgKeyPath) { - throw new Error('No gpg key path specified, unable to get GPG key'); + throw new Error('No path specified in "packageVerification.gpgKeyPath", unable to get GPG key'); } const buffer = await readFile(gpgKeyPath); From 1fe9ecdb868c809ae0677c064c9478ae5520833c Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 21:55:25 +0100 Subject: [PATCH 5/9] improve logging --- src/dev/build/tasks/fleet_download_elastic_gpg_key.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts index d3d66521785dd..74875a7e0bdfc 100644 --- a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts +++ b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts @@ -17,8 +17,8 @@ export const FleetDownloadElasticGpgKey: Task = { async run(config, log, build) { const gpgKeyUrl = ARTIFACTS_URL + GPG_KEY_NAME; - log.info(`Downloading Elastic GPG key from ${gpgKeyUrl}`); const destination = build.resolvePath(BUNDLED_KEYS_DIR, GPG_KEY_NAME); + log.info(`Downloading Elastic GPG key from ${gpgKeyUrl} to ${destination}`); try { await downloadToDisk({ From c10637f94db80209fb9313e736b4c3ce1908bd94 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 21:55:53 +0100 Subject: [PATCH 6/9] return undefined on error --- .../epm/packages/package_verification.test.ts | 36 +++++++++++++++---- .../epm/packages/package_verification.ts | 27 ++++++++++---- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts index 74d623dc6d945..7bb5ee2ba7aff 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.test.ts @@ -7,11 +7,17 @@ import { readFile } from 'fs/promises'; -import { getGpgKey } from './package_verification'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +const mockLoggerFactory = loggingSystemMock.create(); +const mockLogger = mockLoggerFactory.get('mock logger'); +import { getGpgKeyOrUndefined, _readGpgKey } from './package_verification'; + +const mockGetConfig = jest.fn(); jest.mock('../../app_context', () => ({ appContextService: { - getConfig: () => ({ packageVerification: { gpgKeyPath: 'somePath' } }), + getConfig: () => mockGetConfig(), + getLogger: () => mockLogger, }, })); @@ -21,14 +27,32 @@ jest.mock('fs/promises', () => ({ const mockedReadFile = readFile as jest.MockedFunction; -describe('getGpgKey', () => { +beforeEach(() => { + jest.resetAllMocks(); +}); +describe('getGpgKeyOrUndefined', () => { it('should cache the gpg key after reading file once', async () => { const keyContent = 'this is the gpg key'; mockedReadFile.mockResolvedValue(Buffer.from(keyContent)); - - expect(await getGpgKey()).toEqual(keyContent); - expect(await getGpgKey()).toEqual(keyContent); + mockGetConfig.mockReturnValue({ packageVerification: { gpgKeyPath: 'somePath' } }); + expect(await getGpgKeyOrUndefined()).toEqual(keyContent); + expect(await getGpgKeyOrUndefined()).toEqual(keyContent); expect(mockedReadFile).toHaveBeenCalledWith('somePath'); expect(mockedReadFile).toHaveBeenCalledTimes(1); }); }); + +describe('_readGpgKey', () => { + it('should return undefined if the key file isnt configured', async () => { + mockedReadFile.mockResolvedValue(Buffer.from('this is the gpg key')); + mockGetConfig.mockReturnValue({ packageVerification: {} }); + + expect(await _readGpgKey()).toEqual(undefined); + }); + it('should return undefined if there is an error reading the file', async () => { + mockedReadFile.mockRejectedValue(new Error('some error')); + mockGetConfig.mockReturnValue({ packageVerification: { gpgKeyPath: 'somePath' } }); + expect(await _readGpgKey()).toEqual(undefined); + expect(mockedReadFile).toHaveBeenCalledWith('somePath'); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts index cf4e89368a1b8..017596a8a26b0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts @@ -9,21 +9,34 @@ import { readFile } from 'fs/promises'; import { appContextService } from '../../app_context'; -let cachedKey: string = ''; +let cachedKey: string | undefined | null = null; -export async function getGpgKey(): Promise { - if (cachedKey) return cachedKey; +export async function getGpgKeyOrUndefined(): Promise { + if (cachedKey !== null) return cachedKey; - const gpgKeyPath = appContextService.getConfig()?.packageVerification?.gpgKeyPath; + cachedKey = await _readGpgKey(); + return cachedKey; +} + +export async function _readGpgKey(): Promise { + const config = appContextService.getConfig(); + const logger = appContextService.getLogger(); + const gpgKeyPath = config?.packageVerification?.gpgKeyPath; if (!gpgKeyPath) { - throw new Error('No path specified in "packageVerification.gpgKeyPath", unable to get GPG key'); + logger.warn('GPG key path not configured at "xpack.fleet.packageVerification.gpgKeyPath"'); + return undefined; } - const buffer = await readFile(gpgKeyPath); + let buffer: Buffer; + try { + buffer = await readFile(gpgKeyPath); + } catch (e) { + logger.warn(`Unable to retrieve GPG key from '${gpgKeyPath}': ${e}`); + return undefined; + } const key = buffer.toString(); - cachedKey = key; return key; } From aa82bcc2349fcaf13ddda8002dae97da98bb16e5 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 21 Jun 2022 22:03:10 +0100 Subject: [PATCH 7/9] log error code instead of msg --- .../fleet/server/services/epm/packages/package_verification.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts index 017596a8a26b0..9f1a07243a7d3 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts @@ -32,7 +32,7 @@ export async function _readGpgKey(): Promise { try { buffer = await readFile(gpgKeyPath); } catch (e) { - logger.warn(`Unable to retrieve GPG key from '${gpgKeyPath}': ${e}`); + logger.warn(`Unable to retrieve GPG key from '${gpgKeyPath}': ${e.code}`); return undefined; } From 900833b585d2b7e3ea771fe1422c943d284f8a35 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 22 Jun 2022 16:03:04 +0100 Subject: [PATCH 8/9] perform checksum check on GPG key --- src/dev/build/tasks/fleet_download_elastic_gpg_key.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts index 74875a7e0bdfc..102631aa69f4b 100644 --- a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts +++ b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts @@ -11,6 +11,8 @@ import { Task, downloadToDisk } from '../lib'; const BUNDLED_KEYS_DIR = 'x-pack/plugins/fleet/target/keys'; const ARTIFACTS_URL = 'https://artifacts.elastic.co/'; const GPG_KEY_NAME = 'GPG-KEY-elasticsearch'; +const GPG_KEY_SHA512 = + '84ee193cc337344d9a7da9021daf3f5ede83f5f1ab049d169f3634921529dcd096abf7a91eec7f26f3a6913e5e38f88f69a5e2ce79ad155d46edc75705a648c6'; export const FleetDownloadElasticGpgKey: Task = { description: 'Downloading Elastic GPG key for Fleet', @@ -25,9 +27,9 @@ export const FleetDownloadElasticGpgKey: Task = { log, url: gpgKeyUrl, destination, - shaChecksum: '', + shaChecksum: GPG_KEY_SHA512, shaAlgorithm: 'sha512', - skipChecksumCheck: true, + skipChecksumCheck: false, maxAttempts: 3, }); } catch (error) { From 1e2f48d06ef02c5f48bcf7e38c79fffd4881352e Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 23 Jun 2022 10:32:40 +0100 Subject: [PATCH 9/9] fail build if GPG key download fails --- src/dev/build/tasks/fleet_download_elastic_gpg_key.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts index 102631aa69f4b..8d52c9166d25c 100644 --- a/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts +++ b/src/dev/build/tasks/fleet_download_elastic_gpg_key.ts @@ -33,8 +33,8 @@ export const FleetDownloadElasticGpgKey: Task = { maxAttempts: 3, }); } catch (error) { - log.warning(`Failed to download Elastic GPG key`); - log.warning(error); + log.error(`Error downloading Elastic GPG key from ${gpgKeyUrl} to ${destination}`); + throw error; } }, };