From c46cff2ea350de55ade622f1b1aaa2b75970ea09 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Nov 2024 11:51:08 +0100 Subject: [PATCH 01/16] bump policy if SO different from full policy --- ...et_server_policies_enrollment_keys.test.ts | 52 +++++++++++++++++++ .../fleet_server_policies_enrollment_keys.ts | 34 ++++++++++++ 2 files changed, 86 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts index 07ec6593ec9e5..910fb86e02c7f 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts @@ -48,6 +48,7 @@ describe('ensureAgentPoliciesFleetServerKeysAndPolicies', () => { }, ], } as any); + mockedAgentPolicyService.bumpRevision.mockReset(); }); it('should do nothing with policies already deployed', async () => { @@ -163,4 +164,55 @@ describe('ensureAgentPoliciesFleetServerKeysAndPolicies', () => { expect.anything() ); }); + + it('should bump policy if same revision but content differs, ignore signature', async () => { + const logger = loggingSystemMock.createLogger(); + const esClient = elasticsearchServiceMock.createInternalClient(); + const soClient = savedObjectsClientMock.create(); + mockedAgentPolicyService.getLatestFleetPolicy.mockImplementation(async (_, agentPolicyId) => { + if (agentPolicyId === 'policy1') { + return { + revision_idx: 1, + data: { + signed: { + signature: 'signature', + data: 'data', + }, + outputs: {}, + inputs: [], + }, + } as any; + } + + if (agentPolicyId === 'policy2') { + return { + revision_idx: 1, + } as any; + } + + return null; + }); + + mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => + Promise.resolve({ + id, + revision: 1, + signed: { + signature: 'signature2', + data: 'data', + }, + outputs: {}, + inputs: [], + ...(id === 'policy1' ? { namespaces: [] } : {}), + }) + ); + + await ensureAgentPoliciesFleetServerKeysAndPolicies({ + logger, + esClient, + soClient, + }); + + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(1); + }); }); diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts index f5ed816d96e61..b6cacc07b4c4c 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts @@ -7,12 +7,14 @@ import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from '@kbn/core/server'; import pMap from 'p-map'; +import { isEqual, omit } from 'lodash'; import { agentPolicyService } from '../agent_policy'; import { ensureDefaultEnrollmentAPIKeyForAgentPolicy } from '../api_keys'; import { SO_SEARCH_LIMIT } from '../../constants'; import { appContextService } from '../app_context'; import { scheduleDeployAgentPoliciesTask } from '../agent_policies/deploy_agent_policies_task'; +import type { AgentPolicy, FleetServerPolicy } from '../../types'; export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ logger, @@ -48,6 +50,8 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ if ((latestFleetPolicy?.revision_idx ?? -1) < agentPolicy.revision) { outdatedAgentPolicyIds.push({ id: agentPolicy.id, spaceId: agentPolicy.space_ids?.[0] }); + } else if ((latestFleetPolicy?.revision_idx ?? -1) === agentPolicy.revision) { + bumpPolicyIfDiffers(logger, soClient, esClient, agentPolicy, latestFleetPolicy); } }, { @@ -75,3 +79,33 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ }); } } + +async function bumpPolicyIfDiffers( + logger: Logger, + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agentPolicy: AgentPolicy, + latestFleetPolicy: FleetServerPolicy | null | undefined +) { + // if revision matches, check if SO changed (migrations/backfills) + const latestFullAgentPolicyFromSO = await agentPolicyService.getFullAgentPolicy( + soClient, + agentPolicy.id + ); + // skip signature comparison, it differes even if there was no change in the SO + if ( + !isEqual( + omit(latestFleetPolicy?.data, 'signed.signature'), + omit(latestFullAgentPolicyFromSO, 'signed.signature') + ) + ) { + logger.info( + `Agent policy ${agentPolicy.id} SO has matching revision, but content changed, bumping revision.` + ); + await agentPolicyService.bumpRevision(soClient, esClient, agentPolicy.id, { + asyncDeploy: true, + }); + } else { + logger.debug(`Agent policy ${agentPolicy.id} SO has matching revision and content, skipping.`); + } +} From 0985863f79b649b5d5ae791a8e9e6a8fd19a5d01 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Nov 2024 13:49:45 +0100 Subject: [PATCH 02/16] move logic to async task --- x-pack/plugins/fleet/server/plugin.ts | 2 + .../bump_agent_policies_task.test.ts | 89 ++++++++++++ .../bump_agent_policies_task.ts | 133 ++++++++++++++++++ ...et_server_policies_enrollment_keys.test.ts | 53 +------ .../fleet_server_policies_enrollment_keys.ts | 72 ++++------ 5 files changed, 250 insertions(+), 99 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts create mode 100644 x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 4c5cdf7070530..7716d6b21d9e3 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -143,6 +143,7 @@ import { registerFieldsMetadataExtractors } from './services/register_fields_met import { registerUpgradeManagedPackagePoliciesTask } from './services/setup/managed_package_policies'; import { registerDeployAgentPoliciesTask } from './services/agent_policies/deploy_agent_policies_task'; import { DeleteUnenrolledAgentsTask } from './tasks/delete_unenrolled_agents_task'; +import { registerBumpAgentPoliciesTask } from './services/agent_policies/bump_agent_policies_task'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -619,6 +620,7 @@ export class FleetPlugin // Register task registerUpgradeManagedPackagePoliciesTask(deps.taskManager); registerDeployAgentPoliciesTask(deps.taskManager); + registerBumpAgentPoliciesTask(deps.taskManager); this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core); this.checkDeletedFilesTask = new CheckDeletedFilesTask({ diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts new file mode 100644 index 0000000000000..9d0198a400ec1 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + +import { agentPolicyService } from '../agent_policy'; + +import { _bumpPolicyIfDiffers } from './bump_agent_policies_task'; + +jest.mock('../app_context'); +jest.mock('../agent_policy'); + +const mockedAgentPolicyService = jest.mocked(agentPolicyService); + +describe('_bumpPolicyIfDiffers', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedAgentPolicyService.getLatestFleetPolicy.mockImplementation(async (_, agentPolicyId) => { + if (agentPolicyId === 'policy1') { + return { + revision_idx: 1, + data: { + id: agentPolicyId, + revision: 1, + signed: { + signature: 'signature', + data: 'data', + }, + outputs: {}, + inputs: [], + }, + } as any; + } + return null; + }); + }); + + it('should bump policy if same revision but content differs, ignore signature', async () => { + const logger = loggingSystemMock.createLogger(); + const esClient = elasticsearchServiceMock.createInternalClient(); + const soClient = savedObjectsClientMock.create(); + + mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => + Promise.resolve({ + id, + revision: 1, + signed: { + signature: 'signature2', + data: 'data', + }, + outputs: {}, + inputs: [], + ...(id === 'policy1' ? { namespaces: [] } : {}), + }) + ); + + await _bumpPolicyIfDiffers(logger, soClient, esClient, 'policy1'); + + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(1); + }); + + it('should not bump policy if same revision and content', async () => { + const logger = loggingSystemMock.createLogger(); + const esClient = elasticsearchServiceMock.createInternalClient(); + const soClient = savedObjectsClientMock.create(); + + mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => + Promise.resolve({ + id, + revision: 1, + signed: { + signature: 'signature', + data: 'data', + }, + outputs: {}, + inputs: [], + }) + ); + + await _bumpPolicyIfDiffers(logger, soClient, esClient, 'policy1'); + + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(0); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts new file mode 100644 index 0000000000000..6a98637fb64bd --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from '@kbn/core/server'; +import type { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { v4 as uuidv4 } from 'uuid'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; + +import pMap from 'p-map'; + +import { isEqual, omit } from 'lodash'; + +import { agentPolicyService, appContextService } from '..'; +import { runWithCache } from '../epm/packages/cache'; + +const TASK_TYPE = 'fleet:bump_agent_policies'; + +export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetupContract) { + taskManagerSetup.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Fleet Bump policies', + timeout: '5m', + maxAttempts: 3, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> = + taskInstance.params.agentPolicyIdsWithSpace; + let cancelled = false; + return { + async run() { + if (!agentPolicyIdsWithSpace.length) { + return; + } + appContextService + .getLogger() + .debug(`Checking ${agentPolicyIdsWithSpace.length} policies if bump needed`); + const agentPoliciesIdsIndexedBySpace = agentPolicyIdsWithSpace.reduce( + (acc, { id, spaceId = DEFAULT_SPACE_ID }) => { + if (!acc[spaceId]) { + acc[spaceId] = []; + } + + acc[spaceId].push(id); + + return acc; + }, + {} as { [k: string]: string[] } + ); + + await runWithCache(async () => { + for (const [spaceId, agentPolicyIds] of Object.entries( + agentPoliciesIdsIndexedBySpace + )) { + if (cancelled) { + throw new Error('Task has been cancelled'); + } + await pMap( + agentPolicyIds, + async (agentPolicyId) => { + _bumpPolicyIfDiffers( + appContextService.getLogger(), + appContextService.getInternalUserSOClientForSpaceId(spaceId), + appContextService.getInternalUserESClient(), + agentPolicyId + ); + }, + { concurrency: 20 } + ); + } + }); + }, + async cancel() { + cancelled = true; + }, + }; + }, + }, + }); +} + +export async function _bumpPolicyIfDiffers( + logger: Logger, + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agentPolicyId: string +) { + // if revision matches, check if SO changed (migrations/backfills) + const latestFullAgentPolicyFromSO = await agentPolicyService.getFullAgentPolicy( + soClient, + agentPolicyId + ); + const latestFleetPolicy = await agentPolicyService.getLatestFleetPolicy(esClient, agentPolicyId); + // skip signature comparison, it differes even if there was no change in the SO + if ( + !isEqual( + omit(latestFleetPolicy?.data, 'signed.signature'), + omit(latestFullAgentPolicyFromSO, 'signed.signature') + ) + ) { + logger.info( + `Agent policy ${agentPolicyId} SO has matching revision, but content changed, bumping revision.` + ); + await agentPolicyService.bumpRevision(soClient, esClient, agentPolicyId, { + asyncDeploy: false, + }); + } else { + logger.debug(`Agent policy ${agentPolicyId} SO has matching revision and content, skipping.`); + } +} + +export async function scheduleBumpAgentPoliciesTask( + taskManagerStart: TaskManagerStartContract, + agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> +) { + if (!agentPolicyIdsWithSpace.length) { + return; + } + + await taskManagerStart.ensureScheduled({ + id: `${TASK_TYPE}:${uuidv4()}`, + scope: ['fleet'], + params: { agentPolicyIdsWithSpace }, + taskType: TASK_TYPE, + runAt: new Date(Date.now() + 3 * 1000), + state: {}, + }); +} diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts index 910fb86e02c7f..9fb30ad75ab7e 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.test.ts @@ -17,6 +17,7 @@ import { ensureAgentPoliciesFleetServerKeysAndPolicies } from './fleet_server_po jest.mock('../app_context'); jest.mock('../agent_policy'); jest.mock('../api_keys'); +jest.mock('../agent_policies/bump_agent_policies_task'); const mockedEnsureDefaultEnrollmentAPIKeyForAgentPolicy = jest.mocked( ensureDefaultEnrollmentAPIKeyForAgentPolicy @@ -48,7 +49,6 @@ describe('ensureAgentPoliciesFleetServerKeysAndPolicies', () => { }, ], } as any); - mockedAgentPolicyService.bumpRevision.mockReset(); }); it('should do nothing with policies already deployed', async () => { @@ -164,55 +164,4 @@ describe('ensureAgentPoliciesFleetServerKeysAndPolicies', () => { expect.anything() ); }); - - it('should bump policy if same revision but content differs, ignore signature', async () => { - const logger = loggingSystemMock.createLogger(); - const esClient = elasticsearchServiceMock.createInternalClient(); - const soClient = savedObjectsClientMock.create(); - mockedAgentPolicyService.getLatestFleetPolicy.mockImplementation(async (_, agentPolicyId) => { - if (agentPolicyId === 'policy1') { - return { - revision_idx: 1, - data: { - signed: { - signature: 'signature', - data: 'data', - }, - outputs: {}, - inputs: [], - }, - } as any; - } - - if (agentPolicyId === 'policy2') { - return { - revision_idx: 1, - } as any; - } - - return null; - }); - - mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => - Promise.resolve({ - id, - revision: 1, - signed: { - signature: 'signature2', - data: 'data', - }, - outputs: {}, - inputs: [], - ...(id === 'policy1' ? { namespaces: [] } : {}), - }) - ); - - await ensureAgentPoliciesFleetServerKeysAndPolicies({ - logger, - esClient, - soClient, - }); - - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(1); - }); }); diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts index b6cacc07b4c4c..8260e1d42f393 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts @@ -7,14 +7,13 @@ import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from '@kbn/core/server'; import pMap from 'p-map'; -import { isEqual, omit } from 'lodash'; import { agentPolicyService } from '../agent_policy'; import { ensureDefaultEnrollmentAPIKeyForAgentPolicy } from '../api_keys'; import { SO_SEARCH_LIMIT } from '../../constants'; import { appContextService } from '../app_context'; import { scheduleDeployAgentPoliciesTask } from '../agent_policies/deploy_agent_policies_task'; -import type { AgentPolicy, FleetServerPolicy } from '../../types'; +import { scheduleBumpAgentPoliciesTask } from '../agent_policies/bump_agent_policies_task'; export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ logger, @@ -39,6 +38,7 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ }); const outdatedAgentPolicyIds: Array<{ id: string; spaceId?: string }> = []; + const matchingRevisionAgentPolicyIds: Array<{ id: string; spaceId?: string }> = []; await pMap( agentPolicies, @@ -51,7 +51,10 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ if ((latestFleetPolicy?.revision_idx ?? -1) < agentPolicy.revision) { outdatedAgentPolicyIds.push({ id: agentPolicy.id, spaceId: agentPolicy.space_ids?.[0] }); } else if ((latestFleetPolicy?.revision_idx ?? -1) === agentPolicy.revision) { - bumpPolicyIfDiffers(logger, soClient, esClient, agentPolicy, latestFleetPolicy); + matchingRevisionAgentPolicyIds.push({ + id: agentPolicy.id, + spaceId: agentPolicy.space_ids?.[0], + }); } }, { @@ -59,53 +62,28 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ } ); - if (!outdatedAgentPolicyIds.length) { - return; - } - - if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { - return scheduleDeployAgentPoliciesTask( + if (matchingRevisionAgentPolicyIds.length) { + scheduleBumpAgentPoliciesTask( appContextService.getTaskManagerStart()!, - outdatedAgentPolicyIds + matchingRevisionAgentPolicyIds ); - } else { - return agentPolicyService - .deployPolicies( - soClient, - outdatedAgentPolicyIds.map(({ id }) => id) - ) - .catch((error) => { - logger.warn(`Error deploying policies: ${error.message}`, { error }); - }); } -} -async function bumpPolicyIfDiffers( - logger: Logger, - soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - agentPolicy: AgentPolicy, - latestFleetPolicy: FleetServerPolicy | null | undefined -) { - // if revision matches, check if SO changed (migrations/backfills) - const latestFullAgentPolicyFromSO = await agentPolicyService.getFullAgentPolicy( - soClient, - agentPolicy.id - ); - // skip signature comparison, it differes even if there was no change in the SO - if ( - !isEqual( - omit(latestFleetPolicy?.data, 'signed.signature'), - omit(latestFullAgentPolicyFromSO, 'signed.signature') - ) - ) { - logger.info( - `Agent policy ${agentPolicy.id} SO has matching revision, but content changed, bumping revision.` - ); - await agentPolicyService.bumpRevision(soClient, esClient, agentPolicy.id, { - asyncDeploy: true, - }); - } else { - logger.debug(`Agent policy ${agentPolicy.id} SO has matching revision and content, skipping.`); + if (outdatedAgentPolicyIds.length) { + if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { + return scheduleDeployAgentPoliciesTask( + appContextService.getTaskManagerStart()!, + outdatedAgentPolicyIds + ); + } else { + return agentPolicyService + .deployPolicies( + soClient, + outdatedAgentPolicyIds.map(({ id }) => id) + ) + .catch((error) => { + logger.warn(`Error deploying policies: ${error.message}`, { error }); + }); + } } } From 7cd7cf5560c519f99eafe2dba4ccdfcc6f927a6c Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Nov 2024 17:02:56 +0100 Subject: [PATCH 03/16] use new field bump_agent_policy_revision to trigger revision bump --- .../common/types/models/package_policy.ts | 1 + .../fleet/server/saved_objects/index.ts | 21 +++ .../bump_agent_policies_task.test.ts | 90 ++++--------- .../bump_agent_policies_task.ts | 122 ++++++------------ .../fleet_server_policies_enrollment_keys.ts | 14 +- 5 files changed, 93 insertions(+), 155 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 354834d2571dc..6a24e7ede12c9 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -112,6 +112,7 @@ export interface PackagePolicy extends Omit { updated_by: string; created_at: string; created_by: string; + bump_agent_policy_revision?: boolean; } export type DryRunPackagePolicy = NewPackagePolicy & { diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 706d0686b9845..4be407c703a49 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -619,6 +619,7 @@ export const getSavedObjectTypes = ( updated_by: { type: 'keyword' }, created_at: { type: 'date' }, created_by: { type: 'keyword' }, + bump_agent_policy_revision: { type: 'boolean' }, }, }, modelVersions: { @@ -763,6 +764,26 @@ export const getSavedObjectTypes = ( }, ], }, + '15': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + bump_agent_policy_revision: { type: 'boolean' }, + }, + }, + ], + }, + // '16': { + // changes: [ + // { + // type: 'data_backfill', + // backfillFn: (doc) => { + // return { attributes: { ...doc.attributes, bump_agent_policy_revision: true } }; + // }, + // }, + // ], + // }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts index 9d0198a400ec1..12c1916cd96c6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -6,84 +6,50 @@ */ import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; import { agentPolicyService } from '../agent_policy'; -import { _bumpPolicyIfDiffers } from './bump_agent_policies_task'; +import { packagePolicyService } from '../package_policy'; +import type { PackagePolicy } from '../../types'; + +import { _updatePackagePoliciesThatNeedBump } from './bump_agent_policies_task'; jest.mock('../app_context'); jest.mock('../agent_policy'); +jest.mock('../package_policy'); const mockedAgentPolicyService = jest.mocked(agentPolicyService); +const mockedPackagePolicyService = jest.mocked(packagePolicyService); -describe('_bumpPolicyIfDiffers', () => { +describe('_updatePackagePoliciesThatNeedBump', () => { beforeEach(() => { jest.clearAllMocks(); - mockedAgentPolicyService.getLatestFleetPolicy.mockImplementation(async (_, agentPolicyId) => { - if (agentPolicyId === 'policy1') { - return { - revision_idx: 1, - data: { - id: agentPolicyId, - revision: 1, - signed: { - signature: 'signature', - data: 'data', - }, - outputs: {}, - inputs: [], - }, - } as any; - } - return null; + mockedPackagePolicyService.list.mockResolvedValueOnce({ + total: 1, + items: [ + { + id: 'packagePolicy1', + bump_agent_policy_revision: true, + } as PackagePolicy, + ], + page: 1, + perPage: 100, + }); + mockedPackagePolicyService.list.mockResolvedValueOnce({ + total: 0, + items: [], + page: 1, + perPage: 100, }); }); - it('should bump policy if same revision but content differs, ignore signature', async () => { - const logger = loggingSystemMock.createLogger(); - const esClient = elasticsearchServiceMock.createInternalClient(); - const soClient = savedObjectsClientMock.create(); - - mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => - Promise.resolve({ - id, - revision: 1, - signed: { - signature: 'signature2', - data: 'data', - }, - outputs: {}, - inputs: [], - ...(id === 'policy1' ? { namespaces: [] } : {}), - }) - ); - - await _bumpPolicyIfDiffers(logger, soClient, esClient, 'policy1'); - - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(1); - }); - - it('should not bump policy if same revision and content', async () => { + it('should update package policy if bump agent policy revision needed', async () => { const logger = loggingSystemMock.createLogger(); - const esClient = elasticsearchServiceMock.createInternalClient(); - const soClient = savedObjectsClientMock.create(); - - mockedAgentPolicyService.getFullAgentPolicy.mockImplementation((_, id) => - Promise.resolve({ - id, - revision: 1, - signed: { - signature: 'signature', - data: 'data', - }, - outputs: {}, - inputs: [], - }) - ); - await _bumpPolicyIfDiffers(logger, soClient, esClient, 'policy1'); + await _updatePackagePoliciesThatNeedBump(logger); - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledTimes(0); + expect(mockedPackagePolicyService.bulkUpdate).toHaveBeenCalledWith(undefined, undefined, [ + { bump_agent_policy_revision: false, id: 'packagePolicy1' }, + ]); }); }); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts index 6a98637fb64bd..f94a504d2a973 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts @@ -4,24 +4,19 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from '@kbn/core/server'; +import type { Logger } from '@kbn/core/server'; import type { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import { v4 as uuidv4 } from 'uuid'; -import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; -import pMap from 'p-map'; - -import { isEqual, omit } from 'lodash'; - -import { agentPolicyService, appContextService } from '..'; +import { appContextService, packagePolicyService } from '..'; import { runWithCache } from '../epm/packages/cache'; const TASK_TYPE = 'fleet:bump_agent_policies'; - +const BATCH_SIZE = 100; export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetupContract) { taskManagerSetup.registerTaskDefinitions({ [TASK_TYPE]: { @@ -29,50 +24,17 @@ export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetup timeout: '5m', maxAttempts: 3, createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - const agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> = - taskInstance.params.agentPolicyIdsWithSpace; let cancelled = false; return { async run() { - if (!agentPolicyIdsWithSpace.length) { - return; + if (cancelled) { + throw new Error('Task has been cancelled'); } - appContextService - .getLogger() - .debug(`Checking ${agentPolicyIdsWithSpace.length} policies if bump needed`); - const agentPoliciesIdsIndexedBySpace = agentPolicyIdsWithSpace.reduce( - (acc, { id, spaceId = DEFAULT_SPACE_ID }) => { - if (!acc[spaceId]) { - acc[spaceId] = []; - } - - acc[spaceId].push(id); - - return acc; - }, - {} as { [k: string]: string[] } - ); await runWithCache(async () => { - for (const [spaceId, agentPolicyIds] of Object.entries( - agentPoliciesIdsIndexedBySpace - )) { - if (cancelled) { - throw new Error('Task has been cancelled'); - } - await pMap( - agentPolicyIds, - async (agentPolicyId) => { - _bumpPolicyIfDiffers( - appContextService.getLogger(), - appContextService.getInternalUserSOClientForSpaceId(spaceId), - appContextService.getInternalUserESClient(), - agentPolicyId - ); - }, - { concurrency: 20 } - ); - } + await _updatePackagePoliciesThatNeedBump(appContextService.getLogger()); + + // TODO agent policies }); }, async cancel() { @@ -84,48 +46,48 @@ export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetup }); } -export async function _bumpPolicyIfDiffers( - logger: Logger, - soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - agentPolicyId: string -) { - // if revision matches, check if SO changed (migrations/backfills) - const latestFullAgentPolicyFromSO = await agentPolicyService.getFullAgentPolicy( - soClient, - agentPolicyId +async function getPackagePoliciesToBump() { + return await packagePolicyService.list(appContextService.getInternalUserSOClient(), { + kuery: 'ingest-package-policies.bump_agent_policy_revision:true', + perPage: BATCH_SIZE, + }); +} + +export async function _updatePackagePoliciesThatNeedBump(logger: Logger) { + // TODO spaces? + let packagePoliciesToBump = await getPackagePoliciesToBump(); + + logger.info( + `Found ${packagePoliciesToBump.total} package policies that need agent policy revision bump` ); - const latestFleetPolicy = await agentPolicyService.getLatestFleetPolicy(esClient, agentPolicyId); - // skip signature comparison, it differes even if there was no change in the SO - if ( - !isEqual( - omit(latestFleetPolicy?.data, 'signed.signature'), - omit(latestFullAgentPolicyFromSO, 'signed.signature') - ) - ) { - logger.info( - `Agent policy ${agentPolicyId} SO has matching revision, but content changed, bumping revision.` + + while (packagePoliciesToBump.total > 0) { + const start = Date.now(); + // resetting the flag will trigger a revision bump + await packagePolicyService.bulkUpdate( + appContextService.getInternalUserSOClient(), + appContextService.getInternalUserESClient(), + packagePoliciesToBump.items.map((item) => ({ + ...item, + bump_agent_policy_revision: false, + })) ); - await agentPolicyService.bumpRevision(soClient, esClient, agentPolicyId, { - asyncDeploy: false, - }); - } else { - logger.debug(`Agent policy ${agentPolicyId} SO has matching revision and content, skipping.`); - } -} + const updatedCount = packagePoliciesToBump.items.length; -export async function scheduleBumpAgentPoliciesTask( - taskManagerStart: TaskManagerStartContract, - agentPolicyIdsWithSpace: Array<{ id: string; spaceId?: string }> -) { - if (!agentPolicyIdsWithSpace.length) { - return; + packagePoliciesToBump = await getPackagePoliciesToBump(); + logger.debug( + `Updated ${updatedCount} package policies in ${Date.now() - start}ms, ${ + packagePoliciesToBump.total + } remaining` + ); } +} +export async function scheduleBumpAgentPoliciesTask(taskManagerStart: TaskManagerStartContract) { await taskManagerStart.ensureScheduled({ id: `${TASK_TYPE}:${uuidv4()}`, scope: ['fleet'], - params: { agentPolicyIdsWithSpace }, + params: {}, taskType: TASK_TYPE, runAt: new Date(Date.now() + 3 * 1000), state: {}, diff --git a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts index 8260e1d42f393..cd7e91fb81ac4 100644 --- a/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts +++ b/x-pack/plugins/fleet/server/services/setup/fleet_server_policies_enrollment_keys.ts @@ -38,8 +38,6 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ }); const outdatedAgentPolicyIds: Array<{ id: string; spaceId?: string }> = []; - const matchingRevisionAgentPolicyIds: Array<{ id: string; spaceId?: string }> = []; - await pMap( agentPolicies, async (agentPolicy) => { @@ -50,11 +48,6 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ if ((latestFleetPolicy?.revision_idx ?? -1) < agentPolicy.revision) { outdatedAgentPolicyIds.push({ id: agentPolicy.id, spaceId: agentPolicy.space_ids?.[0] }); - } else if ((latestFleetPolicy?.revision_idx ?? -1) === agentPolicy.revision) { - matchingRevisionAgentPolicyIds.push({ - id: agentPolicy.id, - spaceId: agentPolicy.space_ids?.[0], - }); } }, { @@ -62,12 +55,7 @@ export async function ensureAgentPoliciesFleetServerKeysAndPolicies({ } ); - if (matchingRevisionAgentPolicyIds.length) { - scheduleBumpAgentPoliciesTask( - appContextService.getTaskManagerStart()!, - matchingRevisionAgentPolicyIds - ); - } + await scheduleBumpAgentPoliciesTask(appContextService.getTaskManagerStart()!); if (outdatedAgentPolicyIds.length) { if (appContextService.getExperimentalFeatures().asyncDeployPolicies) { From 33e2a3742983ae64723dc6e6711a02ad1c8882e1 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:26:23 +0000 Subject: [PATCH 04/16] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- packages/kbn-check-mappings-update-cli/current_fields.json | 1 + packages/kbn-check-mappings-update-cli/current_mappings.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 5493b8dc3bbdb..813bd7188dbad 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -690,6 +690,7 @@ "version" ], "ingest-package-policies": [ + "bump_agent_policy_revision", "created_at", "created_by", "description", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 726b6e9e1d4c5..c793d232d298c 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2292,6 +2292,9 @@ }, "ingest-package-policies": { "properties": { + "bump_agent_policy_revision": { + "type": "boolean" + }, "created_at": { "type": "date" }, From 65eee0f7457b998a43142b4409e2aed1db971016 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:27:10 +0000 Subject: [PATCH 05/16] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 02f7007b51202..79907d915e95e 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -123,7 +123,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "5e95e539826a40ad08fd0c1d161da0a4d86ffc6d", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "55988d5f778bbe0e76caa7e6468707a0a056bdd8", - "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", + "ingest-package-policies": "dfa7b1045a2667a822181f40f012786724492439", "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", From 600590a5ed3d6f055805dcf9addb1da1c6541d19 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Nov 2024 13:03:47 +0100 Subject: [PATCH 06/16] refactor --- .../common/types/models/package_policy.ts | 2 +- .../fleet/server/saved_objects/index.ts | 23 +++-- .../bump_agent_policies_task.test.ts | 6 +- .../bump_agent_policies_task.ts | 97 +++++++++++++------ .../server/services/package_policies/utils.ts | 15 ++- .../fleet/server/services/package_policy.ts | 56 ++++------- .../fleet/server/types/rest_spec/settings.ts | 4 +- .../fleet/server/types/so_attributes.ts | 1 + 8 files changed, 116 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 6a24e7ede12c9..d811810677813 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -112,7 +112,7 @@ export interface PackagePolicy extends Omit { updated_by: string; created_at: string; created_by: string; - bump_agent_policy_revision?: boolean; + // bump_agent_policy_revision?: boolean; } export type DryRunPackagePolicy = NewPackagePolicy & { diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 4be407c703a49..5173ada3f5418 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -774,16 +774,6 @@ export const getSavedObjectTypes = ( }, ], }, - // '16': { - // changes: [ - // { - // type: 'data_backfill', - // backfillFn: (doc) => { - // return { attributes: { ...doc.attributes, bump_agent_policy_revision: true } }; - // }, - // }, - // ], - // }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, @@ -844,6 +834,19 @@ export const getSavedObjectTypes = ( updated_by: { type: 'keyword' }, created_at: { type: 'date' }, created_by: { type: 'keyword' }, + bump_agent_policy_revision: { type: 'boolean' }, + }, + }, + modelVersions: { + '1': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + bump_agent_policy_revision: { type: 'boolean' }, + }, + }, + ], }, }, }, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts index 12c1916cd96c6..03944866ac139 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -7,8 +7,6 @@ import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { agentPolicyService } from '../agent_policy'; - import { packagePolicyService } from '../package_policy'; import type { PackagePolicy } from '../../types'; @@ -18,7 +16,6 @@ jest.mock('../app_context'); jest.mock('../agent_policy'); jest.mock('../package_policy'); -const mockedAgentPolicyService = jest.mocked(agentPolicyService); const mockedPackagePolicyService = jest.mocked(packagePolicyService); describe('_updatePackagePoliciesThatNeedBump', () => { @@ -29,7 +26,6 @@ describe('_updatePackagePoliciesThatNeedBump', () => { items: [ { id: 'packagePolicy1', - bump_agent_policy_revision: true, } as PackagePolicy, ], page: 1, @@ -46,7 +42,7 @@ describe('_updatePackagePoliciesThatNeedBump', () => { it('should update package policy if bump agent policy revision needed', async () => { const logger = loggingSystemMock.createLogger(); - await _updatePackagePoliciesThatNeedBump(logger); + await _updatePackagePoliciesThatNeedBump(logger, false); expect(mockedPackagePolicyService.bulkUpdate).toHaveBeenCalledWith(undefined, undefined, [ { bump_agent_policy_revision: false, id: 'packagePolicy1' }, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts index f94a504d2a973..2bb5b2a1a1e1b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts @@ -11,12 +11,20 @@ import type { TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import { v4 as uuidv4 } from 'uuid'; +import { uniq } from 'lodash'; -import { appContextService, packagePolicyService } from '..'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; + +import { agentPolicyService, appContextService } from '..'; import { runWithCache } from '../epm/packages/cache'; +import { getPackagePolicySavedObjectType } from '../package_policy'; +import { mapPackagePolicySavedObjectToPackagePolicy } from '../package_policies'; +import type { PackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { normalizeKuery } from '../saved_object'; +import { SO_SEARCH_LIMIT } from '../../constants'; const TASK_TYPE = 'fleet:bump_agent_policies'; -const BATCH_SIZE = 100; + export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetupContract) { taskManagerSetup.registerTaskDefinitions({ [TASK_TYPE]: { @@ -32,9 +40,7 @@ export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetup } await runWithCache(async () => { - await _updatePackagePoliciesThatNeedBump(appContextService.getLogger()); - - // TODO agent policies + await _updatePackagePoliciesThatNeedBump(appContextService.getLogger(), cancelled); }); }, async cancel() { @@ -46,39 +52,76 @@ export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetup }); } -async function getPackagePoliciesToBump() { - return await packagePolicyService.list(appContextService.getInternalUserSOClient(), { - kuery: 'ingest-package-policies.bump_agent_policy_revision:true', - perPage: BATCH_SIZE, - }); +async function getPackagePoliciesToBump(savedObjectType: string) { + const result = await appContextService + .getInternalUserSOClientWithoutSpaceExtension() + .find({ + type: savedObjectType, + filter: normalizeKuery(savedObjectType, `${savedObjectType}.bump_agent_policy_revision:true`), + perPage: SO_SEARCH_LIMIT, + namespaces: ['*'], + fields: ['id', 'namespaces', 'policy_ids'], + }); + return { + total: result.total, + items: result.saved_objects.map((so) => + mapPackagePolicySavedObjectToPackagePolicy(so, so.namespaces) + ), + }; } -export async function _updatePackagePoliciesThatNeedBump(logger: Logger) { - // TODO spaces? - let packagePoliciesToBump = await getPackagePoliciesToBump(); +export async function _updatePackagePoliciesThatNeedBump(logger: Logger, cancelled: boolean) { + const savedObjectType = await getPackagePolicySavedObjectType(); + const packagePoliciesToBump = await getPackagePoliciesToBump(savedObjectType); logger.info( `Found ${packagePoliciesToBump.total} package policies that need agent policy revision bump` ); - while (packagePoliciesToBump.total > 0) { - const start = Date.now(); - // resetting the flag will trigger a revision bump - await packagePolicyService.bulkUpdate( - appContextService.getInternalUserSOClient(), - appContextService.getInternalUserESClient(), - packagePoliciesToBump.items.map((item) => ({ - ...item, - bump_agent_policy_revision: false, + const packagePoliciesIndexedBySpace = packagePoliciesToBump.items.reduce((acc, policy) => { + const spaceId = policy.spaceIds?.[0] ?? DEFAULT_SPACE_ID; + if (!acc[spaceId]) { + acc[spaceId] = []; + } + + acc[spaceId].push(policy); + + return acc; + }, {} as { [k: string]: PackagePolicy[] }); + + const start = Date.now(); + + for (const [spaceId, packagePolicies] of Object.entries(packagePoliciesIndexedBySpace)) { + if (cancelled) { + throw new Error('Task has been cancelled'); + } + + const soClient = appContextService.getInternalUserSOClientForSpaceId(spaceId); + const esClient = appContextService.getInternalUserESClient(); + + await soClient.bulkUpdate( + packagePolicies.map((item) => ({ + type: savedObjectType, + id: item.id, + attributes: { + bump_agent_policy_revision: false, + }, })) ); - const updatedCount = packagePoliciesToBump.items.length; - packagePoliciesToBump = await getPackagePoliciesToBump(); + const updatedCount = packagePolicies.length; + + const agentPoliciesToBump = uniq(packagePolicies.map((item) => item.policy_ids).flat()); + + // TODO bump at once + for (const agentPolicyId of agentPoliciesToBump) { + await agentPolicyService.bumpRevision(soClient, esClient, agentPolicyId); + } + logger.debug( - `Updated ${updatedCount} package policies in ${Date.now() - start}ms, ${ - packagePoliciesToBump.total - } remaining` + `Updated ${updatedCount} package policies in space ${spaceId} in ${ + Date.now() - start + }ms, bump ${agentPoliciesToBump.length} agent policies` ); } } diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index ef59c643a8b35..a27ac5943f80f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -28,17 +28,16 @@ import { licenseService } from '../license'; import { outputService } from '../output'; import { appContextService } from '../app_context'; -export const mapPackagePolicySavedObjectToPackagePolicy = ({ - id, - version, - attributes, - namespaces, -}: SavedObject): PackagePolicy => { +export const mapPackagePolicySavedObjectToPackagePolicy = ( + { id, version, attributes }: SavedObject, + namespaces?: string[] +): PackagePolicy => { + const { bump_agent_policy_revision: bumpAgentPolicyRevision, ...restAttributes } = attributes; return { id, version, - spaceIds: namespaces, - ...attributes, + ...(namespaces ? { spaceIds: namespaces } : {}), + ...restAttributes, }; }; diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 48dc3956d6984..31747952173ce 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -20,6 +20,7 @@ import type { RequestHandlerContext, SavedObjectsBulkCreateObject, SavedObjectsBulkUpdateObject, + SavedObject, } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; import { v4 as uuidv4 } from 'uuid'; @@ -446,7 +447,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } - const createdPackagePolicy = { id: newSo.id, version: newSo.version, ...newSo.attributes }; + const createdPackagePolicy = mapPackagePolicySavedObjectToPackagePolicy(newSo); logger.debug(`Created new package policy with id ${newSo.id} and version ${newSo.version}`); return packagePolicyService.runExternalCallbacks( @@ -668,11 +669,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } logger.debug(`Created new package policies`); return { - created: newSos.map((newSo) => ({ - id: newSo.id, - version: newSo.version, - ...newSo.attributes, - })), + created: newSos.map((newSo) => mapPackagePolicySavedObjectToPackagePolicy(newSo)), failed: failedPolicies, }; } @@ -754,11 +751,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } - const response = { - id: packagePolicySO.id, - version: packagePolicySO.version, - ...packagePolicySO.attributes, - }; + const response = mapPackagePolicySavedObjectToPackagePolicy(packagePolicySO); // If possible, return the experimental features map for the package policy's `package` field if (experimentalFeatures && response.package) { @@ -788,11 +781,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { return []; } - const packagePolicies = packagePolicySO.saved_objects.map((so) => ({ - id: so.id, - version: so.version, - ...so.attributes, - })); + const packagePolicies = packagePolicySO.saved_objects.map((so) => + mapPackagePolicySavedObjectToPackagePolicy(so) + ); for (const packagePolicy of packagePolicies) { auditLoggingService.writeCustomSoAuditLog({ @@ -835,11 +826,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } - return { - id: so.id, - version: so.version, - ...so.attributes, - }; + return mapPackagePolicySavedObjectToPackagePolicy(so); }) .filter((packagePolicy): packagePolicy is PackagePolicy => packagePolicy !== null); @@ -889,12 +876,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } return { - items: packagePolicies?.saved_objects.map((packagePolicySO) => ({ - id: packagePolicySO.id, - version: packagePolicySO.version, - ...packagePolicySO.attributes, - spaceIds: packagePolicySO.namespaces, - })), + items: packagePolicies?.saved_objects.map((so) => + mapPackagePolicySavedObjectToPackagePolicy(so, so.namespaces) + ), total: packagePolicies?.total, page, perPage, @@ -1392,13 +1376,10 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const updatedPoliciesSuccess = updatedPolicies .filter((policy) => !policy.error && policy.attributes) - .map( - (soPolicy) => - ({ - id: soPolicy.id, - version: soPolicy.version, - ...soPolicy.attributes, - } as PackagePolicy) + .map((soPolicy) => + mapPackagePolicySavedObjectToPackagePolicy( + soPolicy as SavedObject + ) ); return { updatedPolicies: updatedPoliciesSuccess, failedPolicies }; @@ -2189,7 +2170,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { perPage: SO_SEARCH_LIMIT, namespaces: ['*'], }) - ).saved_objects.map(mapPackagePolicySavedObjectToPackagePolicy); + ).saved_objects.map((so) => mapPackagePolicySavedObjectToPackagePolicy(so, so.namespaces)); if (packagePolicies.length > 0) { const getPackagePolicyUpdate = (packagePolicy: PackagePolicy) => ({ @@ -2306,7 +2287,10 @@ class PackagePolicyClientImpl implements PackagePolicyClient { savedObjectType, }); - return mapPackagePolicySavedObjectToPackagePolicy(packagePolicySO); + return mapPackagePolicySavedObjectToPackagePolicy( + packagePolicySO, + packagePolicySO.namespaces + ); }); }, }); diff --git a/x-pack/plugins/fleet/server/types/rest_spec/settings.ts b/x-pack/plugins/fleet/server/types/rest_spec/settings.ts index c40dcc0b63596..64684e51350d4 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/settings.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/settings.ts @@ -61,7 +61,9 @@ export const SettingsResponseSchema = schema.object({ use_space_awareness_migration_status: schema.maybe( schema.oneOf([schema.literal('pending'), schema.literal('success'), schema.literal('error')]) ), - use_space_awareness_migration_started_at: schema.maybe(schema.string()), + use_space_awareness_migration_started_at: schema.maybe( + schema.oneOf([schema.literal(null), schema.string()]) + ), delete_unenrolled_agents: schema.maybe( schema.object({ enabled: schema.boolean(), diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 31207dda64bc8..6f415eea9eb61 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -137,6 +137,7 @@ export interface PackagePolicySOAttributes { }; agents?: number; overrides?: any | null; + bump_agent_policy_revision?: boolean; } interface OutputSoBaseAttributes { From 58c89a9933754509d30160f4d7addce4165f3992 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:35:16 +0000 Subject: [PATCH 07/16] [CI] Auto-commit changed files from 'node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions --include-path /api/security/role --include-path /api/spaces --include-path /api/fleet --update' --- oas_docs/bundle.json | 2 ++ oas_docs/bundle.serverless.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json index d30ac3c4552e2..074d44f40006d 100644 --- a/oas_docs/bundle.json +++ b/oas_docs/bundle.json @@ -36515,6 +36515,7 @@ "type": "boolean" }, "use_space_awareness_migration_started_at": { + "nullable": true, "type": "string" }, "use_space_awareness_migration_status": { @@ -36716,6 +36717,7 @@ "type": "boolean" }, "use_space_awareness_migration_started_at": { + "nullable": true, "type": "string" }, "use_space_awareness_migration_status": { diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json index 4b56e3581c66f..c64cd83e3e4c6 100644 --- a/oas_docs/bundle.serverless.json +++ b/oas_docs/bundle.serverless.json @@ -36515,6 +36515,7 @@ "type": "boolean" }, "use_space_awareness_migration_started_at": { + "nullable": true, "type": "string" }, "use_space_awareness_migration_status": { @@ -36716,6 +36717,7 @@ "type": "boolean" }, "use_space_awareness_migration_started_at": { + "nullable": true, "type": "string" }, "use_space_awareness_migration_status": { From 409bbf77fe1acf30cf0a47d7659cf417735da066 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:35:34 +0000 Subject: [PATCH 08/16] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- packages/kbn-check-mappings-update-cli/current_fields.json | 1 + packages/kbn-check-mappings-update-cli/current_mappings.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 813bd7188dbad..a072d45249596 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -509,6 +509,7 @@ ], "fleet-message-signing-keys": [], "fleet-package-policies": [ + "bump_agent_policy_revision", "created_at", "created_by", "description", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index c793d232d298c..c40661540da30 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1707,6 +1707,9 @@ }, "fleet-package-policies": { "properties": { + "bump_agent_policy_revision": { + "type": "boolean" + }, "created_at": { "type": "date" }, From 32dc285c039c71d88c772239d7e161726aab1a1d Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Nov 2024 13:49:49 +0100 Subject: [PATCH 09/16] fix test --- .../common/types/models/package_policy.ts | 1 - .../bump_agent_policies_task.test.ts | 87 +++++++++++++++---- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index d811810677813..354834d2571dc 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -112,7 +112,6 @@ export interface PackagePolicy extends Omit { updated_by: string; created_at: string; created_by: string; - // bump_agent_policy_revision?: boolean; } export type DryRunPackagePolicy = NewPackagePolicy & { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts index 03944866ac139..312bc0f91800e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -7,8 +7,10 @@ import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { packagePolicyService } from '../package_policy'; -import type { PackagePolicy } from '../../types'; +import { agentPolicyService } from '../agent_policy'; + +import { appContextService } from '..'; +import { getPackagePolicySavedObjectType } from '../package_policy'; import { _updatePackagePoliciesThatNeedBump } from './bump_agent_policies_task'; @@ -16,27 +18,50 @@ jest.mock('../app_context'); jest.mock('../agent_policy'); jest.mock('../package_policy'); -const mockedPackagePolicyService = jest.mocked(packagePolicyService); +const mockedAgentPolicyService = jest.mocked(agentPolicyService); +const mockedAppContextService = jest.mocked(appContextService); +const mockSoClient = { + find: jest.fn(), + bulkUpdate: jest.fn(), +} as any; +const mockGetPackagePolicySavedObjectType = jest.mocked(getPackagePolicySavedObjectType); describe('_updatePackagePoliciesThatNeedBump', () => { beforeEach(() => { jest.clearAllMocks(); - mockedPackagePolicyService.list.mockResolvedValueOnce({ - total: 1, - items: [ + mockSoClient.find.mockResolvedValue({ + total: 3, + saved_objects: [ { id: 'packagePolicy1', - } as PackagePolicy, + namespaces: ['default'], + attributes: { + policy_ids: ['policy1'], + }, + }, + { + id: 'packagePolicy2', + namespaces: ['space'], + attributes: { + policy_ids: ['policy2'], + }, + }, + { + id: 'packagePolicy3', + namespaces: ['space'], + attributes: { + policy_ids: ['policy3'], + }, + }, ], page: 1, perPage: 100, }); - mockedPackagePolicyService.list.mockResolvedValueOnce({ - total: 0, - items: [], - page: 1, - perPage: 100, - }); + mockedAppContextService.getInternalUserSOClientWithoutSpaceExtension.mockReturnValue( + mockSoClient + ); + mockedAppContextService.getInternalUserSOClientForSpaceId.mockReturnValue(mockSoClient); + mockGetPackagePolicySavedObjectType.mockResolvedValue('fleet-package-policies'); }); it('should update package policy if bump agent policy revision needed', async () => { @@ -44,8 +69,40 @@ describe('_updatePackagePoliciesThatNeedBump', () => { await _updatePackagePoliciesThatNeedBump(logger, false); - expect(mockedPackagePolicyService.bulkUpdate).toHaveBeenCalledWith(undefined, undefined, [ - { bump_agent_policy_revision: false, id: 'packagePolicy1' }, + expect(mockSoClient.bulkUpdate).toHaveBeenCalledWith([ + { + attributes: { bump_agent_policy_revision: false }, + id: 'packagePolicy1', + type: 'fleet-package-policies', + }, ]); + expect(mockSoClient.bulkUpdate).toHaveBeenCalledWith([ + { + attributes: { bump_agent_policy_revision: false }, + id: 'packagePolicy2', + type: 'fleet-package-policies', + }, + { + attributes: { bump_agent_policy_revision: false }, + id: 'packagePolicy3', + type: 'fleet-package-policies', + }, + ]); + + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( + expect.anything(), + undefined, + 'policy1' + ); + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( + expect.anything(), + undefined, + 'policy2' + ); + expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( + expect.anything(), + undefined, + 'policy3' + ); }); }); From 92a2a6254d193b68698665314ffd0b225c7594e1 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Nov 2024 13:54:46 +0100 Subject: [PATCH 10/16] add missing header to /enable_space_awareness doc --- x-pack/plugins/fleet/dev_docs/space_awareness.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/dev_docs/space_awareness.md b/x-pack/plugins/fleet/dev_docs/space_awareness.md index fc4ce3a0acd06..90371d1f8bb08 100644 --- a/x-pack/plugins/fleet/dev_docs/space_awareness.md +++ b/x-pack/plugins/fleet/dev_docs/space_awareness.md @@ -15,7 +15,7 @@ xpack.fleet.enableExperimental: ['useSpaceAwareness', 'subfeaturePrivileges'] After the feature flag is enabled you will have to do another step to opt-in for the feature, that call will migrate the current space agnostic saved objects to new space aware saved objects. ```shell -curl -u elastic:changeme -XPOST "http://localhost:5601/internal/fleet/enable_space_awareness" -H "kbn-xsrf: reporting" -H 'elastic-api-version: 1' +curl -u elastic:changeme -XPOST "http://localhost:5601/internal/fleet/enable_space_awareness" -H "kbn-xsrf: reporting" -H 'elastic-api-version: 1' -H 'x-elastic-internal-origin: 1' ``` ## Space aware entities in Fleet From 6704eb9c265be7edb8ec9cad4cd181deab22c992 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:07:23 +0000 Subject: [PATCH 11/16] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 2 ++ oas_docs/output/kibana.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 55dd5277d1d93..a3c347b1f58d5 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -29721,6 +29721,7 @@ paths: secret_storage_requirements_met: type: boolean use_space_awareness_migration_started_at: + nullable: true type: string use_space_awareness_migration_status: enum: @@ -29854,6 +29855,7 @@ paths: secret_storage_requirements_met: type: boolean use_space_awareness_migration_started_at: + nullable: true type: string use_space_awareness_migration_status: enum: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index b2c3ae00be9d0..8604ff6bbbad2 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -32487,6 +32487,7 @@ paths: secret_storage_requirements_met: type: boolean use_space_awareness_migration_started_at: + nullable: true type: string use_space_awareness_migration_status: enum: @@ -32619,6 +32620,7 @@ paths: secret_storage_requirements_met: type: boolean use_space_awareness_migration_started_at: + nullable: true type: string use_space_awareness_migration_status: enum: From 34095292483bf5c9c8b5c38527ee088375de20ae Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:09:39 +0000 Subject: [PATCH 12/16] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 79907d915e95e..147b8fdea2684 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () => "fleet-agent-policies": "f57d3b70e4175a19a18f18ee72a379ceec82e1fc", "fleet-fleet-server-host": "69be15f6b6f2a2875ad3c7050ddea7a87f505417", "fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f", - "fleet-package-policies": "2f4d524adb49a5281d3af0b66bb3003ba0ff2e44", + "fleet-package-policies": "8be2cabfed89e103e0d413f2900e9cf6cd31bc68", "fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e", "fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d", "fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde", From 2878cbd8151f586db8c38cebe83a37c2935a6736 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Nov 2024 14:47:40 +0100 Subject: [PATCH 13/16] bump agent policies in bulk --- .../bump_agent_policies_task.test.ts | 25 ++++++++++++------- .../bump_agent_policies_task.ts | 5 +--- .../fleet/server/services/agent_policy.ts | 21 ++++++++++++++++ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts index 312bc0f91800e..eca5219c5f57e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -39,6 +39,13 @@ describe('_updatePackagePoliciesThatNeedBump', () => { policy_ids: ['policy1'], }, }, + { + id: 'packagePolicy12', + namespaces: ['default'], + attributes: { + policy_ids: ['policy1'], + }, + }, { id: 'packagePolicy2', namespaces: ['space'], @@ -75,6 +82,11 @@ describe('_updatePackagePoliciesThatNeedBump', () => { id: 'packagePolicy1', type: 'fleet-package-policies', }, + { + attributes: { bump_agent_policy_revision: false }, + id: 'packagePolicy12', + type: 'fleet-package-policies', + }, ]); expect(mockSoClient.bulkUpdate).toHaveBeenCalledWith([ { @@ -89,20 +101,15 @@ describe('_updatePackagePoliciesThatNeedBump', () => { }, ]); - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( - expect.anything(), - undefined, - 'policy1' - ); - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( + expect(mockedAgentPolicyService.bumpAgentPoliciesByIds).toHaveBeenCalledWith( expect.anything(), undefined, - 'policy2' + ['policy1'] ); - expect(mockedAgentPolicyService.bumpRevision).toHaveBeenCalledWith( + expect(mockedAgentPolicyService.bumpAgentPoliciesByIds).toHaveBeenCalledWith( expect.anything(), undefined, - 'policy3' + ['policy2', 'policy3'] ); }); }); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts index 2bb5b2a1a1e1b..dd1e6cbc3a963 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts @@ -113,10 +113,7 @@ export async function _updatePackagePoliciesThatNeedBump(logger: Logger, cancell const agentPoliciesToBump = uniq(packagePolicies.map((item) => item.policy_ids).flat()); - // TODO bump at once - for (const agentPolicyId of agentPoliciesToBump) { - await agentPolicyService.bumpRevision(soClient, esClient, agentPolicyId); - } + await agentPolicyService.bumpAgentPoliciesByIds(soClient, esClient, agentPoliciesToBump); logger.debug( `Updated ${updatedCount} package policies in space ${spaceId} in ${ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index cada1c8e64452..21614d2a97481 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -1646,6 +1646,27 @@ class AgentPolicyService { ); } + public async bumpAgentPoliciesByIds( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agentPolicyIds: string[], + options?: { user?: AuthenticatedUser } + ): Promise> { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + const savedObjectType = await getAgentPolicySavedObjectType(); + + const objects = agentPolicyIds.map((id: string) => ({ id, type: savedObjectType })); + const bulkGetResponse = await soClient.bulkGet(objects); + + return this._bumpPolicies( + internalSoClientWithoutSpaceExtension, + esClient, + bulkGetResponse.saved_objects, + options + ); + } + public async getInactivityTimeouts(): Promise< Array<{ policyIds: string[]; inactivityTimeout: number }> > { From eeb2c5427a233059c1f44b69c1dff88c633ff8b2 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Nov 2024 16:05:00 +0100 Subject: [PATCH 14/16] fix test, updated create_agent_policies script to use spaces --- .../create_agent_policies.ts | 33 ++++++++----------- .../scripts/create_agent_policies/fixtures.ts | 9 +++-- .../scripts/create_agent_policies/index.js | 2 +- .../check_registered_task_types.ts | 1 + 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts index db47c056ce12d..e000640a477e3 100644 --- a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts +++ b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts @@ -9,8 +9,7 @@ import { ToolingLog } from '@kbn/tooling-log'; import yargs from 'yargs'; import { chunk } from 'lodash'; -import { LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants'; -import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants'; import { packagePolicyFixture } from './fixtures'; @@ -30,20 +29,18 @@ const printUsage = () => const INDEX_BULK_OP = '{ "index":{ "_id": "{{id}}" } }\n'; +const space = 'default'; function getPolicyId(idx: number | string) { - return `test-policy-${idx}`; + return `test-policy-${space}-${idx}`; } async function createAgentPoliciesDocsBulk(range: number[]) { const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); const body = range .flatMap((idx) => [ - INDEX_BULK_OP.replace( - /{{id}}/, - `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}` - ), + INDEX_BULK_OP.replace(/{{id}}/, `${AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}`), JSON.stringify({ - [LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE]: { + [AGENT_POLICY_SAVED_OBJECT_TYPE]: { namespace: 'default', monitoring_enabled: ['logs', 'metrics', 'traces'], name: `Test Policy ${idx}`, @@ -60,11 +57,11 @@ async function createAgentPoliciesDocsBulk(range: number[]) { schema_version: '1.1.1', is_protected: false, }, - type: LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE, + namespaces: [space], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, references: [], managed: false, coreMigrationVersion: '8.8.0', - typeMigrationVersion: '10.3.0', created_at: new Date().toISOString(), updated_at: new Date().toISOString(), }) + '\n', @@ -81,7 +78,7 @@ async function createAgentPoliciesDocsBulk(range: number[]) { const data = await res.json(); if (!data.items) { - logger.error('Error creating agent policies docs: ' + JSON.stringify(data)); + logger.error('Error creating agent policy docs: ' + JSON.stringify(data)); process.exit(1); } return data; @@ -91,14 +88,14 @@ async function createEnrollmentToken(range: number[]) { const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); const body = range .flatMap((idx) => [ - INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${idx}`), + INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${space}-${idx}`), JSON.stringify({ active: true, api_key_id: 'faketest123', api_key: 'test==', name: `Test Policy ${idx}`, policy_id: `${getPolicyId(idx)}`, - namespaces: [], + namespaces: [space], created_at: new Date().toISOString(), }) + '\n', ]) @@ -115,7 +112,7 @@ async function createEnrollmentToken(range: number[]) { const data = await res.json(); if (!data.items) { - logger.error('Error creating agent policies docs: ' + JSON.stringify(data)); + logger.error('Error creating enrollment key docs: ' + JSON.stringify(data)); process.exit(1); } return data; @@ -125,14 +122,12 @@ async function createPackagePolicies(range: number[]) { const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64'); const body = range .flatMap((idx) => [ - INDEX_BULK_OP.replace( - /{{id}}/, - `${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}` - ), + INDEX_BULK_OP.replace(/{{id}}/, `fleet-package-policies:test-policy-${space}-${idx}`), JSON.stringify( packagePolicyFixture({ idx, agentPolicyId: getPolicyId(idx), + space, }) ) + '\n', ]) @@ -150,7 +145,7 @@ async function createPackagePolicies(range: number[]) { const data = await res.json(); if (!data.items) { - logger.error('Error creating agent policies docs: ' + JSON.stringify(data)); + logger.error('Error creating package policy docs: ' + JSON.stringify(data)); process.exit(1); } return data; diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts index b10f412ac43fe..ddaa226c0729e 100644 --- a/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts +++ b/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts @@ -8,11 +8,13 @@ export const packagePolicyFixture = ({ agentPolicyId, idx, + space, }: { idx: number; agentPolicyId: string; + space: string; }) => ({ - 'ingest-package-policies': { + 'fleet-package-policies': { name: `system-test-${idx}`, namespace: '', description: '', @@ -790,11 +792,12 @@ export const packagePolicyFixture = ({ updated_at: '2024-08-30T13:45:51.197Z', updated_by: 'system', }, - type: 'ingest-package-policies', + namespaces: [space], + type: 'fleet-package-policies', references: [], managed: false, coreMigrationVersion: '8.8.0', - typeMigrationVersion: '10.14.0', + typeMigrationVersion: '10.1.0', updated_at: '2024-08-30T13:45:51.197Z', created_at: '2024-08-30T13:45:51.197Z', }); diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/index.js b/x-pack/plugins/fleet/scripts/create_agent_policies/index.js index a51859ee684c6..a61ed0e7e54d4 100644 --- a/x-pack/plugins/fleet/scripts/create_agent_policies/index.js +++ b/x-pack/plugins/fleet/scripts/create_agent_policies/index.js @@ -12,6 +12,6 @@ require('./create_agent_policies').run(); Usage: cd x-pack/plugins/fleet -node scripts/create_agents/index.js +node scripts/create_agent_policies/index.js */ diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 88ef256b353e6..bf731cba14237 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -140,6 +140,7 @@ export default function ({ getService }: FtrProviderContext) { 'endpoint:metadata-check-transforms-task', 'endpoint:user-artifact-packager', 'entity_store:field_retention:enrichment', + 'fleet:bump_agent_policies', 'fleet:check-deleted-files-task', 'fleet:delete-unenrolled-agents-task', 'fleet:deploy_agent_policies', From 26e46d4dbbd8ce4bdc9c1e81ad81533dbc02ef9f Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Fri, 22 Nov 2024 15:39:09 +0100 Subject: [PATCH 15/16] use isCancelled instead of cancelled --- .../agent_policies/bump_agent_policies_task.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts index dd1e6cbc3a963..d134fdbfae04f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.ts @@ -33,14 +33,15 @@ export function registerBumpAgentPoliciesTask(taskManagerSetup: TaskManagerSetup maxAttempts: 3, createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { let cancelled = false; + const isCancelled = () => cancelled; return { async run() { - if (cancelled) { + if (isCancelled()) { throw new Error('Task has been cancelled'); } await runWithCache(async () => { - await _updatePackagePoliciesThatNeedBump(appContextService.getLogger(), cancelled); + await _updatePackagePoliciesThatNeedBump(appContextService.getLogger(), isCancelled); }); }, async cancel() { @@ -70,7 +71,10 @@ async function getPackagePoliciesToBump(savedObjectType: string) { }; } -export async function _updatePackagePoliciesThatNeedBump(logger: Logger, cancelled: boolean) { +export async function _updatePackagePoliciesThatNeedBump( + logger: Logger, + isCancelled: () => boolean +) { const savedObjectType = await getPackagePolicySavedObjectType(); const packagePoliciesToBump = await getPackagePoliciesToBump(savedObjectType); @@ -92,7 +96,7 @@ export async function _updatePackagePoliciesThatNeedBump(logger: Logger, cancell const start = Date.now(); for (const [spaceId, packagePolicies] of Object.entries(packagePoliciesIndexedBySpace)) { - if (cancelled) { + if (isCancelled()) { throw new Error('Task has been cancelled'); } From d8c9711075b402e508852d5bbc41f62a67b7df31 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Fri, 22 Nov 2024 16:25:34 +0100 Subject: [PATCH 16/16] fix test --- .../services/agent_policies/bump_agent_policies_task.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts index eca5219c5f57e..2e7305c5c5717 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/bump_agent_policies_task.test.ts @@ -74,7 +74,7 @@ describe('_updatePackagePoliciesThatNeedBump', () => { it('should update package policy if bump agent policy revision needed', async () => { const logger = loggingSystemMock.createLogger(); - await _updatePackagePoliciesThatNeedBump(logger, false); + await _updatePackagePoliciesThatNeedBump(logger, () => false); expect(mockSoClient.bulkUpdate).toHaveBeenCalledWith([ {