-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Fleet] Adjust background File cleanup task to also process
to-host…
… indexes (#161138) ## Summary - Updates the `fleet:check-deleted-files-task` to include the indexes that store files for delivery to the Host (currently used only by Endpoint integration)
- Loading branch information
1 parent
0c03f10
commit 9e5d6b3
Showing
7 changed files
with
198 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import { getFileDataIndexName, getFileMetadataIndexName } from '../../../common'; | ||
|
|
||
| import { parseFileStorageIndex } from './utils'; | ||
|
|
||
| describe('Files service utils', () => { | ||
| describe('parseFileStorageIndex()', () => { | ||
| it.each([ | ||
| [ | ||
| 'tohost meta', | ||
| '.ds-.fleet-fileds-tohost-meta-endpoint-2023.07.03-000001', | ||
| { | ||
| index: getFileMetadataIndexName('endpoint', true), | ||
| integration: 'endpoint', | ||
| direction: 'to-host', | ||
| type: 'meta', | ||
| }, | ||
| ], | ||
| [ | ||
| 'tohost data', | ||
| '.ds-.fleet-fileds-tohost-data-agent-2023.07.03-000001', | ||
| { | ||
| index: getFileDataIndexName('agent', true), | ||
| integration: 'agent', | ||
| direction: 'to-host', | ||
| type: 'data', | ||
| }, | ||
| ], | ||
| [ | ||
| 'fromhost meta', | ||
| '.ds-.fleet-fileds-fromhost-meta-agent-2023.07.03-000001', | ||
| { | ||
| index: getFileMetadataIndexName('agent'), | ||
| integration: 'agent', | ||
| direction: 'from-host', | ||
| type: 'meta', | ||
| }, | ||
| ], | ||
| [ | ||
| 'fromhost data', | ||
| '.ds-.fleet-fileds-fromhost-data-endpoint-2023.07.03-000001', | ||
| { | ||
| index: getFileDataIndexName('endpoint'), | ||
| integration: 'endpoint', | ||
| direction: 'from-host', | ||
| type: 'data', | ||
| }, | ||
| ], | ||
| ])('should parse index %s', (_, index, result) => { | ||
| expect(parseFileStorageIndex(index)).toEqual(result); | ||
| }); | ||
|
|
||
| it('should error if index does not match a known pattern', () => { | ||
| expect(() => parseFileStorageIndex('foo')).toThrow( | ||
| 'Unable to parse index [foo]. Does not match a known index pattern: [.fleet-fileds-fromhost-meta-* | ' + | ||
| '.fleet-fileds-fromhost-data-* | .fleet-fileds-tohost-meta-* | .fleet-fileds-tohost-data-*]' | ||
| ); | ||
| }); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import { getFileDataIndexName, getFileMetadataIndexName } from '../../../common/services'; | ||
|
|
||
| import { | ||
| FILE_STORAGE_DATA_INDEX_PATTERN, | ||
| FILE_STORAGE_METADATA_INDEX_PATTERN, | ||
| FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN, | ||
| FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN, | ||
| } from '../../../common/constants'; | ||
|
|
||
| interface ParsedFileStorageIndex { | ||
| index: string; | ||
| integration: string; | ||
| type: 'meta' | 'data'; | ||
| direction: 'to-host' | 'from-host'; | ||
| } | ||
|
|
||
| /** | ||
| * Given a document index (from either a file's metadata doc or a file's chunk doc), utility will | ||
| * parse it and return information about that index | ||
| * @param index | ||
| */ | ||
| export const parseFileStorageIndex = (index: string): ParsedFileStorageIndex => { | ||
| const response: ParsedFileStorageIndex = { | ||
| index: '', | ||
| integration: '', | ||
| type: 'meta', | ||
| direction: 'from-host', | ||
| }; | ||
|
|
||
| const fileStorageIndexPatterns = [ | ||
| FILE_STORAGE_METADATA_INDEX_PATTERN, | ||
| FILE_STORAGE_DATA_INDEX_PATTERN, | ||
|
|
||
| FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN, | ||
| FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN, | ||
| ]; | ||
|
|
||
| for (const indexPattern of fileStorageIndexPatterns) { | ||
| const indexPrefix = indexPattern.substring(0, indexPattern.indexOf('*')); | ||
|
|
||
| if (index.includes(indexPrefix)) { | ||
| const isDeliveryToHost = index.includes('-tohost-'); | ||
| const isDataIndex = index.includes('host-data-'); | ||
| const integrationPosition = indexPattern.split('-').indexOf('*'); | ||
| const integration = index | ||
| .replace(/^\.ds-/, '') | ||
| .split('-') | ||
| .at(integrationPosition); | ||
|
|
||
| if (!integration) { | ||
| throw new Error(`Index name ${index} does not seem to be a File storage index`); | ||
| } | ||
|
|
||
| response.direction = isDeliveryToHost ? 'to-host' : 'from-host'; | ||
| response.type = isDataIndex ? 'data' : 'meta'; | ||
| response.integration = integration; | ||
| response.index = isDataIndex | ||
| ? getFileDataIndexName(response.integration, isDeliveryToHost) | ||
| : getFileMetadataIndexName(response.integration, isDeliveryToHost); | ||
|
|
||
| return response; | ||
| } | ||
| } | ||
|
|
||
| throw new Error( | ||
| `Unable to parse index [${index}]. Does not match a known index pattern: [${fileStorageIndexPatterns.join( | ||
| ' | ' | ||
| )}]` | ||
| ); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters