From 00f06ba8bee67fce9d1e83212e5aaa59f2fe7884 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 31 Mar 2021 15:44:28 -0700 Subject: [PATCH 01/16] Adjust registry typings for new fields added to support integrations & input groups --- .../plugins/fleet/common/types/models/epm.ts | 29 +++++++++++++++---- .../fleet/common/types/models/package_spec.ts | 4 ++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 80fabd51613ae..97417d9e0188e 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -19,7 +19,12 @@ import type { } from '../../constants'; import type { ValueOf } from '../../types'; -import type { PackageSpecManifest, PackageSpecScreenshot } from './package_spec'; +import type { + PackageSpecManifest, + PackageSpecIcon, + PackageSpecScreenshot, + PackageSpecCategory, +} from './package_spec'; export type InstallationStatus = typeof installationStatuses; @@ -111,18 +116,18 @@ interface RegistryOverridePropertyValue { } export type RegistryRelease = PackageSpecManifest['release']; -export interface RegistryImage { - src: string; +export interface RegistryImage extends PackageSpecIcon { path: string; - title?: string; - size?: string; - type?: string; } export enum RegistryPolicyTemplateKeys { name = 'name', title = 'title', description = 'description', + icons = 'icons', + screenshots = 'screenshots', + categories = 'categories', + data_streams = 'data_streams', inputs = 'inputs', multiple = 'multiple', } @@ -131,6 +136,10 @@ export interface RegistryPolicyTemplate { [RegistryPolicyTemplateKeys.name]: string; [RegistryPolicyTemplateKeys.title]: string; [RegistryPolicyTemplateKeys.description]: string; + [RegistryPolicyTemplateKeys.icons]?: RegistryImage[]; + [RegistryPolicyTemplateKeys.screenshots]?: RegistryImage[]; + [RegistryPolicyTemplateKeys.categories]?: Array; + [RegistryPolicyTemplateKeys.data_streams]?: string[]; [RegistryPolicyTemplateKeys.inputs]?: RegistryInput[]; [RegistryPolicyTemplateKeys.multiple]?: boolean; } @@ -141,6 +150,7 @@ export enum RegistryInputKeys { description = 'description', template_path = 'template_path', condition = 'condition', + input_group = 'input_group', vars = 'vars', } @@ -150,9 +160,16 @@ export interface RegistryInput { [RegistryInputKeys.description]: string; [RegistryInputKeys.template_path]?: string; [RegistryInputKeys.condition]?: string; + [RegistryInputKeys.input_group]?: string; [RegistryInputKeys.vars]?: RegistryVarsEntry[]; } +export interface RegistryInputGroup { + name: string; + title: string; + description: string; +} + export enum RegistryStreamKeys { input = 'input', title = 'title', diff --git a/x-pack/plugins/fleet/common/types/models/package_spec.ts b/x-pack/plugins/fleet/common/types/models/package_spec.ts index 65be72cbb7b6b..3001679f66a1b 100644 --- a/x-pack/plugins/fleet/common/types/models/package_spec.ts +++ b/x-pack/plugins/fleet/common/types/models/package_spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RegistryPolicyTemplate } from './epm'; +import type { RegistryPolicyTemplate, RegistryVarsEntry, RegistryInputGroup } from './epm'; // Based on https://github.com/elastic/package-spec/blob/master/versions/1/manifest.spec.yml#L8 export interface PackageSpecManifest { @@ -21,7 +21,9 @@ export interface PackageSpecManifest { conditions?: PackageSpecConditions; icons?: PackageSpecIcon[]; screenshots?: PackageSpecScreenshot[]; + input_groups?: RegistryInputGroup[]; policy_templates?: RegistryPolicyTemplate[]; + vars?: RegistryVarsEntry[]; owner: { github: string }; } From 72b563c41f814cbe6dd58bb0f43bc619c94c6111 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 31 Mar 2021 18:06:18 -0700 Subject: [PATCH 02/16] Initial pass at showing integration tiles (search doesn't quite work right yet) --- .../sections/epm/components/package_card.tsx | 20 +++- .../epm/components/package_list_grid.tsx | 96 +++++++++++++------ .../epm/screens/detail/overview/overview.tsx | 18 +++- .../fleet/sections/epm/screens/home/index.tsx | 5 +- 4 files changed, 103 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx index b4592a2283d6d..39bdfe3ced31a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx @@ -15,7 +15,9 @@ import { PackageIcon } from '../../../components/package_icon'; import { RELEASE_BADGE_LABEL, RELEASE_BADGE_DESCRIPTION } from './release_badge'; -type PackageCardProps = PackageListItem | PackageInfo; +type PackageCardProps = (PackageListItem | PackageInfo) & { + integration?: string; +}; // adding the `href` causes EuiCard to use a `a` instead of a `button` // `a` tags use `euiLinkColor` which results in blueish Badge text @@ -31,6 +33,7 @@ export function PackageCard({ release, status, icons, + integration, ...restProps }: PackageCardProps) { const { getHref } = useLink(); @@ -39,13 +42,24 @@ export function PackageCard({ if ('savedObject' in restProps) { urlVersion = restProps.savedObject.attributes.version || version; } + const packageOrIntegrationName = integration ? `${name}-${integration}` : name; return ( } - href={getHref('integration_details_overview', { pkgkey: `${name}-${urlVersion}` })} + icon={ + + } + href={getHref('integration_details_overview', { + pkgkey: `${name}-${urlVersion}`, + ...(integration ? { integration } : {}), + })} betaBadgeLabel={release && release !== 'ga' ? RELEASE_BADGE_LABEL[release] : undefined} betaBadgeTooltipContent={ release && release !== 'ga' ? RELEASE_BADGE_DESCRIPTION[release] : undefined diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx index 9ce223ea8e456..dfc0875b03503 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx @@ -6,7 +6,7 @@ */ import type { ReactNode } from 'react'; -import React, { Fragment, useState } from 'react'; +import React, { Fragment, useState, useMemo } from 'react'; import type { Query } from '@elastic/eui'; import { EuiFlexGrid, @@ -24,23 +24,53 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Loading } from '../../../components'; import type { PackageList } from '../../../types'; import { useLocalSearch, searchIdField } from '../hooks'; -import { pkgKeyFromPackageInfo } from '../../../services/pkg_key_from_package_info'; import { PackageCard } from './package_card'; -interface ListProps { +interface PackageListProps { isLoading?: boolean; controls?: ReactNode; title: string; - list: PackageList; + packages: PackageList; + showIntegrations?: boolean; } -export function PackageListGrid({ isLoading, controls, title, list }: ListProps) { +export function PackageListGrid({ + isLoading, + controls, + title, + packages, + showIntegrations = true, +}: PackageListProps) { const initialQuery = EuiSearchBar.Query.MATCH_ALL; const [query, setQuery] = useState(initialQuery); const [searchTerm, setSearchTerm] = useState(''); - const localSearchRef = useLocalSearch(list); + const packagesAndIntegrations = useMemo( + () => + showIntegrations + ? packages.reduce((acc: PackageList, pkg) => { + return [ + ...acc, + pkg, + ...(pkg.policy_templates?.length + ? pkg.policy_templates.map((integration) => { + const { name, title: integrationTitle, description, icons } = integration; + return { + ...pkg, + title: integrationTitle, + description, + integration: name, + icons: icons || pkg.icons, + }; + }) + : []), + ]; + }, []) + : packages, + [packages, showIntegrations] + ); + const localSearchRef = useLocalSearch(packagesAndIntegrations); const onQueryChange = ({ // eslint-disable-next-line @typescript-eslint/no-shadow @@ -58,21 +88,29 @@ export function PackageListGrid({ isLoading, controls, title, list }: ListProps) } }; - const controlsContent = ; - let gridContent: JSX.Element; + const controlsContent = useMemo(() => , [ + controls, + title, + ]); + + const filteredPackages = useMemo( + () => + !isLoading && searchTerm + ? packagesAndIntegrations.filter((pkg) => + (localSearchRef.current!.search(searchTerm) as PackageList) + .map((match) => match[searchIdField]) + .includes(pkg[searchIdField]) + ) + : packagesAndIntegrations, + [isLoading, localSearchRef, packagesAndIntegrations, searchTerm] + ); - if (isLoading || !localSearchRef.current) { - gridContent = ; - } else { - const filteredList = searchTerm - ? list.filter((item) => - (localSearchRef.current!.search(searchTerm) as PackageList) - .map((match) => match[searchIdField]) - .includes(item[searchIdField]) - ) - : list; - gridContent = ; - } + const gridContent = useMemo(() => { + if (isLoading || !localSearchRef.current) { + return ; + } + return ; + }, [filteredPackages, isLoading, localSearchRef]); return ( @@ -116,18 +154,20 @@ function ControlsColumn({ controls, title }: ControlsColumnProps) { } interface GridColumnProps { - list: PackageList; + packages: PackageList; } -function GridColumn({ list }: GridColumnProps) { +function GridColumn({ packages }: GridColumnProps) { return ( - {list.length ? ( - list.map((item) => ( - - - - )) + {packages.length ? ( + packages.map((pkg, i) => { + return ( + + + + ); + }) ) : ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx index bd6bece14f3e3..9058bc90e9862 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { memo } from 'react'; +import React, { memo, useMemo } from 'react'; import styled from 'styled-components'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; @@ -26,6 +26,18 @@ const LeftColumn = styled(EuiFlexItem)` `; export const OverviewPage: React.FC = memo(({ packageInfo }: Props) => { + // Collect all package-level and integration-level screenshots + const allScreenshots = useMemo( + () => + (packageInfo.policy_templates || []).reduce( + (screenshots, policyTemplate) => { + return [...screenshots, ...(policyTemplate.screenshots || [])]; + }, + [...(packageInfo.screenshots || [])] + ), + [packageInfo.policy_templates, packageInfo.screenshots] + ); + return ( @@ -40,10 +52,10 @@ export const OverviewPage: React.FC = memo(({ packageInfo }: Props) => { - {packageInfo.screenshots && packageInfo.screenshots.length ? ( + {allScreenshots.length ? ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx index e1aafde8772e1..236a88a3e5235 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx @@ -113,7 +113,8 @@ function InstalledPackages() { isLoading={isLoadingPackages} controls={controls} title={title} - list={selectedCategory === 'updates_available' ? updatablePackages : allInstalledPackages} + packages={selectedCategory === 'updates_available' ? updatablePackages : allInstalledPackages} + showIntegrations={false} /> ); } @@ -166,7 +167,7 @@ function AvailablePackages() { isLoading={isLoadingCategoryPackages} title={title} controls={controls} - list={packages} + packages={packages} /> ); } From 21d5c0940bcc81525f35769b18fb5fb2e220ffa7 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 1 Apr 2021 16:06:07 -0700 Subject: [PATCH 03/16] Second pass at integration tiles, search now works too --- .../plugins/fleet/common/types/models/epm.ts | 5 +- .../fleet/common/types/rest_spec/epm.ts | 5 +- .../fleet/components/package_icon.tsx | 4 +- .../fleet/hooks/use_package_icon_type.ts | 24 +-- .../sections/epm/components/package_card.tsx | 10 +- .../epm/components/package_list_grid.tsx | 55 ++---- .../fleet/sections/epm/hooks/index.tsx | 2 +- .../sections/epm/hooks/use_local_search.tsx | 7 +- .../epm/screens/detail/overview/overview.tsx | 4 +- .../fleet/sections/epm/screens/home/index.tsx | 166 ++++++++++++------ .../fleet/server/services/epm/packages/get.ts | 2 +- 11 files changed, 155 insertions(+), 129 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 97417d9e0188e..f5ea53bdf3c96 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -340,7 +340,10 @@ type Merge = Omit; +export type PackageListItem = Installable & { + integration?: string; + id: string; +}; export type PackagesGroupedByStatus = Record, PackageList>; export type PackageInfo = | Installable> diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 3c7a32265d20a..0508ef2ecf802 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -8,8 +8,7 @@ import type { AssetReference, CategorySummaryList, - Installable, - RegistrySearchResult, + PackageList, PackageInfo, PackageUsageStats, } from '../models/epm'; @@ -32,7 +31,7 @@ export interface GetPackagesRequest { } export interface GetPackagesResponse { - response: Array>; + response: PackageList; } export interface GetLimitedPackagesResponse { diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/package_icon.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/package_icon.tsx index cb0b02527f756..bb84a79056978 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/package_icon.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/package_icon.tsx @@ -14,7 +14,7 @@ import { usePackageIconType } from '../hooks'; export const PackageIcon: React.FunctionComponent< UsePackageIconType & Omit -> = ({ packageName, version, icons, tryApi, ...euiIconProps }) => { - const iconType = usePackageIconType({ packageName, version, icons, tryApi }); +> = ({ packageName, integrationName, version, icons, tryApi, ...euiIconProps }) => { + const iconType = usePackageIconType({ packageName, integrationName, version, icons, tryApi }); return ; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/hooks/use_package_icon_type.ts b/x-pack/plugins/fleet/public/applications/fleet/hooks/use_package_icon_type.ts index 654cfc70ab418..5701dd100bfed 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/hooks/use_package_icon_type.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/hooks/use_package_icon_type.ts @@ -16,7 +16,8 @@ import { sendGetPackageInfoByKey } from './index'; type Package = PackageInfo | PackageListItem; export interface UsePackageIconType { - packageName: Package['name']; + packageName: string; + integrationName?: string; version: Package['version']; icons?: Package['icons']; tryApi?: boolean; // should it call API to try to find missing icons? @@ -26,6 +27,7 @@ const CACHED_ICONS = new Map(); export const usePackageIconType = ({ packageName, + integrationName, version, icons: paramIcons, tryApi = false, @@ -33,13 +35,13 @@ export const usePackageIconType = ({ const { toPackageImage } = useLinks(); const [iconList, setIconList] = useState(); const [iconType, setIconType] = useState(''); // FIXME: use `empty` icon during initialization - see: https://github.com/elastic/kibana/issues/60622 - const pkgKey = `${packageName}-${version}`; + const cacheKey = `${packageName}-${version}${integrationName ? `-${integrationName}` : ''}`; // Generates an icon path or Eui Icon name based on an icon list from the package // or by using the package name against logo icons from Eui useEffect(() => { - if (CACHED_ICONS.has(pkgKey)) { - setIconType(CACHED_ICONS.get(pkgKey) || ''); + if (CACHED_ICONS.has(cacheKey)) { + setIconType(CACHED_ICONS.get(cacheKey) || ''); return; } const svgIcons = (paramIcons || iconList)?.filter( @@ -48,29 +50,29 @@ export const usePackageIconType = ({ const localIconSrc = Array.isArray(svgIcons) && toPackageImage(svgIcons[0], packageName, version); if (localIconSrc) { - CACHED_ICONS.set(pkgKey, localIconSrc); - setIconType(CACHED_ICONS.get(pkgKey) || ''); + CACHED_ICONS.set(cacheKey, localIconSrc); + setIconType(CACHED_ICONS.get(cacheKey) || ''); return; } const euiLogoIcon = ICON_TYPES.find((key) => key.toLowerCase() === `logo${packageName}`); if (euiLogoIcon) { - CACHED_ICONS.set(pkgKey, euiLogoIcon); + CACHED_ICONS.set(cacheKey, euiLogoIcon); setIconType(euiLogoIcon); return; } if (tryApi && !paramIcons && !iconList) { - sendGetPackageInfoByKey(pkgKey) + sendGetPackageInfoByKey(cacheKey) .catch((error) => undefined) // Ignore API errors .then((res) => { - CACHED_ICONS.delete(pkgKey); + CACHED_ICONS.delete(cacheKey); setIconList(res?.data?.response?.icons); }); } - CACHED_ICONS.set(pkgKey, 'package'); + CACHED_ICONS.set(cacheKey, 'package'); setIconType('package'); - }, [paramIcons, pkgKey, toPackageImage, iconList, packageName, iconType, tryApi, version]); + }, [paramIcons, cacheKey, toPackageImage, iconList, packageName, iconType, tryApi, version]); return iconType; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx index 39bdfe3ced31a..be76f05b0f5a1 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_card.tsx @@ -9,15 +9,13 @@ import React from 'react'; import styled from 'styled-components'; import { EuiCard } from '@elastic/eui'; -import type { PackageInfo, PackageListItem } from '../../../types'; +import type { PackageListItem } from '../../../types'; import { useLink } from '../../../hooks'; import { PackageIcon } from '../../../components/package_icon'; import { RELEASE_BADGE_LABEL, RELEASE_BADGE_DESCRIPTION } from './release_badge'; -type PackageCardProps = (PackageListItem | PackageInfo) & { - integration?: string; -}; +type PackageCardProps = PackageListItem; // adding the `href` causes EuiCard to use a `a` instead of a `button` // `a` tags use `euiLinkColor` which results in blueish Badge text @@ -42,7 +40,6 @@ export function PackageCard({ if ('savedObject' in restProps) { urlVersion = restProps.savedObject.attributes.version || version; } - const packageOrIntegrationName = integration ? `${name}-${integration}` : name; return ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx index dfc0875b03503..d6c0ae4d9150e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/components/package_list_grid.tsx @@ -23,7 +23,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Loading } from '../../../components'; import type { PackageList } from '../../../types'; -import { useLocalSearch, searchIdField } from '../hooks'; +import { useLocalSearch } from '../hooks'; import { PackageCard } from './package_card'; @@ -35,42 +35,12 @@ interface PackageListProps { showIntegrations?: boolean; } -export function PackageListGrid({ - isLoading, - controls, - title, - packages, - showIntegrations = true, -}: PackageListProps) { +export function PackageListGrid({ isLoading, controls, title, packages }: PackageListProps) { const initialQuery = EuiSearchBar.Query.MATCH_ALL; const [query, setQuery] = useState(initialQuery); const [searchTerm, setSearchTerm] = useState(''); - const packagesAndIntegrations = useMemo( - () => - showIntegrations - ? packages.reduce((acc: PackageList, pkg) => { - return [ - ...acc, - pkg, - ...(pkg.policy_templates?.length - ? pkg.policy_templates.map((integration) => { - const { name, title: integrationTitle, description, icons } = integration; - return { - ...pkg, - title: integrationTitle, - description, - integration: name, - icons: icons || pkg.icons, - }; - }) - : []), - ]; - }, []) - : packages, - [packages, showIntegrations] - ); - const localSearchRef = useLocalSearch(packagesAndIntegrations); + const localSearchRef = useLocalSearch(packages); const onQueryChange = ({ // eslint-disable-next-line @typescript-eslint/no-shadow @@ -93,16 +63,17 @@ export function PackageListGrid({ title, ]); + // useEffect(() => { + // console.log(packages); + // console.log(localSearchRef.current!.search(searchTerm)); + // }, [localSearchRef, searchTerm]); + const filteredPackages = useMemo( () => !isLoading && searchTerm - ? packagesAndIntegrations.filter((pkg) => - (localSearchRef.current!.search(searchTerm) as PackageList) - .map((match) => match[searchIdField]) - .includes(pkg[searchIdField]) - ) - : packagesAndIntegrations, - [isLoading, localSearchRef, packagesAndIntegrations, searchTerm] + ? (localSearchRef.current!.search(searchTerm) as PackageList) + : packages, + [isLoading, localSearchRef, packages, searchTerm] ); const gridContent = useMemo(() => { @@ -161,9 +132,9 @@ function GridColumn({ packages }: GridColumnProps) { return ( {packages.length ? ( - packages.map((pkg, i) => { + packages.map((pkg) => { return ( - + ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/index.tsx index 45f189afe350a..61a74d3eb1d92 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/index.tsx @@ -6,7 +6,7 @@ */ export { useLinks } from './use_links'; -export { useLocalSearch, searchIdField } from './use_local_search'; +export { useLocalSearch } from './use_local_search'; export { PackageInstallProvider, useUninstallPackage, diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/use_local_search.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/use_local_search.tsx index efdc2f3884542..43db5657b0615 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/use_local_search.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/hooks/use_local_search.tsx @@ -8,11 +8,10 @@ import { Search as LocalSearch } from 'js-search'; import { useEffect, useRef } from 'react'; -import type { PackageList, PackageListItem } from '../../../types'; +import type { PackageList } from '../../../types'; -export type SearchField = keyof PackageListItem; -export const searchIdField: SearchField = 'name'; -export const fieldsToSearch: SearchField[] = ['description', 'name', 'title']; +export const searchIdField = 'id'; +export const fieldsToSearch = ['description', 'name', 'title']; export function useLocalSearch(packageList: PackageList) { const localSearchRef = useRef(null); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx index 9058bc90e9862..8cafe5d7f2e1e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/overview/overview.tsx @@ -8,7 +8,7 @@ import React, { memo, useMemo } from 'react'; import styled from 'styled-components'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import type { PackageInfo } from '../../../../../types'; +import type { PackageInfo, ScreenshotItem } from '../../../../../types'; import { Screenshots } from './screenshots'; import { Readme } from './readme'; @@ -30,7 +30,7 @@ export const OverviewPage: React.FC = memo(({ packageInfo }: Props) => { const allScreenshots = useMemo( () => (packageInfo.policy_templates || []).reduce( - (screenshots, policyTemplate) => { + (screenshots: ScreenshotItem[], policyTemplate) => { return [...screenshots, ...(policyTemplate.screenshots || [])]; }, [...(packageInfo.screenshots || [])] diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx index 236a88a3e5235..8b7b3f1989d44 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx @@ -5,8 +5,9 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { memo, useState, useMemo } from 'react'; import { useRouteMatch, Switch, Route, useLocation, useHistory } from 'react-router-dom'; +import semverLt from 'semver/functions/lt'; import type { Props as EuiTabProps } from '@elastic/eui/src/components/tabs/tab'; import { i18n } from '@kbn/i18n'; @@ -14,13 +15,13 @@ import { installationStatuses } from '../../../../../../../common/constants'; import { PAGE_ROUTING_PATHS } from '../../../../constants'; import { useLink, useGetCategories, useGetPackages, useBreadcrumbs } from '../../../../hooks'; import { WithHeaderLayout } from '../../../../layouts'; -import type { CategorySummaryItem } from '../../../../types'; +import type { CategorySummaryItem, PackageList } from '../../../../types'; import { PackageListGrid } from '../../components/package_list_grid'; import { CategoryFacets } from './category_facets'; import { HeroCopy, HeroImage } from './header'; -export function EPMHomePage() { +export const EPMHomePage: React.FC = memo(() => { const { params: { tabId }, } = useRouteMatch<{ tabId?: string }>(); @@ -61,51 +62,95 @@ export function EPMHomePage() { ); -} - -function InstalledPackages() { +}); + +// Packages can export multiple integrations, aka `policy_templates` +// In the case where packages ship >1 `policy_templates`, we flatten out the +// list of packages by bringing all integrations to top-level so that +// each integration is displayed as its own tile +const packageListToIntegrationsList = (packages: PackageList): PackageList => { + return packages.reduce((acc: PackageList, pkg) => { + const { policy_templates: policyTemplates = [], ...restOfPackage } = pkg; + return [ + ...acc, + ...(policyTemplates.length > 1 + ? policyTemplates.map((integration) => { + const { name, title, description, icons } = integration; + return { + ...restOfPackage, + id: `${restOfPackage}-${name}`, + integration: name, + title, + description, + icons: icons || restOfPackage.icons, + }; + }) + : [restOfPackage]), + ]; + }, []); +}; + +const InstalledPackages: React.FC = memo(() => { useBreadcrumbs('integrations_installed'); const { data: allPackages, isLoading: isLoadingPackages } = useGetPackages({ experimental: true, }); const [selectedCategory, setSelectedCategory] = useState(''); - const title = i18n.translate('xpack.fleet.epmList.installedTitle', { - defaultMessage: 'Installed integrations', - }); - - const allInstalledPackages = - allPackages && allPackages.response - ? allPackages.response.filter((pkg) => pkg.status === installationStatuses.Installed) - : []; + const allInstalledPackages = useMemo( + () => + packageListToIntegrationsList( + (allPackages?.response || []).filter((pkg) => pkg.status === installationStatuses.Installed) + ), + [allPackages?.response] + ); - const updatablePackages = allInstalledPackages.filter( - (item) => 'savedObject' in item && item.version > item.savedObject.attributes.version + const updatablePackages = useMemo( + () => + allInstalledPackages.filter( + (item) => + 'savedObject' in item && semverLt(item.savedObject.attributes.version, item.version) + ), + [allInstalledPackages] ); - const categories = [ - { - id: '', - title: i18n.translate('xpack.fleet.epmList.allFilterLinkText', { - defaultMessage: 'All', - }), - count: allInstalledPackages.length, - }, - { - id: 'updates_available', - title: i18n.translate('xpack.fleet.epmList.updatesAvailableFilterLinkText', { - defaultMessage: 'Updates available', + const title = useMemo( + () => + i18n.translate('xpack.fleet.epmList.installedTitle', { + defaultMessage: 'Installed integrations', }), - count: updatablePackages.length, - }, - ]; + [] + ); - const controls = ( - setSelectedCategory(id)} - /> + const categories = useMemo( + () => [ + { + id: '', + title: i18n.translate('xpack.fleet.epmList.allFilterLinkText', { + defaultMessage: 'All', + }), + count: allInstalledPackages.length, + }, + { + id: 'updates_available', + title: i18n.translate('xpack.fleet.epmList.updatesAvailableFilterLinkText', { + defaultMessage: 'Updates available', + }), + count: updatablePackages.length, + }, + ], + [allInstalledPackages.length, updatablePackages.length] + ); + + const controls = useMemo( + () => ( + setSelectedCategory(id)} + /> + ), + [categories, selectedCategory] ); return ( @@ -114,12 +159,11 @@ function InstalledPackages() { controls={controls} title={title} packages={selectedCategory === 'updates_available' ? updatablePackages : allInstalledPackages} - showIntegrations={false} /> ); -} +}); -function AvailablePackages() { +const AvailablePackages: React.FC = memo(() => { useBreadcrumbs('integrations_all'); const history = useHistory(); const queryParams = new URLSearchParams(useLocation().search); @@ -130,23 +174,33 @@ function AvailablePackages() { category: selectedCategory, }); const { data: categoriesRes, isLoading: isLoadingCategories } = useGetCategories(); - const packages = - categoryPackagesRes && categoryPackagesRes.response ? categoryPackagesRes.response : []; - - const title = i18n.translate('xpack.fleet.epmList.allTitle', { - defaultMessage: 'Browse by category', - }); + const packages = useMemo( + () => packageListToIntegrationsList(categoryPackagesRes?.response || []), + [categoryPackagesRes] + ); - const categories = [ - { - id: '', - title: i18n.translate('xpack.fleet.epmList.allPackagesFilterLinkText', { - defaultMessage: 'All', + const title = useMemo( + () => + i18n.translate('xpack.fleet.epmList.allTitle', { + defaultMessage: 'Browse by category', }), - count: allPackagesRes?.response?.length || 0, - }, - ...(categoriesRes ? categoriesRes.response : []), - ]; + [] + ); + + const categories = useMemo( + () => [ + { + id: '', + title: i18n.translate('xpack.fleet.epmList.allPackagesFilterLinkText', { + defaultMessage: 'All', + }), + count: allPackagesRes?.response?.length || 0, + }, + ...(categoriesRes ? categoriesRes.response : []), + ], + [allPackagesRes?.response?.length, categoriesRes] + ); + const controls = categories ? ( ); -} +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 98dbd3bd57162..593f661222b6b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -48,7 +48,7 @@ export async function getPackages( const { savedObjectsClient, experimental, category } = options; const registryItems = await Registry.fetchList({ category, experimental }).then((items) => { return items.map((item) => - Object.assign({}, item, { title: item.title || nameAsTitle(item.name) }) + Object.assign({}, item, { title: item.title || nameAsTitle(item.name) }, { id: item.name }) ); }); // get the installed packages From 88c27ae6739fb48af2ce0995ec7a33daf6281d0e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 1 Apr 2021 16:14:28 -0700 Subject: [PATCH 04/16] Pass integration name from tiles through package details page and policy editor --- .../fleet/constants/page_paths.ts | 17 ++++++++++------ .../sections/epm/screens/detail/index.tsx | 20 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/constants/page_paths.ts b/x-pack/plugins/fleet/public/applications/fleet/constants/page_paths.ts index bcb450d5ec94e..27df7a4ebf11d 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/constants/page_paths.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/constants/page_paths.ts @@ -56,7 +56,7 @@ export const PAGE_ROUTING_PATHS = { policy_details: '/policies/:policyId/:tabId?', policy_details_settings: '/policies/:policyId/settings', add_integration_from_policy: '/policies/:policyId/add-integration', - add_integration_to_policy: '/integrations/:pkgkey/add-integration', + add_integration_to_policy: '/integrations/:pkgkey/add-integration/:integration?', edit_integration: '/policies/:policyId/edit-integration/:packagePolicyId', fleet: '/fleet', fleet_agent_list: '/fleet/agents', @@ -77,17 +77,22 @@ export const pagePathGetters: { integrations: () => '/integrations', integrations_all: () => '/integrations', integrations_installed: () => '/integrations/installed', - integration_details_overview: ({ pkgkey }) => `/integrations/detail/${pkgkey}/overview`, - integration_details_policies: ({ pkgkey }) => `/integrations/detail/${pkgkey}/policies`, - integration_details_settings: ({ pkgkey }) => `/integrations/detail/${pkgkey}/settings`, - integration_details_custom: ({ pkgkey }) => `/integrations/detail/${pkgkey}/custom`, + integration_details_overview: ({ pkgkey, integration }) => + `/integrations/detail/${pkgkey}/overview${integration ? `?integration=${integration}` : ''}`, + integration_details_policies: ({ pkgkey, integration }) => + `/integrations/detail/${pkgkey}/policies${integration ? `?integration=${integration}` : ''}`, + integration_details_settings: ({ pkgkey, integration }) => + `/integrations/detail/${pkgkey}/settings${integration ? `?integration=${integration}` : ''}`, + integration_details_custom: ({ pkgkey, integration }) => + `/integrations/detail/${pkgkey}/custom${integration ? `?integration=${integration}` : ''}`, integration_policy_edit: ({ packagePolicyId }) => `/integrations/edit-integration/${packagePolicyId}`, policies: () => '/policies', policies_list: () => '/policies', policy_details: ({ policyId, tabId }) => `/policies/${policyId}${tabId ? `/${tabId}` : ''}`, add_integration_from_policy: ({ policyId }) => `/policies/${policyId}/add-integration`, - add_integration_to_policy: ({ pkgkey }) => `/integrations/${pkgkey}/add-integration`, + add_integration_to_policy: ({ pkgkey, integration }) => + `/integrations/${pkgkey}/add-integration${integration ? `/${integration}` : ''}`, edit_integration: ({ policyId, packagePolicyId }) => `/policies/${policyId}/edit-integration/${packagePolicyId}`, fleet: () => '/fleet', diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx index a4f465cd3d619..e6c886ee42d29 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx @@ -74,7 +74,9 @@ export function Detail() { const { getHref, getPath } = useLink(); const hasWriteCapabilites = useCapabilities().write; const history = useHistory(); - const location = useLocation(); + const { pathname, search, hash } = useLocation(); + const queryParams = useMemo(() => new URLSearchParams(search), [search]); + const integration = useMemo(() => queryParams.get('integration'), [queryParams]); // Package info state const [packageInfo, setPackageInfo] = useState(null); @@ -184,9 +186,9 @@ export function Detail() { // The object below, given to `createHref` is explicitly accessing keys of `location` in order // to ensure that dependencies to this `useCallback` is set correctly (because `location` is mutable) const currentPath = history.createHref({ - pathname: location.pathname, - search: location.search, - hash: location.hash, + pathname, + search, + hash, }); const redirectToPath: CreatePackagePolicyRouteState['onSaveNavigateTo'] & CreatePackagePolicyRouteState['onCancelNavigateTo'] = [ @@ -204,11 +206,12 @@ export function Detail() { history.push({ pathname: getPath('add_integration_to_policy', { pkgkey, + ...(integration ? { integration } : {}), }), state: redirectBackRouteState, }); }, - [getPath, history, location.hash, location.pathname, location.search, pkgkey] + [getPath, history, hash, pathname, search, pkgkey, integration] ); const headerRightContent = useMemo( @@ -255,6 +258,7 @@ export function Detail() { iconType="plusInCircle" href={getHref('add_integration_to_policy', { pkgkey, + ...(integration ? { integration } : {}), })} onClick={handleAddIntegrationPolicyClick} data-test-subj="addIntegrationPolicyButton" @@ -316,6 +320,7 @@ export function Detail() { 'data-test-subj': `tab-overview`, href: getHref('integration_details_overview', { pkgkey: packageInfoKey, + ...(integration ? { integration } : {}), }), }, ]; @@ -333,6 +338,7 @@ export function Detail() { 'data-test-subj': `tab-policies`, href: getHref('integration_details_policies', { pkgkey: packageInfoKey, + ...(integration ? { integration } : {}), }), }); } @@ -349,6 +355,7 @@ export function Detail() { 'data-test-subj': `tab-settings`, href: getHref('integration_details_settings', { pkgkey: packageInfoKey, + ...(integration ? { integration } : {}), }), }); @@ -365,12 +372,13 @@ export function Detail() { 'data-test-subj': `tab-custom`, href: getHref('integration_details_custom', { pkgkey: packageInfoKey, + ...(integration ? { integration } : {}), }), }); } return tabs; - }, [getHref, packageInfo, panel, showCustomTab, packageInstallStatus]); + }, [packageInfo, panel, getHref, integration, packageInstallStatus, showCustomTab]); return ( Date: Thu, 1 Apr 2021 16:15:46 -0700 Subject: [PATCH 05/16] Fix missing dep --- .../applications/fleet/sections/epm/screens/detail/index.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx index e6c886ee42d29..31d4d8725f62a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/detail/index.tsx @@ -294,6 +294,7 @@ export function Detail() { getHref, handleAddIntegrationPolicyClick, hasWriteCapabilites, + integration, packageInfo, packageInstallStatus, pkgkey, From 53aea478bac0ff45da689655ca97ad1921703e89 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 1 Apr 2021 16:23:52 -0700 Subject: [PATCH 06/16] Create common service `doesPackageHaveIntegrations()` --- x-pack/plugins/fleet/common/services/index.ts | 1 + .../common/services/packages_with_integrations.ts | 11 +++++++++++ x-pack/plugins/fleet/common/types/models/epm.ts | 2 +- .../fleet/sections/epm/screens/home/index.tsx | 3 ++- .../fleet/public/applications/fleet/services/index.ts | 1 + 5 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/fleet/common/services/packages_with_integrations.ts diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts index cee34db4c7ec4..1fea5033e645c 100644 --- a/x-pack/plugins/fleet/common/services/index.ts +++ b/x-pack/plugins/fleet/common/services/index.ts @@ -16,3 +16,4 @@ export { isValidNamespace } from './is_valid_namespace'; export { isDiffPathProtocol } from './is_diff_path_protocol'; export { LicenseService } from './license'; export { isAgentUpgradeable } from './is_agent_upgradeable'; +export { doesPackageHaveIntegrations } from './packages_with_integrations'; diff --git a/x-pack/plugins/fleet/common/services/packages_with_integrations.ts b/x-pack/plugins/fleet/common/services/packages_with_integrations.ts new file mode 100644 index 0000000000000..d63c7518f4edb --- /dev/null +++ b/x-pack/plugins/fleet/common/services/packages_with_integrations.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PackageInfo, PackageListItem } from '../types'; + +export const doesPackageHaveIntegrations = (pkgInfo: PackageInfo | PackageListItem) => { + return (pkgInfo.policy_templates || []).length > 1; +}; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index f5ea53bdf3c96..c2b9e81c67459 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -339,11 +339,11 @@ type Merge = Omit & { integration?: string; id: string; }; + export type PackagesGroupedByStatus = Record, PackageList>; export type PackageInfo = | Installable> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx index 8b7b3f1989d44..eee7383137694 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/screens/home/index.tsx @@ -14,6 +14,7 @@ import { i18n } from '@kbn/i18n'; import { installationStatuses } from '../../../../../../../common/constants'; import { PAGE_ROUTING_PATHS } from '../../../../constants'; import { useLink, useGetCategories, useGetPackages, useBreadcrumbs } from '../../../../hooks'; +import { doesPackageHaveIntegrations } from '../../../../services'; import { WithHeaderLayout } from '../../../../layouts'; import type { CategorySummaryItem, PackageList } from '../../../../types'; import { PackageListGrid } from '../../components/package_list_grid'; @@ -73,7 +74,7 @@ const packageListToIntegrationsList = (packages: PackageList): PackageList => { const { policy_templates: policyTemplates = [], ...restOfPackage } = pkg; return [ ...acc, - ...(policyTemplates.length > 1 + ...(doesPackageHaveIntegrations(pkg) ? policyTemplates.map((integration) => { const { name, title, description, icons } = integration; return { diff --git a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts index 8d59496d8f9d0..7ac5305d40785 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts @@ -28,4 +28,5 @@ export { isValidNamespace, LicenseService, isAgentUpgradeable, + doesPackageHaveIntegrations, } from '../../../../common'; From 067d95ffb2a3516ead1121462a31fee2430338a5 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 1 Apr 2021 17:45:52 -0700 Subject: [PATCH 07/16] Initial pass at supporting package-level vars --- .../package_to_package_policy.test.ts | 6 +- .../services/package_to_package_policy.ts | 38 +- .../common/types/models/package_policy.ts | 1 + .../step_define_package_policy.tsx | 328 +++++++++++------- .../applications/fleet/services/index.ts | 1 + .../server/services/package_policy.test.ts | 80 +++++ .../fleet/server/services/package_policy.ts | 22 +- 7 files changed, 323 insertions(+), 153 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index a4cca4455a274..e67217f497d23 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -378,10 +378,11 @@ describe('Fleet - packageToPackagePolicy', () => { }, }); }); - it('returns package policy with inputs', () => { + it('returns package policy with inputs and package-level vars', () => { const mockPackageWithPolicyTemplates = ({ ...mockPackage, policy_templates: [{ inputs: [{ type: 'foo' }] }], + vars: [{ default: 'foo-var-value', name: 'var-name', type: 'text' }], } as unknown) as PackageInfo; expect( @@ -390,7 +391,6 @@ describe('Fleet - packageToPackagePolicy', () => { policy_id: '1', namespace: 'default', enabled: true, - inputs: [{ type: 'foo', enabled: true, streams: [] }], name: 'pkgPolicy-1', output_id: '2', package: { @@ -398,6 +398,8 @@ describe('Fleet - packageToPackagePolicy', () => { title: 'Mock package', version: '0.0.0', }, + inputs: [{ type: 'foo', enabled: true, streams: [] }], + vars: { 'var-name': { value: 'foo-var-value', type: 'text' } }, }); }); }); diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 0dfeb63f3b261..90707b79470ef 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -40,6 +40,21 @@ const getStreamsForInputType = ( return streams; }; +// Reduces registry var def into config object entry +const varsReducer = ( + configObject: PackagePolicyConfigRecord, + registryVar: RegistryVarsEntry +): PackagePolicyConfigRecord => { + const configEntry: PackagePolicyConfigRecordEntry = { + value: !registryVar.default && registryVar.multi ? [] : registryVar.default, + }; + if (registryVar.type) { + configEntry.type = registryVar.type; + } + configObject![registryVar.name] = configEntry; + return configObject; +}; + /* * This service creates a package policy inputs definition from defaults provided in package info */ @@ -58,21 +73,6 @@ export const packageToPackagePolicyInputs = ( if (packagePolicyTemplate?.inputs?.length) { // Map each package package policy input to agent policy package policy input packagePolicyTemplate.inputs.forEach((packageInput) => { - // Reduces registry var def into config object entry - const varsReducer = ( - configObject: PackagePolicyConfigRecord, - registryVar: RegistryVarsEntry - ): PackagePolicyConfigRecord => { - const configEntry: PackagePolicyConfigRecordEntry = { - value: !registryVar.default && registryVar.multi ? [] : registryVar.default, - }; - if (registryVar.type) { - configEntry.type = registryVar.type; - } - configObject![registryVar.name] = configEntry; - return configObject; - }; - // Map each package input stream into package policy input stream const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( packageInput.type, @@ -121,7 +121,7 @@ export const packageToPackagePolicy = ( packagePolicyName?: string, description?: string ): NewPackagePolicy => { - return { + const packagePolicy: NewPackagePolicy = { name: packagePolicyName || `${packageInfo.name}-1`, namespace, description, @@ -135,4 +135,10 @@ export const packageToPackagePolicy = ( output_id: outputId, inputs: packageToPackagePolicyInputs(packageInfo), }; + + if (packageInfo.vars && packageInfo.vars.length) { + packagePolicy.vars = packageInfo.vars.reduce(varsReducer, {}); + } + + return packagePolicy; }; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index cb84c0a2fc09a..920b8ff13c89e 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -55,6 +55,7 @@ export interface NewPackagePolicy { output_id: string; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; + vars?: PackagePolicyConfigRecord; } export interface UpdatePackagePolicy extends NewPackagePolicy { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx index 25234e5da3f81..4e18950fc55de 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect, useState } from 'react'; +import React, { memo, useEffect, useState, useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFormRow, @@ -19,12 +19,20 @@ import { EuiFlexItem, } from '@elastic/eui'; -import type { AgentPolicy, PackageInfo, PackagePolicy, NewPackagePolicy } from '../../../types'; -import { packageToPackagePolicyInputs } from '../../../services'; +import type { + AgentPolicy, + PackageInfo, + PackagePolicy, + NewPackagePolicy, + RegistryVarsEntry, +} from '../../../types'; +import { packageToPackagePolicy } from '../../../services'; import { Loading } from '../../../components'; import { pkgKeyFromPackageInfo } from '../../../services/pkg_key_from_package_info'; +import { isAdvancedVar } from './services'; import type { PackagePolicyValidationResults } from './services'; +import { PackagePolicyInputVarField } from './components'; export const StepDefinePackagePolicy: React.FunctionComponent<{ agentPolicy: AgentPolicy; @@ -32,10 +40,24 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ packagePolicy: NewPackagePolicy; updatePackagePolicy: (fields: Partial) => void; validationResults: PackagePolicyValidationResults; -}> = ({ agentPolicy, packageInfo, packagePolicy, updatePackagePolicy, validationResults }) => { +}> = memo(({ agentPolicy, packageInfo, packagePolicy, updatePackagePolicy, validationResults }) => { // Form show/hide states const [isShowingAdvanced, setIsShowingAdvanced] = useState(false); + // Package-level vars + const requiredVars: RegistryVarsEntry[] = []; + const advancedVars: RegistryVarsEntry[] = []; + + if (packageInfo.vars) { + packageInfo.vars.forEach((varDef) => { + if (isAdvancedVar(varDef)) { + advancedVars.push(varDef); + } else { + requiredVars.push(varDef); + } + }); + } + // Update package policy's package and agent policy info useEffect(() => { const pkg = packagePolicy.package; @@ -51,20 +73,21 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ .map((ds) => parseInt(ds.name.match(pkgPoliciesNamePattern)![1], 10)) .sort((a, b) => a - b); - updatePackagePolicy({ - // FIXME: Improve package policies name uniqueness - https://github.com/elastic/kibana/issues/72948 - name: `${packageInfo.name}-${ - pkgPoliciesWithMatchingNames.length - ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 - : 1 - }`, - package: { - name: packageInfo.name, - title: packageInfo.title, - version: packageInfo.version, - }, - inputs: packageToPackagePolicyInputs(packageInfo), - }); + updatePackagePolicy( + packageToPackagePolicy( + packageInfo, + agentPolicy.id, + packagePolicy.output_id, + packagePolicy.namespace, + // FIXME: Improve package policies name uniqueness - https://github.com/elastic/kibana/issues/72948 + `${packageInfo.name}-${ + pkgPoliciesWithMatchingNames.length + ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 + : 1 + }`, + packagePolicy.description + ) + ); } // If agent policy has changed, update package policy's agent policy ID and namespace @@ -74,13 +97,7 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ namespace: agentPolicy.namespace, }); } - }, [ - packagePolicy.package, - packagePolicy.policy_id, - agentPolicy, - packageInfo, - updatePackagePolicy, - ]); + }, [packagePolicy, agentPolicy, packageInfo, updatePackagePolicy]); return validationResults ? ( } > - <> + {/* Name */} - - } - > - - updatePackagePolicy({ - name: e.target.value, - }) + + } - data-test-subj="packagePolicyNameInput" - /> - + > + + updatePackagePolicy({ + name: e.target.value, + }) + } + data-test-subj="packagePolicyNameInput" + /> + + {/* Description */} - - } - labelAppend={ - + + - - } - isInvalid={!!validationResults.description} - error={validationResults.description} - > - - updatePackagePolicy({ - description: e.target.value, - }) } - data-test-subj="packagePolicyDescriptionInput" - /> - - + labelAppend={ + + + + } + isInvalid={!!validationResults.description} + error={validationResults.description} + > + + updatePackagePolicy({ + description: e.target.value, + }) + } + data-test-subj="packagePolicyDescriptionInput" + /> + + - {/* Advanced options toggle */} - - - setIsShowingAdvanced(!isShowingAdvanced)} - flush="left" - > - { + const { name: varName, type: varType } = varDef; + if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; + const value = packagePolicy.vars[varName].value; + return ( + + { + updatePackagePolicy({ + vars: { + ...packagePolicy.vars, + [varName]: { + type: varType, + value: newValue, + }, + }, + }); + }} /> - - - {!isShowingAdvanced && !!validationResults.namespace ? ( + + ); + })} + + {/* Advanced options toggle */} + + - + setIsShowingAdvanced(!isShowingAdvanced)} + flush="left" + > - + - ) : null} - + {!isShowingAdvanced && !!validationResults.namespace ? ( + + + + + + ) : null} + + {/* Advanced options content */} {/* Todo: Populate list of existing namespaces */} {isShowingAdvanced ? ( - <> - - - } - > - { - updatePackagePolicy({ - namespace: newNamespace, - }); - }} - onChange={(newNamespaces: Array<{ label: string }>) => { - updatePackagePolicy({ - namespace: newNamespaces.length ? newNamespaces[0].label : '', - }); - }} - /> - - + + + + + } + > + { + updatePackagePolicy({ + namespace: newNamespace, + }); + }} + onChange={(newNamespaces: Array<{ label: string }>) => { + updatePackagePolicy({ + namespace: newNamespaces.length ? newNamespaces[0].label : '', + }); + }} + /> + + + {/* Advanced vars */} + {advancedVars.map((varDef) => { + const { name: varName, type: varType } = varDef; + if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; + const value = packagePolicy.vars![varName].value; + return ( + + { + updatePackagePolicy({ + vars: { + ...packagePolicy.vars, + [varName]: { + type: varType, + value: newValue, + }, + }, + }); + }} + /> + + ); + })} + + ) : null} - + ) : ( ); -}; +}); diff --git a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts index 7ac5305d40785..07fb04628ea2e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts @@ -20,6 +20,7 @@ export { outputRoutesService, settingsRoutesService, appRoutesService, + packageToPackagePolicy, packageToPackagePolicyInputs, storedPackagePoliciesToAgentInputs, fullAgentPolicyToYaml, diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index b3e726bdf7c9e..af44e1514d801 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -34,6 +34,12 @@ paths: {{#each paths}} - {{this}} {{/each}} +{{#if hosts}} +hosts: +{{#each hosts}} +- {{this}} +{{/each}} +{{/if}} `), }, ]; @@ -86,6 +92,7 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, [ { type: 'log', @@ -147,6 +154,73 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, + [ + { + type: 'log', + enabled: true, + vars: { + paths: { + value: ['/var/log/set.log'], + }, + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + }, + ], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + vars: { + paths: { + value: ['/var/log/set.log'], + }, + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + compiled_stream: { + metricset: ['dataset1'], + paths: ['/var/log/set.log'], + type: 'log', + }, + }, + ], + }, + ]); + }); + + it('should work with config variables at the package level', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [ + { + dataset: 'package.dataset1', + type: 'logs', + streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + }, + ], + policy_templates: [ + { + inputs: [{ type: 'log' }], + }, + ], + } as unknown) as PackageInfo, + { + hosts: { + value: ['localhost'], + }, + }, [ { type: 'log', @@ -185,6 +259,7 @@ describe('Package policy service', () => { metricset: ['dataset1'], paths: ['/var/log/set.log'], type: 'log', + hosts: ['localhost'], }, }, ], @@ -202,6 +277,7 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, [ { type: 'log', @@ -249,6 +325,7 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, [ { type: 'log', @@ -295,6 +372,7 @@ describe('Package policy service', () => { compiled_stream: { metricset: ['dataset1'], paths: ['/var/log/set.log'], + hosts: ['localhost'], type: 'log', }, }, @@ -312,6 +390,7 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, [] ); @@ -327,6 +406,7 @@ describe('Package policy service', () => { }, ], } as unknown) as PackageInfo, + {}, [] ); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 418a10225edad..92b3ec68dcae0 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -112,7 +112,7 @@ class PackagePolicyService { } } - inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs); } const isoDate = new Date().toISOString(); @@ -341,7 +341,7 @@ class PackagePolicyService { pkgVersion: packagePolicy.package.version, }); - inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs); } await soClient.update( @@ -435,12 +435,13 @@ class PackagePolicyService { public async compilePackagePolicyInputs( pkgInfo: PackageInfo, + vars: PackagePolicy['vars'], inputs: PackagePolicyInput[] ): Promise { const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); const inputsPromises = inputs.map(async (input) => { - const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, input); - const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, input); + const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, vars, input); + const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, vars, input); return { ...input, compiled_input: compiledInput, @@ -490,6 +491,7 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI async function _compilePackagePolicyInput( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, + vars: PackagePolicy['vars'], input: PackagePolicyInput ) { if ((!input.enabled || !pkgInfo.policy_templates?.[0]?.inputs?.length) ?? 0 > 0) { @@ -515,8 +517,8 @@ async function _compilePackagePolicyInput( } return compileTemplate( - // Populate template variables from input vars - Object.assign({}, input.vars), + // Populate template variables from package- and input-level vars + Object.assign({}, vars, input.vars), pkgInputTemplate.buffer.toString() ); } @@ -524,10 +526,11 @@ async function _compilePackagePolicyInput( async function _compilePackageStreams( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, + vars: PackagePolicy['vars'], input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _compilePackageStream(registryPkgInfo, pkgInfo, input, stream) + _compilePackageStream(registryPkgInfo, pkgInfo, vars, input, stream) ); return await Promise.all(streamsPromises); @@ -536,6 +539,7 @@ async function _compilePackageStreams( async function _compilePackageStream( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, + vars: PackagePolicy['vars'], input: PackagePolicyInput, stream: PackagePolicyInputStream ) { @@ -579,8 +583,8 @@ async function _compilePackageStream( } const yaml = compileTemplate( - // Populate template variables from input vars and stream vars - Object.assign({}, input.vars, stream.vars), + // Populate template variables from package-, input-, and stream-level vars + Object.assign({}, vars, input.vars, stream.vars), pkgStreamTemplate.buffer.toString() ); From aab68b68b4f3af0e07d9e1ad5b42aa88aac078f8 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 5 Apr 2021 12:40:28 -0700 Subject: [PATCH 08/16] First pass at building package policy from package with integrations info --- .../package_to_package_policy.test.ts | 437 ++++++++++++++++++ .../services/package_to_package_policy.ts | 131 ++++-- 2 files changed, 526 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index e67217f497d23..1a4e46a43e0ba 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -338,6 +338,7 @@ describe('Fleet - packageToPackagePolicy', () => { }, }); }); + it('returns package policy with custom name', () => { expect(packageToPackagePolicy(mockPackage, '1', '2', 'default', 'pkgPolicy-1')).toEqual({ policy_id: '1', @@ -353,6 +354,7 @@ describe('Fleet - packageToPackagePolicy', () => { }, }); }); + it('returns package policy with namespace and description', () => { expect( packageToPackagePolicy( @@ -378,6 +380,7 @@ describe('Fleet - packageToPackagePolicy', () => { }, }); }); + it('returns package policy with inputs and package-level vars', () => { const mockPackageWithPolicyTemplates = ({ ...mockPackage, @@ -402,5 +405,439 @@ describe('Fleet - packageToPackagePolicy', () => { vars: { 'var-name': { value: 'foo-var-value', type: 'text' } }, }); }); + + it('returns package policy with multiple policy templates correctly', () => { + const mockPackageWithPolicyTemplates = ({ + ...mockPackage, + data_streams: [ + { + type: 'logs', + dataset: 'aws.cloudtrail', + title: 'AWS CloudTrail logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudTrail logs', + description: 'Collect AWS CloudTrail logs using s3 input', + enabled: true, + }, + { + input: 'httpjson', + vars: [ + { + name: 'interval', + type: 'text', + title: 'Interval to query Splunk Enterprise REST API', + description: 'Go Duration syntax (eg. 10s)', + multi: false, + required: true, + show_user: true, + default: '10s', + }, + { + name: 'search', + type: 'text', + title: 'Splunk search string', + multi: false, + required: true, + show_user: true, + default: 'search sourcetype=aws:cloudtrail', + }, + { + name: 'tags', + type: 'text', + title: 'Tags', + multi: true, + required: false, + show_user: false, + default: ['forwarded'], + }, + ], + template_path: 'httpjson.yml.hbs', + title: 'AWS CloudTrail logs via Splunk Enterprise REST API', + description: 'Collect AWS CloudTrail logs via Splunk Enterprise REST API', + enabled: false, + }, + ], + package: 'aws', + path: 'cloudtrail', + }, + { + type: 'logs', + dataset: 'aws.cloudwatch_logs', + title: 'AWS CloudWatch logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudWatch logs', + description: 'Collect AWS CloudWatch logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_logs', + }, + { + type: 'metrics', + dataset: 'aws.cloudwatch_metrics', + title: 'AWS CloudWatch metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '300s', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'metrics', + type: 'yaml', + title: 'Metrics', + multi: false, + required: true, + show_user: true, + default: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS CloudWatch metrics', + description: 'Collect AWS CloudWatch metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_metrics', + }, + ], + policy_templates: [ + { + name: 'cloudtrail', + title: 'AWS Cloudtrail', + description: 'Collect logs from AWS Cloudtrail', + data_streams: ['cloudtrail'], + inputs: [ + { + type: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + ], + title: 'Collect logs from Cloudtrail service', + description: 'Collecting Cloudtrail logs using S3 input', + input_group: 'logs', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudtrail.svg', + path: '/package/aws/0.5.2/img/logo_cloudtrail.svg', + title: 'AWS Cloudtrail logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/filebeat-aws-cloudtrail.png', + path: '/package/aws/0.5.2/img/filebeat-aws-cloudtrail.png', + title: 'filebeat aws cloudtrail', + size: '1702x1063', + type: 'image/png', + }, + ], + }, + { + name: 'cloudwatch', + title: 'AWS CloudWatch', + description: 'Collect logs and metrics from CloudWatch', + data_streams: ['cloudwatch_logs', 'cloudwatch_metrics'], + inputs: [ + { + type: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + ], + title: 'Collect logs from CloudWatch', + description: 'Collecting logs from CloudWatch using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + vars: [ + { + name: 'metrics', + type: 'yaml', + title: 'Metrics', + multi: false, + required: true, + show_user: true, + default: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + }, + ], + title: 'Collect metrics from CloudWatch', + description: 'Collecting metrics from AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudwatch.svg', + path: '/package/aws/0.5.2/img/logo_cloudwatch.svg', + title: 'AWS CloudWatch logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + ], + vars: [ + { + name: 'shared_credential_file', + type: 'text', + title: 'Shared Credential File', + description: 'Directory of the shared credentials file.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'credential_profile_name', + type: 'text', + title: 'Credential Profile Name', + multi: false, + required: false, + show_user: true, + }, + { + name: 'access_key_id', + type: 'text', + title: 'Access Key ID', + multi: false, + required: false, + show_user: false, + }, + { + name: 'secret_access_key', + type: 'text', + title: 'Secret Access Key', + multi: false, + required: false, + show_user: false, + }, + { + name: 'session_token', + type: 'text', + title: 'Session Token', + multi: false, + required: false, + show_user: false, + }, + { + name: 'role_arn', + type: 'text', + title: 'Role ARN', + multi: false, + required: false, + show_user: false, + }, + { + name: 'endpoint', + type: 'text', + title: 'Endpoint', + description: 'URL of the entry point for an AWS web service.', + multi: false, + required: false, + show_user: false, + default: 'amazonaws.com', + }, + ], + } as unknown) as PackageInfo; + + expect( + packageToPackagePolicy( + mockPackageWithPolicyTemplates, + 'some-policy-id', + 'some-output-id', + 'default' + ) + ).toEqual({ + name: 'mock-package-1', + namespace: 'default', + package: { name: 'mock-package', title: 'Mock package', version: '0.0.0' }, + enabled: true, + policy_id: 'some-policy-id', + output_id: 'some-output-id', + inputs: [ + { + type: 'cloudtrail', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + vars: { + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + }, + }, + ], + }, + { + type: 'cloudwatch', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, + vars: { + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, + vars: { + period: { value: '300s', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + metrics: { + value: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + type: 'yaml', + }, + }, + }, + ], + }, + ], + vars: { + shared_credential_file: { type: 'text' }, + credential_profile_name: { type: 'text' }, + access_key_id: { type: 'text' }, + secret_access_key: { type: 'text' }, + session_token: { type: 'text' }, + role_arn: { type: 'text' }, + endpoint: { value: 'amazonaws.com', type: 'text' }, + }, + }); + }); }); }); diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 90707b79470ef..1307d6f56781e 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -7,7 +7,6 @@ import type { PackageInfo, - RegistryPolicyTemplate, RegistryVarsEntry, RegistryStream, PackagePolicyConfigRecord, @@ -17,13 +16,15 @@ import type { PackagePolicyConfigRecordEntry, } from '../types'; -const getStreamsForInputType = ( +import { doesPackageHaveIntegrations } from './'; + +export const getStreamsForInputType = ( inputType: string, - packageInfo: PackageInfo + dataStreams: PackageInfo['data_streams'] = [] ): Array => { const streams: Array = []; - (packageInfo.data_streams || []).forEach((dataStream) => { + dataStreams.forEach((dataStream) => { (dataStream.streams || []).forEach((stream) => { if (stream.input === inputType) { streams.push({ @@ -61,48 +62,94 @@ const varsReducer = ( export const packageToPackagePolicyInputs = ( packageInfo: PackageInfo ): NewPackagePolicy['inputs'] => { - const inputs: NewPackagePolicy['inputs'] = []; - - // Assume package will only ever ship one package policy template for now - const packagePolicyTemplate: RegistryPolicyTemplate | null = - packageInfo.policy_templates && packageInfo.policy_templates[0] - ? packageInfo.policy_templates[0] - : null; - - // Create package policy input property - if (packagePolicyTemplate?.inputs?.length) { - // Map each package package policy input to agent policy package policy input - packagePolicyTemplate.inputs.forEach((packageInput) => { - // Map each package input stream into package policy input stream - const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( - packageInput.type, - packageInfo - ).map((packageStream) => { - const stream: NewPackagePolicyInputStream = { - enabled: packageStream.enabled === false ? false : true, - data_stream: packageStream.data_stream, - }; - if (packageStream.vars && packageStream.vars.length) { - stream.vars = packageStream.vars.reduce(varsReducer, {}); - } - return stream; - }); + const inputsByTypeOrPolicyTemplate: { [key: string]: NewPackagePolicyInput } = {}; + const packageDataStreams = packageInfo.data_streams || []; + const packagePolicyTemplates = packageInfo.policy_templates || []; + const hasIntegrations = doesPackageHaveIntegrations(packageInfo); - const input: NewPackagePolicyInput = { - type: packageInput.type, - enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, - streams, - }; + packagePolicyTemplates.forEach( + ({ + inputs: policyTemplateInputs = [], + data_streams: policyTemplateDataStreams = [], + name: policyTemplateName, + }) => { + // If policy template is has data streams defined, only look for inputs on those data streams + // Otherwise look in all of the package data streams + const dataStreamsToSearchForInputs = policyTemplateDataStreams.length + ? packageDataStreams.filter((dataStream) => + // TODO: is `path` field safe to use? + policyTemplateDataStreams.includes(dataStream.path) + ) + : packageDataStreams; - if (packageInput.vars && packageInput.vars.length) { - input.vars = packageInput.vars.reduce(varsReducer, {}); - } + // For each input on the policy template, look for matching streams + policyTemplateInputs.forEach((policyTemplateInput) => { + const { type: inputType } = policyTemplateInput; + const inputVars = policyTemplateInput.vars?.length + ? policyTemplateInput.vars.reduce(varsReducer, {}) + : {}; - inputs.push(input); - }); - } + // Map each package input stream into package policy input stream + const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( + inputType, + dataStreamsToSearchForInputs + ).map((packageStream) => { + const stream: NewPackagePolicyInputStream = { + enabled: packageStream.enabled === false ? false : true, + data_stream: packageStream.data_stream, + }; + const streamVars = packageStream.vars?.length + ? packageStream.vars.reduce(varsReducer, {}) + : {}; + + if (hasIntegrations) { + if (Object.entries(streamVars).length || Object.entries(inputVars).length) { + stream.vars = { ...inputVars, ...streamVars }; + } + } else { + if (Object.entries(streamVars).length) { + stream.vars = streamVars; + } + } + return stream; + }); + + const inputKey = hasIntegrations ? policyTemplateName : inputType; + const input: NewPackagePolicyInput = { + type: inputKey, + enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, + streams, + }; + + if (!hasIntegrations && Object.entries(inputVars).length) { + input.vars = inputVars; + } + + if (inputsByTypeOrPolicyTemplate[inputKey]) { + const existingInput = inputsByTypeOrPolicyTemplate[inputKey]; + inputsByTypeOrPolicyTemplate[inputKey] = { + ...existingInput, + enabled: existingInput.enabled || input.enabled, + streams: [...existingInput.streams, ...input.streams], + }; + if ( + !hasIntegrations && + (Object.entries(existingInput.vars || {}).length || + Object.entries(input.vars || {}).length) + ) { + inputsByTypeOrPolicyTemplate[inputKey].vars = { + ...existingInput.vars, + ...input.vars, + }; + } + } else { + inputsByTypeOrPolicyTemplate[inputKey] = input; + } + }); + } + ); - return inputs; + return Object.values(inputsByTypeOrPolicyTemplate); }; /** From 61968e5ef31050de5eceac45d87a0928112ddd33 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 5 Apr 2021 19:32:52 -0700 Subject: [PATCH 09/16] Add new field to keep track of integrations instead --- .../package_to_package_policy.test.ts | 40 +- .../services/package_to_package_policy.ts | 71 +- .../common/types/models/package_policy.ts | 7 + .../services/validate_package_policy.test.ts | 1401 ++++++++++++----- .../services/validate_package_policy.ts | 45 +- .../step_configure_package.tsx | 20 +- .../edit_package_policy_page/index.tsx | 3 - .../fleet/server/saved_objects/index.ts | 7 + 8 files changed, 1075 insertions(+), 519 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index 1a4e46a43e0ba..631c78b524a79 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -782,35 +782,35 @@ describe('Fleet - packageToPackagePolicy', () => { output_id: 'some-output-id', inputs: [ { - type: 'cloudtrail', + type: 's3', enabled: true, streams: [ { enabled: true, data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, - vars: { - queue_url: { type: 'text' }, - fips_enabled: { value: false, type: 'bool' }, - visibility_timeout: { type: 'text' }, - api_timeout: { type: 'text' }, - }, + vars: { queue_url: { type: 'text' }, fips_enabled: { value: false, type: 'bool' } }, }, ], + integration: 'cloudtrail', + vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, }, { - type: 'cloudwatch', + type: 's3', enabled: true, streams: [ { enabled: true, data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, - vars: { - queue_url: { type: 'text' }, - fips_enabled: { value: false, type: 'bool' }, - visibility_timeout: { type: 'text' }, - api_timeout: { type: 'text' }, - }, + vars: { queue_url: { type: 'text' }, fips_enabled: { value: false, type: 'bool' } }, }, + ], + integration: 'cloudwatch', + vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, + }, + { + type: 'aws/metrics', + enabled: true, + streams: [ { enabled: true, data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, @@ -826,8 +826,20 @@ describe('Fleet - packageToPackagePolicy', () => { }, }, ], + integration: 'cloudwatch', + vars: { + metrics: { + value: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + type: 'yaml', + }, + }, }, ], + integrations: [ + { name: 'cloudtrail', enabled: true }, + { name: 'cloudwatch', enabled: true }, + ], vars: { shared_credential_file: { type: 'text' }, credential_profile_name: { type: 'text' }, diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 1307d6f56781e..2113d63839af1 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -62,7 +62,7 @@ const varsReducer = ( export const packageToPackagePolicyInputs = ( packageInfo: PackageInfo ): NewPackagePolicy['inputs'] => { - const inputsByTypeOrPolicyTemplate: { [key: string]: NewPackagePolicyInput } = {}; + const inputs: NewPackagePolicy['inputs'] = []; const packageDataStreams = packageInfo.data_streams || []; const packagePolicyTemplates = packageInfo.policy_templates || []; const hasIntegrations = doesPackageHaveIntegrations(packageInfo); @@ -73,83 +73,51 @@ export const packageToPackagePolicyInputs = ( data_streams: policyTemplateDataStreams = [], name: policyTemplateName, }) => { - // If policy template is has data streams defined, only look for inputs on those data streams + // If policy template has data streams defined, only look for inputs on those data streams // Otherwise look in all of the package data streams const dataStreamsToSearchForInputs = policyTemplateDataStreams.length ? packageDataStreams.filter((dataStream) => - // TODO: is `path` field safe to use? policyTemplateDataStreams.includes(dataStream.path) ) : packageDataStreams; - // For each input on the policy template, look for matching streams - policyTemplateInputs.forEach((policyTemplateInput) => { - const { type: inputType } = policyTemplateInput; - const inputVars = policyTemplateInput.vars?.length - ? policyTemplateInput.vars.reduce(varsReducer, {}) - : {}; - + // Map each package package policy input to agent policy package policy input + policyTemplateInputs.forEach((packageInput) => { // Map each package input stream into package policy input stream const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( - inputType, + packageInput.type, dataStreamsToSearchForInputs ).map((packageStream) => { const stream: NewPackagePolicyInputStream = { enabled: packageStream.enabled === false ? false : true, data_stream: packageStream.data_stream, }; - const streamVars = packageStream.vars?.length - ? packageStream.vars.reduce(varsReducer, {}) - : {}; - - if (hasIntegrations) { - if (Object.entries(streamVars).length || Object.entries(inputVars).length) { - stream.vars = { ...inputVars, ...streamVars }; - } - } else { - if (Object.entries(streamVars).length) { - stream.vars = streamVars; - } + if (packageStream.vars && packageStream.vars.length) { + stream.vars = packageStream.vars.reduce(varsReducer, {}); } return stream; }); - const inputKey = hasIntegrations ? policyTemplateName : inputType; const input: NewPackagePolicyInput = { - type: inputKey, + type: packageInput.type, enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, streams, }; - if (!hasIntegrations && Object.entries(inputVars).length) { - input.vars = inputVars; + if (hasIntegrations) { + input.integration = policyTemplateName; } - if (inputsByTypeOrPolicyTemplate[inputKey]) { - const existingInput = inputsByTypeOrPolicyTemplate[inputKey]; - inputsByTypeOrPolicyTemplate[inputKey] = { - ...existingInput, - enabled: existingInput.enabled || input.enabled, - streams: [...existingInput.streams, ...input.streams], - }; - if ( - !hasIntegrations && - (Object.entries(existingInput.vars || {}).length || - Object.entries(input.vars || {}).length) - ) { - inputsByTypeOrPolicyTemplate[inputKey].vars = { - ...existingInput.vars, - ...input.vars, - }; - } - } else { - inputsByTypeOrPolicyTemplate[inputKey] = input; + if (packageInput.vars && packageInput.vars.length) { + input.vars = packageInput.vars.reduce(varsReducer, {}); } + + inputs.push(input); }); } ); - return Object.values(inputsByTypeOrPolicyTemplate); + return inputs; }; /** @@ -183,7 +151,14 @@ export const packageToPackagePolicy = ( inputs: packageToPackagePolicyInputs(packageInfo), }; - if (packageInfo.vars && packageInfo.vars.length) { + if (doesPackageHaveIntegrations(packageInfo)) { + packagePolicy.integrations = packageInfo.policy_templates?.map((policyTemplate) => ({ + name: policyTemplate.name, + enabled: true, + })); + } + + if (packageInfo.vars?.length) { packagePolicy.vars = packageInfo.vars.reduce(varsReducer, {}); } diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 920b8ff13c89e..36a878a48bed3 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -35,6 +35,7 @@ export interface PackagePolicyInputStream extends NewPackagePolicyInputStream { export interface NewPackagePolicyInput { type: string; + integration?: string; enabled: boolean; vars?: PackagePolicyConfigRecord; config?: PackagePolicyConfigRecord; @@ -46,6 +47,11 @@ export interface PackagePolicyInput extends Omit { - const mockPackage = ({ - name: 'mock-package', - title: 'Mock package', - version: '0.0.0', - description: 'description', - type: 'mock', - categories: [], - requirement: { kibana: { versions: '' }, elasticsearch: { versions: '' } }, - format_version: '', - download: '', - path: '', - assets: { - kibana: { - dashboard: [], - visualization: [], - search: [], - 'index-pattern': [], - }, - }, - status: installationStatuses.NotInstalled, - data_streams: [ - { - dataset: 'foo', - streams: [ - { - input: 'foo', - title: 'Foo', - vars: [{ name: 'var-name', type: 'yaml' }], - }, - ], - }, - { - dataset: 'bar', - streams: [ - { - input: 'bar', - title: 'Bar', - vars: [{ name: 'var-name', type: 'yaml', required: true }], - }, - { - input: 'with-no-stream-vars', - title: 'Bar stream no vars', - enabled: true, - }, - ], - }, - { - dataset: 'bar2', - streams: [ - { - input: 'bar', - title: 'Bar 2', - vars: [{ default: 'bar2-var-value', name: 'var-name', type: 'text' }], - }, - ], - }, - { - dataset: 'disabled', - streams: [ - { - input: 'with-disabled-streams', - title: 'Disabled', - enabled: false, - vars: [{ multi: true, required: true, name: 'var-name', type: 'text' }], - }, - ], - }, - { - dataset: 'disabled2', - streams: [ - { - input: 'with-disabled-streams', - title: 'Disabled 2', - enabled: false, - }, - ], - }, - ], - policy_templates: [ - { - name: 'pkgPolicy1', - title: 'Package policy 1', - description: 'test package policy', - inputs: [ - { - type: 'foo', - title: 'Foo', - vars: [ - { default: 'foo-input-var-value', name: 'foo-input-var-name', type: 'text' }, - { - default: 'foo-input2-var-value', - name: 'foo-input2-var-name', - required: true, - type: 'text', - }, - { name: 'foo-input3-var-name', type: 'text', required: true, multi: true }, - ], - }, - { - type: 'bar', - title: 'Bar', - vars: [ - { - default: ['value1', 'value2'], - name: 'bar-input-var-name', - type: 'text', - multi: true, - }, - { name: 'bar-input2-var-name', required: true, type: 'text' }, - ], - }, - { - type: 'with-no-config-or-streams', - title: 'With no config or streams', - }, - { - type: 'with-disabled-streams', - title: 'With disabled streams', - }, - { - type: 'with-no-stream-vars', - enabled: true, - vars: [{ required: true, name: 'var-name', type: 'text' }], - }, - ], + describe('works for packages without integrations', () => { + const mockPackage = ({ + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + description: 'description', + type: 'mock', + categories: [], + requirement: { kibana: { versions: '' }, elasticsearch: { versions: '' } }, + format_version: '', + download: '', + path: '', + assets: { + kibana: { + dashboard: [], + visualization: [], + search: [], + 'index-pattern': [], + }, }, - ], - } as unknown) as PackageInfo; + status: installationStatuses.NotInstalled, + data_streams: [ + { + dataset: 'foo', + streams: [ + { + input: 'foo', + title: 'Foo', + vars: [{ name: 'var-name', type: 'yaml' }], + }, + ], + }, + { + dataset: 'bar', + streams: [ + { + input: 'bar', + title: 'Bar', + vars: [{ name: 'var-name', type: 'yaml', required: true }], + }, + { + input: 'with-no-stream-vars', + title: 'Bar stream no vars', + enabled: true, + }, + ], + }, + { + dataset: 'bar2', + streams: [ + { + input: 'bar', + title: 'Bar 2', + vars: [{ default: 'bar2-var-value', name: 'var-name', type: 'text' }], + }, + ], + }, + { + dataset: 'disabled', + streams: [ + { + input: 'with-disabled-streams', + title: 'Disabled', + enabled: false, + vars: [{ multi: true, required: true, name: 'var-name', type: 'text' }], + }, + ], + }, + { + dataset: 'disabled2', + streams: [ + { + input: 'with-disabled-streams', + title: 'Disabled 2', + enabled: false, + }, + ], + }, + ], + policy_templates: [ + { + name: 'pkgPolicy1', + title: 'Package policy 1', + description: 'test package policy', + inputs: [ + { + type: 'foo', + title: 'Foo', + vars: [ + { default: 'foo-input-var-value', name: 'foo-input-var-name', type: 'text' }, + { + default: 'foo-input2-var-value', + name: 'foo-input2-var-name', + required: true, + type: 'text', + }, + { name: 'foo-input3-var-name', type: 'text', required: true, multi: true }, + ], + }, + { + type: 'bar', + title: 'Bar', + vars: [ + { + default: ['value1', 'value2'], + name: 'bar-input-var-name', + type: 'text', + multi: true, + }, + { name: 'bar-input2-var-name', required: true, type: 'text' }, + ], + }, + { + type: 'with-no-config-or-streams', + title: 'With no config or streams', + }, + { + type: 'with-disabled-streams', + title: 'With disabled streams', + }, + { + type: 'with-no-stream-vars', + enabled: true, + vars: [{ required: true, name: 'var-name', type: 'text' }], + }, + ], + }, + ], + } as unknown) as PackageInfo; - const validPackagePolicy: NewPackagePolicy = { - name: 'pkgPolicy1-1', - namespace: 'default', - policy_id: 'test-policy', - enabled: true, - output_id: 'test-output', - inputs: [ - { - type: 'foo', - enabled: true, - vars: { - 'foo-input-var-name': { value: 'foo-input-var-value', type: 'text' }, - 'foo-input2-var-name': { value: 'foo-input2-var-value', type: 'text' }, - 'foo-input3-var-name': { value: ['test'], type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'foo', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, - }, - ], - }, - { - type: 'bar', - enabled: true, - vars: { - 'bar-input-var-name': { value: ['value1', 'value2'], type: 'text' }, - 'bar-input2-var-name': { value: 'test', type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'bar', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, - }, - { - data_stream: { dataset: 'bar2', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: undefined, type: 'text' } }, - }, - ], - }, - { - type: 'with-no-config-or-streams', - enabled: true, - streams: [], - }, - { - type: 'with-disabled-streams', - enabled: true, - streams: [ - { - data_stream: { dataset: 'disabled', type: 'logs' }, - enabled: false, - vars: { 'var-name': { value: undefined, type: 'text' } }, + const validPackagePolicy: NewPackagePolicy = { + name: 'pkgPolicy1-1', + namespace: 'default', + policy_id: 'test-policy', + enabled: true, + output_id: 'test-output', + inputs: [ + { + type: 'foo', + enabled: true, + vars: { + 'foo-input-var-name': { value: 'foo-input-var-value', type: 'text' }, + 'foo-input2-var-name': { value: 'foo-input2-var-value', type: 'text' }, + 'foo-input3-var-name': { value: ['test'], type: 'text' }, }, - { - data_stream: { dataset: 'disabled2', type: 'logs' }, - enabled: false, + streams: [ + { + data_stream: { dataset: 'foo', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, + }, + ], + }, + { + type: 'bar', + enabled: true, + vars: { + 'bar-input-var-name': { value: ['value1', 'value2'], type: 'text' }, + 'bar-input2-var-name': { value: 'test', type: 'text' }, }, - ], - }, - { - type: 'with-no-stream-vars', - enabled: true, - vars: { - 'var-name': { value: 'test', type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, - enabled: true, + streams: [ + { + data_stream: { dataset: 'bar', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, + }, + { + data_stream: { dataset: 'bar2', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: undefined, type: 'text' } }, + }, + ], + }, + { + type: 'with-no-config-or-streams', + enabled: true, + streams: [], + }, + { + type: 'with-disabled-streams', + enabled: true, + streams: [ + { + data_stream: { dataset: 'disabled', type: 'logs' }, + enabled: false, + vars: { 'var-name': { value: undefined, type: 'text' } }, + }, + { + data_stream: { dataset: 'disabled2', type: 'logs' }, + enabled: false, + }, + ], + }, + { + type: 'with-no-stream-vars', + enabled: true, + vars: { + 'var-name': { value: 'test', type: 'text' }, }, - ], - }, - ], - }; + streams: [ + { + data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, + enabled: true, + }, + ], + }, + ], + }; - const invalidPackagePolicy: NewPackagePolicy = { - ...validPackagePolicy, - name: '', - inputs: [ - { - type: 'foo', - enabled: true, - vars: { - 'foo-input-var-name': { value: undefined, type: 'text' }, - 'foo-input2-var-name': { value: '', type: 'text' }, - 'foo-input3-var-name': { value: [], type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'foo', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } }, - }, - ], - }, - { - type: 'bar', - enabled: true, - vars: { - 'bar-input-var-name': { value: 'invalid value for multi', type: 'text' }, - 'bar-input2-var-name': { value: undefined, type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'bar', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: ' \n\n', type: 'yaml' } }, + const invalidPackagePolicy: NewPackagePolicy = { + ...validPackagePolicy, + name: '', + inputs: [ + { + type: 'foo', + enabled: true, + vars: { + 'foo-input-var-name': { value: undefined, type: 'text' }, + 'foo-input2-var-name': { value: '', type: 'text' }, + 'foo-input3-var-name': { value: [], type: 'text' }, }, - { - data_stream: { dataset: 'bar2', type: 'logs' }, - enabled: true, - vars: { 'var-name': { value: undefined, type: 'text' } }, + streams: [ + { + data_stream: { dataset: 'foo', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } }, + }, + ], + }, + { + type: 'bar', + enabled: true, + vars: { + 'bar-input-var-name': { value: 'invalid value for multi', type: 'text' }, + 'bar-input2-var-name': { value: undefined, type: 'text' }, }, - ], - }, - { - type: 'with-no-config-or-streams', - enabled: true, - streams: [], - }, - { - type: 'with-disabled-streams', - enabled: true, - streams: [ - { - data_stream: { dataset: 'disabled', type: 'logs' }, - enabled: false, - vars: { - 'var-name': { - value: 'invalid value but not checked due to not enabled', - type: 'text', + streams: [ + { + data_stream: { dataset: 'bar', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: ' \n\n', type: 'yaml' } }, + }, + { + data_stream: { dataset: 'bar2', type: 'logs' }, + enabled: true, + vars: { 'var-name': { value: undefined, type: 'text' } }, + }, + ], + }, + { + type: 'with-no-config-or-streams', + enabled: true, + streams: [], + }, + { + type: 'with-disabled-streams', + enabled: true, + streams: [ + { + data_stream: { dataset: 'disabled', type: 'logs' }, + enabled: false, + vars: { + 'var-name': { + value: 'invalid value but not checked due to not enabled', + type: 'text', + }, }, }, - }, - { - data_stream: { dataset: 'disabled2', type: 'logs' }, - enabled: false, - }, - ], - }, - { - type: 'with-no-stream-vars', - enabled: true, - vars: { - 'var-name': { value: undefined, type: 'text' }, - }, - streams: [ - { - data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, - enabled: true, - }, - ], - }, - ], - }; - - const noErrorsValidationResults = { - name: null, - description: null, - namespace: null, - inputs: { - foo: { - vars: { - 'foo-input-var-name': null, - 'foo-input2-var-name': null, - 'foo-input3-var-name': null, - }, - streams: { foo: { vars: { 'var-name': null } } }, - }, - bar: { - vars: { 'bar-input-var-name': null, 'bar-input2-var-name': null }, - streams: { - bar: { vars: { 'var-name': null } }, - bar2: { vars: { 'var-name': null } }, + { + data_stream: { dataset: 'disabled2', type: 'logs' }, + enabled: false, + }, + ], }, - }, - 'with-disabled-streams': { - streams: { - disabled: { - vars: { 'var-name': null }, + { + type: 'with-no-stream-vars', + enabled: true, + vars: { + 'var-name': { value: undefined, type: 'text' }, }, - disabled2: {}, - }, - }, - 'with-no-stream-vars': { - streams: { - 'with-no-stream-vars-bar': {}, + streams: [ + { + data_stream: { dataset: 'with-no-stream-vars-bar', type: 'logs' }, + enabled: true, + }, + ], }, - vars: { 'var-name': null }, - }, - }, - }; + ], + }; - it('returns no errors for valid package policy', () => { - expect(validatePackagePolicy(validPackagePolicy, mockPackage)).toEqual( - noErrorsValidationResults - ); - }); - - it('returns errors for invalid package policy', () => { - expect(validatePackagePolicy(invalidPackagePolicy, mockPackage)).toEqual({ - name: ['Name is required'], + const noErrorsValidationResults = { + name: null, description: null, namespace: null, inputs: { foo: { vars: { 'foo-input-var-name': null, - 'foo-input2-var-name': ['foo-input2-var-name is required'], - 'foo-input3-var-name': ['foo-input3-var-name is required'], + 'foo-input2-var-name': null, + 'foo-input3-var-name': null, }, - streams: { foo: { vars: { 'var-name': ['Invalid YAML format'] } } }, + streams: { foo: { vars: { 'var-name': null } } }, }, bar: { - vars: { - 'bar-input-var-name': ['Invalid format'], - 'bar-input2-var-name': ['bar-input2-var-name is required'], - }, + vars: { 'bar-input-var-name': null, 'bar-input2-var-name': null }, streams: { - bar: { vars: { 'var-name': ['var-name is required'] } }, + bar: { vars: { 'var-name': null } }, bar2: { vars: { 'var-name': null } }, }, }, 'with-disabled-streams': { streams: { - disabled: { vars: { 'var-name': null } }, + disabled: { + vars: { 'var-name': null }, + }, disabled2: {}, }, }, 'with-no-stream-vars': { - vars: { - 'var-name': ['var-name is required'], + streams: { + 'with-no-stream-vars-bar': {}, }, - streams: { 'with-no-stream-vars-bar': {} }, + vars: { 'var-name': null }, }, }, + }; + + it('returns no errors for valid package policy', () => { + expect(validatePackagePolicy(validPackagePolicy, mockPackage)).toEqual( + noErrorsValidationResults + ); }); - }); - it('returns no errors for disabled inputs', () => { - const disabledInputs = invalidPackagePolicy.inputs.map((input) => ({ - ...input, - enabled: false, - })); - expect( - validatePackagePolicy({ ...validPackagePolicy, inputs: disabledInputs }, mockPackage) - ).toEqual(noErrorsValidationResults); + it('returns errors for invalid package policy', () => { + expect(validatePackagePolicy(invalidPackagePolicy, mockPackage)).toEqual({ + name: ['Name is required'], + description: null, + namespace: null, + inputs: { + foo: { + vars: { + 'foo-input-var-name': null, + 'foo-input2-var-name': ['foo-input2-var-name is required'], + 'foo-input3-var-name': ['foo-input3-var-name is required'], + }, + streams: { foo: { vars: { 'var-name': ['Invalid YAML format'] } } }, + }, + bar: { + vars: { + 'bar-input-var-name': ['Invalid format'], + 'bar-input2-var-name': ['bar-input2-var-name is required'], + }, + streams: { + bar: { vars: { 'var-name': ['var-name is required'] } }, + bar2: { vars: { 'var-name': null } }, + }, + }, + 'with-disabled-streams': { + streams: { + disabled: { vars: { 'var-name': null } }, + disabled2: {}, + }, + }, + 'with-no-stream-vars': { + vars: { + 'var-name': ['var-name is required'], + }, + streams: { 'with-no-stream-vars-bar': {} }, + }, + }, + }); + }); + + it('returns no errors for disabled inputs', () => { + const disabledInputs = invalidPackagePolicy.inputs.map((input) => ({ + ...input, + enabled: false, + })); + expect( + validatePackagePolicy({ ...validPackagePolicy, inputs: disabledInputs }, mockPackage) + ).toEqual(noErrorsValidationResults); + }); + + it('returns only package policy and input-level errors for disabled streams', () => { + const inputsWithDisabledStreams = invalidPackagePolicy.inputs.map((input) => + input.streams + ? { + ...input, + streams: input.streams.map((stream) => ({ ...stream, enabled: false })), + } + : input + ); + expect( + validatePackagePolicy( + { ...invalidPackagePolicy, inputs: inputsWithDisabledStreams }, + mockPackage + ) + ).toEqual({ + name: ['Name is required'], + description: null, + namespace: null, + inputs: { + foo: { + vars: { + 'foo-input-var-name': null, + 'foo-input2-var-name': ['foo-input2-var-name is required'], + 'foo-input3-var-name': ['foo-input3-var-name is required'], + }, + streams: { foo: { vars: { 'var-name': null } } }, + }, + bar: { + vars: { + 'bar-input-var-name': ['Invalid format'], + 'bar-input2-var-name': ['bar-input2-var-name is required'], + }, + streams: { + bar: { vars: { 'var-name': null } }, + bar2: { vars: { 'var-name': null } }, + }, + }, + 'with-disabled-streams': { + streams: { + disabled: { + vars: { 'var-name': null }, + }, + disabled2: {}, + }, + }, + 'with-no-stream-vars': { + vars: { + 'var-name': ['var-name is required'], + }, + streams: { 'with-no-stream-vars-bar': {} }, + }, + }, + }); + }); + + it('returns no errors for packages with no package policies', () => { + expect( + validatePackagePolicy(validPackagePolicy, { + ...mockPackage, + policy_templates: undefined, + }) + ).toEqual({ + name: null, + description: null, + namespace: null, + inputs: null, + }); + expect( + validatePackagePolicy(validPackagePolicy, { + ...mockPackage, + policy_templates: [], + }) + ).toEqual({ + name: null, + description: null, + namespace: null, + inputs: null, + }); + }); + + it('returns no errors for packages with no inputs', () => { + expect( + validatePackagePolicy(validPackagePolicy, { + ...mockPackage, + policy_templates: [{} as RegistryPolicyTemplate], + }) + ).toEqual({ + name: null, + description: null, + namespace: null, + inputs: null, + }); + expect( + validatePackagePolicy(validPackagePolicy, { + ...mockPackage, + policy_templates: [({ inputs: [] } as unknown) as RegistryPolicyTemplate], + }) + ).toEqual({ + name: null, + description: null, + namespace: null, + inputs: null, + }); + }); }); - it('returns only package policy and input-level errors for disabled streams', () => { - const inputsWithDisabledStreams = invalidPackagePolicy.inputs.map((input) => - input.streams - ? { - ...input, - streams: input.streams.map((stream) => ({ ...stream, enabled: false })), - } - : input - ); - expect( - validatePackagePolicy( - { ...invalidPackagePolicy, inputs: inputsWithDisabledStreams }, - mockPackage - ) - ).toEqual({ - name: ['Name is required'], + describe('works for packages with integrations (multiple policy templates)', () => { + const mockPackage = ({ + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + latestVersion: '0.0.0', + description: 'description', + type: 'integration', + categories: [], + conditions: { kibana: { version: '' } }, + format_version: '', + download: '', + path: '', + assets: { + kibana: { + dashboard: [], + visualization: [], + search: [], + index_pattern: [], + map: [], + lens: [], + ml_module: [], + }, + elasticsearch: { + ingest_pipeline: [], + component_template: [], + index_template: [], + transform: [], + ilm_policy: [], + data_stream_ilm_policy: [], + }, + }, + status: 'not_installed', + release: 'experimental', + owner: { + github: 'elastic/fleet', + }, + data_streams: [ + { + type: 'logs', + dataset: 'aws.cloudtrail', + title: 'AWS CloudTrail logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudTrail logs', + description: 'Collect AWS CloudTrail logs using s3 input', + enabled: true, + }, + { + input: 'httpjson', + vars: [ + { + name: 'interval', + type: 'text', + title: 'Interval to query Splunk Enterprise REST API', + description: 'Go Duration syntax (eg. 10s)', + multi: false, + required: true, + show_user: true, + default: '10s', + }, + { + name: 'search', + type: 'text', + title: 'Splunk search string', + multi: false, + required: true, + show_user: true, + default: 'search sourcetype=aws:cloudtrail', + }, + { + name: 'tags', + type: 'text', + title: 'Tags', + multi: true, + required: false, + show_user: false, + default: ['forwarded'], + }, + ], + template_path: 'httpjson.yml.hbs', + title: 'AWS CloudTrail logs via Splunk Enterprise REST API', + description: 'Collect AWS CloudTrail logs via Splunk Enterprise REST API', + enabled: false, + }, + ], + package: 'aws', + path: 'cloudtrail', + }, + { + type: 'logs', + dataset: 'aws.cloudwatch_logs', + title: 'AWS CloudWatch logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudWatch logs', + description: 'Collect AWS CloudWatch logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_logs', + }, + { + type: 'metrics', + dataset: 'aws.cloudwatch_metrics', + title: 'AWS CloudWatch metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '300s', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'metrics', + type: 'yaml', + title: 'Metrics', + multi: false, + required: true, + show_user: true, + default: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS CloudWatch metrics', + description: 'Collect AWS CloudWatch metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_metrics', + }, + ], + policy_templates: [ + { + name: 'cloudtrail', + title: 'AWS Cloudtrail', + description: 'Collect logs from AWS Cloudtrail', + data_streams: ['cloudtrail'], + inputs: [ + { + type: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + ], + title: 'Collect logs from Cloudtrail service', + description: 'Collecting Cloudtrail logs using S3 input', + input_group: 'logs', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudtrail.svg', + path: '/package/aws/0.5.2/img/logo_cloudtrail.svg', + title: 'AWS Cloudtrail logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/filebeat-aws-cloudtrail.png', + path: '/package/aws/0.5.2/img/filebeat-aws-cloudtrail.png', + title: 'filebeat aws cloudtrail', + size: '1702x1063', + type: 'image/png', + }, + ], + }, + { + name: 'cloudwatch', + title: 'AWS CloudWatch', + description: 'Collect logs and metrics from CloudWatch', + data_streams: ['cloudwatch_logs', 'cloudwatch_metrics'], + inputs: [ + { + type: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + ], + title: 'Collect logs from CloudWatch', + description: 'Collecting logs from CloudWatch using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + vars: [ + { + name: 'metrics', + type: 'yaml', + title: 'Metrics', + multi: false, + required: true, + show_user: true, + default: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + }, + ], + title: 'Collect metrics from CloudWatch', + description: 'Collecting metrics from AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudwatch.svg', + path: '/package/aws/0.5.2/img/logo_cloudwatch.svg', + title: 'AWS CloudWatch logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + ], + vars: [ + { + name: 'shared_credential_file', + type: 'text', + title: 'Shared Credential File', + description: 'Directory of the shared credentials file.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'credential_profile_name', + type: 'text', + title: 'Credential Profile Name', + multi: false, + required: false, + show_user: true, + }, + { + name: 'access_key_id', + type: 'text', + title: 'Access Key ID', + multi: false, + required: false, + show_user: false, + }, + { + name: 'secret_access_key', + type: 'text', + title: 'Secret Access Key', + multi: false, + required: false, + show_user: false, + }, + { + name: 'session_token', + type: 'text', + title: 'Session Token', + multi: false, + required: false, + show_user: false, + }, + { + name: 'role_arn', + type: 'text', + title: 'Role ARN', + multi: false, + required: false, + show_user: false, + }, + { + name: 'endpoint', + type: 'text', + title: 'Endpoint', + description: 'URL of the entry point for an AWS web service.', + multi: false, + required: false, + show_user: false, + default: 'amazonaws.com', + }, + ], + } as unknown) as PackageInfo; + + const validPackagePolicy: NewPackagePolicy = { + name: 'mock-package-1', + namespace: 'default', + package: { name: 'mock-package', title: 'Mock package', version: '0.0.0' }, + enabled: true, + policy_id: 'some-policy-id', + output_id: 'some-output-id', + inputs: [ + { + type: 's3', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + vars: { + queue_url: { type: 'text', value: 'localhost' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + ], + integration: 'cloudtrail', + vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, + }, + { + type: 's3', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, + vars: { + queue_url: { type: 'text', value: 'localhost' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + ], + integration: 'cloudwatch', + vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, + }, + { + type: 'aws/metrics', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, + vars: { + period: { value: '300s', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + metrics: { + value: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + type: 'yaml', + }, + }, + }, + ], + integration: 'cloudwatch', + vars: { + metrics: { + value: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + type: 'yaml', + }, + }, + }, + ], + integrations: [ + { name: 'cloudtrail', enabled: true }, + { name: 'cloudwatch', enabled: true }, + ], + vars: { + shared_credential_file: { type: 'text' }, + credential_profile_name: { type: 'text' }, + access_key_id: { type: 'text' }, + secret_access_key: { type: 'text' }, + session_token: { type: 'text' }, + role_arn: { type: 'text' }, + endpoint: { value: 'amazonaws.com', type: 'text' }, + }, + }; + + const noErrorsValidationResults = { + name: null, description: null, namespace: null, inputs: { - foo: { - vars: { - 'foo-input-var-name': null, - 'foo-input2-var-name': ['foo-input2-var-name is required'], - 'foo-input3-var-name': ['foo-input3-var-name is required'], + 's3-cloudtrail': { + streams: { + 'aws.cloudtrail': { + vars: { + queue_url: null, + fips_enabled: null, + }, + }, }, - streams: { foo: { vars: { 'var-name': null } } }, - }, - bar: { vars: { - 'bar-input-var-name': ['Invalid format'], - 'bar-input2-var-name': ['bar-input2-var-name is required'], + visibility_timeout: null, + api_timeout: null, }, + }, + 's3-cloudwatch': { streams: { - bar: { vars: { 'var-name': null } }, - bar2: { vars: { 'var-name': null } }, + 'aws.cloudwatch_logs': { + vars: { + queue_url: null, + fips_enabled: null, + }, + }, + }, + vars: { + visibility_timeout: null, + api_timeout: null, }, }, - 'with-disabled-streams': { + 'aws/metrics-cloudwatch': { streams: { - disabled: { - vars: { 'var-name': null }, + 'aws.cloudwatch_metrics': { + vars: { + period: null, + regions: null, + latency: null, + metrics: null, + }, }, - disabled2: {}, }, - }, - 'with-no-stream-vars': { vars: { - 'var-name': ['var-name is required'], + metrics: null, }, - streams: { 'with-no-stream-vars-bar': {} }, }, }, - }); - }); - - it('returns no errors for packages with no package policies', () => { - expect( - validatePackagePolicy(validPackagePolicy, { - ...mockPackage, - policy_templates: undefined, - }) - ).toEqual({ - name: null, - description: null, - namespace: null, - inputs: null, - }); - expect( - validatePackagePolicy(validPackagePolicy, { - ...mockPackage, - policy_templates: [], - }) - ).toEqual({ - name: null, - description: null, - namespace: null, - inputs: null, - }); - }); + vars: { + shared_credential_file: null, + credential_profile_name: null, + access_key_id: null, + secret_access_key: null, + session_token: null, + role_arn: null, + endpoint: null, + }, + }; - it('returns no errors for packages with no inputs', () => { - expect( - validatePackagePolicy(validPackagePolicy, { - ...mockPackage, - policy_templates: [{} as RegistryPolicyTemplate], - }) - ).toEqual({ - name: null, - description: null, - namespace: null, - inputs: null, - }); - expect( - validatePackagePolicy(validPackagePolicy, { - ...mockPackage, - policy_templates: [({ inputs: [] } as unknown) as RegistryPolicyTemplate], - }) - ).toEqual({ - name: null, - description: null, - namespace: null, - inputs: null, + it('returns no errors for valid package policy', () => { + expect(validatePackagePolicy(validPackagePolicy, mockPackage)).toEqual( + noErrorsValidationResults + ); }); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts index e36a4b46039f4..c5c9df089f0c4 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts @@ -7,8 +7,14 @@ import { i18n } from '@kbn/i18n'; import { safeLoad } from 'js-yaml'; +import { keyBy } from 'lodash'; + +import { + getFlattenedObject, + isValidNamespace, + doesPackageHaveIntegrations, +} from '../../../../services'; -import { getFlattenedObject, isValidNamespace } from '../../../../services'; import type { NewPackagePolicy, PackagePolicyInput, @@ -32,12 +38,12 @@ export type PackagePolicyInputValidationResults = PackagePolicyConfigValidationR streams?: Record; }; -export interface PackagePolicyValidationResults { +export type PackagePolicyValidationResults = { name: Errors; description: Errors; namespace: Errors; inputs: Record | null; -} +} & PackagePolicyConfigValidationResults; /* * Returns validation information for a given package policy and package info @@ -47,6 +53,7 @@ export const validatePackagePolicy = ( packagePolicy: NewPackagePolicy, packageInfo: PackageInfo ): PackagePolicyValidationResults => { + const hasIntegrations = doesPackageHaveIntegrations(packageInfo); const validationResults: PackagePolicyValidationResults = { name: null, description: null, @@ -67,6 +74,16 @@ export const validatePackagePolicy = ( validationResults.namespace = [namespaceValidation.error]; } + // Validate package-level vars + const packageVarsByName = keyBy(packageInfo.vars || [], 'name'); + const packageVars = Object.entries(packagePolicy.vars || {}); + if (packageVars.length) { + validationResults.vars = packageVars.reduce((results, [name, configEntry]) => { + results[name] = validatePackagePolicyConfig(configEntry, packageVarsByName[name]); + return results; + }, {} as ValidationEntry); + } + if ( !packageInfo.policy_templates || packageInfo.policy_templates.length === 0 || @@ -78,11 +95,14 @@ export const validatePackagePolicy = ( return validationResults; } - const registryInputsByType: Record< + const registryInputsByTypeAndIntegration: Record< string, RegistryInput - > = packageInfo.policy_templates[0].inputs.reduce((inputs, registryInput) => { - inputs[registryInput.type] = registryInput; + > = packageInfo.policy_templates.reduce((inputs, policyTemplate) => { + (policyTemplate.inputs || []).forEach((input) => { + inputs[hasIntegrations ? `${input.type}-${policyTemplate.name}` : input.type] = input; + }); + return inputs; }, {} as Record); @@ -99,18 +119,13 @@ export const validatePackagePolicy = ( return; } + const inputKey = hasIntegrations ? `${input.type}-${input.integration}` : input.type; const inputValidationResults: PackagePolicyInputValidationResults = { vars: undefined, streams: {}, }; - const inputVarsByName = (registryInputsByType[input.type].vars || []).reduce( - (vars, registryVar) => { - vars[registryVar.name] = registryVar; - return vars; - }, - {} as Record - ); + const inputVarsByName = keyBy(registryInputsByTypeAndIntegration[inputKey].vars || [], 'name'); // Validate input-level config fields const inputConfigs = Object.entries(input.vars || {}); @@ -161,7 +176,7 @@ export const validatePackagePolicy = ( } if (inputValidationResults.vars || inputValidationResults.streams) { - validationResults.inputs![input.type] = inputValidationResults; + validationResults.inputs![inputKey] = inputValidationResults; } }); @@ -263,7 +278,7 @@ export const countValidationErrors = ( | PackagePolicyInputValidationResults | PackagePolicyConfigValidationResults ): number => { - const flattenedValidation = getFlattenedObject(validationResults); + const flattenedValidation = getFlattenedObject(validationResults || {}); const errors = Object.values(flattenedValidation).filter((value) => Boolean(value)) || []; return errors.length; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx index 87c5af56f31b2..e8b82786caa84 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx @@ -50,22 +50,12 @@ const findStreamsForInputType = ( }; export const StepConfigurePackagePolicy: React.FunctionComponent<{ - from?: CreatePackagePolicyFrom; packageInfo: PackageInfo; packagePolicy: NewPackagePolicy; - packagePolicyId?: string; updatePackagePolicy: (fields: Partial) => void; validationResults: PackagePolicyValidationResults; submitAttempted: boolean; -}> = ({ - from = 'policy', - packageInfo, - packagePolicy, - packagePolicyId, - updatePackagePolicy, - validationResults, - submitAttempted, -}) => { +}> = ({ packageInfo, packagePolicy, updatePackagePolicy, validationResults, submitAttempted }) => { // Configure inputs (and their streams) // Assume packages only export one config template for now const renderConfigureInputs = () => @@ -100,7 +90,13 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{ inputs: newInputs, }); }} - inputValidationResults={validationResults!.inputs![packagePolicyInput.type]} + inputValidationResults={ + validationResults!.inputs![ + packagePolicyInput.integration + ? `${packagePolicyInput.type}-${packagePolicyInput.integration}` + : packagePolicyInput.type + ] + } forceShowErrors={submitAttempted} /> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx index 8df8b7ebcd4cf..79de168c738af 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx @@ -356,10 +356,8 @@ export const EditPackagePolicyForm = memo<{ {/* Only show the out-of-box configuration step if a UI extension is NOT registered */} {!ExtensionView && ( Date: Tue, 6 Apr 2021 13:30:24 -0700 Subject: [PATCH 10/16] Revert storing `integration` key on inputs, group by input groups instead --- .../__snapshots__/group_inputs.test.ts.snap | 1055 ++++++++++ .../package_to_package_policy.test.ts.snap | 623 ++++++ .../common/services/fixtures/packages.ts | 1858 +++++++++++++++++ .../common/services/group_inputs.test.ts | 14 + .../fleet/common/services/group_inputs.ts | 63 + x-pack/plugins/fleet/common/services/index.ts | 8 +- .../package_to_package_policy.test.ts | 440 +--- .../services/package_to_package_policy.ts | 93 +- .../services/packages_with_integrations.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 4 +- .../common/types/models/package_policy.ts | 7 - .../services/index.ts | 1 + .../step_configure_package.tsx | 1 - .../applications/fleet/services/index.ts | 2 + .../public/applications/fleet/types/index.ts | 1 + .../fleet/server/saved_objects/index.ts | 7 - 16 files changed, 3665 insertions(+), 514 deletions(-) create mode 100644 x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap create mode 100644 x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap create mode 100644 x-pack/plugins/fleet/common/services/fixtures/packages.ts create mode 100644 x-pack/plugins/fleet/common/services/group_inputs.test.ts create mode 100644 x-pack/plugins/fleet/common/services/group_inputs.ts diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap new file mode 100644 index 0000000000000..b9b35675ab13b --- /dev/null +++ b/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap @@ -0,0 +1,1055 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`groupInputs() should group inputs for a package with multiple policy templates and input groups correctly 1`] = ` +Array [ + Object { + "description": "Collect logs for AWS services", + "name": "logs", + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.cloudtrail", + "type": "logs", + }, + "description": "Collect AWS CloudTrail logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS CloudTrail logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_logs", + "type": "logs", + }, + "description": "Collect AWS CloudWatch logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS CloudWatch logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_logs", + "type": "logs", + }, + "description": "Collect AWS EC2 logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS EC2 logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_logs", + "type": "logs", + }, + "description": "Collect AWS ELB logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS ELB logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.s3access", + "type": "logs", + }, + "description": "Collect AWS s3access logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS s3access logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.vpcflow", + "type": "logs", + }, + "description": "Collect AWS vpcflow logs using s3 input", + "enabled": true, + "input": "s3", + "template_path": "s3.yml.hbs", + "title": "AWS vpcflow logs", + "vars": Array [ + Object { + "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", + "multi": false, + "name": "visibility_timeout", + "required": false, + "show_user": false, + "title": "Visibility Timeout", + "type": "text", + }, + Object { + "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", + "multi": false, + "name": "api_timeout", + "required": false, + "show_user": false, + "title": "API Timeout", + "type": "text", + }, + Object { + "description": "URL of the AWS SQS queue that messages will be received from.", + "multi": false, + "name": "queue_url", + "required": true, + "show_user": true, + "title": "Queue URL", + "type": "text", + }, + Object { + "default": false, + "description": "Enabling this option changes the service name from \`s3\` to \`s3-fips\` for connecting to the correct service endpoint.", + "multi": false, + "name": "fips_enabled", + "required": false, + "show_user": false, + "title": "Enable S3 FIPS", + "type": "bool", + }, + ], + }, + ], + "title": "Logs", + }, + Object { + "description": "Collect metrics for AWS services", + "name": "metrics", + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.billing", + "type": "metrics", + }, + "description": "Collect AWS billing metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS Billing metrics", + "vars": Array [ + Object { + "default": "12h", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": Array [ + "AZ", + "INSTANCE_TYPE", + "SERVICE", + ], + "multi": true, + "name": "cost_explorer_config.group_by_dimension_keys", + "required": false, + "show_user": true, + "title": "Cost Explorer Group By Dimension Keys", + "type": "text", + }, + Object { + "default": Array [ + "aws:createdBy", + ], + "multi": true, + "name": "cost_explorer_config.group_by_tag_keys", + "required": false, + "show_user": true, + "title": "Cost Explorer Group By Tag Keys", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_metrics", + "type": "metrics", + }, + "description": "Collect AWS CloudWatch metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS CloudWatch metrics", + "vars": Array [ + Object { + "default": "300s", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "- namespace: AWS/EC2 + resource_type: ec2:instance + name: + - CPUUtilization + - DiskWriteOps + statistic: + - Average + - Maximum + # dimensions: + # - name: InstanceId + # value: i-123456 + # tags: + # - key: created-by + # value: foo +", + "multi": false, + "name": "metrics", + "required": true, + "show_user": true, + "title": "Metrics", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.dynamodb", + "type": "metrics", + }, + "description": "Collect AWS DynamoDB metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS DynamoDB metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.ebs", + "type": "metrics", + }, + "description": "Collect AWS EBS metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS EBS metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_metrics", + "type": "metrics", + }, + "description": "Collect AWS EC2 metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS EC2 metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_metrics", + "type": "metrics", + }, + "description": "Collect AWS ELB metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS ELB metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.lambda", + "type": "metrics", + }, + "description": "Collect AWS Lambda metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS Lambda metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.natgateway", + "type": "metrics", + }, + "description": "Collect AWS NAT gateway metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS NAT gateway metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.rds", + "type": "metrics", + }, + "description": "Collect AWS RDS metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS RDS metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_daily_storage", + "type": "metrics", + }, + "description": "Collect AWS S3 daily storage metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS S3 daily storage metrics", + "vars": Array [ + Object { + "default": "24h", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_request", + "type": "metrics", + }, + "description": "Collect AWS S3 request metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS S3 request metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.sns", + "type": "metrics", + }, + "description": "Collect AWS SNS metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS SNS metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.sqs", + "type": "metrics", + }, + "description": "Collect AWS SQS metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS SQS metrics", + "vars": Array [ + Object { + "default": "5m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.transitgateway", + "type": "metrics", + }, + "description": "Collect AWS Transit Gateway metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS Transit Gateway metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.usage", + "type": "metrics", + }, + "description": "Collect AWS usage metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS usage metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + ], + }, + Object { + "data_stream": Object { + "dataset": "aws.vpn", + "type": "metrics", + }, + "description": "Collect AWS VPN metrics", + "enabled": true, + "input": "aws/metrics", + "template_path": "stream.yml.hbs", + "title": "AWS VPN metrics", + "vars": Array [ + Object { + "default": "1m", + "multi": false, + "name": "period", + "required": true, + "show_user": true, + "title": "Period", + "type": "text", + }, + Object { + "multi": true, + "name": "regions", + "required": false, + "show_user": true, + "title": "Regions", + "type": "text", + }, + Object { + "multi": false, + "name": "latency", + "required": false, + "show_user": false, + "title": "Latency", + "type": "text", + }, + Object { + "default": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + "multi": false, + "name": "tags_filter", + "required": false, + "show_user": false, + "title": "Tags Filter", + "type": "yaml", + }, + ], + }, + ], + "title": "Metrics", + }, +] +`; diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap new file mode 100644 index 0000000000000..4d53b1a6013e8 --- /dev/null +++ b/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap @@ -0,0 +1,623 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Fleet - packageToPackagePolicy packageToPackagePolicy returns package policy with multiple policy templates correctly 1`] = ` +Object { + "description": undefined, + "enabled": true, + "inputs": Array [ + Object { + "enabled": true, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.cloudtrail", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_logs", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_logs", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_logs", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3access", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.vpcflow", + "type": "logs", + }, + "enabled": true, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + ], + "type": "logs", + }, + Object { + "enabled": true, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.billing", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "cost_explorer_config.group_by_dimension_keys": Object { + "type": "text", + "value": Array [ + "AZ", + "INSTANCE_TYPE", + "SERVICE", + ], + }, + "cost_explorer_config.group_by_tag_keys": Object { + "type": "text", + "value": Array [ + "aws:createdBy", + ], + }, + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "12h", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_metrics", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "metrics": Object { + "type": "yaml", + "value": "- namespace: AWS/EC2 + resource_type: ec2:instance + name: + - CPUUtilization + - DiskWriteOps + statistic: + - Average + - Maximum + # dimensions: + # - name: InstanceId + # value: i-123456 + # tags: + # - key: created-by + # value: foo +", + }, + "period": Object { + "type": "text", + "value": "300s", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.dynamodb", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ebs", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_metrics", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_metrics", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.lambda", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.natgateway", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.rds", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_daily_storage", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "24h", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_request", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.sns", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.sqs", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.transitgateway", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.usage", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.vpn", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + ], + "type": "metrics", + }, + ], + "name": "aws-1", + "namespace": "default", + "output_id": "some-output-id", + "package": Object { + "name": "aws", + "title": "AWS", + "version": "0.5.2", + }, + "policy_id": "some-policy-id", + "vars": Object { + "access_key_id": Object { + "type": "text", + "value": undefined, + }, + "credential_profile_name": Object { + "type": "text", + "value": undefined, + }, + "endpoint": Object { + "type": "text", + "value": "amazonaws.com", + }, + "role_arn": Object { + "type": "text", + "value": undefined, + }, + "secret_access_key": Object { + "type": "text", + "value": undefined, + }, + "session_token": Object { + "type": "text", + "value": undefined, + }, + "shared_credential_file": Object { + "type": "text", + "value": undefined, + }, + }, +} +`; diff --git a/x-pack/plugins/fleet/common/services/fixtures/packages.ts b/x-pack/plugins/fleet/common/services/fixtures/packages.ts new file mode 100644 index 0000000000000..0cd2f590c8c09 --- /dev/null +++ b/x-pack/plugins/fleet/common/services/fixtures/packages.ts @@ -0,0 +1,1858 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PackageInfo } from '../../../../../types'; + +export const packageWithIntegrations: PackageInfo = { + name: 'aws', + title: 'AWS', + version: '0.5.2', + release: 'beta', + description: 'AWS Integration', + type: 'integration', + download: '/epr/aws/aws-0.5.2.zip', + path: '/package/aws/0.5.2', + icons: [ + { + src: '/img/logo_aws.svg', + path: '/package/aws/0.5.2/img/logo_aws.svg', + title: 'logo aws', + size: '32x32', + type: 'image/svg+xml', + }, + ], + format_version: '1.0.0', + readme: '/package/aws/0.5.2/docs/README.md', + license: 'basic', + categories: ['aws', 'cloud', 'network', 'security'], + screenshots: [ + { + src: '/img/metricbeat-aws-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-overview.png', + title: 'metricbeat aws overview', + size: '3848x2440', + type: 'image/png', + }, + ], + // @ts-expect-error + assets: {}, + input_groups: [ + { name: 'logs', title: 'Logs', description: 'Collect logs for AWS services' }, + { name: 'metrics', title: 'Metrics', description: 'Collect metrics for AWS services' }, + ], + policy_templates: [ + { + name: 'billing', + title: 'AWS Billing', + description: 'Collect AWS billing metrics', + data_streams: ['billing'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect billing metrics', + description: 'Collect billing metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_billing.svg', + path: '/package/aws/0.5.2/img/logo_billing.svg', + title: 'AWS Billing logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-billing-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-billing-overview.png', + title: 'metricbeat aws billing overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'cloudtrail', + title: 'AWS Cloudtrail', + description: 'Collect logs from AWS Cloudtrail', + data_streams: ['cloudtrail'], + inputs: [ + { + type: 's3', + title: 'Collect logs from Cloudtrail service', + description: 'Collecting Cloudtrail logs using S3 input', + input_group: 'logs', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudtrail.svg', + path: '/package/aws/0.5.2/img/logo_cloudtrail.svg', + title: 'AWS Cloudtrail logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/filebeat-aws-cloudtrail.png', + path: '/package/aws/0.5.2/img/filebeat-aws-cloudtrail.png', + title: 'filebeat aws cloudtrail', + size: '1702x1063', + type: 'image/png', + }, + ], + }, + { + name: 'cloudwatch', + title: 'AWS CloudWatch', + description: 'Collect logs and metrics from CloudWatch', + data_streams: ['cloudwatch_logs', 'cloudwatch_metrics'], + inputs: [ + { + type: 's3', + title: 'Collect logs from CloudWatch', + description: 'Collecting logs from CloudWatch using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + title: 'Collect metrics from CloudWatch', + description: 'Collecting metrics from AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_cloudwatch.svg', + path: '/package/aws/0.5.2/img/logo_cloudwatch.svg', + title: 'AWS CloudWatch logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + { + name: 'dynamodb', + title: 'AWS DynamoDB', + description: 'Collect AWS DynamoDB metrics', + data_streams: ['dynamodb'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect dynamodb metrics', + description: 'Collect dynamodb metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_dynamodb.svg', + path: '/package/aws/0.5.2/img/logo_dynamodb.svg', + title: 'AWS DynamoDB logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + categories: ['datastore'], + screenshots: [ + { + src: '/img/metricbeat-aws-dynamodb-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-dynamodb-overview.png', + title: 'metricbeat aws dynamodb overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'ebs', + title: 'AWS EBS', + description: 'Collect AWS EBS metrics', + data_streams: ['ebs'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect EBS metrics', + description: 'Collect EBS metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_ebs.svg', + path: '/package/aws/0.5.2/img/logo_ebs.svg', + title: 'AWS EBS logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + categories: ['datastore'], + screenshots: [ + { + src: '/img/metricbeat-aws-ebs-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-ebs-overview.png', + title: 'metricbeat aws ebs overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'ec2', + title: 'AWS EC2', + description: 'Collect logs and metrics from EC2 service', + data_streams: ['ec2_logs', 'ec2_metrics'], + inputs: [ + { + type: 's3', + title: 'Collect logs from EC2 service', + description: 'Collecting EC2 logs using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + title: 'Collect metrics from EC2 service', + description: 'Collecting EC2 metrics using AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_ec2.svg', + path: '/package/aws/0.5.2/img/logo_ec2.svg', + title: 'AWS EC2 logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-ec2-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-ec2-overview.png', + title: 'metricbeat aws ec2 overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'elb', + title: 'AWS ELB', + description: 'Collect logs and metrics from ELB service', + data_streams: ['elb_logs', 'elb_metrics'], + inputs: [ + { + type: 's3', + title: 'Collect logs from ELB service', + description: 'Collecting ELB logs using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + title: 'Collect metrics from ELB service', + description: 'Collecting ELB metrics using AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_elb.svg', + path: '/package/aws/0.5.2/img/logo_elb.svg', + title: 'AWS ELB logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-elb-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-elb-overview.png', + title: 'metricbeat aws elb overview', + size: '2640x2240', + type: 'image/png', + }, + { + src: '/img/filebeat-aws-elb-overview.png', + path: '/package/aws/0.5.2/img/filebeat-aws-elb-overview.png', + title: 'filebeat aws elb overview', + size: '1684x897', + type: 'image/png', + }, + ], + }, + { + name: 'lambda', + title: 'AWS Lambda', + description: 'Collect AWS Lambda metrics', + data_streams: ['lambda'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect Lambda metrics', + description: 'Collect Lambda metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_lambda.svg', + path: '/package/aws/0.5.2/img/logo_lambda.svg', + title: 'AWS Lambda logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-lambda-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-lambda-overview.png', + title: 'metricbeat aws lambda overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'natgateway', + title: 'AWS NATGateway', + description: 'Collect AWS NATGateway metrics', + data_streams: ['natgateway'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect NATGateway metrics', + description: 'Collect NATGateway metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_natgateway.svg', + path: '/package/aws/0.5.2/img/logo_natgateway.svg', + title: 'AWS NATGateway logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + { + name: 'rds', + title: 'AWS RDS', + description: 'Collect AWS RDS metrics', + data_streams: ['rds'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect RDS metrics', + description: 'Collect RDS metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_rds.svg', + path: '/package/aws/0.5.2/img/logo_rds.svg', + title: 'AWS RDS logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + categories: ['datastore'], + screenshots: [ + { + src: '/img/metricbeat-aws-rds-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-rds-overview.png', + title: 'metricbeat aws rds overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 's3', + title: 'AWS S3', + description: 'Collect AWS S3 metrics', + data_streams: ['s3_daily_storage', 's3_request', 's3access'], + inputs: [ + { + type: 's3', + title: 'Collect S3 access logs', + description: 'Collecting S3 access logs using S3 input', + input_group: 'logs', + }, + { + type: 'aws/metrics', + title: 'Collect metrics from S3', + description: 'Collecting S3 metrics using AWS CloudWatch', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_s3.svg', + path: '/package/aws/0.5.2/img/logo_s3.svg', + title: 'AWS S3 logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + categories: ['datastore'], + screenshots: [ + { + src: '/img/metricbeat-aws-s3-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-s3-overview.png', + title: 'metricbeat aws s3 overview', + size: '2640x2240', + type: 'image/png', + }, + { + src: '/img/filebeat-aws-s3access-overview.png', + path: '/package/aws/0.5.2/img/filebeat-aws-s3access-overview.png', + title: 'filebeat aws s3access overview', + size: '1684x897', + type: 'image/png', + }, + ], + }, + { + name: 'sns', + title: 'AWS SNS', + description: 'Collect AWS SNS metrics', + data_streams: ['sns'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect SNS metrics', + description: 'Collect SNS metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_sns.svg', + path: '/package/aws/0.5.2/img/logo_sns.svg', + title: 'AWS SNS logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-sns-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-sns-overview.png', + title: 'metricbeat aws sns overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'sqs', + title: 'AWS SQS', + description: 'Collect AWS SQS metrics', + data_streams: ['sqs'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect SQS metrics', + description: 'Collect SQS metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_sqs.svg', + path: '/package/aws/0.5.2/img/logo_sqs.svg', + title: 'AWS SQS logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + screenshots: [ + { + src: '/img/metricbeat-aws-sqs-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-sqs-overview.png', + title: 'metricbeat aws sqs overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'transitgateway', + title: 'AWS Transit Gateway', + description: 'Collect AWS Transit Gateway metrics', + data_streams: ['transitgateway'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect Transit Gateway metrics', + description: 'Collect Transit Gateway metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_transitgateway.svg', + path: '/package/aws/0.5.2/img/logo_transitgateway.svg', + title: 'AWS Transit Gateway logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + { + name: 'usage', + title: 'AWS Usage', + description: 'Collect AWS Usage metrics', + data_streams: ['usage'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect Usage metrics', + description: 'Collect Usage metrics', + input_group: 'metrics', + }, + ], + multiple: true, + screenshots: [ + { + src: '/img/metricbeat-aws-usage-overview.png', + path: '/package/aws/0.5.2/img/metricbeat-aws-usage-overview.png', + title: 'metricbeat aws sns overview', + size: '2640x2240', + type: 'image/png', + }, + ], + }, + { + name: 'vpcflow', + title: 'AWS VPC Flow', + description: 'Collect AWS vpcflow logs', + data_streams: ['vpcflow'], + inputs: [ + { + type: 's3', + title: 'Collect VPC Flow logs', + description: 'Collecting VPC Flow logs using S3 input', + input_group: 'logs', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_vpcflow.svg', + path: '/package/aws/0.5.2/img/logo_vpcflow.svg', + title: 'AWS VPC logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + }, + { + name: 'vpn', + title: 'AWS VPN', + description: 'Collect AWS VPN metrics', + data_streams: ['vpn'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect VPN metrics', + description: 'Collect VPN metrics', + input_group: 'metrics', + }, + ], + multiple: true, + icons: [ + { + src: '/img/logo_vpn.svg', + path: '/package/aws/0.5.2/img/logo_vpn.svg', + title: 'AWS VPN logo', + size: '32x32', + type: 'image/svg+xml', + }, + ], + categories: ['network'], + }, + ], + data_streams: [ + { + type: 'metrics', + dataset: 'aws.billing', + title: 'AWS billing metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '12h', + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'cost_explorer_config.group_by_dimension_keys', + type: 'text', + title: 'Cost Explorer Group By Dimension Keys', + multi: true, + required: false, + show_user: true, + default: ['AZ', 'INSTANCE_TYPE', 'SERVICE'], + }, + { + name: 'cost_explorer_config.group_by_tag_keys', + type: 'text', + title: 'Cost Explorer Group By Tag Keys', + multi: true, + required: false, + show_user: true, + default: ['aws:createdBy'], + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS Billing metrics', + description: 'Collect AWS billing metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'billing', + }, + { + type: 'logs', + dataset: 'aws.cloudtrail', + title: 'AWS CloudTrail logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudTrail logs', + description: 'Collect AWS CloudTrail logs using s3 input', + enabled: true, + }, + { + input: 'httpjson', + vars: [ + { + name: 'interval', + type: 'text', + title: 'Interval to query Splunk Enterprise REST API', + description: 'Go Duration syntax (eg. 10s)', + multi: false, + required: true, + show_user: true, + default: '10s', + }, + { + name: 'search', + type: 'text', + title: 'Splunk search string', + multi: false, + required: true, + show_user: true, + default: 'search sourcetype=aws:cloudtrail', + }, + { + name: 'tags', + type: 'text', + title: 'Tags', + multi: true, + required: false, + show_user: false, + default: ['forwarded'], + }, + ], + template_path: 'httpjson.yml.hbs', + title: 'AWS CloudTrail logs via Splunk Enterprise REST API', + description: 'Collect AWS CloudTrail logs via Splunk Enterprise REST API', + enabled: false, + }, + ], + package: 'aws', + path: 'cloudtrail', + }, + { + type: 'logs', + dataset: 'aws.cloudwatch_logs', + title: 'AWS CloudWatch logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS CloudWatch logs', + description: 'Collect AWS CloudWatch logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_logs', + }, + { + type: 'metrics', + dataset: 'aws.cloudwatch_metrics', + title: 'AWS CloudWatch metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '300s', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'metrics', + type: 'yaml', + title: 'Metrics', + multi: false, + required: true, + show_user: true, + default: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS CloudWatch metrics', + description: 'Collect AWS CloudWatch metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'cloudwatch_metrics', + }, + { + type: 'metrics', + dataset: 'aws.dynamodb', + title: 'AWS DynamoDB metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS DynamoDB metrics', + description: 'Collect AWS DynamoDB metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'dynamodb', + }, + { + type: 'metrics', + dataset: 'aws.ebs', + title: 'AWS EBS metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS EBS metrics', + description: 'Collect AWS EBS metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'ebs', + }, + { + type: 'logs', + dataset: 'aws.ec2_logs', + title: 'AWS EC2 logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS EC2 logs', + description: 'Collect AWS EC2 logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'ec2_logs', + }, + { + type: 'metrics', + dataset: 'aws.ec2_metrics', + title: 'AWS EC2 metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS EC2 metrics', + description: 'Collect AWS EC2 metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'ec2_metrics', + }, + { + type: 'logs', + dataset: 'aws.elb_logs', + title: 'AWS ELB logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS ELB logs', + description: 'Collect AWS ELB logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'elb_logs', + }, + { + type: 'metrics', + dataset: 'aws.elb_metrics', + title: 'AWS ELB metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS ELB metrics', + description: 'Collect AWS ELB metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'elb_metrics', + }, + { + type: 'metrics', + dataset: 'aws.lambda', + title: 'AWS Lambda metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS Lambda metrics', + description: 'Collect AWS Lambda metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'lambda', + }, + { + type: 'metrics', + dataset: 'aws.natgateway', + title: 'AWS NAT gateway metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS NAT gateway metrics', + description: 'Collect AWS NAT gateway metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'natgateway', + }, + { + type: 'metrics', + dataset: 'aws.rds', + title: 'AWS RDS metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS RDS metrics', + description: 'Collect AWS RDS metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'rds', + }, + { + type: 'metrics', + dataset: 'aws.s3_daily_storage', + title: 'AWS S3 daily storage metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '24h', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS S3 daily storage metrics', + description: 'Collect AWS S3 daily storage metrics', + enabled: true, + }, + ], + package: 'aws', + path: 's3_daily_storage', + }, + { + type: 'metrics', + dataset: 'aws.s3_request', + title: 'AWS S3 request metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS S3 request metrics', + description: 'Collect AWS S3 request metrics', + enabled: true, + }, + ], + package: 'aws', + path: 's3_request', + }, + { + type: 'logs', + dataset: 'aws.s3access', + title: 'AWS s3access logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS s3access logs', + description: 'Collect AWS s3access logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 's3access', + }, + { + type: 'metrics', + dataset: 'aws.sns', + title: 'AWS SNS metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS SNS metrics', + description: 'Collect AWS SNS metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'sns', + }, + { + type: 'metrics', + dataset: 'aws.sqs', + title: 'AWS SQS metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '5m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS SQS metrics', + description: 'Collect AWS SQS metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'sqs', + }, + { + type: 'metrics', + dataset: 'aws.transitgateway', + title: 'AWS Transit Gateway metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS Transit Gateway metrics', + description: 'Collect AWS Transit Gateway metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'transitgateway', + }, + { + type: 'metrics', + dataset: 'aws.usage', + title: 'AWS usage metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS usage metrics', + description: 'Collect AWS usage metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'usage', + }, + { + type: 'logs', + dataset: 'aws.vpcflow', + title: 'AWS vpcflow logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'visibility_timeout', + type: 'text', + title: 'Visibility Timeout', + description: + 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'api_timeout', + type: 'text', + title: 'API Timeout', + description: + 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'queue_url', + type: 'text', + title: 'Queue URL', + description: 'URL of the AWS SQS queue that messages will be received from.', + multi: false, + required: true, + show_user: true, + }, + { + name: 'fips_enabled', + type: 'bool', + title: 'Enable S3 FIPS', + description: + 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', + multi: false, + required: false, + show_user: false, + default: false, + }, + ], + template_path: 's3.yml.hbs', + title: 'AWS vpcflow logs', + description: 'Collect AWS vpcflow logs using s3 input', + enabled: true, + }, + ], + package: 'aws', + path: 'vpcflow', + }, + { + type: 'metrics', + dataset: 'aws.vpn', + title: 'AWS VPN metrics', + release: 'beta', + streams: [ + { + input: 'aws/metrics', + vars: [ + { + name: 'period', + type: 'text', + title: 'Period', + multi: false, + required: true, + show_user: true, + default: '1m', + }, + { + name: 'regions', + type: 'text', + title: 'Regions', + multi: true, + required: false, + show_user: true, + }, + { + name: 'latency', + type: 'text', + title: 'Latency', + multi: false, + required: false, + show_user: false, + }, + { + name: 'tags_filter', + type: 'yaml', + title: 'Tags Filter', + multi: false, + required: false, + show_user: false, + default: '# - key: "created-by"\n # value: "foo"\n', + }, + ], + template_path: 'stream.yml.hbs', + title: 'AWS VPN metrics', + description: 'Collect AWS VPN metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'vpn', + }, + ], + owner: { github: 'elastic/integrations' }, + vars: [ + { + name: 'shared_credential_file', + type: 'text', + title: 'Shared Credential File', + description: 'Directory of the shared credentials file.', + multi: false, + required: false, + show_user: false, + }, + { + name: 'credential_profile_name', + type: 'text', + title: 'Credential Profile Name', + multi: false, + required: false, + show_user: true, + }, + { + name: 'access_key_id', + type: 'text', + title: 'Access Key ID', + multi: false, + required: false, + show_user: false, + }, + { + name: 'secret_access_key', + type: 'text', + title: 'Secret Access Key', + multi: false, + required: false, + show_user: false, + }, + { + name: 'session_token', + type: 'text', + title: 'Session Token', + multi: false, + required: false, + show_user: false, + }, + { + name: 'role_arn', + type: 'text', + title: 'Role ARN', + multi: false, + required: false, + show_user: false, + }, + { + name: 'endpoint', + type: 'text', + title: 'Endpoint', + description: 'URL of the entry point for an AWS web service.', + multi: false, + required: false, + show_user: false, + default: 'amazonaws.com', + }, + ], + latestVersion: '0.5.2', + removable: true, + status: 'not_installed', +}; diff --git a/x-pack/plugins/fleet/common/services/group_inputs.test.ts b/x-pack/plugins/fleet/common/services/group_inputs.test.ts new file mode 100644 index 0000000000000..143fc99e6135f --- /dev/null +++ b/x-pack/plugins/fleet/common/services/group_inputs.test.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { groupInputs } from './group_inputs'; +import { packageWithIntegrations } from './fixtures/packages'; + +describe('groupInputs()', () => { + it('should group inputs for a package with multiple policy templates and input groups correctly', () => { + expect(groupInputs(packageWithIntegrations)).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/fleet/common/services/group_inputs.ts b/x-pack/plugins/fleet/common/services/group_inputs.ts new file mode 100644 index 0000000000000..47eaa97ddccbd --- /dev/null +++ b/x-pack/plugins/fleet/common/services/group_inputs.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PackageInfo, RegistryInputGroup, RegistryVarsEntry } from '../types'; + +import { doesPackageHaveIntegrations, getStreamsForInputType, findDataStreamsByNames } from './'; + +interface InputGroupWithStreams extends RegistryInputGroup { + vars?: RegistryVarsEntry[]; + streams?: ReturnType; +} + +export const groupInputs = (packageInfo: PackageInfo): InputGroupWithStreams[] => { + const inputGroups: InputGroupWithStreams[] = [...(packageInfo.input_groups || [])]; + const hasIntegrations = doesPackageHaveIntegrations(packageInfo); + const { policy_templates: policyTemplates = [] } = packageInfo; + + // If no policy templates, return no groups + if (policyTemplates.length === 0) { + return inputGroups; + } + + // If no integrations, return first policy template's inputs as groups + if (!hasIntegrations) { + (policyTemplates[0].inputs || []).forEach(({ type, ...input }) => { + inputGroups.push({ + ...input, + name: type, + streams: getStreamsForInputType(type, packageInfo.data_streams), + }); + }); + return inputGroups; + } + + // Otherwise look in each policy template + policyTemplates.forEach((policyTemplate) => { + const policyTemplateDataStreams = findDataStreamsByNames( + policyTemplate.data_streams, + packageInfo.data_streams + ); + + (policyTemplate.inputs || []).forEach(({ type, input_group: inputGroupName }) => { + // Package should already have `input_groups` defined and each input should have an `input_group` + // If either one of these conditions aren't met, skip adding this input + const inputGroup = inputGroupName + ? inputGroups.find((group) => group.name === inputGroupName) + : undefined; + if (!inputGroup) { + return; + } + + inputGroup.streams = [ + ...(inputGroup.streams || []), + ...getStreamsForInputType(type, policyTemplateDataStreams), + ]; + }); + }); + + return inputGroups; +}; diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts index 1fea5033e645c..6b4b91994934b 100644 --- a/x-pack/plugins/fleet/common/services/index.ts +++ b/x-pack/plugins/fleet/common/services/index.ts @@ -7,7 +7,12 @@ export * from './routes'; export * as AgentStatusKueryHelper from './agent_status'; -export { packageToPackagePolicyInputs, packageToPackagePolicy } from './package_to_package_policy'; +export { + packageToPackagePolicyInputs, + packageToPackagePolicy, + getStreamsForInputType, + findDataStreamsByNames, +} from './package_to_package_policy'; export { storedPackagePoliciesToAgentInputs } from './package_policies_to_agent_inputs'; export { fullAgentPolicyToYaml } from './full_agent_policy_to_yaml'; export { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from './limited_package'; @@ -17,3 +22,4 @@ export { isDiffPathProtocol } from './is_diff_path_protocol'; export { LicenseService } from './license'; export { isAgentUpgradeable } from './is_agent_upgradeable'; export { doesPackageHaveIntegrations } from './packages_with_integrations'; +export { groupInputs } from './group_inputs'; diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index 631c78b524a79..d9395e25d6ff8 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -7,6 +7,7 @@ import type { PackageInfo } from '../types'; +import { packageWithIntegrations } from './fixtures/packages'; import { packageToPackagePolicy, packageToPackagePolicyInputs } from './package_to_package_policy'; describe('Fleet - packageToPackagePolicy', () => { @@ -407,449 +408,14 @@ describe('Fleet - packageToPackagePolicy', () => { }); it('returns package policy with multiple policy templates correctly', () => { - const mockPackageWithPolicyTemplates = ({ - ...mockPackage, - data_streams: [ - { - type: 'logs', - dataset: 'aws.cloudtrail', - title: 'AWS CloudTrail logs', - release: 'beta', - ingest_pipeline: 'default', - streams: [ - { - input: 's3', - vars: [ - { - name: 'queue_url', - type: 'text', - title: 'Queue URL', - description: 'URL of the AWS SQS queue that messages will be received from.', - multi: false, - required: true, - show_user: true, - }, - { - name: 'fips_enabled', - type: 'bool', - title: 'Enable S3 FIPS', - description: - 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', - multi: false, - required: false, - show_user: false, - default: false, - }, - ], - template_path: 's3.yml.hbs', - title: 'AWS CloudTrail logs', - description: 'Collect AWS CloudTrail logs using s3 input', - enabled: true, - }, - { - input: 'httpjson', - vars: [ - { - name: 'interval', - type: 'text', - title: 'Interval to query Splunk Enterprise REST API', - description: 'Go Duration syntax (eg. 10s)', - multi: false, - required: true, - show_user: true, - default: '10s', - }, - { - name: 'search', - type: 'text', - title: 'Splunk search string', - multi: false, - required: true, - show_user: true, - default: 'search sourcetype=aws:cloudtrail', - }, - { - name: 'tags', - type: 'text', - title: 'Tags', - multi: true, - required: false, - show_user: false, - default: ['forwarded'], - }, - ], - template_path: 'httpjson.yml.hbs', - title: 'AWS CloudTrail logs via Splunk Enterprise REST API', - description: 'Collect AWS CloudTrail logs via Splunk Enterprise REST API', - enabled: false, - }, - ], - package: 'aws', - path: 'cloudtrail', - }, - { - type: 'logs', - dataset: 'aws.cloudwatch_logs', - title: 'AWS CloudWatch logs', - release: 'beta', - ingest_pipeline: 'default', - streams: [ - { - input: 's3', - vars: [ - { - name: 'queue_url', - type: 'text', - title: 'Queue URL', - description: 'URL of the AWS SQS queue that messages will be received from.', - multi: false, - required: true, - show_user: true, - }, - { - name: 'fips_enabled', - type: 'bool', - title: 'Enable S3 FIPS', - description: - 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', - multi: false, - required: false, - show_user: false, - default: false, - }, - ], - template_path: 's3.yml.hbs', - title: 'AWS CloudWatch logs', - description: 'Collect AWS CloudWatch logs using s3 input', - enabled: true, - }, - ], - package: 'aws', - path: 'cloudwatch_logs', - }, - { - type: 'metrics', - dataset: 'aws.cloudwatch_metrics', - title: 'AWS CloudWatch metrics', - release: 'beta', - streams: [ - { - input: 'aws/metrics', - vars: [ - { - name: 'period', - type: 'text', - title: 'Period', - multi: false, - required: true, - show_user: true, - default: '300s', - }, - { - name: 'regions', - type: 'text', - title: 'Regions', - multi: true, - required: false, - show_user: true, - }, - { - name: 'latency', - type: 'text', - title: 'Latency', - multi: false, - required: false, - show_user: false, - }, - { - name: 'metrics', - type: 'yaml', - title: 'Metrics', - multi: false, - required: true, - show_user: true, - default: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - }, - ], - template_path: 'stream.yml.hbs', - title: 'AWS CloudWatch metrics', - description: 'Collect AWS CloudWatch metrics', - enabled: true, - }, - ], - package: 'aws', - path: 'cloudwatch_metrics', - }, - ], - policy_templates: [ - { - name: 'cloudtrail', - title: 'AWS Cloudtrail', - description: 'Collect logs from AWS Cloudtrail', - data_streams: ['cloudtrail'], - inputs: [ - { - type: 's3', - vars: [ - { - name: 'visibility_timeout', - type: 'text', - title: 'Visibility Timeout', - description: - 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'api_timeout', - type: 'text', - title: 'API Timeout', - description: - 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', - multi: false, - required: false, - show_user: false, - }, - ], - title: 'Collect logs from Cloudtrail service', - description: 'Collecting Cloudtrail logs using S3 input', - input_group: 'logs', - }, - ], - multiple: true, - icons: [ - { - src: '/img/logo_cloudtrail.svg', - path: '/package/aws/0.5.2/img/logo_cloudtrail.svg', - title: 'AWS Cloudtrail logo', - size: '32x32', - type: 'image/svg+xml', - }, - ], - screenshots: [ - { - src: '/img/filebeat-aws-cloudtrail.png', - path: '/package/aws/0.5.2/img/filebeat-aws-cloudtrail.png', - title: 'filebeat aws cloudtrail', - size: '1702x1063', - type: 'image/png', - }, - ], - }, - { - name: 'cloudwatch', - title: 'AWS CloudWatch', - description: 'Collect logs and metrics from CloudWatch', - data_streams: ['cloudwatch_logs', 'cloudwatch_metrics'], - inputs: [ - { - type: 's3', - vars: [ - { - name: 'visibility_timeout', - type: 'text', - title: 'Visibility Timeout', - description: - 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'api_timeout', - type: 'text', - title: 'API Timeout', - description: - 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', - multi: false, - required: false, - show_user: false, - }, - ], - title: 'Collect logs from CloudWatch', - description: 'Collecting logs from CloudWatch using S3 input', - input_group: 'logs', - }, - { - type: 'aws/metrics', - vars: [ - { - name: 'metrics', - type: 'yaml', - title: 'Metrics', - multi: false, - required: true, - show_user: true, - default: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - }, - ], - title: 'Collect metrics from CloudWatch', - description: 'Collecting metrics from AWS CloudWatch', - input_group: 'metrics', - }, - ], - multiple: true, - icons: [ - { - src: '/img/logo_cloudwatch.svg', - path: '/package/aws/0.5.2/img/logo_cloudwatch.svg', - title: 'AWS CloudWatch logo', - size: '32x32', - type: 'image/svg+xml', - }, - ], - }, - ], - vars: [ - { - name: 'shared_credential_file', - type: 'text', - title: 'Shared Credential File', - description: 'Directory of the shared credentials file.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'credential_profile_name', - type: 'text', - title: 'Credential Profile Name', - multi: false, - required: false, - show_user: true, - }, - { - name: 'access_key_id', - type: 'text', - title: 'Access Key ID', - multi: false, - required: false, - show_user: false, - }, - { - name: 'secret_access_key', - type: 'text', - title: 'Secret Access Key', - multi: false, - required: false, - show_user: false, - }, - { - name: 'session_token', - type: 'text', - title: 'Session Token', - multi: false, - required: false, - show_user: false, - }, - { - name: 'role_arn', - type: 'text', - title: 'Role ARN', - multi: false, - required: false, - show_user: false, - }, - { - name: 'endpoint', - type: 'text', - title: 'Endpoint', - description: 'URL of the entry point for an AWS web service.', - multi: false, - required: false, - show_user: false, - default: 'amazonaws.com', - }, - ], - } as unknown) as PackageInfo; - expect( packageToPackagePolicy( - mockPackageWithPolicyTemplates, + packageWithIntegrations, 'some-policy-id', 'some-output-id', 'default' ) - ).toEqual({ - name: 'mock-package-1', - namespace: 'default', - package: { name: 'mock-package', title: 'Mock package', version: '0.0.0' }, - enabled: true, - policy_id: 'some-policy-id', - output_id: 'some-output-id', - inputs: [ - { - type: 's3', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, - vars: { queue_url: { type: 'text' }, fips_enabled: { value: false, type: 'bool' } }, - }, - ], - integration: 'cloudtrail', - vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, - }, - { - type: 's3', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, - vars: { queue_url: { type: 'text' }, fips_enabled: { value: false, type: 'bool' } }, - }, - ], - integration: 'cloudwatch', - vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, - }, - { - type: 'aws/metrics', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, - vars: { - period: { value: '300s', type: 'text' }, - regions: { value: [], type: 'text' }, - latency: { type: 'text' }, - metrics: { - value: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - type: 'yaml', - }, - }, - }, - ], - integration: 'cloudwatch', - vars: { - metrics: { - value: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - type: 'yaml', - }, - }, - }, - ], - integrations: [ - { name: 'cloudtrail', enabled: true }, - { name: 'cloudwatch', enabled: true }, - ], - vars: { - shared_credential_file: { type: 'text' }, - credential_profile_name: { type: 'text' }, - access_key_id: { type: 'text' }, - secret_access_key: { type: 'text' }, - session_token: { type: 'text' }, - role_arn: { type: 'text' }, - endpoint: { value: 'amazonaws.com', type: 'text' }, - }, - }); + ).toMatchSnapshot(); }); }); }); diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 2113d63839af1..88bfb77238133 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -16,13 +16,22 @@ import type { PackagePolicyConfigRecordEntry, } from '../types'; -import { doesPackageHaveIntegrations } from './'; +import { doesPackageHaveIntegrations, groupInputs } from './'; + +export const findDataStreamsByNames = ( + names: string[] = [], + dataStreams: PackageInfo['data_streams'] = [] +): PackageInfo['data_streams'] => { + return names.length + ? dataStreams.filter((dataStream) => names.includes(dataStream.path)) + : dataStreams; +}; export const getStreamsForInputType = ( inputType: string, dataStreams: PackageInfo['data_streams'] = [] -): Array => { - const streams: Array = []; +): Array> => { + const streams: Array> = []; dataStreams.forEach((dataStream) => { (dataStream.streams || []).forEach((stream) => { @@ -63,59 +72,34 @@ export const packageToPackagePolicyInputs = ( packageInfo: PackageInfo ): NewPackagePolicy['inputs'] => { const inputs: NewPackagePolicy['inputs'] = []; - const packageDataStreams = packageInfo.data_streams || []; - const packagePolicyTemplates = packageInfo.policy_templates || []; - const hasIntegrations = doesPackageHaveIntegrations(packageInfo); - - packagePolicyTemplates.forEach( - ({ - inputs: policyTemplateInputs = [], - data_streams: policyTemplateDataStreams = [], - name: policyTemplateName, - }) => { - // If policy template has data streams defined, only look for inputs on those data streams - // Otherwise look in all of the package data streams - const dataStreamsToSearchForInputs = policyTemplateDataStreams.length - ? packageDataStreams.filter((dataStream) => - policyTemplateDataStreams.includes(dataStream.path) - ) - : packageDataStreams; - - // Map each package package policy input to agent policy package policy input - policyTemplateInputs.forEach((packageInput) => { - // Map each package input stream into package policy input stream - const streams: NewPackagePolicyInputStream[] = getStreamsForInputType( - packageInput.type, - dataStreamsToSearchForInputs - ).map((packageStream) => { - const stream: NewPackagePolicyInputStream = { - enabled: packageStream.enabled === false ? false : true, - data_stream: packageStream.data_stream, - }; - if (packageStream.vars && packageStream.vars.length) { - stream.vars = packageStream.vars.reduce(varsReducer, {}); - } - return stream; - }); - - const input: NewPackagePolicyInput = { - type: packageInput.type, - enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, - streams, + const packageInputGroups = groupInputs(packageInfo); + + packageInputGroups.forEach((inputGroup) => { + const streams: NewPackagePolicyInputStream[] = (inputGroup.streams || []).map( + (packageStream) => { + const stream: NewPackagePolicyInputStream = { + enabled: packageStream.enabled === false ? false : true, + data_stream: packageStream.data_stream, }; - - if (hasIntegrations) { - input.integration = policyTemplateName; + if (packageStream.vars && packageStream.vars.length) { + stream.vars = packageStream.vars.reduce(varsReducer, {}); } + return stream; + } + ); - if (packageInput.vars && packageInput.vars.length) { - input.vars = packageInput.vars.reduce(varsReducer, {}); - } + const input: NewPackagePolicyInput = { + type: inputGroup.name, + enabled: streams.length ? !!streams.find((stream) => stream.enabled) : true, + streams, + }; - inputs.push(input); - }); + if (inputGroup.vars && inputGroup.vars.length) { + input.vars = inputGroup.vars.reduce(varsReducer, {}); } - ); + + inputs.push(input); + }); return inputs; }; @@ -151,13 +135,6 @@ export const packageToPackagePolicy = ( inputs: packageToPackagePolicyInputs(packageInfo), }; - if (doesPackageHaveIntegrations(packageInfo)) { - packagePolicy.integrations = packageInfo.policy_templates?.map((policyTemplate) => ({ - name: policyTemplate.name, - enabled: true, - })); - } - if (packageInfo.vars?.length) { packagePolicy.vars = packageInfo.vars.reduce(varsReducer, {}); } diff --git a/x-pack/plugins/fleet/common/services/packages_with_integrations.ts b/x-pack/plugins/fleet/common/services/packages_with_integrations.ts index d63c7518f4edb..e8fe2f9ee769a 100644 --- a/x-pack/plugins/fleet/common/services/packages_with_integrations.ts +++ b/x-pack/plugins/fleet/common/services/packages_with_integrations.ts @@ -7,5 +7,5 @@ import type { PackageInfo, PackageListItem } from '../types'; export const doesPackageHaveIntegrations = (pkgInfo: PackageInfo | PackageListItem) => { - return (pkgInfo.policy_templates || []).length > 1; + return 'input_groups' in pkgInfo || (pkgInfo.policy_templates || []).length > 1; }; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index c2b9e81c67459..7041e0982c84d 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -283,7 +283,7 @@ export interface RegistryDataStream { [RegistryDataStreamKeys.streams]?: RegistryStream[]; [RegistryDataStreamKeys.package]: string; [RegistryDataStreamKeys.path]: string; - [RegistryDataStreamKeys.ingest_pipeline]: string; + [RegistryDataStreamKeys.ingest_pipeline]?: string; [RegistryDataStreamKeys.elasticsearch]?: RegistryElasticsearch; [RegistryDataStreamKeys.dataset_is_prefix]?: boolean; } @@ -317,7 +317,7 @@ export interface RegistryVarsEntry { [RegistryVarsEntryKeys.required]?: boolean; [RegistryVarsEntryKeys.show_user]?: boolean; [RegistryVarsEntryKeys.multi]?: boolean; - [RegistryVarsEntryKeys.default]?: string | string[]; + [RegistryVarsEntryKeys.default]?: string | string[] | boolean; [RegistryVarsEntryKeys.os]?: { [key: string]: { default: string | string[]; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 36a878a48bed3..920b8ff13c89e 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -35,7 +35,6 @@ export interface PackagePolicyInputStream extends NewPackagePolicyInputStream { export interface NewPackagePolicyInput { type: string; - integration?: string; enabled: boolean; vars?: PackagePolicyConfigRecord; config?: PackagePolicyConfigRecord; @@ -47,11 +46,6 @@ export interface PackagePolicyInput extends Omit Date: Tue, 6 Apr 2021 14:00:12 -0700 Subject: [PATCH 11/16] Adjust package policy validation service to match, move it to /common --- .../common/services/fixtures/packages.ts | 257 +++++++- .../common/services/group_inputs.test.ts | 4 +- x-pack/plugins/fleet/common/services/index.ts | 10 + .../package_to_package_policy.test.ts | 4 +- .../services/validate_package_policy.test.ts | 574 +++--------------- .../services/validate_package_policy.ts | 99 ++- .../services/index.ts | 3 +- .../applications/fleet/services/index.ts | 9 +- 8 files changed, 384 insertions(+), 576 deletions(-) rename x-pack/plugins/fleet/{public/applications/fleet/sections/agent_policy/create_package_policy_page => common}/services/validate_package_policy.test.ts (52%) rename x-pack/plugins/fleet/{public/applications/fleet/sections/agent_policy/create_package_policy_page => common}/services/validate_package_policy.ts (72%) diff --git a/x-pack/plugins/fleet/common/services/fixtures/packages.ts b/x-pack/plugins/fleet/common/services/fixtures/packages.ts index 0cd2f590c8c09..d823c9e415315 100644 --- a/x-pack/plugins/fleet/common/services/fixtures/packages.ts +++ b/x-pack/plugins/fleet/common/services/fixtures/packages.ts @@ -4,9 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { PackageInfo } from '../../../../../types'; +import type { PackageInfo, NewPackagePolicy } from '../../types'; -export const packageWithIntegrations: PackageInfo = { +export const packageWithPolicyTemplates: PackageInfo = { name: 'aws', title: 'AWS', version: '0.5.2', @@ -1856,3 +1856,256 @@ export const packageWithIntegrations: PackageInfo = { removable: true, status: 'not_installed', }; + +export const packagePolicyForPolicyTemplates: NewPackagePolicy = { + name: 'aws-1', + namespace: 'default', + package: { name: 'aws', title: 'AWS', version: '0.5.2' }, + enabled: true, + policy_id: 'some-policy-id', + output_id: 'some-output-id', + inputs: [ + { + type: 'logs', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.ec2_logs' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.elb_logs' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.s3access' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + { + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.vpcflow' }, + vars: { + visibility_timeout: { type: 'text' }, + api_timeout: { type: 'text' }, + queue_url: { type: 'text' }, + fips_enabled: { value: false, type: 'bool' }, + }, + }, + ], + }, + { + type: 'metrics', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.billing' }, + vars: { + period: { value: '12h', type: 'text' }, + latency: { type: 'text' }, + 'cost_explorer_config.group_by_dimension_keys': { + value: ['AZ', 'INSTANCE_TYPE', 'SERVICE'], + type: 'text', + }, + 'cost_explorer_config.group_by_tag_keys': { value: ['aws:createdBy'], type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, + vars: { + period: { value: '300s', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + metrics: { + value: + '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', + type: 'yaml', + }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.dynamodb' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.ebs' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.ec2_metrics' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.elb_metrics' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.lambda' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.natgateway' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.rds' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.s3_daily_storage' }, + vars: { + period: { value: '24h', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.s3_request' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.sns' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.sqs' }, + vars: { + period: { value: '5m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.transitgateway' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.usage' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + }, + }, + { + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.vpn' }, + vars: { + period: { value: '1m', type: 'text' }, + regions: { value: [], type: 'text' }, + latency: { type: 'text' }, + tags_filter: { value: '# - key: "created-by"\n # value: "foo"\n', type: 'yaml' }, + }, + }, + ], + }, + ], + vars: { + shared_credential_file: { type: 'text' }, + credential_profile_name: { type: 'text' }, + access_key_id: { type: 'text' }, + secret_access_key: { type: 'text' }, + session_token: { type: 'text' }, + role_arn: { type: 'text' }, + endpoint: { value: 'amazonaws.com', type: 'text' }, + }, +}; diff --git a/x-pack/plugins/fleet/common/services/group_inputs.test.ts b/x-pack/plugins/fleet/common/services/group_inputs.test.ts index 143fc99e6135f..9043be22ad714 100644 --- a/x-pack/plugins/fleet/common/services/group_inputs.test.ts +++ b/x-pack/plugins/fleet/common/services/group_inputs.test.ts @@ -5,10 +5,10 @@ * 2.0. */ import { groupInputs } from './group_inputs'; -import { packageWithIntegrations } from './fixtures/packages'; +import { packageWithPolicyTemplates } from './fixtures/packages'; describe('groupInputs()', () => { it('should group inputs for a package with multiple policy templates and input groups correctly', () => { - expect(groupInputs(packageWithIntegrations)).toMatchSnapshot(); + expect(groupInputs(packageWithPolicyTemplates)).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts index 6b4b91994934b..e6f34a1a50be6 100644 --- a/x-pack/plugins/fleet/common/services/index.ts +++ b/x-pack/plugins/fleet/common/services/index.ts @@ -23,3 +23,13 @@ export { LicenseService } from './license'; export { isAgentUpgradeable } from './is_agent_upgradeable'; export { doesPackageHaveIntegrations } from './packages_with_integrations'; export { groupInputs } from './group_inputs'; + +export { + PackagePolicyValidationResults, + PackagePolicyConfigValidationResults, + PackagePolicyInputValidationResults, + validatePackagePolicy, + validatePackagePolicyConfig, + validationHasErrors, + countValidationErrors, +} from './validate_package_policy'; diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index d9395e25d6ff8..59768e6ac8c85 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -7,7 +7,7 @@ import type { PackageInfo } from '../types'; -import { packageWithIntegrations } from './fixtures/packages'; +import { packageWithPolicyTemplates } from './fixtures/packages'; import { packageToPackagePolicy, packageToPackagePolicyInputs } from './package_to_package_policy'; describe('Fleet - packageToPackagePolicy', () => { @@ -410,7 +410,7 @@ describe('Fleet - packageToPackagePolicy', () => { it('returns package policy with multiple policy templates correctly', () => { expect( packageToPackagePolicy( - packageWithIntegrations, + packageWithPolicyTemplates, 'some-policy-id', 'some-output-id', 'default' diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts b/x-pack/plugins/fleet/common/services/validate_package_policy.test.ts similarity index 52% rename from x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts rename to x-pack/plugins/fleet/common/services/validate_package_policy.test.ts index 1c7683c54fffc..73df96c797252 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/validate_package_policy.test.ts @@ -5,9 +5,10 @@ * 2.0. */ -import { installationStatuses } from '../../../../../../../common/constants'; -import type { PackageInfo, NewPackagePolicy, RegistryPolicyTemplate } from '../../../../types'; +import { installationStatuses } from '../constants'; +import type { PackageInfo, NewPackagePolicy, RegistryPolicyTemplate } from '../types'; +import { packageWithPolicyTemplates, packagePolicyForPolicyTemplates } from './fixtures/packages'; import { validatePackagePolicy, validationHasErrors } from './validate_package_policy'; describe('Fleet - validatePackagePolicy()', () => { @@ -501,529 +502,98 @@ describe('Fleet - validatePackagePolicy()', () => { }); describe('works for packages with integrations (multiple policy templates)', () => { - const mockPackage = ({ - name: 'mock-package', - title: 'Mock package', - version: '0.0.0', - latestVersion: '0.0.0', - description: 'description', - type: 'integration', - categories: [], - conditions: { kibana: { version: '' } }, - format_version: '', - download: '', - path: '', - assets: { - kibana: { - dashboard: [], - visualization: [], - search: [], - index_pattern: [], - map: [], - lens: [], - ml_module: [], - }, - elasticsearch: { - ingest_pipeline: [], - component_template: [], - index_template: [], - transform: [], - ilm_policy: [], - data_stream_ilm_policy: [], - }, - }, - status: 'not_installed', - release: 'experimental', - owner: { - github: 'elastic/fleet', - }, - data_streams: [ - { - type: 'logs', - dataset: 'aws.cloudtrail', - title: 'AWS CloudTrail logs', - release: 'beta', - ingest_pipeline: 'default', - streams: [ - { - input: 's3', - vars: [ - { - name: 'queue_url', - type: 'text', - title: 'Queue URL', - description: 'URL of the AWS SQS queue that messages will be received from.', - multi: false, - required: true, - show_user: true, - }, - { - name: 'fips_enabled', - type: 'bool', - title: 'Enable S3 FIPS', - description: - 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', - multi: false, - required: false, - show_user: false, - default: false, - }, - ], - template_path: 's3.yml.hbs', - title: 'AWS CloudTrail logs', - description: 'Collect AWS CloudTrail logs using s3 input', - enabled: true, - }, - { - input: 'httpjson', - vars: [ - { - name: 'interval', - type: 'text', - title: 'Interval to query Splunk Enterprise REST API', - description: 'Go Duration syntax (eg. 10s)', - multi: false, - required: true, - show_user: true, - default: '10s', - }, - { - name: 'search', - type: 'text', - title: 'Splunk search string', - multi: false, - required: true, - show_user: true, - default: 'search sourcetype=aws:cloudtrail', - }, - { - name: 'tags', - type: 'text', - title: 'Tags', - multi: true, - required: false, - show_user: false, - default: ['forwarded'], - }, - ], - template_path: 'httpjson.yml.hbs', - title: 'AWS CloudTrail logs via Splunk Enterprise REST API', - description: 'Collect AWS CloudTrail logs via Splunk Enterprise REST API', - enabled: false, - }, - ], - package: 'aws', - path: 'cloudtrail', - }, - { - type: 'logs', - dataset: 'aws.cloudwatch_logs', - title: 'AWS CloudWatch logs', - release: 'beta', - ingest_pipeline: 'default', - streams: [ - { - input: 's3', - vars: [ - { - name: 'queue_url', - type: 'text', - title: 'Queue URL', - description: 'URL of the AWS SQS queue that messages will be received from.', - multi: false, - required: true, - show_user: true, - }, - { - name: 'fips_enabled', - type: 'bool', - title: 'Enable S3 FIPS', - description: - 'Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.', - multi: false, - required: false, - show_user: false, - default: false, - }, - ], - template_path: 's3.yml.hbs', - title: 'AWS CloudWatch logs', - description: 'Collect AWS CloudWatch logs using s3 input', - enabled: true, - }, - ], - package: 'aws', - path: 'cloudwatch_logs', - }, - { - type: 'metrics', - dataset: 'aws.cloudwatch_metrics', - title: 'AWS CloudWatch metrics', - release: 'beta', - streams: [ - { - input: 'aws/metrics', - vars: [ - { - name: 'period', - type: 'text', - title: 'Period', - multi: false, - required: true, - show_user: true, - default: '300s', - }, - { - name: 'regions', - type: 'text', - title: 'Regions', - multi: true, - required: false, - show_user: true, - }, - { - name: 'latency', - type: 'text', - title: 'Latency', - multi: false, - required: false, - show_user: false, - }, - { - name: 'metrics', - type: 'yaml', - title: 'Metrics', - multi: false, - required: true, - show_user: true, - default: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - }, - ], - template_path: 'stream.yml.hbs', - title: 'AWS CloudWatch metrics', - description: 'Collect AWS CloudWatch metrics', - enabled: true, - }, - ], - package: 'aws', - path: 'cloudwatch_metrics', - }, - ], - policy_templates: [ - { - name: 'cloudtrail', - title: 'AWS Cloudtrail', - description: 'Collect logs from AWS Cloudtrail', - data_streams: ['cloudtrail'], - inputs: [ - { - type: 's3', - vars: [ - { - name: 'visibility_timeout', - type: 'text', - title: 'Visibility Timeout', - description: - 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'api_timeout', - type: 'text', - title: 'API Timeout', - description: - 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', - multi: false, - required: false, - show_user: false, - }, - ], - title: 'Collect logs from Cloudtrail service', - description: 'Collecting Cloudtrail logs using S3 input', - input_group: 'logs', - }, - ], - multiple: true, - icons: [ - { - src: '/img/logo_cloudtrail.svg', - path: '/package/aws/0.5.2/img/logo_cloudtrail.svg', - title: 'AWS Cloudtrail logo', - size: '32x32', - type: 'image/svg+xml', - }, - ], - screenshots: [ - { - src: '/img/filebeat-aws-cloudtrail.png', - path: '/package/aws/0.5.2/img/filebeat-aws-cloudtrail.png', - title: 'filebeat aws cloudtrail', - size: '1702x1063', - type: 'image/png', - }, - ], - }, - { - name: 'cloudwatch', - title: 'AWS CloudWatch', - description: 'Collect logs and metrics from CloudWatch', - data_streams: ['cloudwatch_logs', 'cloudwatch_metrics'], - inputs: [ - { - type: 's3', - vars: [ - { - name: 'visibility_timeout', - type: 'text', - title: 'Visibility Timeout', - description: - 'The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'api_timeout', - type: 'text', - title: 'API Timeout', - description: - 'The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.', - multi: false, - required: false, - show_user: false, - }, - ], - title: 'Collect logs from CloudWatch', - description: 'Collecting logs from CloudWatch using S3 input', - input_group: 'logs', - }, - { - type: 'aws/metrics', - vars: [ - { - name: 'metrics', - type: 'yaml', - title: 'Metrics', - multi: false, - required: true, - show_user: true, - default: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - }, - ], - title: 'Collect metrics from CloudWatch', - description: 'Collecting metrics from AWS CloudWatch', - input_group: 'metrics', - }, - ], - multiple: true, - icons: [ - { - src: '/img/logo_cloudwatch.svg', - path: '/package/aws/0.5.2/img/logo_cloudwatch.svg', - title: 'AWS CloudWatch logo', - size: '32x32', - type: 'image/svg+xml', - }, - ], - }, - ], - vars: [ - { - name: 'shared_credential_file', - type: 'text', - title: 'Shared Credential File', - description: 'Directory of the shared credentials file.', - multi: false, - required: false, - show_user: false, - }, - { - name: 'credential_profile_name', - type: 'text', - title: 'Credential Profile Name', - multi: false, - required: false, - show_user: true, - }, - { - name: 'access_key_id', - type: 'text', - title: 'Access Key ID', - multi: false, - required: false, - show_user: false, - }, - { - name: 'secret_access_key', - type: 'text', - title: 'Secret Access Key', - multi: false, - required: false, - show_user: false, - }, - { - name: 'session_token', - type: 'text', - title: 'Session Token', - multi: false, - required: false, - show_user: false, - }, - { - name: 'role_arn', - type: 'text', - title: 'Role ARN', - multi: false, - required: false, - show_user: false, - }, - { - name: 'endpoint', - type: 'text', - title: 'Endpoint', - description: 'URL of the entry point for an AWS web service.', - multi: false, - required: false, - show_user: false, - default: 'amazonaws.com', - }, - ], - } as unknown) as PackageInfo; - - const validPackagePolicy: NewPackagePolicy = { - name: 'mock-package-1', - namespace: 'default', - package: { name: 'mock-package', title: 'Mock package', version: '0.0.0' }, - enabled: true, - policy_id: 'some-policy-id', - output_id: 'some-output-id', - inputs: [ - { - type: 's3', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + const noErrorsValidationResults = { + name: null, + description: null, + namespace: null, + inputs: { + logs: { + streams: { + 'aws.cloudtrail': { vars: { - queue_url: { type: 'text', value: 'localhost' }, - fips_enabled: { value: false, type: 'bool' }, + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], + fips_enabled: null, }, }, - ], - integration: 'cloudtrail', - vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, - }, - { - type: 's3', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'logs', dataset: 'aws.cloudwatch_logs' }, + 'aws.cloudwatch_logs': { vars: { - queue_url: { type: 'text', value: 'localhost' }, - fips_enabled: { value: false, type: 'bool' }, + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], + fips_enabled: null, }, }, - ], - integration: 'cloudwatch', - vars: { visibility_timeout: { type: 'text' }, api_timeout: { type: 'text' } }, - }, - { - type: 'aws/metrics', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { type: 'metrics', dataset: 'aws.cloudwatch_metrics' }, + 'aws.ec2_logs': { vars: { - period: { value: '300s', type: 'text' }, - regions: { value: [], type: 'text' }, - latency: { type: 'text' }, - metrics: { - value: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - type: 'yaml', - }, + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], + fips_enabled: null, }, }, - ], - integration: 'cloudwatch', - vars: { - metrics: { - value: - '- namespace: AWS/EC2\n resource_type: ec2:instance\n name:\n - CPUUtilization\n - DiskWriteOps\n statistic:\n - Average\n - Maximum\n # dimensions:\n # - name: InstanceId\n # value: i-123456\n # tags:\n # - key: created-by\n # value: foo\n', - type: 'yaml', + 'aws.elb_logs': { + vars: { + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], + fips_enabled: null, + }, }, - }, - }, - ], - integrations: [ - { name: 'cloudtrail', enabled: true }, - { name: 'cloudwatch', enabled: true }, - ], - vars: { - shared_credential_file: { type: 'text' }, - credential_profile_name: { type: 'text' }, - access_key_id: { type: 'text' }, - secret_access_key: { type: 'text' }, - session_token: { type: 'text' }, - role_arn: { type: 'text' }, - endpoint: { value: 'amazonaws.com', type: 'text' }, - }, - }; - - const noErrorsValidationResults = { - name: null, - description: null, - namespace: null, - inputs: { - 's3-cloudtrail': { - streams: { - 'aws.cloudtrail': { + 'aws.s3access': { vars: { - queue_url: null, + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], fips_enabled: null, }, }, - }, - vars: { - visibility_timeout: null, - api_timeout: null, - }, - }, - 's3-cloudwatch': { - streams: { - 'aws.cloudwatch_logs': { + 'aws.vpcflow': { vars: { - queue_url: null, + visibility_timeout: null, + api_timeout: null, + queue_url: ['Queue URL is required'], fips_enabled: null, }, }, }, - vars: { - visibility_timeout: null, - api_timeout: null, - }, }, - 'aws/metrics-cloudwatch': { + metrics: { streams: { - 'aws.cloudwatch_metrics': { + 'aws.billing': { vars: { period: null, - regions: null, latency: null, - metrics: null, + 'cost_explorer_config.group_by_dimension_keys': null, + 'cost_explorer_config.group_by_tag_keys': null, }, }, - }, - vars: { - metrics: null, + 'aws.cloudwatch_metrics': { + vars: { period: null, regions: null, latency: null, metrics: null }, + }, + 'aws.dynamodb': { + vars: { period: null, regions: null, latency: null, tags_filter: null }, + }, + 'aws.ebs': { vars: { period: null, regions: null, latency: null, tags_filter: null } }, + 'aws.ec2_metrics': { + vars: { period: null, regions: null, latency: null, tags_filter: null }, + }, + 'aws.elb_metrics': { + vars: { period: null, regions: null, latency: null, tags_filter: null }, + }, + 'aws.lambda': { + vars: { period: null, regions: null, latency: null, tags_filter: null }, + }, + 'aws.natgateway': { vars: { period: null, regions: null, latency: null } }, + 'aws.rds': { vars: { period: null, regions: null, latency: null, tags_filter: null } }, + 'aws.s3_daily_storage': { vars: { period: null, regions: null, latency: null } }, + 'aws.s3_request': { vars: { period: null, regions: null, latency: null } }, + 'aws.sns': { vars: { period: null, regions: null, latency: null, tags_filter: null } }, + 'aws.sqs': { vars: { period: null, regions: null, latency: null } }, + 'aws.transitgateway': { vars: { period: null, regions: null, latency: null } }, + 'aws.usage': { vars: { period: null, regions: null, latency: null } }, + 'aws.vpn': { vars: { period: null, regions: null, latency: null, tags_filter: null } }, }, }, }, @@ -1039,9 +609,9 @@ describe('Fleet - validatePackagePolicy()', () => { }; it('returns no errors for valid package policy', () => { - expect(validatePackagePolicy(validPackagePolicy, mockPackage)).toEqual( - noErrorsValidationResults - ); + expect( + validatePackagePolicy(packagePolicyForPolicyTemplates, packageWithPolicyTemplates) + ).toEqual(noErrorsValidationResults); }); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts b/x-pack/plugins/fleet/common/services/validate_package_policy.ts similarity index 72% rename from x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts rename to x-pack/plugins/fleet/common/services/validate_package_policy.ts index c5c9df089f0c4..0c6a1fabf0dd0 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/validate_package_policy.ts @@ -9,11 +9,7 @@ import { i18n } from '@kbn/i18n'; import { safeLoad } from 'js-yaml'; import { keyBy } from 'lodash'; -import { - getFlattenedObject, - isValidNamespace, - doesPackageHaveIntegrations, -} from '../../../../services'; +import { getFlattenedObject } from '@kbn/std'; import type { NewPackagePolicy, @@ -21,10 +17,10 @@ import type { PackagePolicyInputStream, PackagePolicyConfigRecordEntry, PackageInfo, - RegistryInput, - RegistryStream, RegistryVarsEntry, -} from '../../../../types'; +} from '../types'; + +import { isValidNamespace, groupInputs } from './'; type Errors = string[] | null; @@ -53,7 +49,9 @@ export const validatePackagePolicy = ( packagePolicy: NewPackagePolicy, packageInfo: PackageInfo ): PackagePolicyValidationResults => { - const hasIntegrations = doesPackageHaveIntegrations(packageInfo); + // const hasIntegrations = doesPackageHaveIntegrations(packageInfo); + const packageInputs = groupInputs(packageInfo); + const packageInputsByName = keyBy(packageInputs, 'name'); const validationResults: PackagePolicyValidationResults = { name: null, description: null, @@ -78,61 +76,40 @@ export const validatePackagePolicy = ( const packageVarsByName = keyBy(packageInfo.vars || [], 'name'); const packageVars = Object.entries(packagePolicy.vars || {}); if (packageVars.length) { - validationResults.vars = packageVars.reduce((results, [name, configEntry]) => { - results[name] = validatePackagePolicyConfig(configEntry, packageVarsByName[name]); + validationResults.vars = packageVars.reduce((results, [name, varEntry]) => { + results[name] = validatePackagePolicyConfig(varEntry, packageVarsByName[name]); return results; }, {} as ValidationEntry); } - if ( - !packageInfo.policy_templates || - packageInfo.policy_templates.length === 0 || - !packageInfo.policy_templates[0] || - !packageInfo.policy_templates[0].inputs || - packageInfo.policy_templates[0].inputs.length === 0 - ) { + // If no inputs, return empty input validation + if (packageInputs.length === 0) { validationResults.inputs = null; return validationResults; } - const registryInputsByTypeAndIntegration: Record< - string, - RegistryInput - > = packageInfo.policy_templates.reduce((inputs, policyTemplate) => { - (policyTemplate.inputs || []).forEach((input) => { - inputs[hasIntegrations ? `${input.type}-${policyTemplate.name}` : input.type] = input; - }); - - return inputs; - }, {} as Record); - - const registryStreamsByDataset: Record = ( - packageInfo.data_streams || [] - ).reduce((dataStreams, registryDataStream) => { - dataStreams[registryDataStream.dataset] = registryDataStream.streams || []; - return dataStreams; - }, {} as Record); - // Validate each package policy input with either its own config fields or streams - packagePolicy.inputs.forEach((input) => { - if (!input.vars && !input.streams) { + packagePolicy.inputs.forEach((policyInput) => { + if (!policyInput.vars && !policyInput.streams) { return; } - - const inputKey = hasIntegrations ? `${input.type}-${input.integration}` : input.type; + const packageInput = packageInputsByName[policyInput.type]; + const packageInputStreamsByDataset = keyBy( + packageInput.streams, + (stream) => stream.data_stream.dataset + ); + const inputVarsByName = keyBy(packageInput.vars || [], 'name'); const inputValidationResults: PackagePolicyInputValidationResults = { vars: undefined, streams: {}, }; - const inputVarsByName = keyBy(registryInputsByTypeAndIntegration[inputKey].vars || [], 'name'); - - // Validate input-level config fields - const inputConfigs = Object.entries(input.vars || {}); - if (inputConfigs.length) { - inputValidationResults.vars = inputConfigs.reduce((results, [name, configEntry]) => { - results[name] = input.enabled - ? validatePackagePolicyConfig(configEntry, inputVarsByName[name]) + // Validate input-level vars + const inputVars = Object.entries(policyInput.vars || {}); + if (inputVars.length) { + inputValidationResults.vars = inputVars.reduce((results, [name, varEntry]) => { + results[name] = policyInput.enabled + ? validatePackagePolicyConfig(varEntry, inputVarsByName[name]) : null; return results; }, {} as ValidationEntry); @@ -141,27 +118,21 @@ export const validatePackagePolicy = ( } // Validate each input stream with config fields - if (input.streams.length) { - input.streams.forEach((stream) => { + if (policyInput.streams.length) { + policyInput.streams.forEach((stream) => { const streamValidationResults: PackagePolicyConfigValidationResults = {}; // Validate stream-level config fields if (stream.vars) { - const streamVarsByName = ( - ( - registryStreamsByDataset[stream.data_stream.dataset].find( - (registryStream) => registryStream.input === input.type - ) || {} - ).vars || [] - ).reduce((vars, registryVar) => { - vars[registryVar.name] = registryVar; - return vars; - }, {} as Record); + const streamVarsByName = keyBy( + packageInputStreamsByDataset[stream.data_stream.dataset].vars || [], + 'name' + ); streamValidationResults.vars = Object.entries(stream.vars).reduce( - (results, [name, configEntry]) => { + (results, [name, varEntry]) => { results[name] = - input.enabled && stream.enabled - ? validatePackagePolicyConfig(configEntry, streamVarsByName[name]) + policyInput.enabled && stream.enabled + ? validatePackagePolicyConfig(varEntry, streamVarsByName[name]) : null; return results; }, @@ -176,7 +147,7 @@ export const validatePackagePolicy = ( } if (inputValidationResults.vars || inputValidationResults.streams) { - validationResults.inputs![inputKey] = inputValidationResults; + validationResults.inputs![policyInput.type] = inputValidationResults; } }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts index c8f2449fba794..0e1953316fd53 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts @@ -15,5 +15,4 @@ export { validatePackagePolicyConfig, validationHasErrors, countValidationErrors, -} from './validate_package_policy'; -export { groupInputs } from './group_inputs'; +} from '../../../../services'; diff --git a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts index 58385181b280f..c5c187d6aefba 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -export { getFlattenedObject } from '@kbn/std'; - export { AgentStatusKueryHelper, agentPolicyRouteService, @@ -32,4 +30,11 @@ export { doesPackageHaveIntegrations, getStreamsForInputType, findDataStreamsByNames, + PackagePolicyValidationResults, + PackagePolicyConfigValidationResults, + PackagePolicyInputValidationResults, + validatePackagePolicy, + validatePackagePolicyConfig, + validationHasErrors, + countValidationErrors, } from '../../../../common'; From 7e60f3c99afc282d6998b195f1c557391cf50240 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Apr 2021 14:38:05 -0700 Subject: [PATCH 12/16] Adjust UI to show package input groups --- .../components/package_policy_input_panel.tsx | 14 ++- .../step_configure_package.tsx | 113 +++++++----------- .../applications/fleet/services/index.ts | 1 + .../public/applications/fleet/types/index.ts | 1 + 4 files changed, 55 insertions(+), 74 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx index 75d7a7549f721..b1b546d85c872 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx @@ -21,8 +21,10 @@ import { import type { NewPackagePolicyInput, + NewPackagePolicyInputStream, PackagePolicyInputStream, RegistryInput, + RegistryInputKeys, RegistryStream, } from '../../../../types'; import type { PackagePolicyInputValidationResults } from '../services'; @@ -39,7 +41,7 @@ const ShortenedHorizontalRule = styled(EuiHorizontalRule)` `; const shouldShowStreamsByDefault = ( - packageInput: RegistryInput, + packageInput: Pick, packageInputStreams: Array, packagePolicyInput: NewPackagePolicyInput ): boolean => { @@ -62,8 +64,14 @@ const shouldShowStreamsByDefault = ( }; export const PackagePolicyInputPanel: React.FunctionComponent<{ - packageInput: RegistryInput; - packageInputStreams: Array; + packageInput: Pick< + RegistryInput, + | RegistryInputKeys.type + | RegistryInputKeys.title + | RegistryInputKeys.description + | RegistryInputKeys.vars + >; + packageInputStreams: Array>; packagePolicyInput: NewPackagePolicyInput; updatePackagePolicyInput: (updatedInput: Partial) => void; inputValidationResults: PackagePolicyInputValidationResults; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx index 9e319c5912159..ced45419b5bc0 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_configure_package.tsx @@ -4,8 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import React from 'react'; +import React, { useMemo, memo } from 'react'; +import { keyBy } from 'lodash'; import { EuiHorizontalRule, EuiFlexGroup, @@ -15,70 +15,61 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import type { - PackageInfo, - RegistryStream, - NewPackagePolicy, - NewPackagePolicyInput, -} from '../../../types'; +import type { PackageInfo, NewPackagePolicy, NewPackagePolicyInput } from '../../../types'; import { Loading } from '../../../components'; +import { groupInputs } from '../../../services'; import type { PackagePolicyValidationResults } from './services'; import { PackagePolicyInputPanel } from './components'; -const findStreamsForInputType = ( - inputType: string, - packageInfo: PackageInfo -): Array => { - const streams: Array = []; - - (packageInfo.data_streams || []).forEach((dataStream) => { - (dataStream.streams || []).forEach((stream) => { - if (stream.input === inputType) { - streams.push({ - ...stream, - data_stream: { - dataset: dataStream.dataset, - }, - }); - } - }); - }); - - return streams; -}; - export const StepConfigurePackagePolicy: React.FunctionComponent<{ packageInfo: PackageInfo; packagePolicy: NewPackagePolicy; updatePackagePolicy: (fields: Partial) => void; validationResults: PackagePolicyValidationResults; submitAttempted: boolean; -}> = ({ packageInfo, packagePolicy, updatePackagePolicy, validationResults, submitAttempted }) => { - // Configure inputs (and their streams) - // Assume packages only export one config template for now - const renderConfigureInputs = () => - packageInfo.policy_templates && - packageInfo.policy_templates[0] && - packageInfo.policy_templates[0].inputs && - packageInfo.policy_templates[0].inputs.length ? ( +}> = memo( + ({ packageInfo, packagePolicy, updatePackagePolicy, validationResults, submitAttempted }) => { + const groupedPackageInputs = useMemo(() => groupInputs(packageInfo), [packageInfo]); + const policyInputsByType = useMemo(() => keyBy(packagePolicy.inputs, 'type'), [ + packagePolicy.inputs, + ]); + + if (groupedPackageInputs.length === 0) { + return ( + +

+ +

+ + } + /> + ); + } + + return validationResults ? ( <> - {packageInfo.policy_templates[0].inputs.map((packageInput) => { - const packagePolicyInput = packagePolicy.inputs.find( - (input) => input.type === packageInput.type - ); - const packageInputStreams = findStreamsForInputType(packageInput.type, packageInfo); + {groupedPackageInputs.map((packageInput) => { + const inputType = packageInput.name; + const packagePolicyInput = policyInputsByType[inputType]; return packagePolicyInput ? ( - + ) => { const indexOfUpdatedInput = packagePolicy.inputs.findIndex( - (input) => input.type === packageInput.type + (input) => input.type === packageInput.name ); const newInputs = [...packagePolicy.inputs]; newInputs[indexOfUpdatedInput] = { @@ -89,13 +80,7 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{ inputs: newInputs, }); }} - inputValidationResults={ - validationResults!.inputs![ - packagePolicyInput.integration - ? `${packagePolicyInput.type}-${packagePolicyInput.integration}` - : packagePolicyInput.type - ] - } + inputValidationResults={validationResults!.inputs![packagePolicyInput.type]} forceShowErrors={submitAttempted} /> @@ -105,21 +90,7 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{ ) : ( - -

- -

- - } - /> + ); - - return validationResults ? renderConfigureInputs() : ; -}; + } +); diff --git a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts index c5c187d6aefba..3f5d2637d7d89 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts @@ -30,6 +30,7 @@ export { doesPackageHaveIntegrations, getStreamsForInputType, findDataStreamsByNames, + groupInputs, PackagePolicyValidationResults, PackagePolicyConfigValidationResults, PackagePolicyInputValidationResults, diff --git a/x-pack/plugins/fleet/public/applications/fleet/types/index.ts b/x-pack/plugins/fleet/public/applications/fleet/types/index.ts index 1b06e6f7ee08d..d154275f4483d 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/types/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/types/index.ts @@ -125,6 +125,7 @@ export { RegistryRelease, PackageSpecCategory, RegistryInputGroup, + RegistryInputKeys, } from '../../../../common'; export * from './intra_app_route_state'; From 5dad3d76daddfdf48ca7d190dec35bf0f4ade730 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Apr 2021 14:41:49 -0700 Subject: [PATCH 13/16] Allow package-level vars in API schema --- x-pack/plugins/fleet/server/types/models/package_policy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 6248b375f8edb..263afa140e450 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -74,6 +74,7 @@ const PackagePolicyBaseSchema = { ), }) ), + vars: schema.maybe(ConfigRecordSchema), }; export const NewPackagePolicySchema = schema.object({ From 5fc6aa8970f0904e8210857cc220ab5144e69616 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Apr 2021 15:45:48 -0700 Subject: [PATCH 14/16] Make `packageToPackagePolicy` params an object, fix types --- .../package_to_package_policy.test.ts | 52 ++++++++++++------- .../services/package_to_package_policy.ts | 29 +++++++---- .../step_define_package_policy.tsx | 20 ++++--- .../routes/package_policy/handlers.test.ts | 4 +- .../fleet/server/services/agent_policy.ts | 14 ++--- .../ingest_pipeline/ingest_pipelines.test.ts | 2 +- .../fleet/server/services/package_policy.ts | 10 ++-- 7 files changed, 79 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index 59768e6ac8c85..549192413527b 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -325,9 +325,11 @@ describe('Fleet - packageToPackagePolicy', () => { describe('packageToPackagePolicy', () => { it('returns package policy with default name', () => { - expect(packageToPackagePolicy(mockPackage, '1', '2')).toEqual({ + expect( + packageToPackagePolicy({ packageInfo: mockPackage, agentPolicyId: '1', outputId: '2' }) + ).toEqual({ policy_id: '1', - namespace: '', + namespace: 'default', enabled: true, inputs: [], name: 'mock-package-1', @@ -341,7 +343,14 @@ describe('Fleet - packageToPackagePolicy', () => { }); it('returns package policy with custom name', () => { - expect(packageToPackagePolicy(mockPackage, '1', '2', 'default', 'pkgPolicy-1')).toEqual({ + expect( + packageToPackagePolicy({ + name: 'pkgPolicy-1', + packageInfo: mockPackage, + agentPolicyId: '1', + outputId: '2', + }) + ).toEqual({ policy_id: '1', namespace: 'default', enabled: true, @@ -358,14 +367,14 @@ describe('Fleet - packageToPackagePolicy', () => { it('returns package policy with namespace and description', () => { expect( - packageToPackagePolicy( - mockPackage, - '1', - '2', - 'mock-namespace', - 'pkgPolicy-1', - 'Test description' - ) + packageToPackagePolicy({ + name: 'pkgPolicy-1', + namespace: 'mock-namespace', + description: 'Test description', + packageInfo: mockPackage, + agentPolicyId: '1', + outputId: '2', + }) ).toEqual({ policy_id: '1', enabled: true, @@ -383,14 +392,19 @@ describe('Fleet - packageToPackagePolicy', () => { }); it('returns package policy with inputs and package-level vars', () => { - const mockPackageWithPolicyTemplates = ({ + const mockPackageWithPackageVars = ({ ...mockPackage, policy_templates: [{ inputs: [{ type: 'foo' }] }], vars: [{ default: 'foo-var-value', name: 'var-name', type: 'text' }], } as unknown) as PackageInfo; expect( - packageToPackagePolicy(mockPackageWithPolicyTemplates, '1', '2', 'default', 'pkgPolicy-1') + packageToPackagePolicy({ + name: 'pkgPolicy-1', + packageInfo: mockPackageWithPackageVars, + agentPolicyId: '1', + outputId: '2', + }) ).toEqual({ policy_id: '1', namespace: 'default', @@ -409,12 +423,12 @@ describe('Fleet - packageToPackagePolicy', () => { it('returns package policy with multiple policy templates correctly', () => { expect( - packageToPackagePolicy( - packageWithPolicyTemplates, - 'some-policy-id', - 'some-output-id', - 'default' - ) + packageToPackagePolicy({ + name: 'aws-1', + packageInfo: packageWithPolicyTemplates, + agentPolicyId: 'some-policy-id', + outputId: 'some-output-id', + }) ).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 88bfb77238133..386c4a61e7de1 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -16,7 +16,7 @@ import type { PackagePolicyConfigRecordEntry, } from '../types'; -import { doesPackageHaveIntegrations, groupInputs } from './'; +import { groupInputs } from './'; export const findDataStreamsByNames = ( names: string[] = [], @@ -112,16 +112,25 @@ export const packageToPackagePolicyInputs = ( * @param outputId * @param packagePolicyName */ -export const packageToPackagePolicy = ( - packageInfo: PackageInfo, - agentPolicyId: string, - outputId: string, - namespace: string = '', - packagePolicyName?: string, - description?: string -): NewPackagePolicy => { +export const packageToPackagePolicy = (options: { + name?: string; + description?: string; + namespace?: string; + packageInfo: PackageInfo; + agentPolicyId: string; + outputId: string; +}): NewPackagePolicy => { + const { + name, + packageInfo, + agentPolicyId, + outputId, + namespace = 'default', + description, + } = options; + const packagePolicy: NewPackagePolicy = { - name: packagePolicyName || `${packageInfo.name}-1`, + name: name || `${packageInfo.name}-1`, namespace, description, package: { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx index 4e18950fc55de..090f4a86dcfc2 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx @@ -5,13 +5,12 @@ * 2.0. */ -import React, { memo, useEffect, useState, useMemo } from 'react'; +import React, { memo, useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFormRow, EuiFieldText, EuiButtonEmpty, - EuiSpacer, EuiText, EuiComboBox, EuiDescribedFormGroup, @@ -74,19 +73,18 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ .sort((a, b) => a - b); updatePackagePolicy( - packageToPackagePolicy( - packageInfo, - agentPolicy.id, - packagePolicy.output_id, - packagePolicy.namespace, - // FIXME: Improve package policies name uniqueness - https://github.com/elastic/kibana/issues/72948 - `${packageInfo.name}-${ + packageToPackagePolicy({ + name: `${packageInfo.name}-${ pkgPoliciesWithMatchingNames.length ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 : 1 }`, - packagePolicy.description - ) + description: packagePolicy.description, + namespace: packagePolicy.namespace, + packageInfo, + agentPolicyId: agentPolicy.id, + outputId: packagePolicy.output_id, + }) ); } diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index 7f2b9d93e2df7..5aa400d6443e6 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -24,7 +24,9 @@ jest.mock('../../services/package_policy', (): { } => { return { packagePolicyService: { - compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), + compilePackagePolicyInputs: jest.fn((packageInfo, vars, dataInputs) => + Promise.resolve(dataInputs) + ), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), create: jest.fn((soClient, esClient, newData) => diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index be61a70154b11..a197eeeec37f3 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -802,14 +802,14 @@ export async function addPackageToAgentPolicy( pkgVersion: packageToInstall.version, }); - const basePackagePolicy = packageToPackagePolicy( + const basePackagePolicy = packageToPackagePolicy({ + name: packagePolicyName, + description: packagePolicyDescription, + namespace: agentPolicy.namespace ?? 'default', packageInfo, - agentPolicy.id, - defaultOutput.id, - agentPolicy.namespace ?? 'default', - packagePolicyName, - packagePolicyDescription - ); + agentPolicyId: agentPolicy.id, + outputId: defaultOutput.id, + }); const newPackagePolicy = transformPackagePolicy ? transformPackagePolicy(basePackagePolicy) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts index 097f9ce28c7d1..f915e871b98f7 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts @@ -121,7 +121,7 @@ test('getPipelineNameForInstallation gets correct name', () => { const packageVersion = '1.0.1'; const pipelineRefName = 'pipeline-json'; const pipelineEntryNameForInstallation = getPipelineNameForInstallation({ - pipelineName: dataStream.ingest_pipeline, + pipelineName: dataStream.ingest_pipeline!, dataStream, packageVersion, }); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 92b3ec68dcae0..de21452a7fd59 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -416,7 +416,7 @@ class PackagePolicyService { ): Promise { const pkgInstall = await getInstallation({ savedObjectsClient: soClient, pkgName }); if (pkgInstall) { - const [pkgInfo, defaultOutputId] = await Promise.all([ + const [packageInfo, defaultOutputId] = await Promise.all([ getPackageInfo({ savedObjectsClient: soClient, pkgName: pkgInstall.name, @@ -424,11 +424,15 @@ class PackagePolicyService { }), outputService.getDefaultOutputId(soClient), ]); - if (pkgInfo) { + if (packageInfo) { if (!defaultOutputId) { throw new Error('Default output is not set'); } - return packageToPackagePolicy(pkgInfo, '', defaultOutputId); + return packageToPackagePolicy({ + packageInfo, + agentPolicyId: '', + outputId: defaultOutputId, + }); } } } From 09ea207d0aa11ba003fa411f1d01f5358e0a9ac2 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Apr 2021 17:18:13 -0700 Subject: [PATCH 15/16] Pass `integrations` param to policy editor to allow only certain policy_template streams to be enabled --- .../__snapshots__/group_inputs.test.ts.snap | 22 + .../package_to_package_policy.test.ts.snap | 622 ++++++++++++++++++ .../fleet/common/services/group_inputs.ts | 67 +- x-pack/plugins/fleet/common/services/index.ts | 7 +- .../package_to_package_policy.test.ts | 12 + .../services/package_to_package_policy.ts | 53 +- .../create_package_policy_page/index.tsx | 8 +- .../step_define_package_policy.tsx | 465 ++++++------- 8 files changed, 972 insertions(+), 284 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap index b9b35675ab13b..d53f9654e279a 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/group_inputs.test.ts.snap @@ -14,6 +14,7 @@ Array [ "description": "Collect AWS CloudTrail logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "cloudtrail", "template_path": "s3.yml.hbs", "title": "AWS CloudTrail logs", "vars": Array [ @@ -64,6 +65,7 @@ Array [ "description": "Collect AWS CloudWatch logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "cloudwatch", "template_path": "s3.yml.hbs", "title": "AWS CloudWatch logs", "vars": Array [ @@ -114,6 +116,7 @@ Array [ "description": "Collect AWS EC2 logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "ec2", "template_path": "s3.yml.hbs", "title": "AWS EC2 logs", "vars": Array [ @@ -164,6 +167,7 @@ Array [ "description": "Collect AWS ELB logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "elb", "template_path": "s3.yml.hbs", "title": "AWS ELB logs", "vars": Array [ @@ -214,6 +218,7 @@ Array [ "description": "Collect AWS s3access logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "s3", "template_path": "s3.yml.hbs", "title": "AWS s3access logs", "vars": Array [ @@ -264,6 +269,7 @@ Array [ "description": "Collect AWS vpcflow logs using s3 input", "enabled": true, "input": "s3", + "policyTemplate": "vpcflow", "template_path": "s3.yml.hbs", "title": "AWS vpcflow logs", "vars": Array [ @@ -321,6 +327,7 @@ Array [ "description": "Collect AWS billing metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "billing", "template_path": "stream.yml.hbs", "title": "AWS Billing metrics", "vars": Array [ @@ -375,6 +382,7 @@ Array [ "description": "Collect AWS CloudWatch metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "cloudwatch", "template_path": "stream.yml.hbs", "title": "AWS CloudWatch metrics", "vars": Array [ @@ -436,6 +444,7 @@ Array [ "description": "Collect AWS DynamoDB metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "dynamodb", "template_path": "stream.yml.hbs", "title": "AWS DynamoDB metrics", "vars": Array [ @@ -485,6 +494,7 @@ Array [ "description": "Collect AWS EBS metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "ebs", "template_path": "stream.yml.hbs", "title": "AWS EBS metrics", "vars": Array [ @@ -534,6 +544,7 @@ Array [ "description": "Collect AWS EC2 metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "ec2", "template_path": "stream.yml.hbs", "title": "AWS EC2 metrics", "vars": Array [ @@ -583,6 +594,7 @@ Array [ "description": "Collect AWS ELB metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "elb", "template_path": "stream.yml.hbs", "title": "AWS ELB metrics", "vars": Array [ @@ -632,6 +644,7 @@ Array [ "description": "Collect AWS Lambda metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "lambda", "template_path": "stream.yml.hbs", "title": "AWS Lambda metrics", "vars": Array [ @@ -681,6 +694,7 @@ Array [ "description": "Collect AWS NAT gateway metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "natgateway", "template_path": "stream.yml.hbs", "title": "AWS NAT gateway metrics", "vars": Array [ @@ -719,6 +733,7 @@ Array [ "description": "Collect AWS RDS metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "rds", "template_path": "stream.yml.hbs", "title": "AWS RDS metrics", "vars": Array [ @@ -768,6 +783,7 @@ Array [ "description": "Collect AWS S3 daily storage metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "s3", "template_path": "stream.yml.hbs", "title": "AWS S3 daily storage metrics", "vars": Array [ @@ -806,6 +822,7 @@ Array [ "description": "Collect AWS S3 request metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "s3", "template_path": "stream.yml.hbs", "title": "AWS S3 request metrics", "vars": Array [ @@ -844,6 +861,7 @@ Array [ "description": "Collect AWS SNS metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "sns", "template_path": "stream.yml.hbs", "title": "AWS SNS metrics", "vars": Array [ @@ -893,6 +911,7 @@ Array [ "description": "Collect AWS SQS metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "sqs", "template_path": "stream.yml.hbs", "title": "AWS SQS metrics", "vars": Array [ @@ -931,6 +950,7 @@ Array [ "description": "Collect AWS Transit Gateway metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "transitgateway", "template_path": "stream.yml.hbs", "title": "AWS Transit Gateway metrics", "vars": Array [ @@ -969,6 +989,7 @@ Array [ "description": "Collect AWS usage metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "usage", "template_path": "stream.yml.hbs", "title": "AWS usage metrics", "vars": Array [ @@ -1007,6 +1028,7 @@ Array [ "description": "Collect AWS VPN metrics", "enabled": true, "input": "aws/metrics", + "policyTemplate": "vpn", "template_path": "stream.yml.hbs", "title": "AWS VPN metrics", "vars": Array [ diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap index 4d53b1a6013e8..45c667acab67d 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/package_to_package_policy.test.ts.snap @@ -1,5 +1,627 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Fleet - packageToPackagePolicy packageToPackagePolicy only enables streams matching a particular policy template if given 1`] = ` +Object { + "description": undefined, + "enabled": true, + "inputs": Array [ + Object { + "enabled": false, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.cloudtrail", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_logs", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_logs", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_logs", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3access", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.vpcflow", + "type": "logs", + }, + "enabled": false, + "vars": Object { + "api_timeout": Object { + "type": "text", + "value": undefined, + }, + "fips_enabled": Object { + "type": "bool", + "value": false, + }, + "queue_url": Object { + "type": "text", + "value": undefined, + }, + "visibility_timeout": Object { + "type": "text", + "value": undefined, + }, + }, + }, + ], + "type": "logs", + }, + Object { + "enabled": true, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "aws.billing", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "cost_explorer_config.group_by_dimension_keys": Object { + "type": "text", + "value": Array [ + "AZ", + "INSTANCE_TYPE", + "SERVICE", + ], + }, + "cost_explorer_config.group_by_tag_keys": Object { + "type": "text", + "value": Array [ + "aws:createdBy", + ], + }, + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "12h", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.cloudwatch_metrics", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "metrics": Object { + "type": "yaml", + "value": "- namespace: AWS/EC2 + resource_type: ec2:instance + name: + - CPUUtilization + - DiskWriteOps + statistic: + - Average + - Maximum + # dimensions: + # - name: InstanceId + # value: i-123456 + # tags: + # - key: created-by + # value: foo +", + }, + "period": Object { + "type": "text", + "value": "300s", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.dynamodb", + "type": "metrics", + }, + "enabled": true, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ebs", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.ec2_metrics", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.elb_metrics", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.lambda", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.natgateway", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.rds", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_daily_storage", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "24h", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.s3_request", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.sns", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.sqs", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "5m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.transitgateway", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.usage", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + }, + }, + Object { + "data_stream": Object { + "dataset": "aws.vpn", + "type": "metrics", + }, + "enabled": false, + "vars": Object { + "latency": Object { + "type": "text", + "value": undefined, + }, + "period": Object { + "type": "text", + "value": "1m", + }, + "regions": Object { + "type": "text", + "value": Array [], + }, + "tags_filter": Object { + "type": "yaml", + "value": "# - key: \\"created-by\\" + # value: \\"foo\\" +", + }, + }, + }, + ], + "type": "metrics", + }, + ], + "name": "aws-1", + "namespace": "default", + "output_id": "some-output-id", + "package": Object { + "name": "aws", + "title": "AWS", + "version": "0.5.2", + }, + "policy_id": "some-policy-id", + "vars": Object { + "access_key_id": Object { + "type": "text", + "value": undefined, + }, + "credential_profile_name": Object { + "type": "text", + "value": undefined, + }, + "endpoint": Object { + "type": "text", + "value": "amazonaws.com", + }, + "role_arn": Object { + "type": "text", + "value": undefined, + }, + "secret_access_key": Object { + "type": "text", + "value": undefined, + }, + "session_token": Object { + "type": "text", + "value": undefined, + }, + "shared_credential_file": Object { + "type": "text", + "value": undefined, + }, + }, +} +`; + exports[`Fleet - packageToPackagePolicy packageToPackagePolicy returns package policy with multiple policy templates correctly 1`] = ` Object { "description": undefined, diff --git a/x-pack/plugins/fleet/common/services/group_inputs.ts b/x-pack/plugins/fleet/common/services/group_inputs.ts index 47eaa97ddccbd..4432671f3f38f 100644 --- a/x-pack/plugins/fleet/common/services/group_inputs.ts +++ b/x-pack/plugins/fleet/common/services/group_inputs.ts @@ -4,17 +4,59 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { PackageInfo, RegistryInputGroup, RegistryVarsEntry } from '../types'; +import type { + PackageInfo, + RegistryInputGroup, + RegistryVarsEntry, + RegistryStream, + NewPackagePolicyInputStream, +} from '../types'; -import { doesPackageHaveIntegrations, getStreamsForInputType, findDataStreamsByNames } from './'; - -interface InputGroupWithStreams extends RegistryInputGroup { +type InputGroupWithStreams = RegistryInputGroup & { vars?: RegistryVarsEntry[]; - streams?: ReturnType; -} + streams: Array; +}; + +type StreamWithDataStreamMeta = RegistryStream & Pick; + +import { doesPackageHaveIntegrations } from './'; + +const findDataStreamsByNames = ( + names: string[] = [], + dataStreams: PackageInfo['data_streams'] = [] +): PackageInfo['data_streams'] => { + return names.length + ? dataStreams.filter((dataStream) => names.includes(dataStream.path)) + : dataStreams; +}; + +const getStreamsForInputType = ( + inputType: string, + dataStreams: PackageInfo['data_streams'] = [] +): StreamWithDataStreamMeta[] => { + const streams: StreamWithDataStreamMeta[] = []; + + dataStreams.forEach((dataStream) => { + (dataStream.streams || []).forEach((stream) => { + if (stream.input === inputType) { + streams.push({ + ...stream, + data_stream: { + type: dataStream.type, + dataset: dataStream.dataset, + }, + }); + } + }); + }); + + return streams; +}; export const groupInputs = (packageInfo: PackageInfo): InputGroupWithStreams[] => { - const inputGroups: InputGroupWithStreams[] = [...(packageInfo.input_groups || [])]; + const inputGroups: InputGroupWithStreams[] = [ + ...(packageInfo.input_groups?.map((inputGroup) => ({ ...inputGroup, streams: [] })) || []), + ]; const hasIntegrations = doesPackageHaveIntegrations(packageInfo); const { policy_templates: policyTemplates = [] } = packageInfo; @@ -48,14 +90,17 @@ export const groupInputs = (packageInfo: PackageInfo): InputGroupWithStreams[] = const inputGroup = inputGroupName ? inputGroups.find((group) => group.name === inputGroupName) : undefined; + if (!inputGroup) { return; } - inputGroup.streams = [ - ...(inputGroup.streams || []), - ...getStreamsForInputType(type, policyTemplateDataStreams), - ]; + inputGroup.streams.push( + ...getStreamsForInputType(type, policyTemplateDataStreams).map((stream) => ({ + ...stream, + policyTemplate: policyTemplate.name, + })) + ); }); }); diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts index e6f34a1a50be6..92ab65a0fbde5 100644 --- a/x-pack/plugins/fleet/common/services/index.ts +++ b/x-pack/plugins/fleet/common/services/index.ts @@ -7,12 +7,7 @@ export * from './routes'; export * as AgentStatusKueryHelper from './agent_status'; -export { - packageToPackagePolicyInputs, - packageToPackagePolicy, - getStreamsForInputType, - findDataStreamsByNames, -} from './package_to_package_policy'; +export { packageToPackagePolicyInputs, packageToPackagePolicy } from './package_to_package_policy'; export { storedPackagePoliciesToAgentInputs } from './package_policies_to_agent_inputs'; export { fullAgentPolicyToYaml } from './full_agent_policy_to_yaml'; export { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from './limited_package'; diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index 549192413527b..5754064ac3ebb 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -431,5 +431,17 @@ describe('Fleet - packageToPackagePolicy', () => { }) ).toMatchSnapshot(); }); + + it('only enables streams matching a particular policy template if given', () => { + expect( + packageToPackagePolicy({ + name: 'aws-1', + packageInfo: packageWithPolicyTemplates, + agentPolicyId: 'some-policy-id', + outputId: 'some-output-id', + enablePolicyTemplate: 'dynamodb', + }) + ).toMatchSnapshot(); + }); }); }); diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 386c4a61e7de1..2f9b9cebf5891 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -18,38 +18,6 @@ import type { import { groupInputs } from './'; -export const findDataStreamsByNames = ( - names: string[] = [], - dataStreams: PackageInfo['data_streams'] = [] -): PackageInfo['data_streams'] => { - return names.length - ? dataStreams.filter((dataStream) => names.includes(dataStream.path)) - : dataStreams; -}; - -export const getStreamsForInputType = ( - inputType: string, - dataStreams: PackageInfo['data_streams'] = [] -): Array> => { - const streams: Array> = []; - - dataStreams.forEach((dataStream) => { - (dataStream.streams || []).forEach((stream) => { - if (stream.input === inputType) { - streams.push({ - ...stream, - data_stream: { - type: dataStream.type, - dataset: dataStream.dataset, - }, - }); - } - }); - }); - - return streams; -}; - // Reduces registry var def into config object entry const varsReducer = ( configObject: PackagePolicyConfigRecord, @@ -69,8 +37,12 @@ const varsReducer = ( * This service creates a package policy inputs definition from defaults provided in package info */ export const packageToPackagePolicyInputs = ( - packageInfo: PackageInfo + packageInfo: PackageInfo, + options: { + enablePolicyTemplate?: string; + } = {} ): NewPackagePolicy['inputs'] => { + const { enablePolicyTemplate } = options; const inputs: NewPackagePolicy['inputs'] = []; const packageInputGroups = groupInputs(packageInfo); @@ -78,7 +50,10 @@ export const packageToPackagePolicyInputs = ( const streams: NewPackagePolicyInputStream[] = (inputGroup.streams || []).map( (packageStream) => { const stream: NewPackagePolicyInputStream = { - enabled: packageStream.enabled === false ? false : true, + enabled: + enablePolicyTemplate && packageStream.policyTemplate + ? packageStream.policyTemplate === enablePolicyTemplate + : packageStream.enabled !== false, data_stream: packageStream.data_stream, }; if (packageStream.vars && packageStream.vars.length) { @@ -116,17 +91,19 @@ export const packageToPackagePolicy = (options: { name?: string; description?: string; namespace?: string; + enablePolicyTemplate?: string; packageInfo: PackageInfo; agentPolicyId: string; outputId: string; }): NewPackagePolicy => { const { name, + description, + namespace = 'default', + enablePolicyTemplate, packageInfo, agentPolicyId, outputId, - namespace = 'default', - description, } = options; const packagePolicy: NewPackagePolicy = { @@ -141,7 +118,9 @@ export const packageToPackagePolicy = (options: { enabled: true, policy_id: agentPolicyId, output_id: outputId, - inputs: packageToPackagePolicyInputs(packageInfo), + inputs: packageToPackagePolicyInputs(packageInfo, { + enablePolicyTemplate, + }), }; if (packageInfo.vars?.length) { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx index f312220d9faec..56fb23ce15d9a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx @@ -67,8 +67,8 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => { agents: { enabled: isFleetEnabled }, } = useConfig(); const { - params: { policyId, pkgkey }, - } = useRouteMatch<{ policyId: string; pkgkey: string }>(); + params: { policyId, pkgkey, integration }, + } = useRouteMatch<{ policyId: string; pkgkey: string; integration?: string }>(); const { getHref, getPath } = useLink(); const history = useHistory(); const handleNavigateTo = useNavigateToCallback(); @@ -331,6 +331,7 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => { agentPolicy={agentPolicy} packageInfo={packageInfo} packagePolicy={packagePolicy} + integration={integration} updatePackagePolicy={updatePackagePolicy} validationResults={validationResults!} /> @@ -361,10 +362,11 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => { agentPolicy, packageInfo, packagePolicy, + integration, updatePackagePolicy, validationResults, - formState, ExtensionView, + formState, handleExtensionViewOnChange, ] ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx index 090f4a86dcfc2..9d2f7e3a99ceb 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx @@ -37,262 +37,273 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ agentPolicy: AgentPolicy; packageInfo: PackageInfo; packagePolicy: NewPackagePolicy; + integration?: string; updatePackagePolicy: (fields: Partial) => void; validationResults: PackagePolicyValidationResults; -}> = memo(({ agentPolicy, packageInfo, packagePolicy, updatePackagePolicy, validationResults }) => { - // Form show/hide states - const [isShowingAdvanced, setIsShowingAdvanced] = useState(false); +}> = memo( + ({ + agentPolicy, + packageInfo, + packagePolicy, + integration, + updatePackagePolicy, + validationResults, + }) => { + // Form show/hide states + const [isShowingAdvanced, setIsShowingAdvanced] = useState(false); - // Package-level vars - const requiredVars: RegistryVarsEntry[] = []; - const advancedVars: RegistryVarsEntry[] = []; + // Package-level vars + const requiredVars: RegistryVarsEntry[] = []; + const advancedVars: RegistryVarsEntry[] = []; - if (packageInfo.vars) { - packageInfo.vars.forEach((varDef) => { - if (isAdvancedVar(varDef)) { - advancedVars.push(varDef); - } else { - requiredVars.push(varDef); - } - }); - } + if (packageInfo.vars) { + packageInfo.vars.forEach((varDef) => { + if (isAdvancedVar(varDef)) { + advancedVars.push(varDef); + } else { + requiredVars.push(varDef); + } + }); + } - // Update package policy's package and agent policy info - useEffect(() => { - const pkg = packagePolicy.package; - const currentPkgKey = pkg ? pkgKeyFromPackageInfo(pkg) : ''; - const pkgKey = pkgKeyFromPackageInfo(packageInfo); + // Update package policy's package and agent policy info + useEffect(() => { + const pkg = packagePolicy.package; + const currentPkgKey = pkg ? pkgKeyFromPackageInfo(pkg) : ''; + const pkgKey = pkgKeyFromPackageInfo(packageInfo); - // If package has changed, create shell package policy with input&stream values based on package info - if (currentPkgKey !== pkgKey) { - // Existing package policies on the agent policy using the package name, retrieve highest number appended to package policy name - const pkgPoliciesNamePattern = new RegExp(`${packageInfo.name}-(\\d+)`); - const pkgPoliciesWithMatchingNames = (agentPolicy.package_policies as PackagePolicy[]) - .filter((ds) => Boolean(ds.name.match(pkgPoliciesNamePattern))) - .map((ds) => parseInt(ds.name.match(pkgPoliciesNamePattern)![1], 10)) - .sort((a, b) => a - b); + // If package has changed, create shell package policy with input&stream values based on package info + if (currentPkgKey !== pkgKey) { + // Existing package policies on the agent policy using the package name, retrieve highest number appended to package policy name + const pkgPoliciesNamePattern = new RegExp(`${packageInfo.name}-(\\d+)`); + const pkgPoliciesWithMatchingNames = (agentPolicy.package_policies as PackagePolicy[]) + .filter((ds) => Boolean(ds.name.match(pkgPoliciesNamePattern))) + .map((ds) => parseInt(ds.name.match(pkgPoliciesNamePattern)![1], 10)) + .sort((a, b) => a - b); - updatePackagePolicy( - packageToPackagePolicy({ - name: `${packageInfo.name}-${ - pkgPoliciesWithMatchingNames.length - ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 - : 1 - }`, - description: packagePolicy.description, - namespace: packagePolicy.namespace, - packageInfo, - agentPolicyId: agentPolicy.id, - outputId: packagePolicy.output_id, - }) - ); - } + updatePackagePolicy( + packageToPackagePolicy({ + name: `${packageInfo.name}-${ + pkgPoliciesWithMatchingNames.length + ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 + : 1 + }`, + description: packagePolicy.description, + namespace: packagePolicy.namespace, + packageInfo, + agentPolicyId: agentPolicy.id, + outputId: packagePolicy.output_id, + enablePolicyTemplate: integration, + }) + ); + } - // If agent policy has changed, update package policy's agent policy ID and namespace - if (packagePolicy.policy_id !== agentPolicy.id) { - updatePackagePolicy({ - policy_id: agentPolicy.id, - namespace: agentPolicy.namespace, - }); - } - }, [packagePolicy, agentPolicy, packageInfo, updatePackagePolicy]); + // If agent policy has changed, update package policy's agent policy ID and namespace + if (packagePolicy.policy_id !== agentPolicy.id) { + updatePackagePolicy({ + policy_id: agentPolicy.id, + namespace: agentPolicy.namespace, + }); + } + }, [packagePolicy, agentPolicy, packageInfo, updatePackagePolicy, integration]); - return validationResults ? ( - + return validationResults ? ( + + + + } + description={ - - } - description={ - - } - > - - {/* Name */} - - - } - > - - updatePackagePolicy({ - name: e.target.value, - }) - } - data-test-subj="packagePolicyNameInput" - /> - - - - {/* Description */} - - - } - labelAppend={ - + } + > + + {/* Name */} + + - - } - isInvalid={!!validationResults.description} - error={validationResults.description} - > - - updatePackagePolicy({ - description: e.target.value, - }) } - data-test-subj="packagePolicyDescriptionInput" - /> - - - - {/* Required vars */} - {requiredVars.map((varDef) => { - const { name: varName, type: varType } = varDef; - if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; - const value = packagePolicy.vars[varName].value; - return ( - - { + > + updatePackagePolicy({ - vars: { - ...packagePolicy.vars, - [varName]: { - type: varType, - value: newValue, - }, - }, - }); - }} + name: e.target.value, + }) + } + data-test-subj="packagePolicyNameInput" /> - - ); - })} + + - {/* Advanced options toggle */} - - - - setIsShowingAdvanced(!isShowingAdvanced)} - flush="left" - > + {/* Description */} + + - - - {!isShowingAdvanced && !!validationResults.namespace ? ( - - + } + labelAppend={ + + } + isInvalid={!!validationResults.description} + error={validationResults.description} + > + + updatePackagePolicy({ + description: e.target.value, + }) + } + data-test-subj="packagePolicyDescriptionInput" + /> + + + + {/* Required vars */} + {requiredVars.map((varDef) => { + const { name: varName, type: varType } = varDef; + if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; + const value = packagePolicy.vars[varName].value; + return ( + + { + updatePackagePolicy({ + vars: { + ...packagePolicy.vars, + [varName]: { + type: varType, + value: newValue, + }, + }, + }); + }} + /> - ) : null} - - + ); + })} - {/* Advanced options content */} - {/* Todo: Populate list of existing namespaces */} - {isShowingAdvanced ? ( + {/* Advanced options toggle */} - - - - } + + + setIsShowingAdvanced(!isShowingAdvanced)} + flush="left" > - { - updatePackagePolicy({ - namespace: newNamespace, - }); - }} - onChange={(newNamespaces: Array<{ label: string }>) => { - updatePackagePolicy({ - namespace: newNamespaces.length ? newNamespaces[0].label : '', - }); - }} + - + - {/* Advanced vars */} - {advancedVars.map((varDef) => { - const { name: varName, type: varType } = varDef; - if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; - const value = packagePolicy.vars![varName].value; - return ( - - { + {!isShowingAdvanced && !!validationResults.namespace ? ( + + + + + + ) : null} + + + + {/* Advanced options content */} + {/* Todo: Populate list of existing namespaces */} + {isShowingAdvanced ? ( + + + + + } + > + { updatePackagePolicy({ - vars: { - ...packagePolicy.vars, - [varName]: { - type: varType, - value: newValue, - }, - }, + namespace: newNamespace, + }); + }} + onChange={(newNamespaces: Array<{ label: string }>) => { + updatePackagePolicy({ + namespace: newNamespaces.length ? newNamespaces[0].label : '', }); }} /> - - ); - })} - - - ) : null} - - - ) : ( - - ); -}); + + + {/* Advanced vars */} + {advancedVars.map((varDef) => { + const { name: varName, type: varType } = varDef; + if (!packagePolicy.vars || !packagePolicy.vars[varName]) return null; + const value = packagePolicy.vars![varName].value; + return ( + + { + updatePackagePolicy({ + vars: { + ...packagePolicy.vars, + [varName]: { + type: varType, + value: newValue, + }, + }, + }); + }} + /> + + ); + })} + + + ) : null} + + + ) : ( + + ); + } +); From a51a5719d2f64463290ec485d6227f83b840e5e8 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 7 Apr 2021 09:44:46 -0700 Subject: [PATCH 16/16] Fix imports --- .../plugins/fleet/common/services/package_to_package_policy.ts | 1 - .../plugins/fleet/public/applications/fleet/services/index.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts index 2f9b9cebf5891..6ee75216b4a7f 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.ts @@ -8,7 +8,6 @@ import type { PackageInfo, RegistryVarsEntry, - RegistryStream, PackagePolicyConfigRecord, NewPackagePolicyInput, NewPackagePolicyInputStream, diff --git a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts index 3f5d2637d7d89..a6cd2497703c7 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/services/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/services/index.ts @@ -28,8 +28,6 @@ export { LicenseService, isAgentUpgradeable, doesPackageHaveIntegrations, - getStreamsForInputType, - findDataStreamsByNames, groupInputs, PackagePolicyValidationResults, PackagePolicyConfigValidationResults,