[GenAI][Integrations] UI for the custom integration creation with AI (#…

…186304) ## Summary This ticket is the initial implementation for the UI side for the AI-driven custom integration creation. This PR only contains the implementation of the UI, due to the tight timing it will not include tests, everything will be tested manually for 8.15 FF. We'll implement the tests later. #### Enable Feature The new integration assistant plugin is disabled by default, to enable it: ``` xpack.integration_assistant.enabled: true ``` #### Complete tasks - [x] New integration button on the /integrations page - [x] New integration "landing" page with buttons to upload zip and assistant - [x] Upload zip page to install integration - [x] Integration assistant: - [x] Connector selection step - [x] Integration details step - [x] Data stream step - [x] Review and install #### Follow-ups (will be implemented in separate PRs) - [ ] Add RBAC - [ ] Add telemetry - [ ] Documentation - [ ] Add license/productType controls - [ ] Add links to the create integration page - [ ] Improve package name retrieval: #185932 - [ ] Add time estimation on the generation stage - [ ] Add support for multi-valuated "input type" - [ ] Enable Langsmith tracing using AI assistant settings #### Demo https://github.com/elastic/kibana/assets/17747913/b04c21c6-09cf-49bb-be8f-bf4b9d3feb8e ## Files by Code Owner ### elastic/docs * packages/kbn-doc-links/src/get_doc_links.ts * packages/kbn-doc-links/src/types.ts ### elastic/fleet * x-pack/plugins/fleet/kibana.jsonc * x-pack/plugins/fleet/public/applications/integrations/hooks/use_breadcrumbs.tsx * x-pack/plugins/fleet/public/applications/integrations/layouts/default.tsx * x-pack/plugins/fleet/public/applications/integrations/sections/epm/index.tsx * x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/create/index.tsx * x-pack/plugins/fleet/public/components/header.tsx * x-pack/plugins/fleet/public/constants/page_paths.ts * x-pack/plugins/fleet/public/plugin.ts * x-pack/plugins/fleet/tsconfig.json ### elastic/kibana-core * x-pack/plugins/fleet/kibana.jsonc * x-pack/plugins/integration_assistant/kibana.jsonc ### elastic/kibana-operations * packages/kbn-optimizer/limits.yml ### elastic/security-solution * x-pack/plugins/integration_assistant/**/* --------- Co-authored-by: Kibana Machine <[email protected]>
elastic · Jun 21, 2024 · 9b3775e · 9b3775e
1 parent 114b582
commit 9b3775e
Show file tree

Hide file tree

Showing 126 changed files with 4,287 additions and 178 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -1017,6 +1017,7 @@ module.exports = {
         'import/no-nodejs-modules': 'error',
         'no-duplicate-imports': 'off',
         '@typescript-eslint/no-duplicate-imports': 'error',
+        '@typescript-eslint/consistent-type-imports': 'error',
         'no-restricted-imports': [
           'error',
           {

diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts
@@ -28,6 +28,7 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
   const ELASTICSEARCH_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/reference/${DOC_LINK_VERSION}/`;
   const KIBANA_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/`;
   const FLEET_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/fleet/${DOC_LINK_VERSION}/`;
+  const INTEGRATIONS_DEV_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/integrations-developer/${DOC_LINK_VERSION}/`;
   const PLUGIN_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/plugins/${DOC_LINK_VERSION}/`;
   const OBSERVABILITY_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/`;
   const APM_DOCS = `${ELASTIC_WEBSITE_URL}guide/en/apm/`;
@@ -865,6 +866,9 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
       roleAndPrivileges: `${FLEET_DOCS}fleet-roles-and-privileges.html`,
       proxiesSettings: `${FLEET_DOCS}fleet-agent-proxy-support.html`,
     },
+    integrationDeveloper: {
+      upload: `${INTEGRATIONS_DEV_DOCS}upload-a-new-integration.html`,
+    },
     ecs: {
       guide: `${ELASTIC_WEBSITE_URL}guide/en/ecs/${ECS_VERSION}/index.html`,
       dataStreams: `${ELASTIC_WEBSITE_URL}guide/en/ecs/${ECS_VERSION}/ecs-data_stream.html`,

diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts
@@ -555,6 +555,9 @@ export interface DocLinks {
     roleAndPrivileges: string;
     proxiesSettings: string;
   }>;
+  readonly integrationDeveloper: {
+    upload: string;
+  };
   readonly ecs: {
     readonly guide: string;
     readonly dataStreams: string;

diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -82,6 +82,7 @@ pageLoadAssetSize:
   ingestPipelines: 58003
   inputControlVis: 172675
   inspector: 148711
+  integrationAssistant: 19524
   interactiveSetup: 80000
   investigate: 17970
   kibanaOverview: 56279

diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json
@@ -51,6 +51,7 @@
     "xpack.logsShared": "plugins/observability_solution/logs_shared",
     "xpack.fleet": "plugins/fleet",
     "xpack.ingestPipelines": "plugins/ingest_pipelines",
+    "xpack.integrationAssistant": "plugins/integration_assistant",
     "xpack.investigate": "plugins/observability_solution/investigate",
     "xpack.kubernetesSecurity": "plugins/kubernetes_security",
     "xpack.lens": "plugins/lens",

diff --git a/x-pack/plugins/fleet/kibana.jsonc b/x-pack/plugins/fleet/kibana.jsonc
@@ -37,6 +37,7 @@
       "ingestPipelines",
       "spaces",
       "guidedOnboarding",
+      "integrationAssistant",
     ],
     "requiredBundles": [
       "kibanaReact",

diff --git a/x-pack/plugins/fleet/public/applications/integrations/hooks/use_breadcrumbs.tsx b/x-pack/plugins/fleet/public/applications/integrations/hooks/use_breadcrumbs.tsx
@@ -42,6 +42,14 @@ const breadcrumbGetters: {
       }),
     },
   ],
+  integration_create: () => [
+    BASE_BREADCRUMB,
+    {
+      text: i18n.translate('xpack.fleet.breadcrumbs.createIntegrationPageTitle', {
+        defaultMessage: 'Create integration',
+      }),
+    },
+  ],
   integration_details_overview: ({ pkgTitle }) => [BASE_BREADCRUMB, { text: pkgTitle }],
   integration_policy_edit: ({ pkgTitle, pkgkey, policyName }) => [
     BASE_BREADCRUMB,

diff --git a/x-pack/plugins/fleet/public/applications/integrations/layouts/default.tsx b/x-pack/plugins/fleet/public/applications/integrations/layouts/default.tsx
@@ -8,7 +8,7 @@ import React, { memo } from 'react';
 import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiNotificationBadge } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
 
-import { useLink } from '../../../hooks';
+import { useLink, useStartServices } from '../../../hooks';
 import type { Section } from '../sections';
 
 import { WithHeaderLayout } from '.';
@@ -21,6 +21,7 @@ interface Props {
 
 export const DefaultLayout: React.FC<Props> = memo(
   ({ section, children, notificationsBySection }) => {
+    const { integrationAssistant } = useStartServices();
     const { getHref } = useLink();
     const tabs = [
       {
@@ -45,6 +46,8 @@ export const DefaultLayout: React.FC<Props> = memo(
       },
     ];
 
+    const CreateIntegrationCardButton = integrationAssistant?.CreateIntegrationCardButton;
+
     return (
       <WithHeaderLayout
         leftColumn={
@@ -70,8 +73,18 @@ export const DefaultLayout: React.FC<Props> = memo(
                 </p>
               </EuiText>
             </EuiFlexItem>
+
+            <EuiSpacer size="s" />
           </EuiFlexGroup>
         }
+        rightColumnGrow={false}
+        rightColumn={
+          CreateIntegrationCardButton ? (
+            <EuiFlexItem grow={false}>
+              <CreateIntegrationCardButton href={getHref('integration_create')} />
+            </EuiFlexItem>
+          ) : undefined
+        }
         tabs={tabs.map((tab) => {
           const notificationCount = notificationsBySection?.[tab.section];
           return {

diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/index.tsx
@@ -11,14 +11,16 @@ import { Routes, Route } from '@kbn/shared-ux-router';
 import { EuiSkeletonText } from '@elastic/eui';
 
 import { INTEGRATIONS_ROUTING_PATHS } from '../../constants';
-import { IntegrationsStateContextProvider, useBreadcrumbs } from '../../hooks';
+import { IntegrationsStateContextProvider, useBreadcrumbs, useStartServices } from '../../hooks';
 
 import { EPMHomePage } from './screens/home';
 import { Detail } from './screens/detail';
 import { Policy } from './screens/policy';
+import { CreateIntegration } from './screens/create';
 import { CustomLanguagesOverview } from './screens/detail/custom_languages_overview';
 
 export const EPMApp: React.FunctionComponent = () => {
+  const { integrationAssistant } = useStartServices();
   useBreadcrumbs('integrations');
 
   return (
@@ -38,6 +40,11 @@ export const EPMApp: React.FunctionComponent = () => {
           </React.Suspense>
         </IntegrationsStateContextProvider>
       </Route>
+      {integrationAssistant && (
+        <Route path={INTEGRATIONS_ROUTING_PATHS.integrations_create}>
+          <CreateIntegration />
+        </Route>
+      )}
       <Route path={INTEGRATIONS_ROUTING_PATHS.integrations}>
         <EPMHomePage />
       </Route>

diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/create/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/create/index.tsx
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useMemo } from 'react';
+
+import { useStartServices, useBreadcrumbs } from '../../../../hooks';
+
+export const CreateIntegration = React.memo(() => {
+  const { integrationAssistant } = useStartServices();
+  useBreadcrumbs('integration_create');
+
+  const CreateIntegrationAssistant = useMemo(
+    () => integrationAssistant?.CreateIntegration,
+    [integrationAssistant]
+  );
+
+  return CreateIntegrationAssistant ? <CreateIntegrationAssistant /> : null;
+});
diff --git a/x-pack/plugins/fleet/public/components/header.tsx b/x-pack/plugins/fleet/public/components/header.tsx
@@ -83,7 +83,6 @@ export const Header: React.FC<HeaderProps> = ({
       <EuiFlexGroup>
         {tabs ? (
           <EuiFlexItem>
-            <EuiSpacer size="s" />
             <Tabs className={tabsClassName}>
               {tabs.map((props, index) => (
                 <EuiTab {...(props as EuiTabProps)} key={`${props.id}-${index}`}>

diff --git a/x-pack/plugins/fleet/public/constants/page_paths.ts b/x-pack/plugins/fleet/public/constants/page_paths.ts
@@ -11,6 +11,7 @@ export type StaticPage =
   | 'base'
   | 'overview'
   | 'integrations'
+  | 'integration_create'
   | 'policies'
   | 'policies_list'
   | 'enrollment_tokens'
@@ -97,6 +98,7 @@ export const INTEGRATIONS_ROUTING_PATHS = {
   integrations_all: '/browse/:category?/:subcategory?',
   integrations_installed: '/installed/:category?',
   integrations_installed_updates_available: '/installed/updates_available/:category?',
+  integrations_create: '/create',
   integration_details: '/detail/:pkgkey/:panel?',
   integration_details_overview: '/detail/:pkgkey/overview',
   integration_details_policies: '/detail/:pkgkey/policies',
@@ -152,6 +154,7 @@ export const pagePathGetters: {
     const queryParams = query ? `?${INTEGRATIONS_SEARCH_QUERYPARAM}=${query}` : ``;
     return [INTEGRATIONS_BASE_PATH, `/installed/updates_available${categoryPath}${queryParams}`];
   },
+  integration_create: () => [INTEGRATIONS_BASE_PATH, `/create`],
   integration_details_overview: ({ pkgkey, integration }) => [
     INTEGRATIONS_BASE_PATH,
     `/detail/${pkgkey}/overview${integration ? `?integration=${integration}` : ''}`,

diff --git a/x-pack/plugins/fleet/public/plugin.ts b/x-pack/plugins/fleet/public/plugin.ts
@@ -18,6 +18,7 @@ import { DEFAULT_APP_CATEGORIES } from '@kbn/core/public';
 import { i18n } from '@kbn/i18n';
 
 import type { NavigationPublicPluginStart } from '@kbn/navigation-plugin/public';
+import type { IntegrationAssistantPluginStart } from '@kbn/integration-assistant-plugin/public';
 import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
 
 import type {
@@ -130,6 +131,7 @@ export interface FleetStartDeps {
   navigation: NavigationPublicPluginStart;
   customIntegrations: CustomIntegrationsStart;
   share: SharePluginStart;
+  integrationAssistant?: IntegrationAssistantPluginStart;
   cloud?: CloudStart;
   usageCollection?: UsageCollectionStart;
   guidedOnboarding?: GuidedOnboardingPluginStart;
@@ -139,6 +141,7 @@ export interface FleetStartServices extends CoreStart, Exclude<FleetStartDeps, '
   storage: Storage;
   share: SharePluginStart;
   dashboard: DashboardStart;
+  integrationAssistant?: IntegrationAssistantPluginStart;
   cloud?: CloudSetup & CloudStart;
   discover?: DiscoverStart;
   spaces?: SpacesPluginStart;

diff --git a/x-pack/plugins/fleet/tsconfig.json b/x-pack/plugins/fleet/tsconfig.json
@@ -110,5 +110,6 @@
     "@kbn/fields-metadata-plugin",
     "@kbn/test-jest-helpers",
     "@kbn/core-saved-objects-utils-server",
+    "@kbn/integration-assistant-plugin",
   ]
 }
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts
@@ -181,7 +181,7 @@ export const categorizationTestState = {
   exAnswer: 'testanswer',
   lastExecutedChain: 'testchain',
   packageName: 'testpackage',
-  dataStreamName: 'testdatastream',
+  dataStreamName: 'testDataStream',
   errors: { test: 'testerror' },
   pipelineResults: [{ test: 'testresult' }],
   finalized: false,

diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
@@ -443,6 +443,6 @@ export const ecsTestState = {
   rawSamples: ['{"test1": "test1"}'],
   samples: ['{ "test1": "test1" }'],
   packageName: 'testpackage',
-  dataStreamName: 'testdatastream',
+  dataStreamName: 'testDataStream',
   formattedSamples: '{"test1": "test1"}',
 };
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts
@@ -163,7 +163,7 @@ export const relatedTestState = {
   ecs: 'testtypes',
   exAnswer: 'testanswer',
   packageName: 'testpackage',
-  dataStreamName: 'testdatastream',
+  dataStreamName: 'testDataStream',
   errors: { test: 'testerror' },
   pipelineResults: [{ test: 'testresult' }],
   finalized: false,

diff --git a/.../plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml b/.../plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml
@@ -19,15 +19,15 @@ paths:
               type: object
               required:
                 - packageName
-                - datastreamName
+                - dataStreamName
                 - rawSamples
                 - currentPipeline
                 - connectorId
               properties:
                 packageName:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
-                datastreamName:
-                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DatastreamName"
+                dataStreamName:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
                 rawSamples:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/RawSamples"
                 currentPipeline:

diff --git a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts
@@ -9,7 +9,7 @@ import { z } from 'zod';
 
 import {
   Connector,
-  DatastreamName,
+  DataStreamName,
   PackageName,
   Pipeline,
   RawSamples,
@@ -19,7 +19,7 @@ import { CategorizationAPIResponse } from '../model/response_schemas';
 export type CategorizationRequestBody = z.infer<typeof CategorizationRequestBody>;
 export const CategorizationRequestBody = z.object({
   packageName: PackageName,
-  datastreamName: DatastreamName,
+  dataStreamName: DataStreamName,
   rawSamples: RawSamples,
   currentPipeline: Pipeline,
   connectorId: Connector,

diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml
@@ -19,14 +19,14 @@ paths:
               type: object
               required:
                 - packageName
-                - datastreamName
+                - dataStreamName
                 - rawSamples
                 - connectorId
               properties:
                 packageName:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
-                datastreamName:
-                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DatastreamName"
+                dataStreamName:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
                 rawSamples:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/RawSamples"
                 mapping:

diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts
@@ -9,7 +9,7 @@ import { z } from 'zod';
 
 import {
   Connector,
-  DatastreamName,
+  DataStreamName,
   Mapping,
   PackageName,
   RawSamples,
@@ -19,7 +19,7 @@ import { EcsMappingAPIResponse } from '../model/response_schemas';
 export type EcsMappingRequestBody = z.infer<typeof EcsMappingRequestBody>;
 export const EcsMappingRequestBody = z.object({
   packageName: PackageName,
-  datastreamName: DatastreamName,
+  dataStreamName: DataStreamName,
   rawSamples: RawSamples,
   mapping: Mapping.optional(),
   connectorId: Connector,