[9.0] [Security Solution] [AI Assistant] Remove citations feature flag (#212204)

x-pack/platform/packages/shared/kbn-elastic-assistant-common/impl/capabilities/index.ts

@@ -21,5 +21,4 @@ export type AssistantFeatureKey = keyof AssistantFeatures;
 export const defaultAssistantFeatures = Object.freeze({
   assistantModelEvaluation: false,
   defendInsights: true,
-  contentReferencesEnabled: false,
 });

x-pack/platform/packages/shared/kbn-elastic-assistant-common/impl/content_references/content_references_store/prune_content_references.ts

@@ -18,27 +18,21 @@ import { ContentReferencesStore, ContentReferenceBlock } from '../types';
 export const pruneContentReferences = (
   content: string,
   contentReferencesStore: ContentReferencesStore
-): ContentReferences | undefined => {
+): ContentReferences => {
   const fullStore = contentReferencesStore.getStore();
   const prunedStore: Record<string, ContentReference> = {};
   const matches = content.matchAll(/\{reference\([0-9a-zA-Z]+\)\}/g);
-  let isPrunedStoreEmpty = true;
 
   for (const match of matches) {
     const referenceElement = match[0];
     const referenceId = getContentReferenceId(referenceElement as ContentReferenceBlock);
     if (!(referenceId in prunedStore)) {
       const contentReference = fullStore[referenceId];
       if (contentReference) {
-        isPrunedStoreEmpty = false;
         prunedStore[referenceId] = contentReference;
       }
     }
   }
 
-  if (isPrunedStoreEmpty) {
-    return undefined;
-  }
-
   return prunedStore;
 };

x-pack/platform/packages/shared/kbn-elastic-assistant-common/impl/content_references/references/utils.ts

@@ -26,8 +26,11 @@ export const getContentReferenceId = (
  * @returns ContentReferenceBlock
  */
 export const contentReferenceBlock = (
-  contentReference: ContentReference
-): ContentReferenceBlock => {
+  contentReference: ContentReference | undefined
+): ContentReferenceBlock | '' => {
+  if (!contentReference) {
+    return '';
+  }
   return `{reference(${contentReference.id})}`;
 };
 
@@ -36,7 +39,10 @@ export const contentReferenceBlock = (
  * @param contentReference A ContentReference
  * @returns the string: `Reference: <contentReferenceBlock>`
  */
-export const contentReferenceString = (contentReference: ContentReference) => {
+export const contentReferenceString = (contentReference: ContentReference | undefined) => {
+  if (!contentReference) {
+    return '';
+  }
   return `Citation: ${contentReferenceBlock(contentReference)}` as const;
 };
 

x-pack/platform/packages/shared/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts

@@ -19,6 +19,5 @@ import { z } from '@kbn/zod';
 export type GetCapabilitiesResponse = z.infer<typeof GetCapabilitiesResponse>;
 export const GetCapabilitiesResponse = z.object({
   assistantModelEvaluation: z.boolean(),
-  contentReferencesEnabled: z.boolean(),
   defendInsights: z.boolean(),
 });

x-pack/platform/packages/shared/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.schema.yaml

@@ -22,13 +22,10 @@ paths:
                 properties:
                   assistantModelEvaluation:
                     type: boolean
-                  contentReferencesEnabled:
-                    type: boolean
                   defendInsights:
                     type: boolean
                 required:
                   - assistantModelEvaluation
-                  - contentReferencesEnabled
                   - defendInsights
         '400':
           description: Generic Error

x-pack/platform/packages/shared/kbn-elastic-assistant/impl/assistant/index.tsx

@@ -101,7 +101,6 @@ const AssistantComponent: React.FC<Props> = ({
     showAnonymizedValues,
     setContentReferencesVisible,
     setShowAnonymizedValues,
-    assistantFeatures: { contentReferencesEnabled },
   } = useAssistantContext();
 
   const [selectedPromptContexts, setSelectedPromptContexts] = useState<
@@ -408,7 +407,6 @@ const AssistantComponent: React.FC<Props> = ({
             currentUserAvatar,
             systemPromptContent: currentSystemPrompt?.content,
             contentReferencesVisible,
-            contentReferencesEnabled,
           })}
           // Avoid comments going off the flyout
           css={css`
@@ -439,7 +437,6 @@ const AssistantComponent: React.FC<Props> = ({
       contentReferencesVisible,
       euiTheme.size.l,
       selectedPromptContextsCount,
-      contentReferencesEnabled,
     ]
   );
 
@@ -457,9 +454,7 @@ const AssistantComponent: React.FC<Props> = ({
 
   return (
     <>
-      {contentReferencesEnabled && (
-        <AnonymizedValuesAndCitationsTour conversation={currentConversation} />
-      )}
+      <AnonymizedValuesAndCitationsTour conversation={currentConversation} />
       <EuiFlexGroup direction={'row'} wrap={false} gutterSize="none">
         {chatHistoryVisible && (
           <EuiFlexItem

x-pack/platform/packages/shared/kbn-elastic-assistant/impl/assistant/settings/settings_context_menu/settings_context_menu.tsx

@@ -66,7 +66,6 @@ export const SettingsContextMenu: React.FC<Params> = React.memo(
       contentReferencesVisible,
       showAnonymizedValues,
       setShowAnonymizedValues,
-      assistantFeatures: { contentReferencesEnabled },
     } = useAssistantContext();
 
     const [isPopoverOpen, setPopover] = useState(false);
@@ -256,62 +255,60 @@ export const SettingsContextMenu: React.FC<Params> = React.memo(
               </EuiFlexItem>
             </EuiFlexGroup>
           </EuiContextMenuItem>
-
-          {contentReferencesEnabled && (
-            <EuiContextMenuItem
-              aria-label={'show-citations'}
-              key={'show-citations'}
-              data-test-subj={'show-citations'}
-            >
-              <EuiFlexGroup direction="row" gutterSize="s" alignItems="center">
-                <EuiFlexItem grow={false}>
-                  <ConditionalWrap
-                    condition={!selectedConversationHasCitations}
-                    wrap={(children) => (
-                      <EuiToolTip
-                        position="top"
-                        key={'disabled-anonymize-values-tooltip'}
-                        content={
-                          <FormattedMessage
-                            id="xpack.elasticAssistant.assistant.settings.showCitationsLabel.disabled.tooltip"
-                            defaultMessage="This conversation does not contain citations."
-                          />
-                        }
-                      >
-                        {children}
-                      </EuiToolTip>
-                    )}
-                  >
-                    <EuiSwitch
-                      label={i18n.SHOW_CITATIONS}
-                      checked={contentReferencesVisible}
-                      onChange={onChangeContentReferencesVisible}
-                      compressed
-                      disabled={!selectedConversationHasCitations}
+          <EuiContextMenuItem
+            aria-label={'show-citations'}
+            key={'show-citations'}
+            data-test-subj={'show-citations'}
+          >
+            <EuiFlexGroup direction="row" gutterSize="s" alignItems="center">
+              <EuiFlexItem grow={false}>
+                <ConditionalWrap
+                  condition={!selectedConversationHasCitations}
+                  wrap={(children) => (
+                    <EuiToolTip
+                      position="top"
+                      key={'disabled-anonymize-values-tooltip'}
+                      content={
+                        <FormattedMessage
+                          id="xpack.elasticAssistant.assistant.settings.showCitationsLabel.disabled.tooltip"
+                          defaultMessage="This conversation does not contain citations."
+                        />
+                      }
+                    >
+                      {children}
+                    </EuiToolTip>
+                  )}
+                >
+                  <EuiSwitch
+                    label={i18n.SHOW_CITATIONS}
+                    checked={contentReferencesVisible}
+                    onChange={onChangeContentReferencesVisible}
+                    compressed
+                    disabled={!selectedConversationHasCitations}
+                  />
+                </ConditionalWrap>
+              </EuiFlexItem>
+              <EuiFlexItem grow={false}>
+                <EuiToolTip
+                  position="top"
+                  key={'show-citations-tooltip'}
+                  content={
+                    <FormattedMessage
+                      id="xpack.elasticAssistant.assistant.settings.showCitationsLabel.tooltip"
+                      defaultMessage="Keyboard shortcut: <bold>{keyboardShortcut}</bold>"
+                      values={{
+                        keyboardShortcut: isMac ? '⌥ + c' : 'Alt + c',
+                        bold: (str) => <strong>{str}</strong>,
+                      }}
                     />
-                  </ConditionalWrap>
-                </EuiFlexItem>
-                <EuiFlexItem grow={false}>
-                  <EuiToolTip
-                    position="top"
-                    key={'show-citations-tooltip'}
-                    content={
-                      <FormattedMessage
-                        id="xpack.elasticAssistant.assistant.settings.showCitationsLabel.tooltip"
-                        defaultMessage="Keyboard shortcut: <bold>{keyboardShortcut}</bold>"
-                        values={{
-                          keyboardShortcut: isMac ? '⌥ + c' : 'Alt + c',
-                          bold: (str) => <strong>{str}</strong>,
-                        }}
-                      />
-                    }
-                  >
-                    <EuiIcon tabIndex={0} type="iInCircle" />
-                  </EuiToolTip>
-                </EuiFlexItem>
-              </EuiFlexGroup>
-            </EuiContextMenuItem>
-          )}
+                  }
+                >
+                  <EuiIcon tabIndex={0} type="iInCircle" />
+                </EuiToolTip>
+              </EuiFlexItem>
+            </EuiFlexGroup>
+          </EuiContextMenuItem>
+
           <EuiHorizontalRule margin="none" />
           <EuiContextMenuItem
             aria-label={'clear-chat'}
@@ -339,7 +336,6 @@ export const SettingsContextMenu: React.FC<Params> = React.memo(
         handleShowAlertsModal,
         knowledgeBase.latestAlerts,
         showDestroyModal,
-        contentReferencesEnabled,
         euiTheme.size.m,
         euiTheme.size.xs,
         selectedConversationHasCitations,

x-pack/platform/packages/shared/kbn-elastic-assistant/impl/assistant_context/types.tsx

@@ -83,6 +83,5 @@ export type GetAssistantMessages = (commentArgs: {
   currentUserAvatar?: UserAvatar;
   setIsStreaming: (isStreaming: boolean) => void;
   systemPromptContent?: string;
-  contentReferencesVisible?: boolean;
-  contentReferencesEnabled?: boolean;
+  contentReferencesVisible: boolean;
 }) => EuiCommentProps[];

x-pack/solutions/security/plugins/elastic_assistant/scripts/draw_graph_script.ts

@@ -68,7 +68,6 @@ async function getAssistantGraph(logger: Logger): Promise<Drawable> {
     createLlmInstance,
     tools: [],
     replacements: {},
-    contentReferencesEnabled: false,
     savedObjectsClient: savedObjectsClientMock.create(),
   });
   return graph.getGraph();

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/conversations/field_maps_configuration.ts

@@ -97,6 +97,16 @@ export const conversationsFieldMap: FieldMap = {
     array: false,
     required: false,
   },
+  'messages.metadata': {
+    type: 'object',
+    array: false,
+    required: false,
+  },
+  'messages.metadata.content_references': {
+    type: 'flattened',
+    array: false,
+    required: false,
+  },
   replacements: {
     type: 'object',
     array: false,
@@ -168,18 +178,3 @@ export const conversationsFieldMap: FieldMap = {
     required: false,
   },
 } as const;
-
-// Once the `contentReferencesEnabled` feature flag is removed, the properties from the schema bellow should me moved into `conversationsFieldMap`
-export const conversationsContentReferencesFieldMap: FieldMap = {
-  ...conversationsFieldMap,
-  'messages.metadata': {
-    type: 'object',
-    array: false,
-    required: false,
-  },
-  'messages.metadata.content_references': {
-    type: 'flattened',
-    array: false,
-    required: false,
-  },
-} as const;

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/helpers.ts

@@ -153,7 +153,7 @@ export const getStructuredToolForIndexEntry = ({
 }: {
   indexEntry: IndexEntry;
   esClient: ElasticsearchClient;
-  contentReferencesStore: ContentReferencesStore | undefined;
+  contentReferencesStore: ContentReferencesStore;
   logger: Logger;
 }): DynamicStructuredTool => {
   const inputSchema = indexEntry.inputSchema?.reduce((prev, input) => {
@@ -223,22 +223,21 @@ export const getStructuredToolForIndexEntry = ({
         const result = await esClient.search(params);
 
         const kbDocs = result.hits.hits.map((hit) => {
-          const reference =
-            contentReferencesStore && contentReferencesStore.add((p) => createReference(p.id, hit));
+          const reference = contentReferencesStore.add((p) => createReference(p.id, hit));
 
           if (indexEntry.outputFields && indexEntry.outputFields.length > 0) {
             return indexEntry.outputFields.reduce(
               (prev, field) => {
                 // @ts-expect-error
                 return { ...prev, [field]: hit._source[field] };
               },
-              reference ? { citation: contentReferenceBlock(reference) } : {}
+              { citation: contentReferenceBlock(reference) }
             );
           }
 
           return {
             text: hit.highlight?.[indexEntry.field].join('\n --- \n'),
-            ...(reference ? { citation: contentReferenceBlock(reference) } : {}),
+            citation: contentReferenceBlock(reference),
           };
         });
 

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts

@@ -817,7 +817,7 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient {
     contentReferencesStore,
     esClient,
   }: {
-    contentReferencesStore: ContentReferencesStore | undefined;
+    contentReferencesStore: ContentReferencesStore;
     esClient: ElasticsearchClient;
   }): Promise<StructuredTool[]> => {
     const user = this.options.currentUser;