Merge branch 'main' into 163690-ml-aiops-log-rate-text-field-support

config/serverless.security.yml

@@ -35,7 +35,6 @@ xpack.fleet.internal.registry.spec.max: '3.0'
 
 # Serverless security specific options
 xpack.securitySolution.enableExperimental:
-  - discoverInTimeline
   - esqlRulesDisabled
 
 xpack.ml.ad.enabled: true

src/plugins/discover/public/application/main/components/top_nav/discover_topnav.test.tsx

@@ -46,6 +46,11 @@ const mockSearchBarCustomizationWithCustomSearchBar: SearchBarCustomization = {
   CustomSearchBar: MockCustomSearchBar,
 };
 
+const mockSearchBarCustomizationWithHiddenDataViewPicker: SearchBarCustomization = {
+  id: 'search_bar',
+  hideDataViewPicker: true,
+};
+
 let mockUseCustomizations = false;
 
 jest.mock('../../../../customizations', () => ({
@@ -256,5 +261,23 @@ describe('Discover topnav component', () => {
       ).find(mockSearchBarCustomization.CustomDataViewPicker!);
       expect(dataViewPickerOverride.length).toBe(1);
     });
+
+    it('should not render the dataView picker when hideDataViewPicker is true', () => {
+      (useDiscoverCustomization as jest.Mock).mockImplementation((id: DiscoverCustomizationId) => {
+        if (id === 'search_bar') {
+          return mockSearchBarCustomizationWithHiddenDataViewPicker;
+        }
+      });
+
+      const props = getProps();
+      const component = mountWithIntl(
+        <DiscoverMainProvider value={props.stateContainer}>
+          <DiscoverTopNav {...props} />
+        </DiscoverMainProvider>
+      );
+
+      const topNav = component.find(mockDiscoverService.navigation.ui.AggregateQueryTopNavMenu);
+      expect(topNav.prop('dataViewPickerComponentProps')).toBeUndefined();
+    });
   });
 });

src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx

@@ -114,6 +114,7 @@ export const DiscoverTopNav = ({
   }, [dataViewEditor, stateContainer]);
 
   const topNavCustomization = useDiscoverCustomization('top_nav');
+
   const topNavMenu = useMemo(
     () =>
       getTopNavLinks({
@@ -171,6 +172,17 @@ export const DiscoverTopNav = ({
   if (isESQLModeEnabled) {
     supportedTextBasedLanguages.push('ESQL');
   }
+
+  const searchBarCustomization = useDiscoverCustomization('search_bar');
+
+  const SearchBar = useMemo(
+    () => searchBarCustomization?.CustomSearchBar ?? AggregateQueryTopNavMenu,
+    [searchBarCustomization?.CustomSearchBar, AggregateQueryTopNavMenu]
+  );
+
+  const shouldHideDefaultDataviewPicker =
+    !!searchBarCustomization?.CustomDataViewPicker || !!searchBarCustomization?.hideDataViewPicker;
+
   const dataViewPickerProps: DataViewPickerProps = {
     trigger: {
       label: dataView?.getName() || '',
@@ -201,13 +213,6 @@ export const DiscoverTopNav = ({
     [services, stateContainer]
   );
 
-  const searchBarCustomization = useDiscoverCustomization('search_bar');
-
-  const SearchBar = useMemo(
-    () => searchBarCustomization?.CustomSearchBar ?? AggregateQueryTopNavMenu,
-    [searchBarCustomization?.CustomSearchBar, AggregateQueryTopNavMenu]
-  );
-
   return (
     <SearchBar
       appName="discover"
@@ -231,7 +236,7 @@ export const DiscoverTopNav = ({
         ) : undefined
       }
       dataViewPickerComponentProps={
-        searchBarCustomization?.CustomDataViewPicker ? undefined : dataViewPickerProps
+        shouldHideDefaultDataviewPicker ? undefined : dataViewPickerProps
       }
       displayStyle="detached"
       textBasedLanguageModeErrors={

src/plugins/discover/public/customizations/customization_types/search_bar_customization.ts

@@ -15,4 +15,5 @@ export interface SearchBarCustomization {
   CustomDataViewPicker?: ComponentType;
   PrependFilterBar?: ComponentType;
   CustomSearchBar?: ComponentType<TopNavMenuProps<AggregateQuery>>;
+  hideDataViewPicker?: boolean;
 }

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -801,6 +801,58 @@ describe('data telemetry collection tasks', () => {
     });
   });
 
+  describe('custom dashboards', () => {
+    const task = tasks.find((t) => t.name === 'custom_dashboards');
+    const savedObjectsClient = savedObjectsClientMock.create();
+
+    it('returns custom dashboards stats from all spaces', async () => {
+      savedObjectsClient.find.mockResolvedValueOnce({
+        page: 1,
+        per_page: 500,
+        total: 2,
+        saved_objects: [
+          {
+            type: 'apm-custom-dashboards',
+            id: '0b6157f0-44bd-11ed-bdb7-bffab551cd4d',
+            namespaces: ['default'],
+            attributes: {
+              dashboardSavedObjectId: 'foo-id',
+              serviceEnvironmentFilterEnabled: true,
+              serviceNameFilterEnabled: true,
+              kuery: 'service.name: frontend and service.environment: prod',
+            },
+            references: [],
+            score: 1,
+          },
+          {
+            type: 'apm-custom-dashboards',
+            id: '0b6157f0-44bd-11ed-bdb7-bffab551cd4d',
+            namespaces: ['space-1'],
+            attributes: {
+              dashboardSavedObjectId: 'bar-id',
+              serviceEnvironmentFilterEnabled: true,
+              serviceNameFilterEnabled: true,
+              kuery: 'service.name: frontend',
+            },
+            references: [],
+            score: 0,
+          },
+        ],
+      });
+
+      expect(
+        await task?.executor({
+          savedObjectsClient,
+        } as any)
+      ).toEqual({
+        custom_dashboards: {
+          kuery_fields: ['service.name', 'service.environment'],
+          total: 2,
+        },
+      });
+    });
+  });
+
   describe('top_traces', () => {
     const task = tasks.find((t) => t.name === 'top_traces');
 

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -71,6 +71,10 @@ import {
 import { APM_AGENT_CONFIGURATION_INDEX } from '../../../routes/settings/apm_indices/apm_system_index_constants';
 import { IndicesStatsResponse, TelemetryClient } from '../telemetry_client';
 import { RollupInterval } from '../../../../common/rollup';
+import {
+  APM_CUSTOM_DASHBOARDS_SAVED_OBJECT_TYPE,
+  SavedApmCustomDashboard,
+} from '../../../../common/custom_dashboards';
 
 type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
 const TIME_RANGES = ['1d', 'all'] as const;
@@ -1491,6 +1495,32 @@ export const tasks: TelemetryTask[] = [
       };
     },
   },
+  {
+    name: 'custom_dashboards',
+    executor: async ({ savedObjectsClient }) => {
+      const response = await savedObjectsClient.find<SavedApmCustomDashboard>({
+        type: APM_CUSTOM_DASHBOARDS_SAVED_OBJECT_TYPE,
+        page: 1,
+        perPage: 500,
+        sortField: 'updated_at',
+        sortOrder: 'desc',
+        namespaces: ['*'],
+      });
+
+      const kueryNodes = response.saved_objects.map(
+        ({ attributes: { kuery } }) => fromKueryExpression(kuery ?? '')
+      );
+
+      const kueryFields = getKueryFields(kueryNodes);
+
+      return {
+        custom_dashboards: {
+          kuery_fields: uniq(kueryFields),
+          total: response.total ?? 0,
+        },
+      };
+    },
+  },
   {
     name: 'per_service',
     executor: async ({ indices, telemetryClient }) => {

x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts

@@ -1083,6 +1083,25 @@ export const apmSchema: MakeSchemaFrom<APMUsage, true> = {
       },
     },
   },
+  custom_dashboards: {
+    kuery_fields: {
+      type: 'array',
+      items: {
+        type: 'keyword',
+        _meta: {
+          description:
+            'An array of up to 500 unique fields used to create the custom dashboards across all spaces. Example  [service.language.name, service.name] ',
+        },
+      },
+    },
+    total: {
+      type: 'long',
+      _meta: {
+        description:
+          'Total number of custom dashboards retrived from the saved object across all spaces',
+      },
+    },
+  },
   per_service: { type: 'array', items: { ...apmPerServiceSchema } },
   top_traces: {
     max: {
@@ -1263,6 +1282,17 @@ export const apmSchema: MakeSchemaFrom<APMUsage, true> = {
         },
       },
     },
+    custom_dashboards: {
+      took: {
+        ms: {
+          type: 'long',
+          _meta: {
+            description:
+              'Execution time in milliseconds for the "custom_dashboards" task',
+          },
+        },
+      },
+    },
     per_service: {
       took: {
         ms: {

x-pack/plugins/apm/server/lib/apm_telemetry/types.ts

@@ -213,6 +213,10 @@ export interface APMUsage {
     kuery_fields: string[];
     total: number;
   };
+  custom_dashboards: {
+    kuery_fields: string[];
+    total: number;
+  };
   per_service: APMPerService[];
   top_traces: {
     max: number;
@@ -234,6 +238,7 @@ export interface APMUsage {
     | 'cardinality'
     | 'environments'
     | 'service_groups'
+    | 'custom_dashboards'
     | 'per_service'
     | 'top_traces',
     { took: { ms: number } }

x-pack/plugins/osquery/cypress/e2e/all/alerts_cases.cy.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { LIVE_QUERY_EDITOR } from '../../screens/live_query';
+import { OSQUERY_FLYOUT_BODY_EDITOR } from '../../screens/live_query';
 import {
   cleanupCase,
   cleanupPack,
@@ -79,7 +79,7 @@ describe(
         cy.getBySel('osquery-action-item').click();
         cy.contains(/^\d+ agen(t|ts) selected/);
         cy.contains('Run a set of queries in a pack').click();
-        cy.get(LIVE_QUERY_EDITOR).should('not.exist');
+        cy.get(OSQUERY_FLYOUT_BODY_EDITOR).should('not.exist');
         cy.getBySel('select-live-pack').click().type(`${packName}{downArrow}{enter}`);
         submitQuery();
         cy.get('[aria-label="Add to Case"]').first().click();

x-pack/plugins/osquery/cypress/e2e/all/alerts_multiple_agents.cy.ts

@@ -14,6 +14,7 @@ import {
   takeOsqueryActionWithParams,
 } from '../../tasks/live_query';
 import { ServerlessRoleName } from '../../support/roles';
+import { OSQUERY_FLYOUT_BODY_EDITOR } from '../../screens/live_query';
 
 describe(
   'Alert Event Details - dynamic params',
@@ -43,12 +44,14 @@ describe(
     it('should substitute parameters in investigation guide', () => {
       cy.getBySel('expand-event').first().click();
       cy.getBySel('securitySolutionFlyoutInvestigationGuideButton').click();
-      cy.contains('Get processes').click();
-      cy.getBySel('flyout-body-osquery').within(() => {
-        cy.contains("SELECT * FROM os_version where name='Ubuntu';");
-        cy.contains('host.os.platform');
-        cy.contains('platform');
-      });
+      // Flakes at times if the button is only clicked once
+      cy.contains('Get processes').should('be.visible').dblclick({ force: true });
+      // Cypress can properly reads the fields when the codeEditor is interacted with first
+      // This is probably due to the tokenization of the fields when it's inactive
+      cy.get(OSQUERY_FLYOUT_BODY_EDITOR).click();
+      cy.getBySel('flyout-body-osquery').contains("SELECT * FROM os_version where name='Ubuntu';");
+      cy.getBySel('flyout-body-osquery').contains('host.os.platform');
+      cy.getBySel('flyout-body-osquery').contains('platform');
     });
 
     // response-actions-notification doesn't exist in expandable flyout

x-pack/plugins/osquery/cypress/screens/live_query.ts

@@ -8,6 +8,8 @@
 export const AGENT_FIELD = '[data-test-subj="comboBoxInput"]';
 export const ALL_AGENTS_OPTION = '[title="All agents"]';
 export const LIVE_QUERY_EDITOR = '.kibanaCodeEditor';
+export const OSQUERY_FLYOUT_BODY_EDITOR =
+  '[data-test-subj="flyout-body-osquery"] .kibanaCodeEditor';
 export const SUBMIT_BUTTON = '#submit-button';
 
 export const RESULTS_TABLE = 'osqueryResultsTable';

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -108,7 +108,7 @@ export const allowedExperimentalValues = Object.freeze({
    * Enables Discover embedded within timeline
    *
    * */
-  discoverInTimeline: false,
+  discoverInTimeline: true,
 
   /**
    * disables ES|QL rules