[Telemetry] Add possibility of registering exclusive collectors for e…

…ach collection (#62665)

* [Telemetry] Add posibility of regitering exclusive collectors for collections

* [Telemetry] Filter unwanted fields from the kibana.os telemetry payload

* Filter the collectors properly in bulkFetch

* Move "kibana" usage collector from Monitoring to OSS Telemetry

* Remove exclusivity of the "kibana_settings" collector

* Unify "kibana_stats" collector from Monitoring and Legacy

* Remove unused legacy constants

* Proper type for UsageCollectionSetup in monitoring

* Missed one undo

* Add unit tests to the migrated collectors

Co-authored-by: Elastic Machine <[email protected]>

src/legacy/server/status/index.js

@@ -20,15 +20,13 @@
 import ServerStatus from './server_status';
 import { Metrics } from './lib/metrics';
 import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
-import { registerOpsStatsCollector } from './collectors';
 import Oppsy from 'oppsy';
 import { cloneDeep } from 'lodash';
 import { getOSInfo } from './lib/get_os_info';
 
 export function statusMixin(kbnServer, server, config) {
   kbnServer.status = new ServerStatus(kbnServer.server);
   const { usageCollection } = server.newPlatform.setup.plugins;
-  registerOpsStatsCollector(usageCollection, server, kbnServer);
 
   const metrics = new Metrics(config, server);
 

src/plugins/telemetry/common/constants.ts

@@ -80,3 +80,14 @@ export const APPLICATION_USAGE_TYPE = 'application_usage';
  * The type name used within the Monitoring index to publish management stats.
  */
 export const KIBANA_STACK_MANAGEMENT_STATS_TYPE = 'stack_management';
+
+/**
+ * The type name used to publish Kibana usage stats.
+ * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats
+ */
+export const KIBANA_USAGE_TYPE = 'kibana';
+
+/**
+ * The type name used to publish Kibana usage stats in the formatted as bulk.
+ */
+export const KIBANA_STATS_TYPE = 'kibana_stats';

src/plugins/telemetry/server/collectors/index.ts

@@ -22,3 +22,5 @@ export { registerUiMetricUsageCollector } from './ui_metric';
 export { registerTelemetryPluginUsageCollector } from './telemetry_plugin';
 export { registerManagementUsageCollector } from './management';
 export { registerApplicationUsageCollector } from './application_usage';
+export { registerKibanaUsageCollector } from './kibana';
+export { registerOpsStatsCollector } from './ops_stats';

src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getSavedObjectsCounts } from './get_saved_object_counts';
+
+describe('getSavedObjectsCounts', () => {
+  test('Get all the saved objects equal to 0 because no results were found', async () => {
+    const callCluster = jest.fn(() => ({}));
+
+    const results = await getSavedObjectsCounts(callCluster as any, '.kibana');
+    expect(results).toStrictEqual({
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    });
+  });
+
+  test('Merge the zeros with the results', async () => {
+    const callCluster = jest.fn(() => ({
+      aggregations: {
+        types: {
+          buckets: [
+            { key: 'dashboard', doc_count: 1 },
+            { key: 'timelion-sheet', doc_count: 2 },
+            { key: 'index-pattern', value: 2 }, // Malformed on purpose
+            { key: 'graph_workspace', doc_count: 3 }, // already snake_cased
+          ],
+        },
+      },
+    }));
+
+    const results = await getSavedObjectsCounts(callCluster as any, '.kibana');
+    expect(results).toStrictEqual({
+      dashboard: { total: 1 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 3 },
+      timelion_sheet: { total: 2 },
+    });
+  });
+});

src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Moved from /x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts
+ *
+ * The PR https://github.com/elastic/kibana/pull/62665 proved what the issue https://github.com/elastic/kibana/issues/58249
+ * was claiming: the structure and payload for common telemetry bits differs between Monitoring and OSS/X-Pack collections.
+ *
+ * Unifying this logic from Monitoring that makes sense to have in OSS here and we will import it on the monitoring side to reuse it.
+ */
+
+import { snakeCase } from 'lodash';
+import { APICaller } from 'kibana/server';
+
+const TYPES = [
+  'dashboard',
+  'visualization',
+  'search',
+  'index-pattern',
+  'graph-workspace',
+  'timelion-sheet',
+];
+
+export interface KibanaSavedObjectCounts {
+  [pluginName: string]: {
+    total: number;
+  };
+}
+
+export async function getSavedObjectsCounts(
+  callCluster: APICaller,
+  kibanaIndex: string // Typically '.kibana'. We might need a way to obtain it from the SavedObjects client (or the SavedObjects client to provide a way to run aggregations?)
+): Promise<KibanaSavedObjectCounts> {
+  const savedObjectCountSearchParams = {
+    index: kibanaIndex,
+    ignoreUnavailable: true,
+    filterPath: 'aggregations.types.buckets',
+    body: {
+      size: 0,
+      query: {
+        terms: { type: TYPES },
+      },
+      aggs: {
+        types: {
+          terms: { field: 'type', size: TYPES.length },
+        },
+      },
+    },
+  };
+  const resp = await callCluster('search', savedObjectCountSearchParams);
+  const buckets: Array<{ key: string; doc_count: number }> =
+    resp.aggregations?.types?.buckets || [];
+
+  // Initialise the object with all zeros for all the types
+  const allZeros: KibanaSavedObjectCounts = TYPES.reduce(
+    (acc, type) => ({ ...acc, [snakeCase(type)]: { total: 0 } }),
+    {}
+  );
+
+  // Add the doc_count from each bucket
+  return buckets.reduce(
+    (acc, { key, doc_count: total }) => (total ? { ...acc, [snakeCase(key)]: { total } } : acc),
+    allZeros
+  );
+}

src/plugins/telemetry/server/collectors/kibana/index.test.ts

@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { UsageCollectionSetup } from '../../../../../plugins/usage_collection/server';
+import { pluginInitializerContextConfigMock } from '../../../../../core/server/mocks';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { CollectorOptions } from '../../../../../plugins/usage_collection/server/collector/collector';
+
+import { registerKibanaUsageCollector } from './';
+
+describe('telemetry_kibana', () => {
+  let collector: CollectorOptions;
+
+  const usageCollectionMock: jest.Mocked<UsageCollectionSetup> = {
+    makeUsageCollector: jest.fn().mockImplementation(config => (collector = config)),
+    registerCollector: jest.fn(),
+  } as any;
+
+  const legacyConfig$ = pluginInitializerContextConfigMock({}).legacy.globalConfig$;
+  const callCluster = jest.fn().mockImplementation(() => ({}));
+
+  beforeAll(() => registerKibanaUsageCollector(usageCollectionMock, legacyConfig$));
+  afterAll(() => jest.clearAllTimers());
+
+  test('registered collector is set', () => {
+    expect(collector).not.toBeUndefined();
+    expect(collector.type).toBe('kibana');
+  });
+
+  test('fetch', async () => {
+    expect(await collector.fetch(callCluster)).toStrictEqual({
+      index: '.kibana-tests',
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    });
+  });
+
+  test('formatForBulkUpload', async () => {
+    const resultFromFetch = {
+      index: '.kibana-tests',
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    };
+
+    expect(collector.formatForBulkUpload!(resultFromFetch)).toStrictEqual({
+      type: 'kibana_stats',
+      payload: {
+        usage: resultFromFetch,
+      },
+    });
+  });
+});

src/legacy/server/status/constants.js → src/plugins/telemetry/server/collectors/kibana/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export const KIBANA_STATS_TYPE = 'oss_kibana_stats'; // kibana stats per 5s intervals
+export { registerKibanaUsageCollector } from './kibana_usage_collector';

src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Observable } from 'rxjs';
+import { take } from 'rxjs/operators';
+import { SharedGlobalConfig } from 'kibana/server';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { KIBANA_STATS_TYPE, KIBANA_USAGE_TYPE } from '../../../common/constants';
+import { getSavedObjectsCounts } from './get_saved_object_counts';
+
+export function getKibanaUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  legacyConfig$: Observable<SharedGlobalConfig>
+) {
+  return usageCollection.makeUsageCollector({
+    type: KIBANA_USAGE_TYPE,
+    isReady: () => true,
+    async fetch(callCluster) {
+      const {
+        kibana: { index },
+      } = await legacyConfig$.pipe(take(1)).toPromise();
+      return {
+        index,
+        ...(await getSavedObjectsCounts(callCluster, index)),
+      };
+    },
+
+    /*
+     * Format the response data into a model for internal upload
+     * 1. Make this data part of the "kibana_stats" type
+     * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc)
+     */
+    formatForBulkUpload: result => {
+      return {
+        type: KIBANA_STATS_TYPE,
+        payload: {
+          usage: result,
+        },
+      };
+    },
+  });
+}
+
+export function registerKibanaUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  legacyConfig$: Observable<SharedGlobalConfig>
+) {
+  usageCollection.registerCollector(getKibanaUsageCollector(usageCollection, legacyConfig$));
+}