[APM] Improvements to data telemetry

x-pack/plugins/apm/common/agent_name.ts

@@ -15,15 +15,14 @@ import { AgentName } from '../typings/es_schemas/ui/fields/agent';
  */
 
 export const AGENT_NAMES: AgentName[] = [
-  'java',
-  'js-base',
-  'rum-js',
   'dotnet',
   'go',
   'java',
+  'js-base',
   'nodejs',
   'python',
   'ruby',
+  'rum-js',
 ];
 
 export function isAgentName(agentName: string): agentName is AgentName {

x-pack/plugins/apm/common/apm_telemetry.test.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getApmTelemetryMapping } from './apm_telemetry';
+
+describe('getApmTelemetry', () => {
+  it('generates a JSON object with the telemetry mapping', () => {
+    expect(getApmTelemetryMapping()).toMatchSnapshot();
+  });
+});

x-pack/plugins/apm/common/apm_telemetry.ts

@@ -0,0 +1,215 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { AGENT_NAMES } from './agent_name';
+
+/**
+ * Generate an object containing the mapping used for APM telemetry. Can be used
+ * with the `upload-telemetry-data` script or to update the mapping in the
+ * telemetry repository.
+ *
+ * This function breaks things up to make the mapping easier to understand.
+ */
+export function getApmTelemetryMapping() {
+  const keyword = {
+    type: 'keyword',
+    ignore_above: 1024,
+  };
+
+  const long = {
+    type: 'long',
+  };
+
+  const allProperties = {
+    properties: {
+      all: long,
+    },
+  };
+
+  const oneDayProperties = {
+    properties: {
+      '1d': long,
+    },
+  };
+
+  const oneDayAllProperties = {
+    properties: {
+      '1d': long,
+      all: long,
+    },
+  };
+
+  const msProperties = {
+    properties: {
+      ms: long,
+    },
+  };
+
+  const tookProperties = {
+    properties: {
+      took: msProperties,
+    },
+  };
+
+  const compositeNameVersionProperties = {
+    properties: {
+      composite: keyword,
+      name: keyword,
+      version: keyword,
+    },
+  };
+
+  const agentProperties = {
+    properties: { version: keyword },
+  };
+
+  const serviceProperties = {
+    properties: {
+      framework: compositeNameVersionProperties,
+      language: compositeNameVersionProperties,
+      runtime: compositeNameVersionProperties,
+    },
+  };
+
+  return {
+    properties: {
+      agents: {
+        properties: AGENT_NAMES.reduce<Record<string, any>>(
+          (previousValue, currentValue) => {
+            previousValue[currentValue] = {
+              properties: {
+                agent: agentProperties,
+                service: serviceProperties,
+              },
+            };
+
+            return previousValue;
+          },
+          {}
+        ),
+      },
+      counts: {
+        properties: {
+          agent_configuration: allProperties,
+          error: oneDayAllProperties,
+          max_error_groups_per_service: oneDayProperties,
+          max_transaction_groups_per_service: oneDayProperties,
+          metric: oneDayAllProperties,
+          onboarding: oneDayAllProperties,
+          services: oneDayProperties,
+          sourcemap: oneDayAllProperties,
+          span: oneDayAllProperties,
+          traces: oneDayProperties,
+          transaction: oneDayAllProperties,
+        },
+      },
+      cardinality: {
+        properties: {
+          user_agent: {
+            properties: {
+              original: {
+                properties: {
+                  all_agents: oneDayProperties,
+                  rum: oneDayProperties,
+                },
+              },
+            },
+          },
+          transaction: {
+            properties: {
+              name: {
+                properties: {
+                  all_agents: oneDayProperties,
+                  rum: oneDayProperties,
+                },
+              },
+            },
+          },
+        },
+      },
+      has_any_services: {
+        type: 'boolean',
+      },
+      indices: {
+        properties: {
+          all: {
+            properties: {
+              total: {
+                properties: {
+                  docs: {
+                    properties: {
+                      count: long,
+                    },
+                  },
+                  store: {
+                    properties: {
+                      size_in_bytes: long,
+                    },
+                  },
+                },
+              },
+            },
+          },
+          shards: {
+            properties: {
+              total: long,
+            },
+          },
+        },
+      },
+      integrations: {
+        properties: {
+          ml: {
+            properties: {
+              all_jobs_count: long,
+            },
+          },
+        },
+      },
+      retainment: {
+        properties: {
+          error: msProperties,
+          metric: msProperties,
+          onboarding: msProperties,
+          span: msProperties,
+          transaction: msProperties,
+        },
+      },
+      services_per_agent: {
+        properties: AGENT_NAMES.reduce<Record<string, any>>(
+          (previousValue, currentValue) => {
+            previousValue[currentValue] = { ...long, null_value: 0 };
+            return previousValue;
+          },
+          {}
+        ),
+      },
+      tasks: {
+        properties: {
+          agent_configuration: tookProperties,
+          agents: tookProperties,
+          cardinality: tookProperties,
+          groupings: tookProperties,
+          indices_stats: tookProperties,
+          integrations: tookProperties,
+          processor_events: tookProperties,
+          services: tookProperties,
+          versions: tookProperties,
+        },
+      },
+      version: {
+        properties: {
+          apm_server: {
+            properties: {
+              major: long,
+              minor: long,
+              patch: long,
+            },
+          },
+        },
+      },
+    },
+  };
+}

x-pack/plugins/apm/dev_docs/telemetry.md

@@ -0,0 +1,58 @@
+# APM Telemetry
+
+In order to learn about our customers' usage and experience of APM, we collect
+two types of telemetry, which we'll refer to here as "Data Telemetry" and
+"Behavioral Telemetry."
+
+This document will explain how they are collected and how to make changes to
+them.
+
+[The telemetry repository has information about accessing the clusters](https://github.com/elastic/telemetry#kibana-access).
+Telemetry data is uploaded to the "xpack-phone-home" indices.
+
+## Data Telemetry
+
+Information that can be derived from a cluster's APM indices is queried and sent
+to the telemetry cluster using the
+[Usage Collection plugin](../../../../src/plugins/usage_collection/README.md).
+
+During the APM server-side plugin's setup phase a
+[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html)
+for APM telemetry is registered and a
+[task manager](../../task_manager/server/README.md) task is registered and started.
+The task periodically queries the APM indices and saves the results in the Saved
+Object, and the usage collector periodically gets the data from the saved object
+and uploads it to the telemetry cluster.
+
+Once uploaded to the telemetry cluster, the data telemetry is stored in
+`stack_stats.kibana.plugins.apm` in the xpack-phone-home index.
+
+### Generating sample data
+
+The script in `scripts/upload-telemetry-data` can generate sample telemetry data and upload it to a cluster of your choosing.
+
+### Updating Data Telemetry Mappings
+
+In order for fields to be searchable on the telemetry cluster, they need to be
+added to the cluster's mapping. The mapping is defined in
+[the telemetry repository's xpack-phone-home template](https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json).
+
+The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm`.
+
+The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function.
+
+To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository.
+
+If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm:
+
+```bash
+node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json
+```
+
+this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes.
+
+## Behavioral Telemetry
+
+Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin.
+
+Please fill this in with more details.

x-pack/plugins/apm/public/plugin.ts

@@ -66,8 +66,9 @@ export interface ApmPluginStartDeps {
 }
 
 export class ApmPlugin implements Plugin<ApmPluginSetup, ApmPluginStart> {
-  private readonly initializerContext: PluginInitializerContext<ConfigSchema>;
-  constructor(initializerContext: PluginInitializerContext<ConfigSchema>) {
+  constructor(
+    private readonly initializerContext: PluginInitializerContext<ConfigSchema>
+  ) {
     this.initializerContext = initializerContext;
   }
   public setup(core: CoreSetup, plugins: ApmPluginSetupDeps) {

x-pack/plugins/apm/readme.md

@@ -167,3 +167,4 @@ You can access the development environment at http://localhost:9001.
 - [Cypress integration tests](./e2e/README.md)
 - [VSCode setup instructions](./dev_docs/vscode_setup.md)
 - [Github PR commands](./dev_docs/github_commands.md)
+- [Telemetry](./dev_docs/telemetry.md)

x-pack/plugins/apm/scripts/merge-telemetry-mapping.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// compile typescript on the fly
+// eslint-disable-next-line import/no-extraneous-dependencies
+require('@babel/register')({
+  extensions: ['.ts'],
+  plugins: [
+    '@babel/plugin-proposal-optional-chaining',
+    '@babel/plugin-proposal-nullish-coalescing-operator',
+  ],
+  presets: [
+    '@babel/typescript',
+    ['@babel/preset-env', { targets: { node: 'current' } }],
+  ],
+});
+
+require('./merge-telemetry-mapping/index.ts');

x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { readFileSync, truncateSync, writeFileSync } from 'fs';
+import produce from 'immer';
+import { resolve } from 'path';
+import { argv } from 'yargs';
+import { getApmTelemetryMapping } from '../../common/apm_telemetry';
+
+function errorExit(error?: Error) {
+  console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console
+  if (error) {
+    throw error;
+  }
+  process.exit(1);
+}
+
+try {
+  const filename = resolve(argv._[0]);
+  const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8'));
+
+  const newMapping = produce(xpackPhoneHomeMapping, (draft) => {
+    draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping();
+    return draft;
+  });
+
+  truncateSync(filename);
+  writeFileSync(filename, JSON.stringify(newMapping, null, 2));
+} catch (error) {
+  errorExit(error);
+}

x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts

@@ -20,7 +20,7 @@ import { stampLogger } from '../shared/stamp-logger';
 import { CollectTelemetryParams } from '../../server/lib/apm_telemetry/collect_data_telemetry';
 import { downloadTelemetryTemplate } from '../shared/download-telemetry-template';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { apmTelemetry } from '../../server/saved_objects/apm_telemetry';
+import { getApmTelemetryMapping } from '../../server/saved_objects/apm_telemetry';
 import { generateSampleDocuments } from './generate-sample-documents';
 import { readKibanaConfig } from '../shared/read-kibana-config';
 import { getHttpAuth } from '../shared/get-http-auth';
@@ -39,8 +39,7 @@ async function uploadData() {
   const telemetryTemplate = await downloadTelemetryTemplate({
     githubToken,
   });
-
-  const kibanaMapping = apmTelemetry.mappings;
+  const kibanaMapping = getApmTelemetryMapping();
 
   const config = readKibanaConfig();