diff --git a/.buildkite/pipelines/performance/nightly.yml b/.buildkite/pipelines/performance/nightly.yml new file mode 100644 index 0000000000000..aa52fb7a9335c --- /dev/null +++ b/.buildkite/pipelines/performance/nightly.yml @@ -0,0 +1,35 @@ +steps: + - block: ":gear: Performance Tests Configuration" + prompt: "Fill out the details for performance test" + fields: + - text: ":arrows_counterclockwise: Iterations" + key: "performance-test-iteration-count" + hint: "How many times you want to run tests? " + required: true + if: build.env('ITERATION_COUNT_ENV') == null + + - label: ":male-mechanic::skin-tone-2: Pre-Build" + command: .buildkite/scripts/lifecycle/pre_build.sh + + - wait + + - label: ":factory_worker: Build Kibana Distribution and Plugins" + command: .buildkite/scripts/steps/build_kibana.sh + agents: + queue: c2-16 + key: build + + - label: ":muscle: Performance Tests" + command: .buildkite/scripts/steps/functional/performance.sh + agents: + queue: ci-group-6 + depends_on: build + concurrency: 50 + concurrency_group: 'performance-test-group' + + - wait: ~ + continue_on_failure: true + + - label: ":male_superhero::skin-tone-2: Post-Build" + command: .buildkite/scripts/lifecycle/post_build.sh + diff --git a/.buildkite/scripts/steps/functional/performance.sh b/.buildkite/scripts/steps/functional/performance.sh new file mode 100644 index 0000000000000..2f1a77690d153 --- /dev/null +++ b/.buildkite/scripts/steps/functional/performance.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -uo pipefail + +if [ -z "${ITERATION_COUNT_ENV+x}" ]; then + ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)" +else + ITERATION_COUNT=$ITERATION_COUNT_ENV +fi + +tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times" + +cat << EOF | buildkite-agent pipeline upload +steps: + - command: .buildkite/scripts/steps/functional/performance_sub.sh + parallelism: "$ITERATION_COUNT" +EOF + + + diff --git a/.buildkite/scripts/steps/functional/performance_sub.sh b/.buildkite/scripts/steps/functional/performance_sub.sh new file mode 100644 index 0000000000000..d3e6c0ba7304e --- /dev/null +++ b/.buildkite/scripts/steps/functional/performance_sub.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh +.buildkite/scripts/download_build_artifacts.sh + +cd "$XPACK_DIR" + +echo --- Run Performance Tests +checks-reporter-with-killswitch "Run Performance Tests" \ + node scripts/functional_tests \ + --debug --bail \ + --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ + --config test/performance/config.ts; diff --git a/.eslintrc.js b/.eslintrc.js index 6f62f953c9a4c..c6aeb6f94255f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -156,6 +156,78 @@ const DEV_PATTERNS = [ 'x-pack/plugins/*/server/scripts/**/*', ]; +/** Restricted imports with suggested alternatives */ +const RESTRICTED_IMPORTS = [ + { + name: 'lodash', + importNames: ['set', 'setWith'], + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash.set', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash.setwith', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/set', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/setWith', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/fp', + importNames: ['set', 'setWith', 'assoc', 'assocPath'], + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/fp/set', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/fp/setWith', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/fp/assoc', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash/fp/assocPath', + message: 'Please use @elastic/safer-lodash-set instead', + }, + { + name: 'lodash', + importNames: ['template'], + message: 'lodash.template is unsafe, and not compatible with our content security policy.', + }, + { + name: 'lodash.template', + message: 'lodash.template is unsafe, and not compatible with our content security policy.', + }, + { + name: 'lodash/template', + message: 'lodash.template is unsafe, and not compatible with our content security policy.', + }, + { + name: 'lodash/fp', + importNames: ['template'], + message: 'lodash.template is unsafe, and not compatible with our content security policy.', + }, + { + name: 'lodash/fp/template', + message: 'lodash.template is unsafe, and not compatible with our content security policy.', + }, + { + name: 'react-use', + message: 'Please use react-use/lib/{method} instead.', + }, +]; + module.exports = { root: true, @@ -671,81 +743,7 @@ module.exports = { 'no-restricted-imports': [ 2, { - paths: [ - { - name: 'lodash', - importNames: ['set', 'setWith'], - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash.set', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash.setwith', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/set', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/setWith', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/fp', - importNames: ['set', 'setWith', 'assoc', 'assocPath'], - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/fp/set', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/fp/setWith', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/fp/assoc', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash/fp/assocPath', - message: 'Please use @elastic/safer-lodash-set instead', - }, - { - name: 'lodash', - importNames: ['template'], - message: - 'lodash.template is unsafe, and not compatible with our content security policy.', - }, - { - name: 'lodash.template', - message: - 'lodash.template is unsafe, and not compatible with our content security policy.', - }, - { - name: 'lodash/template', - message: - 'lodash.template is unsafe, and not compatible with our content security policy.', - }, - { - name: 'lodash/fp', - importNames: ['template'], - message: - 'lodash.template is unsafe, and not compatible with our content security policy.', - }, - { - name: 'lodash/fp/template', - message: - 'lodash.template is unsafe, and not compatible with our content security policy.', - }, - { - name: 'react-use', - message: 'Please use react-use/lib/{method} instead.', - }, - ], + paths: RESTRICTED_IMPORTS, }, ], 'no-restricted-modules': [ @@ -838,6 +836,23 @@ module.exports = { ], }, }, + { + files: ['**/common/**/*.{js,mjs,ts,tsx}', '**/public/**/*.{js,mjs,ts,tsx}'], + rules: { + 'no-restricted-imports': [ + 2, + { + paths: [ + ...RESTRICTED_IMPORTS, + { + name: 'semver', + message: 'Please use "semver/*/{function}" instead', + }, + ], + }, + ], + }, + }, /** * APM and Observability overrides @@ -1585,8 +1600,8 @@ module.exports = { plugins: ['react', '@typescript-eslint'], files: ['x-pack/plugins/osquery/**/*.{js,mjs,ts,tsx}'], rules: { - // 'arrow-body-style': ['error', 'as-needed'], - // 'prefer-arrow-callback': 'error', + 'arrow-body-style': ['error', 'as-needed'], + 'prefer-arrow-callback': 'error', 'no-unused-vars': 'off', 'react/prop-types': 'off', '@typescript-eslint/explicit-module-boundary-types': 'off', diff --git a/docs/developer/plugin/external-plugin-localization.asciidoc b/docs/developer/plugin/external-plugin-localization.asciidoc index d30dec1a8f46b..656dff90fe0de 100644 --- a/docs/developer/plugin/external-plugin-localization.asciidoc +++ b/docs/developer/plugin/external-plugin-localization.asciidoc @@ -135,27 +135,6 @@ export const Component = () => { Full details are {kib-repo}tree/master/packages/kbn-i18n#react[here]. - - -[discrete] -==== i18n for Angular - -You are encouraged to use `i18n.translate()` by statically importing `i18n` from `@kbn/i18n` wherever possible in your Angular code. Angular wrappers use the translation `service` with the i18n engine under the hood. - -The translation directive has the following syntax: -["source","js"] ------------ - ------------ - -Full details are {kib-repo}tree/master/packages/kbn-i18n#angularjs[here]. - - [discrete] === Resources diff --git a/docs/getting-started/quick-start-guide.asciidoc b/docs/getting-started/quick-start-guide.asciidoc index d614aece5b425..2bddd9bf61452 100644 --- a/docs/getting-started/quick-start-guide.asciidoc +++ b/docs/getting-started/quick-start-guide.asciidoc @@ -138,7 +138,7 @@ image::images/dashboard_sampleDataAddFilter_7.15.0.png[The [eCommerce] Revenue D [[quick-start-whats-next]] == What's next? -*Add your own data.* Ready to add your own data? Go to {fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack] to learn how to ingest your data, or go to <> and learn about all the other ways you can add data. +*Add your own data.* Ready to add your own data? Go to {observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}], or go to <> and learn about all the other ways you can add data. *Explore your own data in Discover.* Ready to learn more about exploring your data in *Discover*? Go to <>. diff --git a/docs/management/connectors/action-types/pagerduty.asciidoc b/docs/management/connectors/action-types/pagerduty.asciidoc index db1c4e3932d14..5e12eddaa5c77 100644 --- a/docs/management/connectors/action-types/pagerduty.asciidoc +++ b/docs/management/connectors/action-types/pagerduty.asciidoc @@ -68,7 +68,7 @@ PagerDuty actions have the following properties. Severity:: The perceived severity of on the affected system. This can be one of `Critical`, `Error`, `Warning` or `Info`(default). Event action:: One of `Trigger` (default), `Resolve`, or `Acknowledge`. See https://v2.developer.pagerduty.com/docs/events-api-v2#event-action[event action] for more details. -Dedup Key:: All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `:`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. +Dedup Key:: All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `:`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. Timestamp:: An *optional* https://v2.developer.pagerduty.com/v2/docs/types#datetime[ISO-8601 format date-time], indicating the time the event was detected or generated. Component:: An *optional* value indicating the component of the source machine that is responsible for the event, for example `mysql` or `eth0`. Group:: An *optional* value indicating the logical grouping of components of a service, for example `app-stack`. diff --git a/docs/management/connectors/action-types/servicenow-sir.asciidoc b/docs/management/connectors/action-types/servicenow-sir.asciidoc index 4556746284d5b..2fa49fe552c2e 100644 --- a/docs/management/connectors/action-types/servicenow-sir.asciidoc +++ b/docs/management/connectors/action-types/servicenow-sir.asciidoc @@ -72,13 +72,11 @@ image::management/connectors/images/servicenow-sir-params-test.png[ServiceNow Se ServiceNow SecOps actions have the following configuration properties. Short description:: A short description for the incident, used for searching the contents of the knowledge base. -Source Ips:: A list of source IPs related to the incident. The IPs will be added as observables to the security incident. -Destination Ips:: A list of destination IPs related to the incident. The IPs will be added as observables to the security incident. -Malware URLs:: A list of malware URLs related to the incident. The URLs will be added as observables to the security incident. -Malware Hashes:: A list of malware hashes related to the incident. The hashes will be added as observables to the security incident. Priority:: The priority of the incident. Category:: The category of the incident. Subcategory:: The subcategory of the incident. +Correlation ID:: All actions sharing this ID will be associated with the same ServiceNow security incident. If an incident exists in ServiceNow with the same correlation ID the security incident will be updated. Default value: `:`. +Correlation Display:: A descriptive label of the alert for correlation purposes in ServiceNow. Description:: The details about the incident. Additional comments:: Additional information for the client, such as how to troubleshoot the issue. diff --git a/docs/management/connectors/action-types/servicenow.asciidoc b/docs/management/connectors/action-types/servicenow.asciidoc index cf5244a9e3f9e..f7c3187f3f024 100644 --- a/docs/management/connectors/action-types/servicenow.asciidoc +++ b/docs/management/connectors/action-types/servicenow.asciidoc @@ -76,6 +76,8 @@ Severity:: The severity of the incident. Impact:: The effect an incident has on business. Can be measured by the number of affected users or by how critical it is to the business in question. Category:: The category of the incident. Subcategory:: The category of the incident. +Correlation ID:: All actions sharing this ID will be associated with the same ServiceNow incident. If an incident exists in ServiceNow with the same correlation ID the incident will be updated. Default value: `:`. +Correlation Display:: A descriptive label of the alert for correlation purposes in ServiceNow. Short description:: A short description for the incident, used for searching the contents of the knowledge base. Description:: The details about the incident. Additional comments:: Additional information for the client, such as how to troubleshoot the issue. diff --git a/docs/management/connectors/images/servicenow-sir-params-test.png b/docs/management/connectors/images/servicenow-sir-params-test.png index 80103a4272bfa..a2bf8761a8824 100644 Binary files a/docs/management/connectors/images/servicenow-sir-params-test.png and b/docs/management/connectors/images/servicenow-sir-params-test.png differ diff --git a/docs/osquery/osquery.asciidoc b/docs/osquery/osquery.asciidoc index 1e4e6604a7c70..a4f3c80463143 100644 --- a/docs/osquery/osquery.asciidoc +++ b/docs/osquery/osquery.asciidoc @@ -365,7 +365,7 @@ The following is an example of an **error response** for an undefined action que == System requirements * {fleet-guide}/fleet-overview.html[Fleet] is enabled on your cluster, and -one or more {fleet-guide}/elastic-agent-installation-configuration.html[Elastic Agents] is enrolled. +one or more {fleet-guide}/elastic-agent-installation.html[Elastic Agents] is enrolled. * The https://docs.elastic.co/en/integrations/osquery_manager[*Osquery Manager*] integration has been added and configured for an agent policy through Fleet. diff --git a/docs/setup/connect-to-elasticsearch.asciidoc b/docs/setup/connect-to-elasticsearch.asciidoc index 7dcc803d3e18b..6988460efadcf 100644 --- a/docs/setup/connect-to-elasticsearch.asciidoc +++ b/docs/setup/connect-to-elasticsearch.asciidoc @@ -47,7 +47,7 @@ so you can quickly get insights into your data, and {fleet} mode offers several image::images/addData_fleet_7.15.0.png[Add data using Fleet] To get started, refer to -{fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack]. +{observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}]. [discrete] [[upload-data-kibana]] diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index 2f4fd4d052dad..6527efd6e38d1 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -291,7 +291,7 @@ that the {kib} server uses to perform maintenance on the {kib} index at startup. is an alternative to `elasticsearch.username` and `elasticsearch.password`. | `enterpriseSearch.host` - | The URL of your Enterprise Search instance + | The http(s) URL of your Enterprise Search instance. For example, in a local self-managed setup, set this to `http://localhost:3002`. Authentication between Kibana and the Enterprise Search host URL, such as via OAuth, is not supported. You can also {enterprise-search-ref}/configure-ssl-tls.html#configure-ssl-tls-in-kibana[configure Kibana to trust your Enterprise Search TLS certificate authority]. | `interpreter.enableInVisualize` | Enables use of interpreter in Visualize. *Default: `true`* diff --git a/package.json b/package.json index e8f6bdbd1c069..bafa13d83dd4a 100644 --- a/package.json +++ b/package.json @@ -341,7 +341,7 @@ "react-moment-proptypes": "^1.7.0", "react-monaco-editor": "^0.41.2", "react-popper-tooltip": "^2.10.1", - "react-query": "^3.21.1", + "react-query": "^3.27.0", "react-redux": "^7.2.0", "react-resizable": "^1.7.5", "react-resize-detector": "^4.2.0", @@ -496,7 +496,6 @@ "@testing-library/react-hooks": "^5.1.1", "@testing-library/user-event": "^13.1.1", "@types/angular": "^1.6.56", - "@types/angular-mocks": "^1.7.0", "@types/apidoc": "^0.22.3", "@types/archiver": "^5.1.0", "@types/babel__core": "^7.1.16", @@ -651,7 +650,6 @@ "@yarnpkg/lockfile": "^1.1.0", "abab": "^2.0.4", "aggregate-error": "^3.1.0", - "angular-mocks": "^1.7.9", "antlr4ts-cli": "^0.5.0-alpha.3", "apidoc": "^0.29.0", "apidoc-markdown": "^6.0.0", diff --git a/packages/elastic-apm-generator/README.md b/packages/elastic-apm-generator/README.md index e69de29bb2d1d..e43187a8155d3 100644 --- a/packages/elastic-apm-generator/README.md +++ b/packages/elastic-apm-generator/README.md @@ -0,0 +1,93 @@ +# @elastic/apm-generator + +`@elastic/apm-generator` is an experimental tool to generate synthetic APM data. It is intended to be used for development and testing of the Elastic APM app in Kibana. + +At a high-level, the module works by modeling APM events/metricsets with [a fluent API](https://en.wikipedia.org/wiki/Fluent_interface). The models can then be serialized and converted to Elasticsearch documents. In the future we might support APM Server as an output as well. + +## Usage + +This section assumes that you've installed Kibana's dependencies by running `yarn kbn bootstrap` in the repository's root folder. + +This library can currently be used in two ways: + +- Imported as a Node.js module, for instance to be used in Kibana's functional test suite. +- With a command line interface, to index data based on some example scenarios. + +### Using the Node.js module + +#### Concepts + +- `Service`: a logical grouping for a monitored service. A `Service` object contains fields like `service.name`, `service.environment` and `agent.name`. +- `Instance`: a single instance of a monitored service. E.g., the workload for a monitored service might be spread across multiple containers. An `Instance` object contains fields like `service.node.name` and `container.id`. +- `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets. +- `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html) + + +#### Example + +```ts +import { service, timerange, toElasticsearchOutput } from '@elastic/apm-generator'; + +const instance = service('synth-go', 'production', 'go') + .instance('instance-a'); + +const from = new Date('2021-01-01T12:00:00.000Z').getTime(); +const to = new Date('2021-01-01T12:00:00.000Z').getTime() - 1; + +const traceEvents = timerange(from, to) + .interval('1m') + .rate(10) + .flatMap(timestamp => instance.transaction('GET /api/product/list') + .timestamp(timestamp) + .duration(1000) + .success() + .children( + instance.span('GET apm-*/_search', 'db', 'elasticsearch') + .timestamp(timestamp + 50) + .duration(900) + .destination('elasticsearch') + .success() + ).serialize() + ); + +const metricsets = timerange(from, to) + .interval('30s') + .rate(1) + .flatMap(timestamp => instance.appMetrics({ + 'system.memory.actual.free': 800, + 'system.memory.total': 1000, + 'system.cpu.total.norm.pct': 0.6, + 'system.process.cpu.total.norm.pct': 0.7, + }).timestamp(timestamp) + .serialize() + ); + +const esEvents = toElasticsearchOutput(traceEvents.concat(metricsets)); +``` + +#### Generating metricsets + +`@elastic/apm-generator` can also automatically generate transaction metrics, span destination metrics and transaction breakdown metrics based on the generated trace events. If we expand on the previous example: + +```ts +import { getTransactionMetrics, getSpanDestinationMetrics, getBreakdownMetrics } from '@elastic/apm-generator'; + +const esEvents = toElasticsearchOutput([ + ...traceEvents, + ...getTransactionMetrics(traceEvents), + ...getSpanDestinationMetrics(traceEvents), + ...getBreakdownMetrics(traceEvents) +]); +``` + +### CLI + +Via the CLI, you can upload examples. The supported examples are listed in `src/lib/es.ts`. A `--target` option that specifies the Elasticsearch URL should be defined when running the `example` command. Here's an example: + +`$ node packages/elastic-apm-generator/src/scripts/es.js example simple-trace --target=http://admin:changeme@localhost:9200` + +The following options are supported: +- `to`: the end of the time range, in ISO format. By default, the current time will be used. +- `from`: the start of the time range, in ISO format. By default, `to` minus 15 minutes will be used. +- `apm-server-version`: the version used in the index names bootstrapped by APM Server, e.g. `7.16.0`. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__ + diff --git a/packages/elastic-apm-generator/src/lib/apm_error.ts b/packages/elastic-apm-generator/src/lib/apm_error.ts new file mode 100644 index 0000000000000..5a48093a26db2 --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/apm_error.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Fields } from './entity'; +import { Serializable } from './serializable'; +import { generateLongId, generateShortId } from './utils/generate_id'; + +export class ApmError extends Serializable { + constructor(fields: Fields) { + super({ + ...fields, + 'processor.event': 'error', + 'processor.name': 'error', + 'error.id': generateShortId(), + }); + } + + serialize() { + const [data] = super.serialize(); + data['error.grouping_key'] = generateLongId( + this.fields['error.grouping_name'] || this.fields['error.exception']?.[0]?.message + ); + return [data]; + } +} diff --git a/packages/elastic-apm-generator/src/lib/base_span.ts b/packages/elastic-apm-generator/src/lib/base_span.ts index 6288c16d339b6..f762bf730a717 100644 --- a/packages/elastic-apm-generator/src/lib/base_span.ts +++ b/packages/elastic-apm-generator/src/lib/base_span.ts @@ -10,7 +10,7 @@ import { Fields } from './entity'; import { Serializable } from './serializable'; import { Span } from './span'; import { Transaction } from './transaction'; -import { generateTraceId } from './utils/generate_id'; +import { generateLongId } from './utils/generate_id'; export class BaseSpan extends Serializable { private readonly _children: BaseSpan[] = []; @@ -19,7 +19,7 @@ export class BaseSpan extends Serializable { super({ ...fields, 'event.outcome': 'unknown', - 'trace.id': generateTraceId(), + 'trace.id': generateLongId(), 'processor.name': 'transaction', }); } diff --git a/packages/elastic-apm-generator/src/lib/entity.ts b/packages/elastic-apm-generator/src/lib/entity.ts index 2a4beee652cf7..bf8fc10efd3a7 100644 --- a/packages/elastic-apm-generator/src/lib/entity.ts +++ b/packages/elastic-apm-generator/src/lib/entity.ts @@ -6,6 +6,19 @@ * Side Public License, v 1. */ +export type ApplicationMetricFields = Partial<{ + 'system.process.memory.size': number; + 'system.memory.actual.free': number; + 'system.memory.total': number; + 'system.cpu.total.norm.pct': number; + 'system.process.memory.rss.bytes': number; + 'system.process.cpu.total.norm.pct': number; +}>; + +export interface Exception { + message: string; +} + export type Fields = Partial<{ '@timestamp': number; 'agent.name': string; @@ -14,6 +27,10 @@ export type Fields = Partial<{ 'ecs.version': string; 'event.outcome': string; 'event.ingested': number; + 'error.id': string; + 'error.exception': Exception[]; + 'error.grouping_name': string; + 'error.grouping_key': string; 'host.name': string; 'metricset.name': string; 'observer.version': string; @@ -46,7 +63,8 @@ export type Fields = Partial<{ 'span.destination.service.response_time.count': number; 'span.self_time.count': number; 'span.self_time.sum.us': number; -}>; +}> & + ApplicationMetricFields; export class Entity { constructor(public readonly fields: Fields) { diff --git a/packages/elastic-apm-generator/src/lib/instance.ts b/packages/elastic-apm-generator/src/lib/instance.ts index 4218a9e23f4b4..3570f497c9055 100644 --- a/packages/elastic-apm-generator/src/lib/instance.ts +++ b/packages/elastic-apm-generator/src/lib/instance.ts @@ -6,7 +6,9 @@ * Side Public License, v 1. */ -import { Entity } from './entity'; +import { ApmError } from './apm_error'; +import { ApplicationMetricFields, Entity } from './entity'; +import { Metricset } from './metricset'; import { Span } from './span'; import { Transaction } from './transaction'; @@ -27,4 +29,20 @@ export class Instance extends Entity { 'span.subtype': spanSubtype, }); } + + error(message: string, type?: string, groupingName?: string) { + return new ApmError({ + ...this.fields, + 'error.exception': [{ message, ...(type ? { type } : {}) }], + 'error.grouping_name': groupingName || message, + }); + } + + appMetrics(metrics: ApplicationMetricFields) { + return new Metricset({ + ...this.fields, + 'metricset.name': 'app', + ...metrics, + }); + } } diff --git a/packages/elastic-apm-generator/src/lib/metricset.ts b/packages/elastic-apm-generator/src/lib/metricset.ts index f7abec6fde958..c1ebbea313123 100644 --- a/packages/elastic-apm-generator/src/lib/metricset.ts +++ b/packages/elastic-apm-generator/src/lib/metricset.ts @@ -6,12 +6,15 @@ * Side Public License, v 1. */ +import { Fields } from './entity'; import { Serializable } from './serializable'; -export class Metricset extends Serializable {} - -export function metricset(name: string) { - return new Metricset({ - 'metricset.name': name, - }); +export class Metricset extends Serializable { + constructor(fields: Fields) { + super({ + 'processor.event': 'metric', + 'processor.name': 'metric', + ...fields, + }); + } } diff --git a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts index b4cae1b41b9a6..d90ce8e01f83d 100644 --- a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts +++ b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts @@ -25,7 +25,8 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string const document = {}; // eslint-disable-next-line guard-for-in for (const key in values) { - set(document, key, values[key as keyof typeof values]); + const val = values[key as keyof typeof values]; + set(document, key, val); } return { _index: `apm-${versionOverride || values['observer.version']}-${values['processor.event']}`, diff --git a/packages/elastic-apm-generator/src/lib/span.ts b/packages/elastic-apm-generator/src/lib/span.ts index 36f7f44816d01..3c8d90f56b78e 100644 --- a/packages/elastic-apm-generator/src/lib/span.ts +++ b/packages/elastic-apm-generator/src/lib/span.ts @@ -8,14 +8,14 @@ import { BaseSpan } from './base_span'; import { Fields } from './entity'; -import { generateEventId } from './utils/generate_id'; +import { generateShortId } from './utils/generate_id'; export class Span extends BaseSpan { constructor(fields: Fields) { super({ ...fields, 'processor.event': 'span', - 'span.id': generateEventId(), + 'span.id': generateShortId(), }); } diff --git a/packages/elastic-apm-generator/src/lib/transaction.ts b/packages/elastic-apm-generator/src/lib/transaction.ts index f615f46710996..3a8d32e1843f8 100644 --- a/packages/elastic-apm-generator/src/lib/transaction.ts +++ b/packages/elastic-apm-generator/src/lib/transaction.ts @@ -6,22 +6,48 @@ * Side Public License, v 1. */ +import { ApmError } from './apm_error'; import { BaseSpan } from './base_span'; import { Fields } from './entity'; -import { generateEventId } from './utils/generate_id'; +import { generateShortId } from './utils/generate_id'; export class Transaction extends BaseSpan { private _sampled: boolean = true; + private readonly _errors: ApmError[] = []; constructor(fields: Fields) { super({ ...fields, 'processor.event': 'transaction', - 'transaction.id': generateEventId(), + 'transaction.id': generateShortId(), 'transaction.sampled': true, }); } + parent(span: BaseSpan) { + super.parent(span); + + this._errors.forEach((error) => { + error.fields['trace.id'] = this.fields['trace.id']; + error.fields['transaction.id'] = this.fields['transaction.id']; + error.fields['transaction.type'] = this.fields['transaction.type']; + }); + + return this; + } + + errors(...errors: ApmError[]) { + errors.forEach((error) => { + error.fields['trace.id'] = this.fields['trace.id']; + error.fields['transaction.id'] = this.fields['transaction.id']; + error.fields['transaction.type'] = this.fields['transaction.type']; + }); + + this._errors.push(...errors); + + return this; + } + duration(duration: number) { this.fields['transaction.duration.us'] = duration * 1000; return this; @@ -35,11 +61,13 @@ export class Transaction extends BaseSpan { serialize() { const [transaction, ...spans] = super.serialize(); + const errors = this._errors.flatMap((error) => error.serialize()); + const events = [transaction]; if (this._sampled) { events.push(...spans); } - return events; + return events.concat(errors); } } diff --git a/packages/elastic-apm-generator/src/lib/utils/generate_id.ts b/packages/elastic-apm-generator/src/lib/utils/generate_id.ts index 6c8b33fc19077..cc372a56209aa 100644 --- a/packages/elastic-apm-generator/src/lib/utils/generate_id.ts +++ b/packages/elastic-apm-generator/src/lib/utils/generate_id.ts @@ -12,14 +12,14 @@ let seq = 0; const namespace = 'f38d5b83-8eee-4f5b-9aa6-2107e15a71e3'; -function generateId() { - return uuidv5(String(seq++), namespace).replace(/-/g, ''); +function generateId(seed?: string) { + return uuidv5(seed ?? String(seq++), namespace).replace(/-/g, ''); } -export function generateEventId() { - return generateId().substr(0, 16); +export function generateShortId(seed?: string) { + return generateId(seed).substr(0, 16); } -export function generateTraceId() { - return generateId().substr(0, 32); +export function generateLongId(seed?: string) { + return generateId(seed).substr(0, 32); } diff --git a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts index 7aae2986919c8..f6aad154532c2 100644 --- a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts +++ b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts @@ -14,11 +14,11 @@ export function simpleTrace(from: number, to: number) { const range = timerange(from, to); - const transactionName = '100rpm (80% success) failed 1000ms'; + const transactionName = '240rpm/60% 1000ms'; const successfulTraceEvents = range - .interval('30s') - .rate(40) + .interval('1s') + .rate(3) .flatMap((timestamp) => instance .transaction(transactionName) @@ -38,21 +38,39 @@ export function simpleTrace(from: number, to: number) { ); const failedTraceEvents = range - .interval('30s') - .rate(10) + .interval('1s') + .rate(1) .flatMap((timestamp) => instance .transaction(transactionName) .timestamp(timestamp) .duration(1000) .failure() + .errors( + instance.error('[ResponseError] index_not_found_exception').timestamp(timestamp + 50) + ) .serialize() ); + const metricsets = range + .interval('30s') + .rate(1) + .flatMap((timestamp) => + instance + .appMetrics({ + 'system.memory.actual.free': 800, + 'system.memory.total': 1000, + 'system.cpu.total.norm.pct': 0.6, + 'system.process.cpu.total.norm.pct': 0.7, + }) + .timestamp(timestamp) + .serialize() + ); const events = successfulTraceEvents.concat(failedTraceEvents); return [ ...events, + ...metricsets, ...getTransactionMetrics(events), ...getSpanDestinationMetrics(events), ...getBreakdownMetrics(events), diff --git a/packages/elastic-apm-generator/src/test/scenarios/05_transactions_with_errors.test.ts b/packages/elastic-apm-generator/src/test/scenarios/05_transactions_with_errors.test.ts new file mode 100644 index 0000000000000..289fdfa6cf565 --- /dev/null +++ b/packages/elastic-apm-generator/src/test/scenarios/05_transactions_with_errors.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { pick } from 'lodash'; +import { service } from '../../index'; +import { Instance } from '../../lib/instance'; + +describe('transactions with errors', () => { + let instance: Instance; + const timestamp = new Date('2021-01-01T00:00:00.000Z').getTime(); + + beforeEach(() => { + instance = service('opbeans-java', 'production', 'java').instance('instance'); + }); + it('generates error events', () => { + const events = instance + .transaction('GET /api') + .timestamp(timestamp) + .errors(instance.error('test error').timestamp(timestamp)) + .serialize(); + + const errorEvents = events.filter((event) => event['processor.event'] === 'error'); + + expect(errorEvents.length).toEqual(1); + + expect( + pick(errorEvents[0], 'processor.event', 'processor.name', 'error.exception', '@timestamp') + ).toEqual({ + 'processor.event': 'error', + 'processor.name': 'error', + '@timestamp': timestamp, + 'error.exception': [{ message: 'test error' }], + }); + }); + + it('sets the transaction and trace id', () => { + const [transaction, error] = instance + .transaction('GET /api') + .timestamp(timestamp) + .errors(instance.error('test error').timestamp(timestamp)) + .serialize(); + + const keys = ['transaction.id', 'trace.id', 'transaction.type']; + + expect(pick(error, keys)).toEqual({ + 'transaction.id': transaction['transaction.id'], + 'trace.id': transaction['trace.id'], + 'transaction.type': 'request', + }); + }); + + it('sets the error grouping key', () => { + const [, error] = instance + .transaction('GET /api') + .timestamp(timestamp) + .errors(instance.error('test error').timestamp(timestamp)) + .serialize(); + + expect(error['error.grouping_name']).toEqual('test error'); + expect(error['error.grouping_key']).toMatchInlineSnapshot(`"8b96fa10a7f85a5d960198627bf50840"`); + }); +}); diff --git a/packages/elastic-apm-generator/src/test/scenarios/06_application_metrics.test.ts b/packages/elastic-apm-generator/src/test/scenarios/06_application_metrics.test.ts new file mode 100644 index 0000000000000..59ca8f0edbe88 --- /dev/null +++ b/packages/elastic-apm-generator/src/test/scenarios/06_application_metrics.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { pick } from 'lodash'; +import { service } from '../../index'; +import { Instance } from '../../lib/instance'; + +describe('application metrics', () => { + let instance: Instance; + const timestamp = new Date('2021-01-01T00:00:00.000Z').getTime(); + + beforeEach(() => { + instance = service('opbeans-java', 'production', 'java').instance('instance'); + }); + it('generates application metricsets', () => { + const events = instance + .appMetrics({ + 'system.memory.actual.free': 80, + 'system.memory.total': 100, + }) + .timestamp(timestamp) + .serialize(); + + const appMetrics = events.filter((event) => event['processor.event'] === 'metric'); + + expect(appMetrics.length).toEqual(1); + + expect( + pick( + appMetrics[0], + '@timestamp', + 'agent.name', + 'container.id', + 'metricset.name', + 'processor.event', + 'processor.name', + 'service.environment', + 'service.name', + 'service.node.name', + 'system.memory.actual.free', + 'system.memory.total' + ) + ).toEqual({ + '@timestamp': timestamp, + 'metricset.name': 'app', + 'processor.event': 'metric', + 'processor.name': 'metric', + 'system.memory.actual.free': 80, + 'system.memory.total': 100, + ...pick( + instance.fields, + 'agent.name', + 'container.id', + 'service.environment', + 'service.name', + 'service.node.name' + ), + }); + }); +}); diff --git a/packages/kbn-i18n/BUILD.bazel b/packages/kbn-i18n/BUILD.bazel index dc98865feb4f1..8ea6c3dd192f4 100644 --- a/packages/kbn-i18n/BUILD.bazel +++ b/packages/kbn-i18n/BUILD.bazel @@ -27,7 +27,6 @@ filegroup( ) NPM_MODULE_EXTRA_FILES = [ - "angular/package.json", "react/package.json", "package.json", "GUIDELINE.md", @@ -47,7 +46,6 @@ TYPES_DEPS = [ "//packages/kbn-babel-preset", "@npm//intl-messageformat", "@npm//tslib", - "@npm//@types/angular", "@npm//@types/intl-relativeformat", "@npm//@types/jest", "@npm//@types/node", diff --git a/packages/kbn-i18n/GUIDELINE.md b/packages/kbn-i18n/GUIDELINE.md index fdb082bb1a067..21dd720f9d1f6 100644 --- a/packages/kbn-i18n/GUIDELINE.md +++ b/packages/kbn-i18n/GUIDELINE.md @@ -93,17 +93,6 @@ The long term plan is to rely on using `FormattedMessage` and `i18n.translate()` Currently, we support the following ReactJS `i18n` tools, but they will be removed in future releases: - Usage of `props.intl.formatmessage()` (where `intl` is passed to `props` by `injectI18n` HOC). -#### In AngularJS - -The long term plan is to rely on using `i18n.translate()` by statically importing `i18n` from the `@kbn/i18n` package. **Avoid using the `i18n` filter and the `i18n` service injected in controllers, directives, services.** - -- Call JS function `i18n.translate()` from the `@kbn/i18n` package. -- Use `i18nId` directive in template. - -Currently, we support the following AngluarJS `i18n` tools, but they will be removed in future releases: -- Usage of `i18n` service in controllers, directives, services by injecting it. -- Usage of `i18n` filter in template for attribute translation. Note: Use one-time binding ("{{:: ... }}") in filters wherever it's possible to prevent unnecessary expression re-evaluation. - #### In JavaScript - Use `i18n.translate()` in NodeJS or any other framework agnostic code, where `i18n` is the I18n engine from `@kbn/i18n` package. @@ -223,7 +212,6 @@ For example: - for button: ```js - @@ -232,11 +220,11 @@ For example: - for dropDown: ```js - +