Merge branch 'master' into modelprocessor-frames

elastic · Jul 14, 2021 · 74403cd · 74403cd
2 parents 5afe6fb + 4ed2761
commit 74403cd
Show file tree

Hide file tree

Showing 91 changed files with 1,469 additions and 608 deletions.
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -576,11 +576,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20210614232151-2871d29be93a
+Version: v7.0.0-alpha2.0.20210713084715-eb758b9b11ad
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20210614232151-2871d29be93a/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20210713084715-eb758b9b11ad/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -599,11 +599,11 @@ License Version 2.0.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.8.0
+Version: v1.10.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.8.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.10.0/LICENSE.txt:
 
 
                                  Apache License
@@ -4630,11 +4630,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : go.uber.org/zap
-Version: v1.17.0
+Version: v1.18.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.17.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.18.1/LICENSE.txt:
 
 Copyright (c) 2016-2017 Uber Technologies, Inc.
 
@@ -4770,11 +4770,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/grpc
-Version: v1.38.0
+Version: v1.39.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.38.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.39.0/LICENSE:
 
 
                                  Apache License
@@ -8812,11 +8812,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-structform
-Version: v0.0.8
+Version: v0.0.9
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].9/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -11423,11 +11423,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/h2non/filetype
-Version: v1.1.1-0.20201130172452-f60988ab73d5
+Version: v1.1.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/h2non/[email protected]-0.20201130172452-f60988ab73d5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/h2non/[email protected]/LICENSE:
 
 The MIT License
 
@@ -15448,11 +15448,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/prometheus/procfs
-Version: v0.6.0
+Version: v0.7.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.6.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.7.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -17870,11 +17870,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.0.0-20210513164829-c07d793c2f9a
+Version: v0.0.0-20210711020723-a769d52b0f97
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20210513164829-c07d793c2f9a/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20210711020723-a769d52b0f97/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -17944,11 +17944,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sys
-Version: v0.0.0-20210615035016-665e8c7367d1
+Version: v0.0.0-20210630005230-0f9fa26af87c
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20210615035016-665e8c7367d1/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20210630005230-0f9fa26af87c/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -18055,11 +18055,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/genproto
-Version: v0.0.0-20210614182748-5b3b54cad159
+Version: v0.0.0-20210713002101-d411969a0d9a
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]20210614182748-5b3b54cad159/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]20210713002101-d411969a0d9a/LICENSE:
 
 
                                  Apache License
@@ -18267,11 +18267,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/protobuf
-Version: v1.26.0
+Version: v1.27.1
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.26.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.27.1/LICENSE:
 
 Copyright (c) 2018 The Go Authors. All rights reserved.
 

diff --git a/_meta/beat.yml b/_meta/beat.yml
@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

diff --git a/apm-server.docker.yml b/apm-server.docker.yml
@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

diff --git a/apm-server.yml b/apm-server.yml
@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

diff --git a/apmpackage/apm/agent/input/template.yml.hbs b/apmpackage/apm/agent/input/template.yml.hbs
@@ -1,5 +1,18 @@
 apm-server:
     auth:
+        anonymous:
+            allow_agent:
+                {{#each anonymous_allow_agent}}
+                - {{this}}
+                {{/each}}
+            allow_service:
+                {{#each anonymous_allow_service}}
+                - {{this}}
+                {{/each}}
+            enabled: {{anonymous_enabled}}
+            rate_limit:
+                event_limit: {{anonymous_rate_limit_event_limit}}
+                ip_limit: {{anonymous_rate_limit_ip_limit}}
         api_key:
             enabled: {{api_key_enabled}}
             limit: {{api_key_limit}}
@@ -24,13 +37,7 @@ apm-server:
         {{#each rum_allow_origins}}
           - {{this}}
         {{/each}}
-        allow_service_names:
-        {{#each rum_allow_service_names}}
-          - {{this}}
-        {{/each}}
         enabled: {{enable_rum}}
-        event_rate.limit: {{rum_event_rate_limit}}
-        event_rate.lru_size: {{rum_event_rate_lru_size}}
         exclude_from_grouping: {{rum_exclude_from_grouping}}
         library_pattern: {{rum_library_pattern}}
         response_headers: {{rum_response_headers}}

diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml
@@ -3,6 +3,9 @@
 # change type can be one of: enhancement, bugfix, breaking-change
 - version: "0.4.0"
   changes:
+    - description: add anonymous auth config, replace some RUM config
+      type: breaking-change
+      link: https://github.com/elastic/apm-server/pull/5623
     - description: use new apm-server.auth config
       type: breaking-change
       link: https://github.com/elastic/apm-server/pull/5691

diff --git a/...ata_stream/app_metrics/elasticsearch/ingest_pipeline/apm_convert_destination_address.json b/...ata_stream/app_metrics/elasticsearch/ingest_pipeline/apm_convert_destination_address.json
@@ -0,0 +1,14 @@
+{
+  "description": "Convert destination.address to an IP, storing in destination.ip",
+  "processors": [
+    {
+      "convert": {
+        "field": "destination.address",
+        "target_field": "destination.ip",
+        "type": "ip",
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json
@@ -21,6 +21,11 @@
         "name": "metrics-apm.app-0.4.0-apm_remove_span_metadata"
       }
     },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.4.0-apm_convert_destination_address"
+      }
+    },
     {
       "pipeline": {
         "name": "metrics-apm.app-0.4.0-apm_error_grouping_name",