diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 11e9a722d808..a6207f32bec0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -432,6 +432,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820] - Added experimental dataset `juniper/netscreen`. {pull}20820[20820] - Added experimental dataset `sophos/utm`. {pull}20820[20820] +- Add Cloud Foundry tags in related events. {pull}21177[21177] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index 3006c472ccd6..3a0f1756b6e5 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -494,10 +494,19 @@ func baseMap(evt Event) common.MapStr { "cloudfoundry": common.MapStr{ "type": evt.String(), "envelope": envelopMap(evt), + "tags": dedotedTags(evt.Tags()), }, } } +func dedotedTags(tags map[string]string) common.MapStr { + result := common.MapStr{} + for name, value := range tags { + result[common.DeDot(name)] = value + } + return result +} + func baseMapWithApp(evt EventWithAppID) common.MapStr { base := baseMap(evt) appID := evt.AppGuid() diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index 911455899693..7dfd9bdcbd7d 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -86,6 +86,9 @@ func TestEventTypeHttpAccess(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "http": common.MapStr{ "response": common.MapStr{ @@ -160,6 +163,9 @@ func TestEventTypeLog(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "log message", "stream": "stdout", @@ -207,6 +213,9 @@ func TestEventCounter(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -252,6 +261,9 @@ func TestEventValueMetric(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -315,6 +327,9 @@ func TestEventContainerMetric(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -358,6 +373,9 @@ func TestEventError(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "message", "code": int32(100), diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml index 0ac4a813d502..33d63e362f81 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml @@ -38,6 +38,12 @@ description: > Cloud Foundry organization name. + - name: tags.* + type: object + object_type: keyword + description: > + Cloud Foundry tags. + - name: envelope type: group fields: diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 389b0bc54f95..16a4f1ef128b 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -2,23 +2,27 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "app": { - "id": "c0874969-2811-4d80-837a-301fd485a7bf" + "id": "3ce55e14-de73-49af-836d-adc93f3fee39" }, "container": { - "cpu.pct": 0.45456369228159, - "disk.bytes": 21970944, - "disk.quota.bytes": 1073741824, - "instance_index": 1, - "memory.bytes": 14101162, - "memory.quota.bytes": 67108864 + "cpu.pct": 0.19431789913648675, + "disk.bytes": 16678912, + "disk.quota.bytes": 33554432, + "instance_index": 0, + "memory.bytes": 8529920, + "memory.quota.bytes": 33554432 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d", - "ip": "192.168.16.31", + "index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668", + "ip": "192.168.16.51", "job": "diego_cell", "origin": "rep" }, + "tags": { + "product": "Pivotal Application Service", + "source_id": "3ce55e14-de73-49af-836d-adc93f3fee39" + }, "type": "container" }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go index e871a5823fcf..605f7145f64a 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json index b9104346eac1..5889e632b6bf 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -3,15 +3,21 @@ "cloudfoundry": { "counter": { "delta": 0, - "name": "sinks.dropped", - "total": 89547093 + "name": "dropped", + "total": 0 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "9adedbd0-3af3-47af-9707-99016050b123", - "ip": "192.168.16.54", - "job": "doppler", - "origin": "loggregator.doppler" + "index": "995eb273-f871-4fea-a834-dbc0a4a72efc", + "ip": "192.168.16.37", + "job": "syslog_adapter", + "origin": "loggregator.metron" + }, + "tags": { + "direction": "ingress", + "metric_version": "2.0", + "product": "Pivotal Application Service", + "source_id": "metron" }, "type": "counter" }, diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go index 44cb4935e70a..f7c13b1a6035 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go index c0756c57cca4..8f98d528f586 100644 --- a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go @@ -7,6 +7,7 @@ package mtest import ( "testing" + "github.com/elastic/beats/v7/libbeat/beat" cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" ) @@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} { return config } + +// CleanFields removes fields that can contain data of real deployments +func CleanFields(e beat.Event) { + e.Fields.Delete("cloudfoundry.tags.system_domain") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json index feeb2ce72ba8..1d17070a200c 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -3,16 +3,19 @@ "cloudfoundry": { "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "439bb519-3e68-4d08-a973-8f013f364d61", - "ip": "192.168.16.18", - "job": "cloud_controller", - "origin": "routing_api" + "index": "dffd1799-d03a-405a-9309-3fcce23f536f", + "ip": "192.168.16.15", + "job": "diego_database", + "origin": "silk-controller" + }, + "tags": { + "source_id": "silk-controller" }, "type": "value", "value": { - "name": "memoryStats.lastGCPauseTimeNS", - "unit": "count", - "value": 77499 + "name": "LeasesIndexRequestTime", + "unit": "ms", + "value": 0.681265 } }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go index 610a0a8e0295..af815a3d4b76 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") }