diff --git a/beater/api/mux.go b/beater/api/mux.go index 109dab79b58..3852007a24e 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -310,7 +310,7 @@ func emptyRequestMetadata(c *request.Context) model.Metadata { } func backendRequestMetadata(c *request.Context) model.Metadata { - return model.Metadata{System: model.System{IP: c.SourceIP}} + return model.Metadata{Host: model.Host{IP: c.SourceIP}} } func rumRequestMetadata(c *request.Context) model.Metadata { diff --git a/beater/beater.go b/beater/beater.go index 0700f9f152b..80da157a1e9 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -443,7 +443,7 @@ func (s *serverRunner) run() error { func (s *serverRunner) wrapRunServerWithPreprocessors(runServer RunServerFunc) RunServerFunc { processors := []model.BatchProcessor{ - modelprocessor.SetSystemHostname{}, + modelprocessor.SetHostHostname{}, modelprocessor.SetServiceNodeName{}, modelprocessor.SetMetricsetName{}, modelprocessor.SetGroupingKey{}, diff --git a/beater/otlp/clientmetadata.go b/beater/otlp/clientmetadata.go index 2774bfd81a5..03ced7503b1 100644 --- a/beater/otlp/clientmetadata.go +++ b/beater/otlp/clientmetadata.go @@ -30,7 +30,7 @@ import ( // // Client metadata is extracted from ctx, injected by interceptors.ClientMetadata. func SetClientMetadata(ctx context.Context, meta *model.Metadata) error { - if meta.Service.Agent.Name != "iOS/swift" { + if meta.Agent.Name != "iOS/swift" { // This is not an event from an agent we would consider to be // running on an end-user device. // diff --git a/beater/otlp/clientmetadata_test.go b/beater/otlp/clientmetadata_test.go index 724d5110b08..1abb5d20152 100644 --- a/beater/otlp/clientmetadata_test.go +++ b/beater/otlp/clientmetadata_test.go @@ -44,18 +44,18 @@ func TestSetClientMetadata(t *testing.T) { }, { ctx: context.Background(), meta: model.Metadata{ - Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}}, - Client: model.Client{IP: ip1234}, + Agent: model.Agent{Name: "iOS/swift"}, + Client: model.Client{IP: ip1234}, }, expectedIP: ip1234, }, { ctx: context.Background(), - meta: model.Metadata{Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}}}, + meta: model.Metadata{Agent: model.Agent{Name: "iOS/swift"}}, }, { ctx: interceptors.ContextWithClientMetadata(context.Background(), interceptors.ClientMetadataValues{ SourceIP: ip5678, }), - meta: model.Metadata{Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}}}, + meta: model.Metadata{Agent: model.Agent{Name: "iOS/swift"}}, expectedIP: ip5678, }} { metaCopy := test.meta diff --git a/beater/processors.go b/beater/processors.go index 83abf7d7667..e1d4091fe18 100644 --- a/beater/processors.go +++ b/beater/processors.go @@ -34,7 +34,7 @@ const ( // is authorized to ingest events for the agent and service name in metadata. func authorizeEventIngest(ctx context.Context, meta *model.Metadata) error { return auth.Authorize(ctx, auth.ActionEventIngest, auth.Resource{ - AgentName: meta.Service.Agent.Name, + AgentName: meta.Agent.Name, ServiceName: meta.Service.Name, }) } diff --git a/model/agent.go b/model/agent.go new file mode 100644 index 00000000000..3fbe6fffb83 --- /dev/null +++ b/model/agent.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "github.com/elastic/beats/v7/libbeat/common" +) + +// Agent describes an Elastic APM agent. +type Agent struct { + Name string + Version string + EphemeralID string +} + +func (a *Agent) fields() common.MapStr { + var agent mapStr + agent.maybeSetString("name", a.Name) + agent.maybeSetString("version", a.Version) + agent.maybeSetString("ephemeral_id", a.EphemeralID) + return common.MapStr(agent) +} diff --git a/model/agent_test.go b/model/agent_test.go new file mode 100644 index 00000000000..5637798f14d --- /dev/null +++ b/model/agent_test.go @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +const ( + agentName, agentVersion = "elastic-node", "1.0.0" +) + +func TestAgentFields(t *testing.T) { + tests := []struct { + Agent Agent + Fields common.MapStr + }{ + { + Agent: Agent{}, + Fields: nil, + }, + { + Agent: Agent{ + Name: agentName, + Version: agentVersion, + }, + Fields: common.MapStr{ + "name": "elastic-node", + "version": "1.0.0", + }, + }, + } + + for _, test := range tests { + assert.Equal(t, test.Fields, test.Agent.fields()) + } +} diff --git a/model/error_test.go b/model/error_test.go index 8d98b5cb094..3595c0aeb4f 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -224,9 +224,9 @@ func TestEvents(t *testing.T) { serviceName, agentName, version := "myservice", "go", "1.0" md := Metadata{ + Agent: Agent{Name: agentName, Version: version}, Service: Service{ Name: serviceName, Version: version, - Agent: Agent{Name: agentName, Version: version}, }, Labels: common.MapStr{"label": 101}, } diff --git a/model/host.go b/model/host.go new file mode 100644 index 00000000000..4c52130bf21 --- /dev/null +++ b/model/host.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "net" + + "github.com/elastic/beats/v7/libbeat/common" +) + +type Host struct { + // Hostname holds the detected hostname of the host. + Hostname string + + // Name holds the user-defined name of the host, or the + // detected hostname. + Name string + + // ID holds a unique ID for the host. + ID string + + // Architecture holds the host machine architecture. + Architecture string + + // Type holds the host type, e.g. cloud instance machine type. + Type string + + // IP holds the host IP address. + // + // TODO(axw) this should be a slice. + IP net.IP + + // OS holds information about the operating system running on the host. + OS OS +} + +func (h *Host) fields() common.MapStr { + if h == nil { + return nil + } + var fields mapStr + fields.maybeSetString("hostname", h.Hostname) + fields.maybeSetString("name", h.Name) + fields.maybeSetString("architecture", h.Architecture) + fields.maybeSetString("type", h.Type) + if h.IP != nil { + fields.set("ip", h.IP.String()) + } + fields.maybeSetMapStr("os", h.OS.fields()) + return common.MapStr(fields) +} diff --git a/model/host_test.go b/model/host_test.go new file mode 100644 index 00000000000..f4a0dc99f2b --- /dev/null +++ b/model/host_test.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "encoding/json" + "net" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/approvaltest" +) + +func TestSystemTransformation(t *testing.T) { + detected, configured := "host", "custom hostname" + + for name, host := range map[string]Host{ + "hostname": {Hostname: detected}, + "ignored hostname": {Name: configured}, + "full hostname info": {Hostname: detected, Name: configured}, + "full": { + Hostname: detected, + Name: configured, + Architecture: "amd", + Type: "t2.medium", + IP: net.ParseIP("127.0.0.1"), + OS: OS{ + Platform: "osx", + Full: "Mac OS Mojave", + Type: "macos", + }, + }, + } { + t.Run(name, func(t *testing.T) { + var fields mapStr + metadata := &Metadata{Host: host} + metadata.set(&fields, nil) + resultJSON, err := json.Marshal(fields["host"]) + require.NoError(t, err) + name := filepath.Join("test_approved", "host", strings.ReplaceAll(name, " ", "_")) + approvaltest.ApproveJSON(t, name, resultJSON) + }) + } +} diff --git a/model/metadata.go b/model/metadata.go index ac47c87d036..557e26e6a46 100644 --- a/model/metadata.go +++ b/model/metadata.go @@ -22,27 +22,31 @@ import ( ) type Metadata struct { - Service Service - Process Process - System System - User User - UserAgent UserAgent - Client Client - Cloud Cloud - Labels common.MapStr + Service Service + Agent Agent + Process Process + Host Host + User User + UserAgent UserAgent + Client Client + Cloud Cloud + Network Network + Container Container + Kubernetes Kubernetes + Labels common.MapStr } func (m *Metadata) set(fields *mapStr, eventLabels common.MapStr) { fields.maybeSetMapStr("service", m.Service.Fields()) - fields.maybeSetMapStr("agent", m.Service.Agent.fields()) - fields.maybeSetMapStr("host", m.System.fields()) + fields.maybeSetMapStr("agent", m.Agent.fields()) + fields.maybeSetMapStr("host", m.Host.fields()) fields.maybeSetMapStr("process", m.Process.fields()) fields.maybeSetMapStr("user", m.User.fields()) fields.maybeSetMapStr("client", m.Client.fields()) fields.maybeSetMapStr("user_agent", m.UserAgent.fields()) - fields.maybeSetMapStr("container", m.System.Container.fields()) - fields.maybeSetMapStr("kubernetes", m.System.Kubernetes.fields()) + fields.maybeSetMapStr("container", m.Container.fields()) + fields.maybeSetMapStr("kubernetes", m.Kubernetes.fields()) fields.maybeSetMapStr("cloud", m.Cloud.fields()) - fields.maybeSetMapStr("network", m.System.Network.fields()) + fields.maybeSetMapStr("network", m.Network.fields()) maybeSetLabels(fields, m.Labels, eventLabels) } diff --git a/model/metadata_test.go b/model/metadata_test.go index ba3b9d6654e..148ead49735 100644 --- a/model/metadata_test.go +++ b/model/metadata_test.go @@ -45,18 +45,19 @@ func TestMetadata_Set(t *testing.T) { }{ { input: Metadata{ + Agent: Agent{ + Name: agentName, + Version: agentVersion, + }, + Container: Container{ID: containerID}, Service: Service{ Name: serviceName, Node: ServiceNode{Name: serviceNodeName}, - Agent: Agent{ - Name: agentName, - Version: agentVersion, - }, }, - System: System{ - ConfiguredHostname: host, - DetectedHostname: hostname, - Container: Container{ID: containerID}}, + Host: Host{ + Hostname: hostname, + Name: host, + }, Process: Process{Pid: pid}, User: User{ID: uid, Email: mail}, }, @@ -108,6 +109,14 @@ func BenchmarkMetadataSet(b *testing.B) { }, }) test(b, "full", Metadata{ + Agent: Agent{Name: "go", Version: "2.0"}, + Container: Container{ID: "docker"}, + Kubernetes: Kubernetes{ + Namespace: "system", + NodeName: "node01", + PodName: "pet", + PodUID: "cattle", + }, Service: Service{ Name: "foo", Version: "1.0", @@ -116,7 +125,6 @@ func BenchmarkMetadataSet(b *testing.B) { Language: Language{Name: "go", Version: "++"}, Runtime: Runtime{Name: "gc", Version: "1.0"}, Framework: Framework{Name: "never", Version: "again"}, - Agent: Agent{Name: "go", Version: "2.0"}, }, Process: Process{ Pid: 123, @@ -124,18 +132,13 @@ func BenchmarkMetadataSet(b *testing.B) { Title: "case", Argv: []string{"apm-server"}, }, - System: System{ - DetectedHostname: "detected", - ConfiguredHostname: "configured", - Architecture: "x86_64", - Platform: "linux", - IP: net.ParseIP("10.1.1.1"), - Container: Container{ID: "docker"}, - Kubernetes: Kubernetes{ - Namespace: "system", - NodeName: "node01", - PodName: "pet", - PodUID: "cattle", + Host: Host{ + Hostname: "detected", + Name: "configured", + Architecture: "x86_64", + IP: net.ParseIP("10.1.1.1"), + OS: OS{ + Platform: "linux", }, }, User: User{ diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index de21ef0bbfe..9608e1ca733 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -200,6 +200,7 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti } // overwrite metadata with event specific information mapToServiceModel(from.Context.Service, &out.Metadata.Service) + mapToAgentModel(from.Context.Service.Agent, &out.Metadata.Agent) overwriteUserInMetadataModel(from.Context.User, &out.Metadata) mapToUserAgentModel(from.Context.Request.Headers, &out.Metadata) @@ -341,10 +342,10 @@ func mapToMetadataModel(m *metadata, out *model.Metadata) { // Service if m.Service.Agent.Name.IsSet() { - out.Service.Agent.Name = m.Service.Agent.Name.Val + out.Agent.Name = m.Service.Agent.Name.Val } if m.Service.Agent.Version.IsSet() { - out.Service.Agent.Version = m.Service.Agent.Version.Val + out.Agent.Version = m.Service.Agent.Version.Val } if m.Service.Environment.IsSet() { out.Service.Environment = m.Service.Environment.Val @@ -485,12 +486,6 @@ func mapToRequestModel(from contextRequest, out *model.HTTPRequest) { } func mapToServiceModel(from contextService, out *model.Service) { - if from.Agent.Name.IsSet() { - out.Agent.Name = from.Agent.Name.Val - } - if from.Agent.Version.IsSet() { - out.Agent.Version = from.Agent.Version.Val - } if from.Environment.IsSet() { out.Environment = from.Environment.Val } @@ -520,6 +515,15 @@ func mapToServiceModel(from contextService, out *model.Service) { } } +func mapToAgentModel(from contextServiceAgent, out *model.Agent) { + if from.Name.IsSet() { + out.Name = from.Name.Val + } + if from.Version.IsSet() { + out.Version = from.Version.Val + } +} + func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, out *model.Span) { // set metadata information for span if metadata != nil { @@ -605,12 +609,6 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, out out.HTTP = &http } if from.Context.Service.IsSet() { - if from.Context.Service.Agent.Name.IsSet() { - out.Metadata.Service.Agent.Name = from.Context.Service.Agent.Name.Val - } - if from.Context.Service.Agent.Version.IsSet() { - out.Metadata.Service.Agent.Version = from.Context.Service.Agent.Version.Val - } if from.Context.Service.Name.IsSet() { out.Metadata.Service.Name = from.Context.Service.Name.Val } @@ -714,6 +712,7 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } // overwrite metadata with event specific information mapToServiceModel(from.Context.Service, &out.Metadata.Service) + mapToAgentModel(from.Context.Service.Agent, &out.Metadata.Agent) overwriteUserInMetadataModel(from.Context.User, &out.Metadata) mapToUserAgentModel(from.Context.Request.Headers, &out.Metadata) diff --git a/model/modeldecoder/rumv3/metadata_test.go b/model/modeldecoder/rumv3/metadata_test.go index 43e84d7ba23..42177c6d6f0 100644 --- a/model/modeldecoder/rumv3/metadata_test.go +++ b/model/modeldecoder/rumv3/metadata_test.go @@ -47,7 +47,17 @@ func initializedMetadata() *model.Metadata { } func metadataExceptions(keys ...string) func(key string) bool { - missing := []string{"Cloud", "System", "Process", "Service.Node", "Service.Agent.EphemeralID"} + missing := []string{ + "Agent", + "Cloud", + "Container", + "Kubernetes", + "Network", + "Process", + "Service.Node", + "Service.Agent.EphemeralID", + "Host", + } exceptions := append(missing, keys...) return func(key string) bool { for _, k := range exceptions { @@ -74,9 +84,10 @@ func TestDecodeNestedMetadata(t *testing.T) { testMinValidMetadata := `{"m":{"se":{"n":"name","a":{"n":"go","ve":"1.0.0"}}}}` dec := decoder.NewJSONDecoder(strings.NewReader(testMinValidMetadata)) require.NoError(t, DecodeNestedMetadata(dec, &out)) - assert.Equal(t, model.Metadata{Service: model.Service{ - Name: "name", - Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + assert.Equal(t, model.Metadata{ + Service: model.Service{Name: "name"}, + Agent: model.Agent{Name: "go", Version: "1.0.0"}, + }, out) err := DecodeNestedMetadata(decoder.NewJSONDecoder(strings.NewReader(`malformed`)), &out) require.Error(t, err) @@ -100,8 +111,8 @@ func TestDecodeMetadataMappingToModel(t *testing.T) { labels.Put(fmt.Sprintf("%s%v", s, i), s) } return &model.Metadata{ + Agent: model.Agent{Name: s, Version: s}, Service: model.Service{Name: s, Version: s, Environment: s, - Agent: model.Agent{Name: s, Version: s}, Language: model.Language{Name: s, Version: s}, Runtime: model.Runtime{Name: s, Version: s}, Framework: model.Framework{Name: s, Version: s}}, diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index eebafca5973..272bf50535f 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -251,6 +251,7 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti } // overwrite metadata with event specific information mapToServiceModel(from.Context.Service, &out.Metadata.Service) + mapToAgentModel(from.Context.Service.Agent, &out.Metadata.Agent) overwriteUserInMetadataModel(from.Context.User, &out.Metadata) mapToUserAgentModel(from.Context.Request.Headers, &out.Metadata) mapToClientModel(from.Context.Request, &out.Metadata) @@ -456,13 +457,13 @@ func mapToMetadataModel(from *metadata, out *model.Metadata) { // Service if from.Service.Agent.EphemeralID.IsSet() { - out.Service.Agent.EphemeralID = from.Service.Agent.EphemeralID.Val + out.Agent.EphemeralID = from.Service.Agent.EphemeralID.Val } if from.Service.Agent.Name.IsSet() { - out.Service.Agent.Name = from.Service.Agent.Name.Val + out.Agent.Name = from.Service.Agent.Name.Val } if from.Service.Agent.Version.IsSet() { - out.Service.Agent.Version = from.Service.Agent.Version.Val + out.Agent.Version = from.Service.Agent.Version.Val } if from.Service.Environment.IsSet() { out.Service.Environment = from.Service.Environment.Val @@ -497,35 +498,35 @@ func mapToMetadataModel(from *metadata, out *model.Metadata) { // System if from.System.Architecture.IsSet() { - out.System.Architecture = from.System.Architecture.Val + out.Host.Architecture = from.System.Architecture.Val } if from.System.ConfiguredHostname.IsSet() { - out.System.ConfiguredHostname = from.System.ConfiguredHostname.Val + out.Host.Name = from.System.ConfiguredHostname.Val } if from.System.Container.ID.IsSet() { - out.System.Container.ID = from.System.Container.ID.Val + out.Container.ID = from.System.Container.ID.Val } if from.System.DetectedHostname.IsSet() { - out.System.DetectedHostname = from.System.DetectedHostname.Val + out.Host.Hostname = from.System.DetectedHostname.Val } if !from.System.ConfiguredHostname.IsSet() && !from.System.DetectedHostname.IsSet() && from.System.DeprecatedHostname.IsSet() { - out.System.DetectedHostname = from.System.DeprecatedHostname.Val + out.Host.Hostname = from.System.DeprecatedHostname.Val } if from.System.Kubernetes.Namespace.IsSet() { - out.System.Kubernetes.Namespace = from.System.Kubernetes.Namespace.Val + out.Kubernetes.Namespace = from.System.Kubernetes.Namespace.Val } if from.System.Kubernetes.Node.Name.IsSet() { - out.System.Kubernetes.NodeName = from.System.Kubernetes.Node.Name.Val + out.Kubernetes.NodeName = from.System.Kubernetes.Node.Name.Val } if from.System.Kubernetes.Pod.Name.IsSet() { - out.System.Kubernetes.PodName = from.System.Kubernetes.Pod.Name.Val + out.Kubernetes.PodName = from.System.Kubernetes.Pod.Name.Val } if from.System.Kubernetes.Pod.UID.IsSet() { - out.System.Kubernetes.PodUID = from.System.Kubernetes.Pod.UID.Val + out.Kubernetes.PodUID = from.System.Kubernetes.Pod.UID.Val } if from.System.Platform.IsSet() { - out.System.Platform = from.System.Platform.Val + out.Host.OS.Platform = from.System.Platform.Val } // User @@ -699,15 +700,6 @@ func mapToResponseModel(from contextResponse, out *model.HTTPResponse) { } func mapToServiceModel(from contextService, out *model.Service) { - if from.Agent.EphemeralID.IsSet() { - out.Agent.EphemeralID = from.Agent.EphemeralID.Val - } - if from.Agent.Name.IsSet() { - out.Agent.Name = from.Agent.Name.Val - } - if from.Agent.Version.IsSet() { - out.Agent.Version = from.Agent.Version.Val - } if from.Environment.IsSet() { out.Environment = from.Environment.Val } @@ -740,6 +732,18 @@ func mapToServiceModel(from contextService, out *model.Service) { } } +func mapToAgentModel(from contextServiceAgent, out *model.Agent) { + if from.Name.IsSet() { + out.Name = from.Name.Val + } + if from.Version.IsSet() { + out.Version = from.Version.Val + } + if from.EphemeralID.IsSet() { + out.EphemeralID = from.EphemeralID.Val + } +} + func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, config modeldecoder.Config, out *model.Span) { // set metadata information for span if metadata != nil { @@ -894,6 +898,7 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, con } if from.Context.Service.IsSet() { mapToServiceModel(from.Context.Service, &out.Metadata.Service) + mapToAgentModel(from.Context.Service.Agent, &out.Metadata.Agent) } if len(from.Context.Tags) > 0 { out.Labels = modeldecoderutil.NormalizeLabelValues(from.Context.Tags.Clone()) @@ -1015,6 +1020,7 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } // overwrite metadata with event specific information mapToServiceModel(from.Context.Service, &out.Metadata.Service) + mapToAgentModel(from.Context.Service.Agent, &out.Metadata.Agent) overwriteUserInMetadataModel(from.Context.User, &out.Metadata) mapToUserAgentModel(from.Context.Request.Headers, &out.Metadata) mapToClientModel(from.Context.Request, &out.Metadata) diff --git a/model/modeldecoder/v2/metadata_test.go b/model/modeldecoder/v2/metadata_test.go index 1353bfdf864..b2fe3b3b564 100644 --- a/model/modeldecoder/v2/metadata_test.go +++ b/model/modeldecoder/v2/metadata_test.go @@ -38,24 +38,24 @@ func isUnmappedMetadataField(key string) bool { "Client.Domain", "Client.IP", "Client.Port", + "Container.Runtime", + "Container.ImageName", + "Container.ImageTag", + "Container.Name", + "Network", + "Network.ConnectionType", + "Network.Carrier", + "Network.Carrier.Name", + "Network.Carrier.MCC", + "Network.Carrier.MNC", + "Network.Carrier.ICC", "Process.CommandLine", "Process.Executable", - "System.Container.Runtime", - "System.Container.ImageName", - "System.Container.ImageTag", - "System.Container.Name", - "System.Network", - "System.Network.ConnectionType", - "System.Network.Carrier", - "System.Network.Carrier.Name", - "System.Network.Carrier.MCC", - "System.Network.Carrier.MNC", - "System.Network.Carrier.ICC", - "System.FullPlatform", - "System.ID", - "System.IP", - "System.OSType", - "System.Type", + "Host.OS.Full", + "Host.OS.Type", + "Host.ID", + "Host.IP", + "Host.Type", "UserAgent", "UserAgent.Name", "UserAgent.Original": @@ -104,9 +104,10 @@ func TestDecodeMetadata(t *testing.T) { var out model.Metadata dec := decoder.NewJSONDecoder(strings.NewReader(tc.input)) require.NoError(t, tc.decodeFn(dec, &out)) - assert.Equal(t, model.Metadata{Service: model.Service{ - Name: "user-service", - Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + assert.Equal(t, model.Metadata{ + Service: model.Service{Name: "user-service"}, + Agent: model.Agent{Name: "go", Version: "1.0.0"}, + }, out) err := tc.decodeFn(decoder.NewJSONDecoder(strings.NewReader(`malformed`)), &out) require.Error(t, err) @@ -179,7 +180,7 @@ func TestDecodeMapToMetadataModel(t *testing.T) { input.Reset() modeldecodertest.SetStructValues(&input, otherVal) mapToMetadataModel(&input, &out2) - out2.System.IP, out2.Client.IP = defaultVal.IP, defaultVal.IP + out2.Host.IP, out2.Client.IP = defaultVal.IP, defaultVal.IP modeldecodertest.AssertStructValues(t, &out2, exceptions, otherVal) modeldecodertest.AssertStructValues(t, &out1, exceptions, defaultVal) }) @@ -193,25 +194,25 @@ func TestDecodeMapToMetadataModel(t *testing.T) { input.System.DetectedHostname.Set("detected-host") input.System.DeprecatedHostname.Set("deprecated-host") mapToMetadataModel(&input, &out) - assert.Equal(t, "configured-host", out.System.ConfiguredHostname) - assert.Equal(t, "detected-host", out.System.DetectedHostname) + assert.Equal(t, "configured-host", out.Host.Name) + assert.Equal(t, "detected-host", out.Host.Hostname) // no detected-host information out = model.Metadata{} input.System.DetectedHostname.Reset() mapToMetadataModel(&input, &out) - assert.Equal(t, "configured-host", out.System.ConfiguredHostname) - assert.Empty(t, out.System.DetectedHostname) + assert.Equal(t, "configured-host", out.Host.Name) + assert.Empty(t, out.Host.Hostname) // no configured-host information out = model.Metadata{} input.System.ConfiguredHostname.Reset() mapToMetadataModel(&input, &out) - assert.Empty(t, out.System.ConfiguredHostname) - assert.Equal(t, "deprecated-host", out.System.DetectedHostname) + assert.Empty(t, out.Host.Name) + assert.Equal(t, "deprecated-host", out.Host.Hostname) // no host information given out = model.Metadata{} input.System.DeprecatedHostname.Reset() - assert.Empty(t, out.System.ConfiguredHostname) - assert.Empty(t, out.System.DetectedHostname) + assert.Empty(t, out.Host.Name) + assert.Empty(t, out.Host.Hostname) }) } diff --git a/model/modelprocessor/hostname.go b/model/modelprocessor/hostname.go index fd8296cd0d8..4452635be88 100644 --- a/model/modelprocessor/hostname.go +++ b/model/modelprocessor/hostname.go @@ -23,29 +23,29 @@ import ( "github.com/elastic/apm-server/model" ) -// SetSystemHostname is a transform.Processor that sets the final +// SetHostHostname is a transform.Processor that sets the final // host.name and host.hostname values, according to whether the -// system is running in Kubernetes or not. -type SetSystemHostname struct{} +// event originated from within Kubernetes or not. +type SetHostHostname struct{} // ProcessBatch sets or overrides the host.name and host.hostname fields for events. -func (SetSystemHostname) ProcessBatch(ctx context.Context, b *model.Batch) error { - return MetadataProcessorFunc(setSystemHostname).ProcessBatch(ctx, b) +func (SetHostHostname) ProcessBatch(ctx context.Context, b *model.Batch) error { + return MetadataProcessorFunc(setHostHostname).ProcessBatch(ctx, b) } -func setSystemHostname(ctx context.Context, meta *model.Metadata) error { +func setHostHostname(ctx context.Context, meta *model.Metadata) error { switch { - case meta.System.Kubernetes.NodeName != "": - // system.kubernetes.node.name is set: set host.hostname to its value. - meta.System.DetectedHostname = meta.System.Kubernetes.NodeName - case meta.System.Kubernetes.PodName != "" || meta.System.Kubernetes.PodUID != "" || meta.System.Kubernetes.Namespace != "": - // system.kubernetes.* is set, but system.kubernetes.node.name is not: don't set host.hostname at all. - meta.System.DetectedHostname = "" + case meta.Kubernetes.NodeName != "": + // host.kubernetes.node.name is set: set host.hostname to its value. + meta.Host.Hostname = meta.Kubernetes.NodeName + case meta.Kubernetes.PodName != "" || meta.Kubernetes.PodUID != "" || meta.Kubernetes.Namespace != "": + // kubernetes.* is set, but kubernetes.node.name is not: don't set host.hostname at all. + meta.Host.Hostname = "" default: // Otherwise use the originally specified host.hostname value. } - if meta.System.ConfiguredHostname == "" { - meta.System.ConfiguredHostname = meta.System.DetectedHostname + if meta.Host.Name == "" { + meta.Host.Name = meta.Host.Hostname } return nil } diff --git a/model/modelprocessor/hostname_test.go b/model/modelprocessor/hostname_test.go index 677b89622e2..271f68a3a2b 100644 --- a/model/modelprocessor/hostname_test.go +++ b/model/modelprocessor/hostname_test.go @@ -24,49 +24,49 @@ import ( "github.com/elastic/apm-server/model/modelprocessor" ) -func TestSetSystemHostname(t *testing.T) { +func TestSetHostHostname(t *testing.T) { withConfiguredHostname := model.Metadata{ - System: model.System{ - ConfiguredHostname: "configured_hostname", - DetectedHostname: "detected_hostname", + Host: model.Host{ + Name: "configured_hostname", + Hostname: "detected_hostname", }, } withDetectedHostname := model.Metadata{ - System: model.System{ - DetectedHostname: "detected_hostname", + Host: model.Host{ + Hostname: "detected_hostname", }, } withKubernetesPodName := withDetectedHostname - withKubernetesPodName.System.Kubernetes.PodName = "kubernetes.pod.name" + withKubernetesPodName.Kubernetes.PodName = "kubernetes.pod.name" withKubernetesNodeName := withKubernetesPodName - withKubernetesNodeName.System.Kubernetes.NodeName = "kubernetes.node.name" + withKubernetesNodeName.Kubernetes.NodeName = "kubernetes.node.name" - processor := modelprocessor.SetSystemHostname{} + processor := modelprocessor.SetHostHostname{} testProcessBatchMetadata(t, processor, withConfiguredHostname, withConfiguredHostname) // unchanged testProcessBatchMetadata(t, processor, withDetectedHostname, - metadataWithConfiguredHostname( - metadataWithDetectedHostname(withDetectedHostname, "detected_hostname"), + metadataWithHostName( + metadataWithHostHostname(withDetectedHostname, "detected_hostname"), "detected_hostname", ), ) testProcessBatchMetadata(t, processor, withKubernetesPodName, - metadataWithDetectedHostname(withKubernetesPodName, ""), + metadataWithHostHostname(withKubernetesPodName, ""), ) testProcessBatchMetadata(t, processor, withKubernetesNodeName, - metadataWithConfiguredHostname( - metadataWithDetectedHostname(withKubernetesNodeName, "kubernetes.node.name"), + metadataWithHostName( + metadataWithHostHostname(withKubernetesNodeName, "kubernetes.node.name"), "kubernetes.node.name", ), ) } -func metadataWithDetectedHostname(in model.Metadata, detectedHostname string) model.Metadata { - in.System.DetectedHostname = detectedHostname +func metadataWithHostHostname(in model.Metadata, detectedHostname string) model.Metadata { + in.Host.Hostname = detectedHostname return in } -func metadataWithConfiguredHostname(in model.Metadata, configuredHostname string) model.Metadata { - in.System.ConfiguredHostname = configuredHostname +func metadataWithHostName(in model.Metadata, configuredHostname string) model.Metadata { + in.Host.Name = configuredHostname return in } diff --git a/model/modelprocessor/nodename.go b/model/modelprocessor/nodename.go index f3327f0336a..2a3defb3365 100644 --- a/model/modelprocessor/nodename.go +++ b/model/modelprocessor/nodename.go @@ -26,8 +26,8 @@ import ( // SetServiceNodeName is a transform.Processor that sets the service // node name value for events without one already set. // -// SetServiceNodeName should be called after SetSystemHostname, to -// ensure ConfiguredHostname is set. +// SetServiceNodeName should be called after SetHostHostname, to +// ensure Name is set. type SetServiceNodeName struct{} // ProcessBatch sets a default service.node.name for events without one already set. @@ -40,9 +40,9 @@ func setServiceNodeName(ctx context.Context, meta *model.Metadata) error { // Already set. return nil } - nodeName := meta.System.Container.ID + nodeName := meta.Container.ID if nodeName == "" { - nodeName = meta.System.ConfiguredHostname + nodeName = meta.Host.Name } meta.Service.Node.Name = nodeName return nil diff --git a/model/modelprocessor/nodename_test.go b/model/modelprocessor/nodename_test.go index 7290f5cc6a7..08a6b731330 100644 --- a/model/modelprocessor/nodename_test.go +++ b/model/modelprocessor/nodename_test.go @@ -33,12 +33,10 @@ func TestSetServiceNodeName(t *testing.T) { }, } withConfiguredHostname := model.Metadata{ - System: model.System{ - ConfiguredHostname: "configured_hostname", - }, + Host: model.Host{Name: "configured_hostname"}, } withContainerID := withConfiguredHostname - withContainerID.System.Container.ID = "container_id" + withContainerID.Container.ID = "container_id" processor := modelprocessor.SetServiceNodeName{} diff --git a/model/os.go b/model/os.go new file mode 100644 index 00000000000..bcd2b12b562 --- /dev/null +++ b/model/os.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "github.com/elastic/beats/v7/libbeat/common" +) + +// OS holds information about the operating system. +type OS struct { + // Platform holds the operating system platform, e.g. centos, ubuntu, windows. + Platform string + + // Full holds the full operating system name, including the version or code name. + Full string + + // Type categorizes the operating system into one of the broad commercial families. + // + // If specified, Type must beone of the following (lowercase): linux, macos, unix, windows. + // If the OS you’re dealing with is not in the list, the field should not be populated. + Type string +} + +func (o *OS) fields() common.MapStr { + var fields mapStr + fields.maybeSetString("platform", o.Platform) + fields.maybeSetString("full", o.Full) + fields.maybeSetString("type", o.Type) + return common.MapStr(fields) +} diff --git a/model/service.go b/model/service.go index 9f959b60a9c..b0d7e4761be 100644 --- a/model/service.go +++ b/model/service.go @@ -29,7 +29,6 @@ type Service struct { Language Language Runtime Runtime Framework Framework - Agent Agent Node ServiceNode } @@ -51,13 +50,6 @@ type Framework struct { Version string } -//Agent has an optional version, name and an ephemeral id -type Agent struct { - Name string - Version string - EphemeralID string -} - type ServiceNode struct { Name string } @@ -106,11 +98,3 @@ func (n *ServiceNode) fields() common.MapStr { } return nil } - -func (a *Agent) fields() common.MapStr { - var agent mapStr - agent.maybeSetString("name", a.Name) - agent.maybeSetString("version", a.Version) - agent.maybeSetString("ephemeral_id", a.EphemeralID) - return common.MapStr(agent) -} diff --git a/model/service_test.go b/model/service_test.go index 181e9788ec8..08ce79a1110 100644 --- a/model/service_test.go +++ b/model/service_test.go @@ -26,25 +26,22 @@ import ( ) var ( - version, environment = "5.1.3", "staging" - langName, langVersion = "ecmascript", "8" - rtName, rtVersion = "node", "8.0.0" - fwName, fwVersion = "Express", "1.2.3" - agentName, agentVersion = "elastic-node", "1.0.0" + version, environment = "5.1.3", "staging" + langName, langVersion = "ecmascript", "8" + rtName, rtVersion = "node", "8.0.0" + fwName, fwVersion = "Express", "1.2.3" ) func TestServiceTransform(t *testing.T) { serviceName, serviceNodeName := "myService", "abc" tests := []struct { - Service Service - Fields common.MapStr - AgentFields common.MapStr + Service Service + Fields common.MapStr }{ { - Service: Service{}, - AgentFields: nil, - Fields: nil, + Service: Service{}, + Fields: nil, }, { Service: Service{ @@ -63,16 +60,8 @@ func TestServiceTransform(t *testing.T) { Name: fwName, Version: fwVersion, }, - Agent: Agent{ - Name: agentName, - Version: agentVersion, - }, Node: ServiceNode{Name: serviceNodeName}, }, - AgentFields: common.MapStr{ - "name": "elastic-node", - "version": "1.0.0", - }, Fields: common.MapStr{ "name": "myService", "version": "5.1.3", @@ -96,6 +85,5 @@ func TestServiceTransform(t *testing.T) { for _, test := range tests { assert.Equal(t, test.Fields, test.Service.Fields()) - assert.Equal(t, test.AgentFields, test.Service.Agent.fields()) } } diff --git a/model/system.go b/model/system.go deleted file mode 100644 index ddbe9f8aa0f..00000000000 --- a/model/system.go +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package model - -import ( - "net" - - "github.com/elastic/beats/v7/libbeat/common" -) - -type System struct { - // DetectedHostname holds the detected hostname. - // - // This will be written to the event as "host.hostname". - // - // TODO(axw) rename this to Hostname. - DetectedHostname string - - // ConfiguredHostname holds the user-defined or detected hostname. - // - // If defined, this will be written to the event as "host.name". - // - // TODO(axw) rename this to Name. - ConfiguredHostname string - - // ID holds a unique ID for the host. - ID string - - Architecture string - Platform string - FullPlatform string // Full operating system name, including version - OSType string - Type string // host type, e.g. cloud instance machine type - IP net.IP - - Container Container - Kubernetes Kubernetes - Network Network -} - -func (s *System) fields() common.MapStr { - if s == nil { - return nil - } - var system mapStr - system.maybeSetString("hostname", s.DetectedHostname) - system.maybeSetString("name", s.ConfiguredHostname) - system.maybeSetString("architecture", s.Architecture) - system.maybeSetString("type", s.Type) - - var os mapStr - os.maybeSetString("platform", s.Platform) - os.maybeSetString("full", s.FullPlatform) - os.maybeSetString("type", s.OSType) - system.maybeSetMapStr("os", common.MapStr(os)) - - if s.IP != nil { - system.set("ip", s.IP.String()) - } - return common.MapStr(system) -} diff --git a/model/system_test.go b/model/system_test.go deleted file mode 100644 index 3a1fbf7cf0e..00000000000 --- a/model/system_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package model - -import ( - "encoding/json" - "net" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/elastic/apm-server/approvaltest" -) - -func TestSystemTransformation(t *testing.T) { - detected, configured := "host", "custom hostname" - namespace := "staging" - nodename, podname, podUID := "a.node", "a.pod", "b.podID" - - for name, system := range map[string]System{ - "hostname": {DetectedHostname: detected}, - "ignored hostname": {ConfiguredHostname: configured}, - "full hostname info": {DetectedHostname: detected, ConfiguredHostname: configured}, - "full": { - DetectedHostname: detected, - ConfiguredHostname: configured, - Architecture: "amd", - Platform: "osx", - FullPlatform: "Mac OS Mojave", - OSType: "macos", - Type: "t2.medium", - IP: net.ParseIP("127.0.0.1"), - Container: Container{ID: "1234"}, - Kubernetes: Kubernetes{Namespace: namespace, NodeName: nodename, PodName: podname, PodUID: podUID}, - }, - } { - t.Run(name, func(t *testing.T) { - var fields mapStr - metadata := &Metadata{System: system} - metadata.set(&fields, nil) - resultJSON, err := json.Marshal(fields["host"]) - require.NoError(t, err) - name := filepath.Join("test_approved", "system", strings.ReplaceAll(name, " ", "_")) - approvaltest.ApproveJSON(t, name, resultJSON) - }) - } -} - -func TestNetworkTransformation(t *testing.T) { - var fields mapStr - metadata := &Metadata{System: System{Network: Network{ConnectionType: "3G", - Carrier: Carrier{Name: "Three", MCC: "100", MNC: "200", ICC: "DK"}}}} - metadata.set(&fields, nil) - resultJSON, err := json.Marshal(fields["network"]) - require.NoError(t, err) - name := filepath.Join("test_approved", "system", "network") - approvaltest.ApproveJSON(t, name, resultJSON) -} diff --git a/model/test_approved/system/full.approved.json b/model/test_approved/host/full.approved.json similarity index 100% rename from model/test_approved/system/full.approved.json rename to model/test_approved/host/full.approved.json diff --git a/model/test_approved/system/full_hostname_info.approved.json b/model/test_approved/host/full_hostname_info.approved.json similarity index 100% rename from model/test_approved/system/full_hostname_info.approved.json rename to model/test_approved/host/full_hostname_info.approved.json diff --git a/model/test_approved/system/hostname.approved.json b/model/test_approved/host/hostname.approved.json similarity index 100% rename from model/test_approved/system/hostname.approved.json rename to model/test_approved/host/hostname.approved.json diff --git a/model/test_approved/system/ignored_hostname.approved.json b/model/test_approved/host/ignored_hostname.approved.json similarity index 100% rename from model/test_approved/system/ignored_hostname.approved.json rename to model/test_approved/host/ignored_hostname.approved.json diff --git a/model/test_approved/system/network.approved.json b/model/test_approved/system/network.approved.json deleted file mode 100644 index 300c8407564..00000000000 --- a/model/test_approved/system/network.approved.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "carrier": { - "icc": "DK", - "mcc": "100", - "mnc": "200", - "name": "Three" - }, - "connection_type": "3G" -} diff --git a/model/transaction_test.go b/model/transaction_test.go index 25f92427529..8616c0a1bc1 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -147,11 +147,11 @@ func TestEventsTransformWithMetadata(t *testing.T) { Version: serviceVersion, Node: ServiceNode{Name: serviceNodeName}, }, - System: System{ - ConfiguredHostname: name, - DetectedHostname: hostname, - Architecture: architecture, - Platform: platform, + Host: Host{ + Name: name, + Hostname: hostname, + Architecture: architecture, + OS: OS{Platform: platform}, }, User: User{ID: id, Name: name}, UserAgent: UserAgent{Original: userAgent}, diff --git a/processor/otel/consumer.go b/processor/otel/consumer.go index f5affeced87..32c9728c9b2 100644 --- a/processor/otel/consumer.go +++ b/processor/otel/consumer.go @@ -240,7 +240,7 @@ func translateTransaction( metadata model.Metadata, tx *transactionBuilder, ) { - isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") + isJaeger := strings.HasPrefix(metadata.Agent.Name, "Jaeger") labels := make(common.MapStr) var ( @@ -346,15 +346,15 @@ func translateTransaction( case conventions.AttributeNetHostName: netHostName = stringval case AttributeNetworkType: - tx.Metadata.System.Network.ConnectionType = stringval + tx.Metadata.Network.ConnectionType = stringval case AttributeNetworkMCC: - tx.Metadata.System.Network.Carrier.MCC = stringval + tx.Metadata.Network.Carrier.MCC = stringval case AttributeNetworkMNC: - tx.Metadata.System.Network.Carrier.MNC = stringval + tx.Metadata.Network.Carrier.MNC = stringval case AttributeNetworkCarrierName: - tx.Metadata.System.Network.Carrier.Name = stringval + tx.Metadata.Network.Carrier.Name = stringval case AttributeNetworkICC: - tx.Metadata.System.Network.Carrier.ICC = stringval + tx.Metadata.Network.Carrier.ICC = stringval // messaging.* case "message_bus.destination", conventions.AttributeMessagingDestination: @@ -407,7 +407,7 @@ func translateTransaction( if httpHost == "" { httpHost = netHostName if httpHost == "" { - httpHost = metadata.System.DetectedHostname + httpHost = metadata.Host.Hostname } } if httpHost != "" && netHostPort > 0 { @@ -451,7 +451,7 @@ func translateTransaction( } func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) { - isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") + isJaeger := strings.HasPrefix(metadata.Agent.Name, "Jaeger") labels := make(common.MapStr) var ( @@ -570,15 +570,15 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) netPeerName = stringval } case AttributeNetworkType: - event.Metadata.System.Network.ConnectionType = stringval + event.Metadata.Network.ConnectionType = stringval case AttributeNetworkMCC: - event.Metadata.System.Network.Carrier.MCC = stringval + event.Metadata.Network.Carrier.MCC = stringval case AttributeNetworkMNC: - event.Metadata.System.Network.Carrier.MNC = stringval + event.Metadata.Network.Carrier.MNC = stringval case AttributeNetworkCarrierName: - event.Metadata.System.Network.Carrier.Name = stringval + event.Metadata.Network.Carrier.Name = stringval case AttributeNetworkICC: - event.Metadata.System.Network.Carrier.ICC = stringval + event.Metadata.Network.Carrier.ICC = stringval // messaging.* case "message_bus.destination", conventions.AttributeMessagingDestination: @@ -785,7 +785,7 @@ func convertSpanEvent( out *model.Batch, ) { var e *model.Error - isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") + isJaeger := strings.HasPrefix(metadata.Agent.Name, "Jaeger") if isJaeger { e = convertJaegerErrorSpanEvent(logger, event) } else { diff --git a/processor/otel/consumer_test.go b/processor/otel/consumer_test.go index 1f373a99514..942fcac86af 100644 --- a/processor/otel/consumer_test.go +++ b/processor/otel/consumer_test.go @@ -575,8 +575,8 @@ func TestSpanNetworkAttributes(t *testing.T) { ICC: "UK", }, } - assert.Equal(t, expected, tx.Metadata.System.Network) - assert.Equal(t, expected, span.Metadata.System.Network) + assert.Equal(t, expected, tx.Metadata.Network) + assert.Equal(t, expected, span.Metadata.Network) } func TestArrayLabels(t *testing.T) { diff --git a/processor/otel/exceptions_test.go b/processor/otel/exceptions_test.go index ea53444f66a..e83f26ae16f 100644 --- a/processor/otel/exceptions_test.go +++ b/processor/otel/exceptions_test.go @@ -330,10 +330,10 @@ func languageOnlyMetadata(language string) model.Metadata { Service: model.Service{ Name: "unknown", Language: model.Language{Name: language}, - Agent: model.Agent{ - Name: "otlp/" + language, - Version: "unknown", - }, + }, + Agent: model.Agent{ + Name: "otlp/" + language, + Version: "unknown", }, } } diff --git a/processor/otel/metadata.go b/processor/otel/metadata.go index c905318c9cd..9ea6350703a 100644 --- a/processor/otel/metadata.go +++ b/processor/otel/metadata.go @@ -56,11 +56,11 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { // telemetry.sdk.* case conventions.AttributeTelemetrySDKName: - out.Service.Agent.Name = truncate(v.StringVal()) + out.Agent.Name = truncate(v.StringVal()) + case conventions.AttributeTelemetrySDKVersion: + out.Agent.Version = truncate(v.StringVal()) case conventions.AttributeTelemetrySDKLanguage: out.Service.Language.Name = truncate(v.StringVal()) - case conventions.AttributeTelemetrySDKVersion: - out.Service.Agent.Version = truncate(v.StringVal()) // cloud.* case conventions.AttributeCloudProvider: @@ -76,35 +76,35 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { // container.* case conventions.AttributeContainerName: - out.System.Container.Name = truncate(v.StringVal()) + out.Container.Name = truncate(v.StringVal()) case conventions.AttributeContainerID: - out.System.Container.ID = truncate(v.StringVal()) + out.Container.ID = truncate(v.StringVal()) case conventions.AttributeContainerImage: - out.System.Container.ImageName = truncate(v.StringVal()) + out.Container.ImageName = truncate(v.StringVal()) case conventions.AttributeContainerTag: - out.System.Container.ImageTag = truncate(v.StringVal()) + out.Container.ImageTag = truncate(v.StringVal()) case "container.runtime": - out.System.Container.Runtime = truncate(v.StringVal()) + out.Container.Runtime = truncate(v.StringVal()) // k8s.* case conventions.AttributeK8sNamespace: - out.System.Kubernetes.Namespace = truncate(v.StringVal()) + out.Kubernetes.Namespace = truncate(v.StringVal()) case conventions.AttributeK8sNodeName: - out.System.Kubernetes.NodeName = truncate(v.StringVal()) + out.Kubernetes.NodeName = truncate(v.StringVal()) case conventions.AttributeK8sPod: - out.System.Kubernetes.PodName = truncate(v.StringVal()) + out.Kubernetes.PodName = truncate(v.StringVal()) case conventions.AttributeK8sPodUID: - out.System.Kubernetes.PodUID = truncate(v.StringVal()) + out.Kubernetes.PodUID = truncate(v.StringVal()) // host.* case conventions.AttributeHostName: - out.System.DetectedHostname = truncate(v.StringVal()) + out.Host.Hostname = truncate(v.StringVal()) case conventions.AttributeHostID: - out.System.ID = truncate(v.StringVal()) + out.Host.ID = truncate(v.StringVal()) case conventions.AttributeHostType: - out.System.Type = truncate(v.StringVal()) + out.Host.Type = truncate(v.StringVal()) case "host.arch": - out.System.Architecture = truncate(v.StringVal()) + out.Host.Architecture = truncate(v.StringVal()) // process.* case conventions.AttributeProcessID: @@ -120,9 +120,9 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { // os.* case conventions.AttributeOSType: - out.System.Platform = strings.ToLower(truncate(v.StringVal())) + out.Host.OS.Platform = strings.ToLower(truncate(v.StringVal())) case conventions.AttributeOSDescription: - out.System.FullPlatform = truncate(v.StringVal()) + out.Host.OS.Full = truncate(v.StringVal()) // Legacy OpenCensus attributes. case "opencensus.exporterversion": @@ -141,13 +141,13 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { // // "One of these following values should be used (lowercase): linux, macos, unix, windows. // If the OS you’re dealing with is not in the list, the field should not be populated." - switch out.System.Platform { + switch out.Host.OS.Platform { case "windows", "linux": - out.System.OSType = out.System.Platform + out.Host.OS.Type = out.Host.OS.Platform case "darwin": - out.System.OSType = "macos" + out.Host.OS.Type = "macos" case "aix", "hpux", "solaris": - out.System.OSType = "unix" + out.Host.OS.Type = "unix" } if strings.HasPrefix(exporterVersion, "Jaeger") { @@ -158,17 +158,17 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { out.Service.Language.Name = versionParts[1] } if v := versionParts[len(versionParts)-1]; v != "" { - out.Service.Agent.Version = v + out.Agent.Version = v } - out.Service.Agent.Name = AgentNameJaeger + out.Agent.Name = AgentNameJaeger // Translate known Jaeger labels. if clientUUID, ok := out.Labels["client-uuid"].(string); ok { - out.Service.Agent.EphemeralID = clientUUID + out.Agent.EphemeralID = clientUUID delete(out.Labels, "client-uuid") } if systemIP, ok := out.Labels["ip"].(string); ok { - out.System.IP = net.ParseIP(systemIP) + out.Host.IP = net.ParseIP(systemIP) delete(out.Labels, "ip") } } @@ -177,16 +177,16 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { // service.name is a required field. out.Service.Name = "unknown" } - if out.Service.Agent.Name == "" { - // service.agent.name is a required field. - out.Service.Agent.Name = "otlp" + if out.Agent.Name == "" { + // agent.name is a required field. + out.Agent.Name = "otlp" } - if out.Service.Agent.Version == "" { - // service.agent.version is a required field. - out.Service.Agent.Version = "unknown" + if out.Agent.Version == "" { + // agent.version is a required field. + out.Agent.Version = "unknown" } if out.Service.Language.Name != "" { - out.Service.Agent.Name = fmt.Sprintf("%s/%s", out.Service.Agent.Name, out.Service.Language.Name) + out.Agent.Name = fmt.Sprintf("%s/%s", out.Agent.Name, out.Service.Language.Name) } else { out.Service.Language.Name = "unknown" } diff --git a/processor/otel/metadata_test.go b/processor/otel/metadata_test.go index 606c6dc16fa..6fd126fca99 100644 --- a/processor/otel/metadata_test.go +++ b/processor/otel/metadata_test.go @@ -28,10 +28,10 @@ import ( ) func TestResourceConventions(t *testing.T) { + defaultAgent := model.Agent{Name: "otlp", Version: "unknown"} defaultService := model.Service{ Name: "unknown", Language: model.Language{Name: "unknown"}, - Agent: model.Agent{Name: "otlp", Version: "unknown"}, } for name, test := range map[string]struct { @@ -40,7 +40,7 @@ func TestResourceConventions(t *testing.T) { }{ "empty": { attrs: nil, - expected: model.Metadata{Service: defaultService}, + expected: model.Metadata{Agent: defaultAgent, Service: defaultService}, }, "service": { attrs: map[string]pdata.AttributeValue{ @@ -49,12 +49,12 @@ func TestResourceConventions(t *testing.T) { "deployment.environment": pdata.NewAttributeValueString("service_environment"), }, expected: model.Metadata{ + Agent: model.Agent{Name: "otlp", Version: "unknown"}, Service: model.Service{ Name: "service_name", Version: "service_version", Environment: "service_environment", Language: model.Language{Name: "unknown"}, - Agent: model.Agent{Name: "otlp", Version: "unknown"}, }, }, }, @@ -65,10 +65,10 @@ func TestResourceConventions(t *testing.T) { "telemetry.sdk.language": pdata.NewAttributeValueString("language_name"), }, expected: model.Metadata{ + Agent: model.Agent{Name: "sdk_name/language_name", Version: "sdk_version"}, Service: model.Service{ Name: "unknown", Language: model.Language{Name: "language_name"}, - Agent: model.Agent{Name: "sdk_name/language_name", Version: "sdk_version"}, }, }, }, @@ -78,10 +78,10 @@ func TestResourceConventions(t *testing.T) { "process.runtime.version": pdata.NewAttributeValueString("runtime_version"), }, expected: model.Metadata{ + Agent: model.Agent{Name: "otlp", Version: "unknown"}, Service: model.Service{ Name: "unknown", Language: model.Language{Name: "unknown"}, - Agent: model.Agent{Name: "otlp", Version: "unknown"}, Runtime: model.Runtime{ Name: "runtime_name", Version: "runtime_version", @@ -98,6 +98,7 @@ func TestResourceConventions(t *testing.T) { "cloud.platform": pdata.NewAttributeValueString("platform_name"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, Cloud: model.Cloud{ Provider: "provider_name", @@ -117,15 +118,14 @@ func TestResourceConventions(t *testing.T) { "container.runtime": pdata.NewAttributeValueString("container_runtime"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, - System: model.System{ - Container: model.Container{ - Name: "container_name", - ID: "container_id", - Runtime: "container_runtime", - ImageName: "container_image_name", - ImageTag: "container_image_tag", - }, + Container: model.Container{ + Name: "container_name", + ID: "container_id", + Runtime: "container_runtime", + ImageName: "container_image_name", + ImageTag: "container_image_tag", }, }, }, @@ -137,14 +137,13 @@ func TestResourceConventions(t *testing.T) { "k8s.pod.uid": pdata.NewAttributeValueString("kubernetes_pod_uid"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, - System: model.System{ - Kubernetes: model.Kubernetes{ - Namespace: "kubernetes_namespace", - NodeName: "kubernetes_node_name", - PodName: "kubernetes_pod_name", - PodUID: "kubernetes_pod_uid", - }, + Kubernetes: model.Kubernetes{ + Namespace: "kubernetes_namespace", + NodeName: "kubernetes_node_name", + PodName: "kubernetes_pod_name", + PodUID: "kubernetes_pod_uid", }, }, }, @@ -156,12 +155,13 @@ func TestResourceConventions(t *testing.T) { "host.arch": pdata.NewAttributeValueString("host_arch"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, - System: model.System{ - DetectedHostname: "host_name", - ID: "host_id", - Type: "host_type", - Architecture: "host_arch", + Host: model.Host{ + Hostname: "host_name", + ID: "host_id", + Type: "host_type", + Architecture: "host_arch", }, }, }, @@ -172,6 +172,7 @@ func TestResourceConventions(t *testing.T) { "process.executable.path": pdata.NewAttributeValueString("executable_path"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, Process: model.Process{ Pid: 123, @@ -186,11 +187,14 @@ func TestResourceConventions(t *testing.T) { "os.description": pdata.NewAttributeValueString("Mac OS Mojave"), }, expected: model.Metadata{ + Agent: defaultAgent, Service: defaultService, - System: model.System{ - Platform: "darwin", - OSType: "macos", - FullPlatform: "Mac OS Mojave", + Host: model.Host{ + OS: model.OS{ + Platform: "darwin", + Type: "macos", + Full: "Mac OS Mojave", + }, }, }, }, diff --git a/processor/otel/metrics_test.go b/processor/otel/metrics_test.go index a2b37880393..9548a101776 100644 --- a/processor/otel/metrics_test.go +++ b/processor/otel/metrics_test.go @@ -150,15 +150,15 @@ func TestConsumeMetrics(t *testing.T) { expectDropped++ metadata := model.Metadata{ + Agent: model.Agent{ + Name: "otlp", + Version: "unknown", + }, Service: model.Service{ Name: "unknown", Language: model.Language{ Name: "unknown", }, - Agent: model.Agent{ - Name: "otlp", - Version: "unknown", - }, }, } @@ -258,15 +258,15 @@ func TestConsumeMetrics_JVM(t *testing.T) { addInt64Gauge("runtime.jvm.memory.area", 42, map[string]string{"area": "heap", "type": "used"}) metadata := model.Metadata{ + Agent: model.Agent{ + Name: "otlp", + Version: "unknown", + }, Service: model.Service{ Name: "unknown", Language: model.Language{ Name: "unknown", }, - Agent: model.Agent{ - Name: "otlp", - Version: "unknown", - }, }, } diff --git a/processor/stream/processor_test.go b/processor/stream/processor_test.go index c1776468eb6..25a23de2b52 100644 --- a/processor/stream/processor_test.go +++ b/processor/stream/processor_test.go @@ -177,7 +177,7 @@ func TestIntegrationESOutput(t *testing.T) { ctx := utility.ContextWithRequestTime(context.Background(), reqTimestamp) batchProcessor := makeApproveEventsBatchProcessor(t, name, &accepted) - reqDecoderMeta := &model.Metadata{System: model.System{IP: net.ParseIP("192.0.0.1")}} + reqDecoderMeta := &model.Metadata{Host: model.Host{IP: net.ParseIP("192.0.0.1")}} p := BackendProcessor(&config.Config{MaxEventSize: 100 * 1024}) var actualResult Result diff --git a/x-pack/apm-server/aggregation/spanmetrics/aggregator.go b/x-pack/apm-server/aggregation/spanmetrics/aggregator.go index 39281c2cfd3..58d97024ec7 100644 --- a/x-pack/apm-server/aggregation/spanmetrics/aggregator.go +++ b/x-pack/apm-server/aggregation/spanmetrics/aggregator.go @@ -201,7 +201,7 @@ func (a *Aggregator) processSpan(span *model.Span) *model.Metricset { key := aggregationKey{ serviceEnvironment: span.Metadata.Service.Environment, serviceName: span.Metadata.Service.Name, - agentName: span.Metadata.Service.Agent.Name, + agentName: span.Metadata.Agent.Name, outcome: span.Outcome, resource: span.DestinationService.Resource, } @@ -262,10 +262,10 @@ func makeMetricset(timestamp time.Time, key aggregationKey, metrics spanMetrics, Timestamp: timestamp, Name: metricsetName, Metadata: model.Metadata{ + Agent: model.Agent{Name: key.agentName}, Service: model.Service{ Name: key.serviceName, Environment: key.serviceEnvironment, - Agent: model.Agent{Name: key.agentName}, }, }, Event: model.MetricsetEventCategorization{ diff --git a/x-pack/apm-server/aggregation/spanmetrics/aggregator_test.go b/x-pack/apm-server/aggregation/spanmetrics/aggregator_test.go index febb711ed72..473991a391e 100644 --- a/x-pack/apm-server/aggregation/spanmetrics/aggregator_test.go +++ b/x-pack/apm-server/aggregation/spanmetrics/aggregator_test.go @@ -120,7 +120,8 @@ func TestAggregatorRun(t *testing.T) { assert.ElementsMatch(t, []*model.Metricset{{ Name: "service_destination", Metadata: model.Metadata{ - Service: model.Service{Name: "service-A", Agent: model.Agent{Name: "java"}}, + Agent: model.Agent{Name: "java"}, + Service: model.Service{Name: "service-A"}, }, Event: model.MetricsetEventCategorization{ Outcome: "success", @@ -136,7 +137,8 @@ func TestAggregatorRun(t *testing.T) { }, { Name: "service_destination", Metadata: model.Metadata{ - Service: model.Service{Name: "service-A", Agent: model.Agent{Name: "java"}}, + Agent: model.Agent{Name: "java"}, + Service: model.Service{Name: "service-A"}, }, Event: model.MetricsetEventCategorization{ Outcome: "failure", @@ -152,7 +154,8 @@ func TestAggregatorRun(t *testing.T) { }, { Name: "service_destination", Metadata: model.Metadata{ - Service: model.Service{Name: "service-A", Agent: model.Agent{Name: "java"}}, + Agent: model.Agent{Name: "java"}, + Service: model.Service{Name: "service-A"}, }, Event: model.MetricsetEventCategorization{ Outcome: "success", @@ -168,7 +171,8 @@ func TestAggregatorRun(t *testing.T) { }, { Name: "service_destination", Metadata: model.Metadata{ - Service: model.Service{Name: "service-B", Agent: model.Agent{Name: "python"}}, + Agent: model.Agent{Name: "python"}, + Service: model.Service{Name: "service-B"}, }, Event: model.MetricsetEventCategorization{ Outcome: "success", @@ -226,7 +230,8 @@ func TestAggregatorOverflow(t *testing.T) { assert.Equal(t, &model.Metricset{ Name: "service_destination", Metadata: model.Metadata{ - Service: model.Service{Name: "service", Agent: model.Agent{Name: "agent"}}, + Agent: model.Agent{Name: "agent"}, + Service: model.Service{Name: "service"}, }, Event: model.MetricsetEventCategorization{ Outcome: "success", @@ -249,7 +254,10 @@ func makeSpan( count float64, ) *model.Span { span := &model.Span{ - Metadata: model.Metadata{Service: model.Service{Name: serviceName, Agent: model.Agent{Name: agentName}}}, + Metadata: model.Metadata{ + Agent: model.Agent{Name: agentName}, + Service: model.Service{Name: serviceName}, + }, Name: serviceName + ":" + destinationServiceResource, Duration: duration.Seconds() * 1000, RepresentativeCount: count, diff --git a/x-pack/apm-server/aggregation/txmetrics/aggregator.go b/x-pack/apm-server/aggregation/txmetrics/aggregator.go index e3f82ca63ad..ec13c2e6789 100644 --- a/x-pack/apm-server/aggregation/txmetrics/aggregator.go +++ b/x-pack/apm-server/aggregation/txmetrics/aggregator.go @@ -347,14 +347,14 @@ func (a *Aggregator) makeTransactionAggregationKey(tx *model.Transaction) transa transactionResult: tx.Result, transactionType: tx.Type, - agentName: tx.Metadata.Service.Agent.Name, + agentName: tx.Metadata.Agent.Name, serviceEnvironment: tx.Metadata.Service.Environment, serviceName: tx.Metadata.Service.Name, serviceVersion: tx.Metadata.Service.Version, - hostname: tx.Metadata.System.DetectedHostname, - containerID: tx.Metadata.System.Container.ID, - kubernetesPodName: tx.Metadata.System.Kubernetes.PodName, + hostname: tx.Metadata.Host.Hostname, + containerID: tx.Metadata.Container.ID, + kubernetesPodName: tx.Metadata.Kubernetes.PodName, } } @@ -366,16 +366,16 @@ func makeMetricset( Timestamp: ts, Name: metricsetName, Metadata: model.Metadata{ + Agent: model.Agent{Name: key.agentName}, + Container: model.Container{ID: key.containerID}, + Kubernetes: model.Kubernetes{PodName: key.kubernetesPodName}, Service: model.Service{ Name: key.serviceName, Version: key.serviceVersion, Environment: key.serviceEnvironment, - Agent: model.Agent{Name: key.agentName}, }, - System: model.System{ - DetectedHostname: key.hostname, - Container: model.Container{ID: key.containerID}, - Kubernetes: model.Kubernetes{PodName: key.kubernetesPodName}, + Host: model.Host{ + Hostname: key.hostname, }, }, Event: model.MetricsetEventCategorization{ diff --git a/x-pack/apm-server/aggregation/txmetrics/aggregator_test.go b/x-pack/apm-server/aggregation/txmetrics/aggregator_test.go index 2f068bec1da..a1ffe6b4373 100644 --- a/x-pack/apm-server/aggregation/txmetrics/aggregator_test.go +++ b/x-pack/apm-server/aggregation/txmetrics/aggregator_test.go @@ -385,12 +385,12 @@ func TestAggregationFields(t *testing.T) { &input.Outcome, &input.Result, &input.Type, - &input.Metadata.Service.Agent.Name, + &input.Metadata.Agent.Name, &input.Metadata.Service.Environment, &input.Metadata.Service.Name, &input.Metadata.Service.Version, - &input.Metadata.System.Container.ID, - &input.Metadata.System.Kubernetes.PodName, + &input.Metadata.Container.ID, + &input.Metadata.Kubernetes.PodName, } var expected []*model.Metricset @@ -429,9 +429,9 @@ func TestAggregationFields(t *testing.T) { // Hostname is complex: if any kubernetes fields are set, then // it is taken from Kubernetes.Node.Name, and DetectedHostname // is ignored. - input.Metadata.System.Kubernetes.PodName = "" + input.Metadata.Kubernetes.PodName = "" for _, value := range []string{"something", "anything"} { - input.Metadata.System.DetectedHostname = value + input.Metadata.Host.Hostname = value assert.Nil(t, agg.AggregateTransaction(&input)) assert.Nil(t, agg.AggregateTransaction(&input)) addExpectedCount(2)