Tidy up model to resembles ECS more closely (#5785)

* model: move agent/container/kubernetes/network up

Tidy up the model so it resembles ECS more closely:

- Move Agent out of Service
- Move Container, Kubernetes, and Network out of System

* Rename model.System to model.Host

Again, rename System to Host to resemble ECS
more closely, and rename hostname fields.
Move host.os fields into a separate "OS" type
and introduce OS as a field of Host.

beater/api/mux.go

@@ -310,7 +310,7 @@ func emptyRequestMetadata(c *request.Context) model.Metadata {
 }
 
 func backendRequestMetadata(c *request.Context) model.Metadata {
-	return model.Metadata{System: model.System{IP: c.SourceIP}}
+	return model.Metadata{Host: model.Host{IP: c.SourceIP}}
 }
 
 func rumRequestMetadata(c *request.Context) model.Metadata {

beater/beater.go

@@ -443,7 +443,7 @@ func (s *serverRunner) run() error {
 
 func (s *serverRunner) wrapRunServerWithPreprocessors(runServer RunServerFunc) RunServerFunc {
 	processors := []model.BatchProcessor{
-		modelprocessor.SetSystemHostname{},
+		modelprocessor.SetHostHostname{},
 		modelprocessor.SetServiceNodeName{},
 		modelprocessor.SetMetricsetName{},
 		modelprocessor.SetGroupingKey{},

beater/otlp/clientmetadata.go

@@ -30,7 +30,7 @@ import (
 //
 // Client metadata is extracted from ctx, injected by interceptors.ClientMetadata.
 func SetClientMetadata(ctx context.Context, meta *model.Metadata) error {
-	if meta.Service.Agent.Name != "iOS/swift" {
+	if meta.Agent.Name != "iOS/swift" {
 		// This is not an event from an agent we would consider to be
 		// running on an end-user device.
 		//

beater/otlp/clientmetadata_test.go

@@ -44,18 +44,18 @@ func TestSetClientMetadata(t *testing.T) {
 	}, {
 		ctx: context.Background(),
 		meta: model.Metadata{
-			Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}},
-			Client:  model.Client{IP: ip1234},
+			Agent:  model.Agent{Name: "iOS/swift"},
+			Client: model.Client{IP: ip1234},
 		},
 		expectedIP: ip1234,
 	}, {
 		ctx:  context.Background(),
-		meta: model.Metadata{Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}}},
+		meta: model.Metadata{Agent: model.Agent{Name: "iOS/swift"}},
 	}, {
 		ctx: interceptors.ContextWithClientMetadata(context.Background(), interceptors.ClientMetadataValues{
 			SourceIP: ip5678,
 		}),
-		meta:       model.Metadata{Service: model.Service{Agent: model.Agent{Name: "iOS/swift"}}},
+		meta:       model.Metadata{Agent: model.Agent{Name: "iOS/swift"}},
 		expectedIP: ip5678,
 	}} {
 		metaCopy := test.meta

beater/processors.go

@@ -34,7 +34,7 @@ const (
 // is authorized to ingest events for the agent and service name in metadata.
 func authorizeEventIngest(ctx context.Context, meta *model.Metadata) error {
 	return auth.Authorize(ctx, auth.ActionEventIngest, auth.Resource{
-		AgentName:   meta.Service.Agent.Name,
+		AgentName:   meta.Agent.Name,
 		ServiceName: meta.Service.Name,
 	})
 }

model/agent.go

@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+// Agent describes an Elastic APM agent.
+type Agent struct {
+	Name        string
+	Version     string
+	EphemeralID string
+}
+
+func (a *Agent) fields() common.MapStr {
+	var agent mapStr
+	agent.maybeSetString("name", a.Name)
+	agent.maybeSetString("version", a.Version)
+	agent.maybeSetString("ephemeral_id", a.EphemeralID)
+	return common.MapStr(agent)
+}

model/agent_test.go

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+const (
+	agentName, agentVersion = "elastic-node", "1.0.0"
+)
+
+func TestAgentFields(t *testing.T) {
+	tests := []struct {
+		Agent  Agent
+		Fields common.MapStr
+	}{
+		{
+			Agent:  Agent{},
+			Fields: nil,
+		},
+		{
+			Agent: Agent{
+				Name:    agentName,
+				Version: agentVersion,
+			},
+			Fields: common.MapStr{
+				"name":    "elastic-node",
+				"version": "1.0.0",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.Fields, test.Agent.fields())
+	}
+}

model/error_test.go

@@ -224,9 +224,9 @@ func TestEvents(t *testing.T) {
 
 	serviceName, agentName, version := "myservice", "go", "1.0"
 	md := Metadata{
+		Agent: Agent{Name: agentName, Version: version},
 		Service: Service{
 			Name: serviceName, Version: version,
-			Agent: Agent{Name: agentName, Version: version},
 		},
 		Labels: common.MapStr{"label": 101},
 	}

model/host.go

@@ -0,0 +1,66 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"net"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+type Host struct {
+	// Hostname holds the detected hostname of the host.
+	Hostname string
+
+	// Name holds the user-defined name of the host, or the
+	// detected hostname.
+	Name string
+
+	// ID holds a unique ID for the host.
+	ID string
+
+	// Architecture holds the host machine architecture.
+	Architecture string
+
+	// Type holds the host type, e.g. cloud instance machine type.
+	Type string
+
+	// IP holds the host IP address.
+	//
+	// TODO(axw) this should be a slice.
+	IP net.IP
+
+	// OS holds information about the operating system running on the host.
+	OS OS
+}
+
+func (h *Host) fields() common.MapStr {
+	if h == nil {
+		return nil
+	}
+	var fields mapStr
+	fields.maybeSetString("hostname", h.Hostname)
+	fields.maybeSetString("name", h.Name)
+	fields.maybeSetString("architecture", h.Architecture)
+	fields.maybeSetString("type", h.Type)
+	if h.IP != nil {
+		fields.set("ip", h.IP.String())
+	}
+	fields.maybeSetMapStr("os", h.OS.fields())
+	return common.MapStr(fields)
+}

model/host_test.go

@@ -0,0 +1,62 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"encoding/json"
+	"net"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-server/approvaltest"
+)
+
+func TestSystemTransformation(t *testing.T) {
+	detected, configured := "host", "custom hostname"
+
+	for name, host := range map[string]Host{
+		"hostname":           {Hostname: detected},
+		"ignored hostname":   {Name: configured},
+		"full hostname info": {Hostname: detected, Name: configured},
+		"full": {
+			Hostname:     detected,
+			Name:         configured,
+			Architecture: "amd",
+			Type:         "t2.medium",
+			IP:           net.ParseIP("127.0.0.1"),
+			OS: OS{
+				Platform: "osx",
+				Full:     "Mac OS Mojave",
+				Type:     "macos",
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var fields mapStr
+			metadata := &Metadata{Host: host}
+			metadata.set(&fields, nil)
+			resultJSON, err := json.Marshal(fields["host"])
+			require.NoError(t, err)
+			name := filepath.Join("test_approved", "host", strings.ReplaceAll(name, " ", "_"))
+			approvaltest.ApproveJSON(t, name, resultJSON)
+		})
+	}
+}

model/metadata.go

@@ -22,27 +22,31 @@ import (
 )
 
 type Metadata struct {
-	Service   Service
-	Process   Process
-	System    System
-	User      User
-	UserAgent UserAgent
-	Client    Client
-	Cloud     Cloud
-	Labels    common.MapStr
+	Service    Service
+	Agent      Agent
+	Process    Process
+	Host       Host
+	User       User
+	UserAgent  UserAgent
+	Client     Client
+	Cloud      Cloud
+	Network    Network
+	Container  Container
+	Kubernetes Kubernetes
+	Labels     common.MapStr
 }
 
 func (m *Metadata) set(fields *mapStr, eventLabels common.MapStr) {
 	fields.maybeSetMapStr("service", m.Service.Fields())
-	fields.maybeSetMapStr("agent", m.Service.Agent.fields())
-	fields.maybeSetMapStr("host", m.System.fields())
+	fields.maybeSetMapStr("agent", m.Agent.fields())
+	fields.maybeSetMapStr("host", m.Host.fields())
 	fields.maybeSetMapStr("process", m.Process.fields())
 	fields.maybeSetMapStr("user", m.User.fields())
 	fields.maybeSetMapStr("client", m.Client.fields())
 	fields.maybeSetMapStr("user_agent", m.UserAgent.fields())
-	fields.maybeSetMapStr("container", m.System.Container.fields())
-	fields.maybeSetMapStr("kubernetes", m.System.Kubernetes.fields())
+	fields.maybeSetMapStr("container", m.Container.fields())
+	fields.maybeSetMapStr("kubernetes", m.Kubernetes.fields())
 	fields.maybeSetMapStr("cloud", m.Cloud.fields())
-	fields.maybeSetMapStr("network", m.System.Network.fields())
+	fields.maybeSetMapStr("network", m.Network.fields())
 	maybeSetLabels(fields, m.Labels, eventLabels)
 }