model, modelindexer: generate JSON encoding

go.mod

@@ -232,3 +232,5 @@ replace (
 replace github.com/golang/glog => ./internal/glog
 
 replace go.opentelemetry.io/collector => ./internal/otel_collector
+
+replace go.elastic.co/fastjson => /home/andrew/go/src/go.elastic.co/fastjson

model/agent.go

@@ -23,9 +23,13 @@ import (
 
 // Agent describes an Elastic APM agent.
 type Agent struct {
-	Name        string
-	Version     string
-	EphemeralID string
+	Name        string `json:"name"`
+	Version     string `json:"version"`
+	EphemeralID string `json:"ephemeral_id"`
+}
+
+func (a Agent) isZero() bool {
+	return a == (Agent{})
 }
 
 func (a *Agent) fields() common.MapStr {

model/apmevent.go

@@ -33,53 +33,55 @@ type APMEvent struct {
 	//
 	// This will have the zero value when APM Server is run
 	// in standalone mode.
-	DataStream DataStream
+	DataStream DataStream `json:"data_stream,omitempty"`
 
-	ECSVersion  string
-	Event       Event
-	Agent       Agent
-	Observer    Observer
-	Container   Container
-	Kubernetes  Kubernetes
-	Service     Service
-	Process     Process
-	Host        Host
-	User        User
-	UserAgent   UserAgent
-	Client      Client
-	Source      Source
-	Destination Destination
-	Cloud       Cloud
-	Network     Network
-	Session     Session
-	URL         URL
-	Processor   Processor
-	Trace       Trace
-	Parent      Parent
-	Child       Child
-	HTTP        HTTP
-	FAAS        FAAS
+	ECSVersion  string      `json:"ecs.version"`
+	Event       Event       `json:"event,omitempty"`
+	Agent       Agent       `json:"agent,omitempty"`
+	Observer    Observer    `json:"observer,omitempty"`
+	Container   Container   `json:"container,omitempty"`
+	Kubernetes  Kubernetes  `json:"kubernetes,omitempty"`
+	Service     Service     `json:"service,omitempty"`
+	Process     Process     `json:"process,omitempty"`
+	Host        Host        `json:"host,omitempty"`
+	User        User        `json:"user,omitempty"`
+	UserAgent   UserAgent   `json:"user_agent,omitempty"`
+	Client      Client      `json:"client,omitempty"`
+	Source      Source      `json:"source,omitempty"`
+	Destination Destination `json:"destination,omitempty"`
+	Cloud       Cloud       `json:"cloud,omitempty"`
+	Network     Network     `json:"network,omitempty"`
+	Session     Session     `json:"session,omitempty"`
+	URL         URL         `json:"url,omitempty"`
+	Processor   Processor   `json:"processor,omitempty"`
+	Trace       Trace       `json:"trace,omitempty"`
+	Parent      Parent      `json:"parent,omitempty"`
+	Child       Child       `json:"child,omitempty"`
+	HTTP        HTTP        `json:"http,omitempty"`
+	FAAS        FAAS        `json:"faas,omitempty"`
 
 	// Timestamp holds the event timestamp.
 	//
 	// See https://www.elastic.co/guide/en/ecs/current/ecs-base.html
-	Timestamp time.Time
+	Timestamp time.Time `json:"@timestamp,omitempty"`
 
 	// Labels holds labels to apply to the event.
 	//
 	// See https://www.elastic.co/guide/en/ecs/current/ecs-base.html
-	Labels common.MapStr
+	Labels common.MapStr `json:"labels,omitempty"`
 
 	// Message holds the message for log events.
 	//
 	// See https://www.elastic.co/guide/en/ecs/current/ecs-base.html
-	Message string
+	Message string `json:"message,omitempty"`
 
-	Transaction   *Transaction
-	Span          *Span
-	Metricset     *Metricset
-	Error         *Error
-	ProfileSample *ProfileSample
+	// TODO(axw) add TimestampNanos or something.
+
+	Transaction   *Transaction   `json:"transaction,omitempty"`
+	Span          *Span          `json:"span,omitempty"`
+	Metricset     *Metricset     `json:"-"` // TODO(axw) fastjson
+	Error         *Error         `json:"error,omitempty"`
+	ProfileSample *ProfileSample `json:"-"` // TODO(axw) fastjson
 }
 
 // BeatEvent converts e to a beat.Event.

model/child.go

@@ -27,6 +27,10 @@ type Child struct {
 	ID []string
 }
 
+func (c *Child) isZero() bool {
+	return len(c.ID) == 0
+}
+
 func (c *Child) fields() common.MapStr {
 	var fields mapStr
 	if len(c.ID) > 0 {

model/client.go

@@ -35,6 +35,10 @@ type Client struct {
 	Port int
 }
 
+func (c *Client) isZero() bool {
+	return c.Domain == "" && c.IP == nil && c.Port <= 0
+}
+
 func (c *Client) fields() common.MapStr {
 	var fields mapStr
 	fields.maybeSetString("domain", c.Domain)

model/cloud.go

@@ -24,26 +24,34 @@ import (
 // Cloud holds information about the cloud computing environment
 // in which a service is running.
 type Cloud struct {
-	AccountID        string
-	AccountName      string
-	AvailabilityZone string
-	InstanceID       string
-	InstanceName     string
-	MachineType      string
-	ProjectID        string
-	ProjectName      string
-	Provider         string
-	Region           string
-	ServiceName      string
+	AccountID        string `json:"account.id,omitempty"`
+	AccountName      string `json:"account.name,omitempty"`
+	AvailabilityZone string `json:"availability_zone,omitempty"`
+	InstanceID       string `json:"instance.id,omitempty"`
+	InstanceName     string `json:"instance.name,omitempty"`
+	MachineType      string `json:"machine.type,omitempty"`
+	ProjectID        string `json:"project.id,omitempty"`
+	ProjectName      string `json:"project.name,omitempty"`
+	Provider         string `json:"provider,omitempty"`
+	Region           string `json:"region,omitempty"`
+	ServiceName      string `json:"service.name,omitempty"`
 
-	Origin *CloudOrigin
+	Origin *CloudOrigin `json:"origin,omitempty"`
+}
+
+func (c Cloud) isZero() bool {
+	return c == (Cloud{})
 }
 
 type CloudOrigin struct {
-	AccountID   string
-	Provider    string
-	Region      string
-	ServiceName string
+	AccountID   string `json:"account_id,omitempty"`
+	Provider    string `json:"provider,omitempty"`
+	Region      string `json:"region,omitempty"`
+	ServiceName string `json:"service.name,omitempty"`
+}
+
+func (co CloudOrigin) isZero() bool {
+	return co == (CloudOrigin{})
 }
 
 func (c *Cloud) fields() common.MapStr {

model/container.go

@@ -19,6 +19,7 @@ package model
 
 import (
 	"github.com/elastic/beats/v7/libbeat/common"
+	"go.elastic.co/fastjson"
 )
 
 type Container struct {
@@ -29,6 +30,10 @@ type Container struct {
 	ImageTag  string
 }
 
+func (c Container) isZero() bool {
+	return c == (Container{})
+}
+
 func (c *Container) fields() common.MapStr {
 	var container mapStr
 	container.maybeSetString("name", c.Name)
@@ -41,3 +46,13 @@ func (c *Container) fields() common.MapStr {
 	container.maybeSetMapStr("image", common.MapStr(image))
 	return common.MapStr(container)
 }
+
+func (c *Container) MarshalFastJSON(w *fastjson.Writer) error {
+	w.RawByte('{')
+
+	w.RawString(`"id":`)
+	w.String(c.ID)
+
+	w.RawByte('}')
+	return nil
+}

model/datastream.go

@@ -17,7 +17,10 @@
 
 package model
 
-import "github.com/elastic/apm-server/datastreams"
+import (
+	"github.com/elastic/apm-server/datastreams"
+	"go.elastic.co/fastjson"
+)
 
 // DataStream identifies the data stream to which an event will be written.
 type DataStream struct {
@@ -31,8 +34,26 @@ type DataStream struct {
 	Namespace string
 }
 
+func (d DataStream) isZero() bool {
+	return d == (DataStream{})
+}
+
 func (d *DataStream) setFields(fields *mapStr) {
 	fields.maybeSetString(datastreams.TypeField, d.Type)
 	fields.maybeSetString(datastreams.DatasetField, d.Dataset)
 	fields.maybeSetString(datastreams.NamespaceField, d.Namespace)
 }
+
+func (d *DataStream) MarshalFastJSON(w *fastjson.Writer) error {
+	w.RawByte('{')
+
+	w.RawString(`"type":`)
+	w.String(d.Type)
+	w.RawString(`,"dataset":`)
+	w.String(d.Dataset)
+	w.RawString(`,"namespace":`)
+	w.String(d.Namespace)
+
+	w.RawByte('}')
+	return nil
+}

model/destination.go

@@ -29,6 +29,10 @@ type Destination struct {
 	Port    int
 }
 
+func (d Destination) isZero() bool {
+	return d == (Destination{})
+}
+
 func (d *Destination) fields() common.MapStr {
 	var fields mapStr
 	if fields.maybeSetString("address", d.Address) {

model/error.go

@@ -31,33 +31,44 @@ const (
 )
 
 type Error struct {
-	ID string
-
-	GroupingKey string
-	Culprit     string
-	Custom      common.MapStr
+	ID          string        `json:"id,omitempty"`
+	GroupingKey string        `json:"grouping_key,omitempty"`
+	Culprit     string        `json:"culprit,omitempty"`
+	Custom      common.MapStr `json:"custom,omitempty"`
+	Exception   *Exception    `json:"exception,omitempty"`
+	Log         *Log          `json:"log,omitempty"`
+}
 
-	Exception *Exception
-	Log       *Log
+func (e *Error) isZero() bool {
+	return e.ID == "" && e.GroupingKey == "" && e.Culprit == "" && e.Custom == nil && e.Exception == nil && e.Log == nil
 }
 
 type Exception struct {
-	Message    string
-	Module     string
-	Code       string
-	Attributes interface{}
-	Stacktrace Stacktrace
-	Type       string
-	Handled    *bool
-	Cause      []Exception
+	Message    string      `json:"message,omitempty"`
+	Module     string      `json:"module,omitempty"`
+	Code       string      `json:"code,omitempty"`
+	Attributes interface{} `json:"attributes,omitempty"`
+	Stacktrace Stacktrace  `json:"stacktrace,omitempty"`
+	Type       string      `json:"type,omitempty"`
+	Handled    *bool       `json:"handled,omitempty"`
+	Cause      []Exception `json:"cause,omitempty"`
+}
+
+func (e *Exception) isZero() bool {
+	return e.Message == "" && e.Module == "" && e.Code == "" && e.Attributes == nil && e.Stacktrace == nil && e.Type == "" &&
+		e.Cause == nil
 }
 
 type Log struct {
-	Message      string
-	Level        string
-	ParamMessage string
-	LoggerName   string
-	Stacktrace   Stacktrace
+	Message      string     `json:"message,omitempty"`
+	Level        string     `json:"level,omitempty"`
+	ParamMessage string     `json:"param_message,omitempty"`
+	LoggerName   string     `json:"logger_name,omitempty"`
+	Stacktrace   Stacktrace `json:"stacktrace,omitempty"`
+}
+
+func (l *Log) isZero() bool {
+	return l.Message == "" && l.Level == "" && l.ParamMessage == "" && l.LoggerName == "" && l.Stacktrace == nil
 }
 
 func (e *Error) setFields(fields *mapStr) {

model/event.go

@@ -32,10 +32,14 @@ type Event struct {
 	// TODO(axw) emit an `event.duration` field with the duration in
 	// nanoseconds, in 8.0. For now we emit event-specific duration fields.
 	// See https://github.com/elastic/apm-server/issues/5999
-	Duration time.Duration
+	Duration time.Duration `json:"duration,omitempty"`
 
 	// Outcome holds the event outcome: "success", "failure", or "unknown".
-	Outcome string
+	Outcome string `json:"outcome,omitempty"`
+}
+
+func (e Event) isZero() bool {
+	return e == (Event{})
 }
 
 func (e *Event) fields() common.MapStr {

model/experience.go

@@ -22,34 +22,40 @@ import (
 )
 
 // UserExperience holds real user (browser) experience metrics.
+//
+// TODO(axw) update ingest pipeline to remove negative metrics.
 type UserExperience struct {
 	// CumulativeLayoutShift holds the Cumulative Layout Shift (CLS) metric value,
 	// or a negative value if CLS is unknown. See https://web.dev/cls/
-	CumulativeLayoutShift float64
+	CumulativeLayoutShift float64 `json:"cls"`
 
 	// FirstInputDelay holds the First Input Delay (FID) metric value,
 	// or a negative value if FID is unknown. See https://web.dev/fid/
-	FirstInputDelay float64
+	FirstInputDelay float64 `json:"fid"`
 
 	// TotalBlockingTime holds the Total Blocking Time (TBT) metric value,
 	// or a negative value if TBT is unknown. See https://web.dev/tbt/
-	TotalBlockingTime float64
+	TotalBlockingTime float64 `json:"tbt"`
 
 	// Longtask holds longtask metrics. If Longtask.Count is negative,
 	// then Longtask is considered unset. See https://www.w3.org/TR/longtasks/
-	Longtask LongtaskMetrics
+	Longtask LongtaskMetrics `json:"longtask,omitempty"`
 }
 
 // LongtaskMetrics holds metrics related to RUM longtasks.
 type LongtaskMetrics struct {
 	// Count holds the number of longtasks, or a negative value if unknown.
-	Count int
+	Count int `json:"count"`
 
 	// Sum holds the sum of longtask durations.
-	Sum float64
+	Sum float64 `json:"sum"`
 
 	// Max holds the maximum longtask duration.
-	Max float64
+	Max float64 `json:"max"`
+}
+
+func (l LongtaskMetrics) isZero() bool {
+	return l.Count == 0
 }
 
 func (u *UserExperience) Fields() common.MapStr {