[Metrics builder] Move resource creation to the generated code (#8555)

This change updates metrics builder to fully control creation of the metrics object. Scraper author don't need to create pdata.Metrics object and pass it to MetricsBuiler.Emit function.

This change also moves definition of resource attributes to metadata.yaml which is used to generate attribute helper that can be used in the scraper.

CHANGELOG.md

@@ -18,6 +18,11 @@
 
 - `resourcedetectionprocessor`: Add attribute allowlist (#8547)
 
+### 💡 Enhancements 💡
+
+- `cmd/mdatagen`: Add resource attributes definition to metadata.yaml and move `pdata.Metrics` creation to the
+  generated code (#5270) 
+
 ## v0.47.0
 
 ### 💡 Enhancements 💡

cmd/mdatagen/documentation.tmpl

@@ -25,7 +25,18 @@ metrics:
 ```
 {{- end }}
 
-## Attributes
+{{- if .ResourceAttributes }}
+
+## Resource attributes
+
+| Name | Description | Type |
+| ---- | ----------- | ---- |
+{{- range $attributeName, $attributeInfo := .ResourceAttributes }}
+| {{ $attributeName }} | {{ $attributeInfo.Description }} | {{ $attributeInfo.Type }} |
+{{- end }}
+{{- end }}
+
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |

cmd/mdatagen/loader.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-playground/validator/v10/non-standard/validators"
 	en_translations "github.com/go-playground/validator/v10/translations/en"
 	"go.opentelemetry.io/collector/config/mapprovider/filemapprovider"
+	"go.opentelemetry.io/collector/model/pdata"
 )
 
 type metricName string
@@ -48,6 +49,56 @@ func (mn attributeName) RenderUnexported() (string, error) {
 	return formatIdentifier(string(mn), false)
 }
 
+// ValueType defines an attribute value type.
+type ValueType struct {
+	// ValueType is type of the metric number, options are "double", "int".
+	ValueType pdata.ValueType
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (mvt *ValueType) UnmarshalText(text []byte) error {
+	switch vtStr := string(text); vtStr {
+	case "":
+		mvt.ValueType = pdata.ValueTypeEmpty
+	case "string":
+		mvt.ValueType = pdata.ValueTypeString
+	case "int":
+		mvt.ValueType = pdata.ValueTypeInt
+	case "double":
+		mvt.ValueType = pdata.ValueTypeDouble
+	case "bool":
+		mvt.ValueType = pdata.ValueTypeDouble
+	case "bytes":
+		mvt.ValueType = pdata.ValueTypeDouble
+	default:
+		return fmt.Errorf("invalid type: %q", vtStr)
+	}
+	return nil
+}
+
+// String returns capitalized name of the ValueType.
+func (mvt ValueType) String() string {
+	return strings.Title(strings.ToLower(mvt.ValueType.String()))
+}
+
+// Primitive returns name of primitive type for the ValueType.
+func (mvt ValueType) Primitive() string {
+	switch mvt.ValueType {
+	case pdata.ValueTypeString:
+		return "string"
+	case pdata.ValueTypeInt:
+		return "int64"
+	case pdata.ValueTypeDouble:
+		return "float64"
+	case pdata.ValueTypeBool:
+		return "bool"
+	case pdata.ValueTypeBytes:
+		return "[]byte"
+	default:
+		return ""
+	}
+}
+
 type metric struct {
 	// Enabled defines whether the metric is enabled by default.
 	Enabled *bool `yaml:"enabled" validate:"required"`
@@ -99,11 +150,15 @@ type attribute struct {
 	Value string
 	// Enum can optionally describe the set of values to which the attribute can belong.
 	Enum []string
+	// Type is an attribute type.
+	Type ValueType `mapstructure:"type"`
 }
 
 type metadata struct {
 	// Name of the component.
 	Name string `validate:"notblank"`
+	// ResourceAttributes that can be emitted by the component.
+	ResourceAttributes map[attributeName]attribute `mapstructure:"resource_attributes" validate:"dive"`
 	// Attributes emitted by one or more metrics.
 	Attributes map[attributeName]attribute `validate:"dive"`
 	// Metrics that can be emitted by the component.

cmd/mdatagen/metric-metadata.yaml

@@ -1,6 +1,14 @@
 # Required: name of the receiver.
 name:
 
+# Optional: map of resource attribute definitions with the key being the attribute name.
+resource_attributes:
+  <attribute.name>:
+    # Required: description of the attribute.
+    description:
+    # Required: attribute type.
+    type: <string|int|double|bool|bytes>
+
 # Optional: map of attribute definitions with the key being the attribute name and value
 # being described below.
 attributes:

cmd/mdatagen/metrics_v2.tmpl

@@ -101,7 +101,10 @@ func newMetric{{ $name.Render }}(settings MetricSettings) metric{{ $name.Render
 // MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations
 // required to produce metric representation defined in metadata and user settings.
 type MetricsBuilder struct {
-	startTime                pdata.Timestamp
+	startTime                pdata.Timestamp // start time that will be applied to all recorded data points.
+	metricsCapacity          int             // maximum observed number of metrics per resource.
+	resourceCapacity         int             // maximum observed number of resource attributes.
+	metricsBuffer            pdata.Metrics   // accumulates metrics data before emitting.
 	{{- range $name, $metric := .Metrics }}
 	metric{{ $name.Render }} metric{{ $name.Render }}
 	{{- end }}
@@ -120,6 +123,7 @@ func WithStartTime(startTime pdata.Timestamp) metricBuilderOption {
 func NewMetricsBuilder(settings MetricsSettings, options ...metricBuilderOption) *MetricsBuilder {
 	mb := &MetricsBuilder{
 		startTime:                pdata.NewTimestampFromTime(time.Now()),
+		metricsBuffer:            pdata.NewMetrics(),
 		{{- range $name, $metric := .Metrics }}
 		metric{{ $name.Render }}: newMetric{{ $name.Render }}(settings.{{ $name.Render }}),
 		{{- end }}
@@ -130,13 +134,58 @@ func NewMetricsBuilder(settings MetricsSettings, options ...metricBuilderOption)
 	return mb
 }
 
-// Emit appends generated metrics to a pdata.MetricsSlice and updates the internal state to be ready for recording
-// another set of data points. This function will be doing all transformations required to produce metric representation
-// defined in metadata and user settings, e.g. delta/cumulative translation.
-func (mb *MetricsBuilder) Emit(metrics pdata.MetricSlice) {
+// updateCapacity updates max length of metrics and resource attributes that will be used for the slice capacity.
+func (mb *MetricsBuilder) updateCapacity(rm pdata.ResourceMetrics) {
+	if mb.metricsCapacity < rm.InstrumentationLibraryMetrics().At(0).Metrics().Len() {
+		mb.metricsCapacity = rm.InstrumentationLibraryMetrics().At(0).Metrics().Len()
+	}
+	if mb.resourceCapacity < rm.Resource().Attributes().Len() {
+		mb.resourceCapacity = rm.Resource().Attributes().Len()
+	}
+}
+
+// ResourceOption applies changes to provided resource.
+type ResourceOption func(pdata.Resource)
+
+{{- range $name, $attr := .ResourceAttributes }}
+// With{{ $name.Render }} sets provided value as "{{ $name }}" attribute for current resource.
+func With{{ $name.Render }}(val {{ $attr.Type.Primitive }}) ResourceOption {
+	return func(r pdata.Resource) {
+		r.Attributes().Upsert{{ $attr.Type }}("{{ $name }}", val)
+	}
+}
+{{ end }}
+
+// EmitForResource saves all the generated metrics under a new resource and updates the internal state to be ready for
+// recording another set of data points as part of another resource. This function can be helpful when one scraper
+// needs to emit metrics from several resources. Otherwise calling this function is not required,
+// just `Emit` function can be called instead. Resource attributes should be provided as ResourceOption arguments.
+func (mb *MetricsBuilder) EmitForResource(ro ...ResourceOption) {
+	rm := pdata.NewResourceMetrics()
+	rm.Resource().Attributes().EnsureCapacity(mb.resourceCapacity)
+	for _, op := range ro {
+		op(rm.Resource())
+	}
+	ils := rm.InstrumentationLibraryMetrics().AppendEmpty()
+	ils.InstrumentationLibrary().SetName("otelcol/{{ .Name }}")
+	ils.Metrics().EnsureCapacity(mb.metricsCapacity)
 	{{- range $name, $metric := .Metrics }}
-	mb.metric{{- $name.Render }}.emit(metrics)
+	mb.metric{{- $name.Render }}.emit(ils.Metrics())
 	{{- end }}
+	if ils.Metrics().Len() > 0 {
+		mb.updateCapacity(rm)
+		rm.MoveTo(mb.metricsBuffer.ResourceMetrics().AppendEmpty())
+	}
+}
+
+// Emit returns all the metrics accumulated by the metrics builder and updates the internal state to be ready for
+// recording another set of metrics. This function will be responsible for applying all the transformations required to
+// produce metric representation defined in metadata and user settings, e.g. delta or cumulative.
+func (mb *MetricsBuilder) Emit(ro ...ResourceOption) pdata.Metrics {
+	mb.EmitForResource(ro...)
+	metrics := pdata.NewMetrics()
+	mb.metricsBuffer.MoveTo(metrics)
+	return metrics
 }
 
 {{ range $name, $metric := .Metrics -}}
@@ -159,16 +208,6 @@ func (mb *MetricsBuilder) Reset(options ...metricBuilderOption) {
 	}
 }
 
-// NewMetricData creates new pdata.Metrics and sets the InstrumentationLibrary
-// name on the ResourceMetrics.
-func (mb *MetricsBuilder) NewMetricData() pdata.Metrics {
-	md := pdata.NewMetrics()
-	rm := md.ResourceMetrics().AppendEmpty()
-	ilm := rm.InstrumentationLibraryMetrics().AppendEmpty()
-	ilm.InstrumentationLibrary().SetName("otelcol/{{ .Name }}")
-	return md
-}
-
 // Attributes contains the possible metric attributes that can be used.
 var Attributes = struct {
 {{- range $name, $info := .Attributes }}

receiver/apachereceiver/documentation.md

@@ -24,7 +24,7 @@ metrics:
     enabled: <true|false>
 ```
 
-## Attributes
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |

receiver/apachereceiver/scraper.go

@@ -103,9 +103,7 @@ func (r *apacheScraper) scrape(context.Context) (pdata.Metrics, error) {
 		}
 	}
 
-	md := r.mb.NewMetricData()
-	r.mb.Emit(md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics())
-	return md, nil
+	return r.mb.Emit(), nil
 }
 
 // GetStats collects metric stats by making a get request at an endpoint.

receiver/couchbasereceiver/documentation.md

@@ -18,7 +18,7 @@ metrics:
     enabled: <true|false>
 ```
 
-## Attributes
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |