Merge branch 'roobert-graphite-template-custom-field'

Closes #4178

CHANGELOG.md

@@ -13,6 +13,8 @@
 - [#4265](https://github.com/influxdb/influxdb/pull/4265): Add statistics for Hinted-Handoff
 - [#4284](https://github.com/influxdb/influxdb/pull/4284): Add exponential backoff for hinted-handoff failures
 - [#4310](https://github.com/influxdb/influxdb/pull/4310): Support dropping non-Raft nodes. Work mostly by @corylanou
+- [#4348](https://github.com/influxdb/influxdb/pull/4348): Public ApplyTemplate function for graphite parser.
+- [#4178](https://github.com/influxdb/influxdb/pull/4178): Support fields in graphite parser. Thanks @roobert!
 
 ### Bugfixes
 - [#4166](https://github.com/influxdb/influxdb/pull/4166): Fix parser error on invalid SHOW

services/graphite/README.md

@@ -48,6 +48,39 @@ Additional tags can be added to a metric that don't exist on the received metric
 * Template: `.host.resource.measurement* region=us-west,zone=1a`
 * Output:  _measurement_ = `loadavg.10` _tags_ = `host=localhost resource=cpu region=us-west zone=1a`
 
+### Fields
+
+A field name can be specified by using the keyword _field_. By default if no _field_ keyword is specified then the metric will be written to a field named _value_.
+
+When using the current default engine _BZ1_, it's recommended to use a single field per value for performance reasons.
+
+When using the _TSM1_ engine it's possible to amend measurement metrics with additional fields, e.g:
+
+Input:
+```
+sensu.metric.net.server0.eth0.rx_packets 461295119435 1444234982
+sensu.metric.net.server0.eth0.tx_bytes 1093086493388480 1444234982
+sensu.metric.net.server0.eth0.rx_bytes 1015633926034834 1444234982
+sensu.metric.net.server0.eth0.tx_errors 0 1444234982
+sensu.metric.net.server0.eth0.rx_errors 0 1444234982
+sensu.metric.net.server0.eth0.tx_dropped 0 1444234982
+sensu.metric.net.server0.eth0.rx_dropped 0 1444234982
+```
+
+With template:
+```
+sensu.metric.* ..measurement.host.interface.field
+```
+
+Becomes database entry:
+```
+> select * from net
+name: net
+---------
+time      host  interface rx_bytes    rx_dropped  rx_errors rx_packets    tx_bytes    tx_dropped  tx_errors
+1444234982000000000 server0  eth0    1.015633926034834e+15 0   0   4.61295119435e+11 1.09308649338848e+15  0 0
+```
+
 ## Multiple Templates
 
 One template may not match all metrics.  For example, using multiple plugins with diamond will produce metrics in different formats.  If you need to use multiple templates, you'll need to define a prefix filter that must match before the template can be applied.
@@ -119,13 +152,16 @@ If you need to add the same set of tags to all metrics, you can define them glob
    separator = "_"
    tags = ["region=us-east", "zone=1c"]
    templates = [
-      # filter + template
-      "*.app env.service.resource.measurement",
+     # filter + template
+     "*.app env.service.resource.measurement",
 
      # filter + template + extra tag
      "stats.* .host.measurement* region=us-west,agent=sensu",
 
-      # default template. Ignore the first graphite component "servers"
+     # filter + template with field name
+     "stats.* .host.measurement.field",
+
+     # default template. Ignore the first graphite component "servers"
      ".measurement*",
  ]
 ```

services/graphite/parser.go

@@ -100,7 +100,10 @@ func (p *Parser) Parse(line string) (models.Point, error) {
 
 	// decode the name and tags
 	template := p.matcher.Match(fields[0])
-	measurement, tags := template.Apply(fields[0])
+	measurement, tags, field, err := template.Apply(fields[0])
+	if err != nil {
+		return nil, err
+	}
 
 	// Could not extract measurement, use the raw value
 	if measurement == "" {
@@ -113,7 +116,12 @@ func (p *Parser) Parse(line string) (models.Point, error) {
 		return nil, fmt.Errorf(`field "%s" value: %s`, fields[0], err)
 	}
 
-	fieldValues := map[string]interface{}{"value": v}
+	fieldValues := map[string]interface{}{}
+	if field != "" {
+		fieldValues[field] = v
+	} else {
+		fieldValues["value"] = v
+	}
 
 	// If no 3rd field, use now as timestamp
 	timestamp := time.Now().UTC()
@@ -149,11 +157,11 @@ func (p *Parser) Parse(line string) (models.Point, error) {
 
 // Apply extracts the template fields form the given line and returns the
 // measurement name and tags
-func (p *Parser) ApplyTemplate(line string) (string, map[string]string) {
+func (p *Parser) ApplyTemplate(line string) (string, map[string]string, string, error) {
 	// Break line into fields (name, value, timestamp), only name is used
 	fields := strings.Fields(line)
 	if len(fields) == 0 {
-		return "", make(map[string]string)
+		return "", make(map[string]string), "", nil
 	}
 	// decode the name and tags
 	template := p.matcher.Match(fields[0])
@@ -198,11 +206,12 @@ func NewTemplate(pattern string, defaultTags models.Tags, separator string) (*te
 
 // Apply extracts the template fields form the given line and returns the measurement
 // name and tags
-func (t *template) Apply(line string) (string, map[string]string) {
+func (t *template) Apply(line string) (string, map[string]string, string, error) {
 	fields := strings.Split(line, ".")
 	var (
 		measurement []string
 		tags        = make(map[string]string)
+		field       string
 	)
 
 	// Set any default tags
@@ -217,6 +226,12 @@ func (t *template) Apply(line string) (string, map[string]string) {
 
 		if tag == "measurement" {
 			measurement = append(measurement, fields[i])
+		} else if tag == "field" {
+			if len(field) != 0 {
+				return "", nil, "", fmt.Errorf("'field' can only be used once in each template: %q", line)
+			} else {
+				field = fields[i]
+			}
 		} else if tag == "measurement*" {
 			measurement = append(measurement, fields[i:]...)
 			break
@@ -225,7 +240,7 @@ func (t *template) Apply(line string) (string, map[string]string) {
 		}
 	}
 
-	return strings.Join(measurement, t.separator), tags
+	return strings.Join(measurement, t.separator), tags, field, nil
 }
 
 // matcher determines which template should be applied to a given metric

services/graphite/parser_test.go

@@ -105,7 +105,7 @@ func TestTemplateApply(t *testing.T) {
 			continue
 		}
 
-		measurement, tags := tmpl.Apply(test.input)
+		measurement, tags, _, _ := tmpl.Apply(test.input)
 		if measurement != test.measurement {
 			t.Fatalf("name parse failer.  expected %v, got %v", test.measurement, measurement)
 		}
@@ -558,7 +558,7 @@ func TestApplyTemplate(t *testing.T) {
 		t.Fatalf("unexpected error creating parser, got %v", err)
 	}
 
-	measurement, _ := p.ApplyTemplate("current.users")
+	measurement, _, _, _ := p.ApplyTemplate("current.users")
 	if measurement != "current_users" {
 		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
 			measurement, "current_users")
@@ -576,7 +576,7 @@ func TestApplyTemplateNoMatch(t *testing.T) {
 		t.Fatalf("unexpected error creating parser, got %v", err)
 	}
 
-	measurement, _ := p.ApplyTemplate("current.users")
+	measurement, _, _, _ := p.ApplyTemplate("current.users")
 	if measurement != "current.users" {
 		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
 			measurement, "current.users")
@@ -597,7 +597,7 @@ func TestApplyTemplateSpecific(t *testing.T) {
 		t.Fatalf("unexpected error creating parser, got %v", err)
 	}
 
-	measurement, tags := p.ApplyTemplate("current.users.facebook")
+	measurement, tags, _, _ := p.ApplyTemplate("current.users.facebook")
 	if measurement != "current_users" {
 		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
 			measurement, "current_users")
@@ -621,7 +621,7 @@ func TestApplyTemplateTags(t *testing.T) {
 		t.Fatalf("unexpected error creating parser, got %v", err)
 	}
 
-	measurement, tags := p.ApplyTemplate("current.users")
+	measurement, tags, _, _ := p.ApplyTemplate("current.users")
 	if measurement != "current_users" {
 		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
 			measurement, "current_users")
@@ -635,3 +635,43 @@ func TestApplyTemplateTags(t *testing.T) {
 		t.Errorf("Expected region='us-west' tag, got region='%s'", region)
 	}
 }
+
+func TestApplyTemplateField(t *testing.T) {
+	o := graphite.Options{
+		Separator: "_",
+		Templates: []string{"current.* measurement.measurement.field"},
+	}
+	p, err := graphite.NewParserWithOptions(o)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	measurement, _, field, err := p.ApplyTemplate("current.users.logged_in")
+
+	if measurement != "current_users" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			measurement, "current_users")
+	}
+
+	if field != "logged_in" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			field, "logged_in")
+	}
+}
+
+func TestApplyTemplateFieldError(t *testing.T) {
+	o := graphite.Options{
+		Separator: "_",
+		Templates: []string{"current.* measurement.field.field"},
+	}
+	p, err := graphite.NewParserWithOptions(o)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	_, _, _, err = p.ApplyTemplate("current.users.logged_in")
+	if err == nil {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s", err,
+			"'field' can only be used once in each template: current.users.logged_in")
+	}
+}