Implement basic ILM handling for APM

ILM can be setup by using the `setup --index-management` or the `run` cmd. The templates
and ilm policies can be exported using the `export` cmd.

fixes #1290

_meta/beat.yml

@@ -149,6 +149,12 @@ apm-server:
     # Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
     #overwrite: true
 
+  # When ilm is set to true the APM Server will take care of ILM setup.
+  # Configurations in output.elasticsearch.index and output.elasticsearch.indices sections will be ignored
+  # ilm can only be enabled for output.elasticsearch
+  #ilm:
+    #enabled: false
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.

apm-server.docker.yml

@@ -149,6 +149,12 @@ apm-server:
     # Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
     #overwrite: true
 
+  # When ilm is set to true the APM Server will take care of ILM setup.
+  # Configurations in output.elasticsearch.index and output.elasticsearch.indices sections will be ignored
+  # ilm can only be enabled for output.elasticsearch
+  #ilm:
+    #enabled: false
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.

apm-server.yml

@@ -149,6 +149,12 @@ apm-server:
     # Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
     #overwrite: true
 
+  # When ilm is set to true the APM Server will take care of ILM setup.
+  # Configurations in output.elasticsearch.index and output.elasticsearch.indices sections will be ignored
+  # ilm can only be enabled for output.elasticsearch
+  #ilm:
+    #enabled: false
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.

changelogs/head.asciidoc

@@ -12,7 +12,8 @@
 - Upgrade Go to 1.12.4 {pull}2132[2132].
 - Add geoip processing to the default ingest pipeline {pull}2177[2177].
 - Add ephemeral_id attribute in the metadata {pull}2179[2179].
-- Add `setup --index-management` cmd {pull}[2180].
+- Add basic ILM support for APM Server, using fixed policies {pull}2099[2099].
+- Add `setup --index-management` cmd, deprecate `setup --template` cmd {pull}2180[2180],{pull}2099[2099].
 
 [float]
 ==== Removed

cmd/root.go

@@ -18,11 +18,12 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/spf13/pflag"
 
 	"github.com/elastic/apm-server/beater"
 	"github.com/elastic/apm-server/idxmgmt"
-	"github.com/elastic/apm-server/idxmgmt/ilm"
 	_ "github.com/elastic/apm-server/include"
 	"github.com/elastic/beats/libbeat/cmd"
 	"github.com/elastic/beats/libbeat/cmd/instance"
@@ -33,6 +34,8 @@ import (
 
 // Name of the beat (apm-server).
 const Name = "apm-server"
+
+// IdxPattern for apm
 const IdxPattern = "apm"
 
 // RootCmd for running apm-server.
@@ -68,7 +71,7 @@ func init() {
 	})
 
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
-	RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{
+	settings := instance.Settings{
 		ConfigOverrides: overrides,
 		Name:            Name,
 		IndexPrefix:     IdxPattern,
@@ -77,27 +80,39 @@ func init() {
 		Monitoring: report.Settings{
 			DefaultUsername: "apm_system",
 		},
-		ILM:             ilm.MakeDefaultSupporter,
 		IndexManagement: idxmgmt.MakeDefaultSupporter,
 		Processing:      processing.MakeDefaultObserverSupport(false),
-	})
-	// remove dashboard from export commands
+	}
+	RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings)
+
 	for _, cmd := range RootCmd.ExportCmd.Commands() {
+
+		// remove `dashboard` from `export` commands
 		if cmd.Name() == "dashboard" {
 			RootCmd.ExportCmd.RemoveCommand(cmd)
+			continue
+		}
+
+		// only add defined flags to `export template` command
+		if cmd.Name() == "template" {
+			cmd.ResetFlags()
+			cmd.Flags().String("es.version", settings.Version, "Elasticsearch version")
+			cmd.Flags().String("dir", "", "Specify directory for printing template files. By default templates are printed to stdout.")
 		}
 	}
 	// only add defined flags to setup command
 	setup := RootCmd.SetupCmd
-	setup.Short = "Setup Elasticsearch index template and pipelines"
+	setup.Short = "Setup Elasticsearch index management components and pipelines"
 	setup.Long = `This command does initial setup of the environment:
 
- * Index mapping template in Elasticsearch to ensure fields are mapped.
+ * Index management including loading Elasticsearch templates, ILM policies and write aliases.
  * Ingest pipelines
 `
 	setup.ResetFlags()
 	//lint:ignore SA1019 Setting up template must still be supported until next major version upgrade.
 	setup.Flags().Bool(cmd.TemplateKey, false, "Setup index template")
+	setup.Flags().MarkDeprecated(cmd.TemplateKey, fmt.Sprintf("please use --%s instead", cmd.IndexManagementKey))
 	setup.Flags().Bool(cmd.IndexManagementKey, false, "Setup Elasticsearch index management")
 	setup.Flags().Bool(cmd.PipelineKey, false, "Setup ingest pipelines")
+
 }

idxmgmt/ilm/config.go

@@ -0,0 +1,32 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ilm
+
+import (
+	"github.com/elastic/beats/libbeat/common/fmtstr"
+)
+
+const pattern = "000001"
+
+//Config for ILM supporter
+type Config struct {
+	Enabled    bool                      `config:"enabled"`
+	PolicyName *fmtstr.EventFormatString `config:"policy_name"`
+	AliasName  *fmtstr.EventFormatString `config:"alias_name"`
+	Event      string                    `config:"event"`
+}

idxmgmt/ilm/policy.go

@@ -0,0 +1,91 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ilm
+
+import "github.com/elastic/beats/libbeat/common"
+
+type m common.MapStr
+
+var eventPolicies = map[string]common.MapStr{
+	"error":       PolicyKeepMedium,
+	"span":        PolicyKeepMedium,
+	"transaction": PolicyKeep,
+	"metric":      PolicyKeep,
+}
+
+//PolicyKeep should be used for indices queried max for 2 months
+var PolicyKeep = common.MapStr{
+	"policy": m{
+		"phases": m{
+			"hot": m{
+				"actions": m{
+					"rollover": m{
+						"max_size": "50gb",
+						"max_age":  "7d",
+					},
+					"set_priority": m{
+						"priority": 100,
+					},
+				},
+			},
+			"warm": m{
+				"min_age": "31d",
+				"actions": m{
+					"set_priority": m{
+						"priority": 50,
+					},
+					"readonly": m{},
+					"forcemerge": m{
+						"max_num_segments": 1,
+					},
+				},
+			},
+		},
+	},
+}
+
+//PolicyKeepMedium should be used for indices that need to be queried max for two weeks
+var PolicyKeepMedium = common.MapStr{
+	"policy": m{
+		"phases": m{
+			"hot": m{
+				"actions": m{
+					"rollover": m{
+						"max_size": "50gb",
+						"max_age":  "1d",
+					},
+					"set_priority": m{
+						"priority": 100,
+					},
+				},
+			},
+			"warm": m{
+				"min_age": "7d",
+				"actions": m{
+					"set_priority": m{
+						"priority": 50,
+					},
+					"readonly": m{},
+					"forcemerge": m{
+						"max_num_segments": 1,
+					},
+				},
+			},
+		},
+	},
+}

idxmgmt/ilm/supporter_factory.go

@@ -18,14 +18,71 @@
 package ilm
 
 import (
+	"time"
+
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
-	"github.com/elastic/beats/libbeat/idxmgmt/ilm"
+	"github.com/elastic/beats/libbeat/common/fmtstr"
+	libilm "github.com/elastic/beats/libbeat/idxmgmt/ilm"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
 // MakeDefaultSupporter creates the ILM supporter for APM that is passed to libbeat.
-// Currently ILM is disabled by using a NoopSupport.
-func MakeDefaultSupporter(log *logp.Logger, info beat.Info, config *common.Config) (ilm.Supporter, error) {
-	return ilm.NoopSupport(log, info, config)
+func MakeDefaultSupporter(log *logp.Logger, info beat.Info, cfg *common.Config) (libilm.Supporter, error) {
+	if log == nil {
+		log = logp.NewLogger("ilm")
+	} else {
+		log = log.Named("ilm")
+	}
+
+	var ilmCfg Config
+	if err := cfg.Unpack(&ilmCfg); err != nil {
+		return nil, err
+	}
+	if ilmCfg.AliasName == nil || ilmCfg.PolicyName == nil {
+		return nil, errors.New("ilm alias and policy must be configured")
+	}
+	aliasName, err := applyStaticFmtstr(info, ilmCfg.AliasName)
+	if err != nil {
+		return nil, err
+	}
+
+	policyName, err := applyStaticFmtstr(info, ilmCfg.PolicyName)
+	if err != nil {
+		return nil, err
+	}
+
+	p, ok := eventPolicies[ilmCfg.Event]
+	if !ok {
+		return nil, errors.Errorf("policy for %s undefined", ilmCfg.Event)
+	}
+	policy := libilm.Policy{
+		Name: policyName,
+		Body: p,
+	}
+	alias := libilm.Alias{
+		Name:    aliasName,
+		Pattern: pattern,
+	}
+
+	return libilm.NewStdSupport(log, libilm.ModeEnabled, alias, policy, false, true), nil
+}
+
+func applyStaticFmtstr(info beat.Info, fmt *fmtstr.EventFormatString) (string, error) {
+	return fmt.Run(&beat.Event{
+		Fields: common.MapStr{
+			// beat object was left in for backward compatibility reason for older configs.
+			"beat": common.MapStr{
+				"name":    info.Beat,
+				"version": info.Version,
+			},
+			"observer": common.MapStr{
+				"name":    info.Beat,
+				"version": info.Version,
+			},
+		},
+		Timestamp: time.Now(),
+	})
 }