From 8b2067ecf634dfe284b6a7ebc683582947fb6215 Mon Sep 17 00:00:00 2001
From: Patrick Hemmer <patrick.hemmer@gmail.com>
Date: Thu, 29 Oct 2020 12:42:26 -0400
Subject: [PATCH] add template-error-fatal

---
 README.md               |  9 +++++++++
 cli.go                  | 16 +++++++++++++++-
 config/config.go        | 19 +++++++++++++++++++
 config/template.go      | 16 ++++++++++++++++
 config/template_test.go |  1 +
 manager/runner.go       | 39 +++++++++++++++++++++++++++++++++------
 template/template.go    | 14 ++++++++++++++
 7 files changed, 107 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 5f34ed522..d65efbe88 100644
--- a/README.md
+++ b/README.md
@@ -374,6 +374,11 @@ log_level = "warn"
 # to the process.
 pid_file = "/path/to/pid"
 
+# This controls whether an error within a template will cause consul-template
+# to immediately exit. This value can be overridden within each template
+# configuration.
+template_error_fatal = true
+
 # This is the quiescence timers; it defines the minimum and maximum amount of
 # time to wait for the cluster to reach a consistent state before rendering a
 # template. This is useful to enable in systems that have a lot of flapping,
@@ -585,6 +590,10 @@ template {
   # retrieving secrets from Vault.
   error_on_missing_key = false
 
+  # This controls whether an error within the template will cause
+  # consul-template to immediately exit.
+  error_fatal = true
+
   # This is the permission to render the file. If this option is left
   # unspecified, Consul Template will attempt to match the permissions of the
   # file that already exists at the destination path. If no file exists at that
diff --git a/cli.go b/cli.go
index f038ff118..00a8f67d1 100644
--- a/cli.go
+++ b/cli.go
@@ -433,6 +433,11 @@ func (cli *CLI) ParseFlags(args []string) (
 		return nil
 	}), "template", "")
 
+	flags.Var((funcBoolVar)(func(b bool) error {
+		c.TemplateErrFatal = config.Bool(b)
+		return nil
+	}), "template-error-fatal", "")
+
 	flags.Var((funcVar)(func(s string) error {
 		c.Vault.Address = config.String(s)
 		return nil
@@ -581,6 +586,11 @@ func loadConfigs(paths []string, o *config.Config) (*config.Config, error) {
 	}
 
 	finalC = finalC.Merge(o)
+	if o.TemplateErrFatal != nil {
+		for _, tmpl := range *finalC.Templates {
+			tmpl.ErrFatal = o.TemplateErrFatal
+		}
+	}
 	finalC.Finalize()
 	return finalC, nil
 }
@@ -746,7 +756,11 @@ Options:
       attribute is supplied, the -syslog flag must also be supplied
 
   -template=<template>
-       Adds a new template to watch on disk in the format 'in:out(:command)'
+      Adds a new template to watch on disk in the format 'in:out(:command)'
+
+  -template-error-fatal=<bool>
+      Control whether template errors cause consul-template to immediately exit.
+      This overrides the per-template setting.
 
   -vault-addr=<address>
       Sets the address of the Vault server
diff --git a/config/config.go b/config/config.go
index a4d31d5a9..48f7bf3cd 100644
--- a/config/config.go
+++ b/config/config.go
@@ -81,6 +81,10 @@ type Config struct {
 	// Templates is the list of templates.
 	Templates *TemplateConfigs `mapstructure:"template"`
 
+	// TemplateErrFatal determines whether template errors should cause the
+	// process to exit, or just log and continue.
+	TemplateErrFatal *bool `mapstructure:"template_error_fatal"`
+
 	// Vault is the configuration for connecting to a vault server.
 	Vault *VaultConfig `mapstructure:"vault"`
 
@@ -139,6 +143,10 @@ func (c *Config) Copy() *Config {
 		o.Templates = c.Templates.Copy()
 	}
 
+	if c.TemplateErrFatal != nil {
+		o.TemplateErrFatal = c.TemplateErrFatal
+	}
+
 	if c.Vault != nil {
 		o.Vault = c.Vault.Copy()
 	}
@@ -214,6 +222,10 @@ func (c *Config) Merge(o *Config) *Config {
 		r.Templates = r.Templates.Merge(o.Templates)
 	}
 
+	if o.TemplateErrFatal != nil {
+		r.TemplateErrFatal = o.TemplateErrFatal
+	}
+
 	if o.Vault != nil {
 		r.Vault = r.Vault.Merge(o.Vault)
 	}
@@ -414,6 +426,7 @@ func (c *Config) GoString() string {
 		"ReloadSignal:%s, "+
 		"Syslog:%#v, "+
 		"Templates:%#v, "+
+		"TemplateErrFatal:%#v"+
 		"Vault:%#v, "+
 		"Wait:%#v,"+
 		"Once:%#v"+
@@ -430,6 +443,7 @@ func (c *Config) GoString() string {
 		SignalGoString(c.ReloadSignal),
 		c.Syslog,
 		c.Templates,
+		c.TemplateErrFatal,
 		c.Vault,
 		c.Wait,
 		c.Once,
@@ -532,6 +546,11 @@ func (c *Config) Finalize() {
 	if c.Templates == nil {
 		c.Templates = DefaultTemplateConfigs()
 	}
+	for _, tmpl := range *c.Templates {
+		if tmpl.ErrFatal == nil {
+			tmpl.ErrFatal = c.TemplateErrFatal
+		}
+	}
 	c.Templates.Finalize()
 
 	if c.Vault == nil {
diff --git a/config/template.go b/config/template.go
index a42e4d87f..7eb77c1c5 100644
--- a/config/template.go
+++ b/config/template.go
@@ -55,6 +55,10 @@ type TemplateConfig struct {
 	// to index a struct or map key that does not exist.
 	ErrMissingKey *bool `mapstructure:"error_on_missing_key"`
 
+	// ErrFatal determines whether template errors should cause the process to
+	// exit, or just log and continue.
+	ErrFatal *bool `mapstructure:"error_fatal"`
+
 	// Exec is the configuration for the command to run when the template renders
 	// successfully.
 	Exec *ExecConfig `mapstructure:"exec"`
@@ -123,6 +127,8 @@ func (c *TemplateConfig) Copy() *TemplateConfig {
 
 	o.ErrMissingKey = c.ErrMissingKey
 
+	o.ErrFatal = c.ErrFatal
+
 	if c.Exec != nil {
 		o.Exec = c.Exec.Copy()
 	}
@@ -197,6 +203,10 @@ func (c *TemplateConfig) Merge(o *TemplateConfig) *TemplateConfig {
 		r.ErrMissingKey = o.ErrMissingKey
 	}
 
+	if o.ErrFatal != nil {
+		r.ErrFatal = o.ErrFatal
+	}
+
 	if o.Exec != nil {
 		r.Exec = r.Exec.Merge(o.Exec)
 	}
@@ -267,6 +277,10 @@ func (c *TemplateConfig) Finalize() {
 		c.ErrMissingKey = Bool(false)
 	}
 
+	if c.ErrFatal == nil {
+		c.ErrFatal = Bool(true)
+	}
+
 	if c.Exec == nil {
 		c.Exec = DefaultExecConfig()
 	}
@@ -327,6 +341,7 @@ func (c *TemplateConfig) GoString() string {
 		"CreateDestDirs:%s, "+
 		"Destination:%s, "+
 		"ErrMissingKey:%s, "+
+		"ErrFatal:%s, "+
 		"Exec:%#v, "+
 		"Perms:%s, "+
 		"Source:%s, "+
@@ -343,6 +358,7 @@ func (c *TemplateConfig) GoString() string {
 		BoolGoString(c.CreateDestDirs),
 		StringGoString(c.Destination),
 		BoolGoString(c.ErrMissingKey),
+		BoolGoString(c.ErrFatal),
 		c.Exec,
 		FileModeGoString(c.Perms),
 		StringGoString(c.Source),
diff --git a/config/template_test.go b/config/template_test.go
index e233e25c5..247df738c 100644
--- a/config/template_test.go
+++ b/config/template_test.go
@@ -427,6 +427,7 @@ func TestTemplateConfig_Finalize(t *testing.T) {
 				CreateDestDirs: Bool(true),
 				Destination:    String(""),
 				ErrMissingKey:  Bool(false),
+				ErrFatal:       Bool(true),
 				Exec: &ExecConfig{
 					Command: String(""),
 					Enabled: Bool(false),
diff --git a/manager/runner.go b/manager/runner.go
index 7b0d6dbcd..6eb6adfca 100644
--- a/manager/runner.go
+++ b/manager/runner.go
@@ -10,15 +10,16 @@ import (
 	"sync"
 	"time"
 
+	multierror "github.com/hashicorp/go-multierror"
+	shellwords "github.com/mattn/go-shellwords"
+	"github.com/pkg/errors"
+
 	"github.com/hashicorp/consul-template/child"
 	"github.com/hashicorp/consul-template/config"
 	dep "github.com/hashicorp/consul-template/dependency"
 	"github.com/hashicorp/consul-template/renderer"
 	"github.com/hashicorp/consul-template/template"
 	"github.com/hashicorp/consul-template/watch"
-	multierror "github.com/hashicorp/go-multierror"
-	shellwords "github.com/mattn/go-shellwords"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -167,6 +168,9 @@ type RenderEvent struct {
 	// been rendered we need to know if the event is triggered by quiesence
 	// and if we can skip evaluating it as a render event for those purposes
 	ForQuiescence bool
+
+	// Error contains the error encountered while rendering the template.
+	Error error
 }
 
 // NewRunner accepts a slice of TemplateConfigs and returns a pointer to the new
@@ -639,7 +643,8 @@ type templateRunCtx struct {
 // template to run and a shared run context that allows sharing of information
 // between templates. The run returns a potentially nil render event and any
 // error that occured. The render event is nil in the case that the template has
-// been already rendered and is a once template or if there is an error.
+// been already rendered and is a once template or if there is an error and
+// fatal errors are enabled.
 func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (*RenderEvent, error) {
 	log.Printf("[DEBUG] (runner) checking template %s", tmpl.ID())
 
@@ -686,7 +691,23 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (*
 		Env:   r.childEnv(),
 	})
 	if err != nil {
-		return nil, errors.Wrap(err, tmpl.Source())
+		if tmpl.ErrFatal() {
+			return nil, errors.Wrap(err, tmpl.Source())
+		}
+		log.Printf("[ERR] (runner) %s: %v", tmpl.Source(), err)
+		event.Error = err
+
+		if lastEvent != nil {
+			// Keep watching our dependencies so that we retry when they update.
+			for _, d := range lastEvent.UsedDeps.List() {
+				if _, ok := runCtx.depsMap[d.String()]; !ok {
+					runCtx.depsMap[d.String()] = d
+				}
+			}
+			event.UsedDeps = lastEvent.UsedDeps
+		}
+
+		return event, nil
 	}
 
 	// Grab the list of used and missing dependencies.
@@ -781,7 +802,12 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (*
 			Perms:          config.FileModeVal(templateConfig.Perms),
 		})
 		if err != nil {
-			return nil, errors.Wrap(err, "error rendering "+templateConfig.Display())
+			if tmpl.ErrFatal() {
+				return nil, errors.Wrap(err, "error rendering "+templateConfig.Display())
+			}
+			log.Printf("[ERR] (runner) error rendering: %s: %v", templateConfig.Display(), err)
+			event.Error = err
+			return event, nil
 		}
 
 		renderTime := time.Now().UTC()
@@ -887,6 +913,7 @@ func (r *Runner) init() error {
 			Source:           config.StringVal(ctmpl.Source),
 			Contents:         config.StringVal(ctmpl.Contents),
 			ErrMissingKey:    config.BoolVal(ctmpl.ErrMissingKey),
+			ErrFatal:         config.BoolVal(ctmpl.ErrFatal),
 			LeftDelim:        leftDelim,
 			RightDelim:       rightDelim,
 			FunctionDenylist: ctmpl.FunctionDenylist,
diff --git a/template/template.go b/template/template.go
index 7b70bca55..a4ba35bde 100644
--- a/template/template.go
+++ b/template/template.go
@@ -46,6 +46,10 @@ type Template struct {
 	// is indexed with a key that does not exist.
 	errMissingKey bool
 
+	// errFatal determines whether template errors should cause the process to
+	// exit, or just log and continue.
+	errFatal bool
+
 	// functionDenylist are functions not permitted to be executed
 	// when we render this template
 	functionDenylist []string
@@ -68,6 +72,10 @@ type NewTemplateInput struct {
 	// when a map is indexed with a key that does not exist.
 	ErrMissingKey bool
 
+	// ErrFatal determines whether template errors should cause the process to
+	// exit, or just log and continue.
+	ErrFatal bool
+
 	// LeftDelim and RightDelim are the template delimiters.
 	LeftDelim  string
 	RightDelim string
@@ -104,6 +112,7 @@ func NewTemplate(i *NewTemplateInput) (*Template, error) {
 	t.leftDelim = i.LeftDelim
 	t.rightDelim = i.RightDelim
 	t.errMissingKey = i.ErrMissingKey
+	t.errFatal = i.ErrFatal
 	t.functionDenylist = i.FunctionDenylist
 	t.sandboxPath = i.SandboxPath
 
@@ -140,6 +149,11 @@ func (t *Template) Source() string {
 	return t.source
 }
 
+// ErrFatal indicates whether errors in this template should be fatal.
+func (t *Template) ErrFatal() bool {
+	return t.errFatal
+}
+
 // ExecuteInput is used as input to the template's execute function.
 type ExecuteInput struct {
 	// Brain is the brain where data for the template is stored.