Continuously retry updating the cert secret

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1
 	github.com/hashicorp/go-hclog v0.9.2
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/hashicorp/vault/sdk v0.1.14-0.20191205220236-47cffd09f972

go.sum

@@ -49,6 +49,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
+github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

helper/cert/source_gen.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault-k8s/leader"
 	v1 "k8s.io/api/core/v1"
@@ -61,6 +62,12 @@ type GenSource struct {
 	SecretsCache  informerv1.SecretInformer
 	LeaderElector *leader.LeaderElector
 
+	// UpdateCancel and StopUpdate are used to control and sync the goroutine
+	// that continuously attempts to update the K8s Secret with the new
+	// certificate
+	UpdateCancel context.CancelFunc
+	StopUpdate   chan struct{}
+
 	Log hclog.Logger
 }
 
@@ -143,9 +150,19 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro
 		result.Key = []byte(key)
 
 		if s.LeaderElector != nil {
-			if err := s.updateSecret(ctx, result); err != nil {
-				return result, fmt.Errorf("failed to update Secret: %s", err)
+			// Cancel and wait for the previous retryUpdateSecret to quit
+			if s.UpdateCancel != nil && s.StopUpdate != nil {
+				s.Log.Trace("cancelling cert secret update context")
+				s.UpdateCancel()
+				<-s.StopUpdate
+			}
+			// Start a new update goroutine
+			var updateCtx context.Context
+			updateCtx, s.UpdateCancel = context.WithCancel(ctx)
+			if s.StopUpdate == nil {
+				s.StopUpdate = make(chan struct{})
 			}
+			go s.retryUpdateSecret(updateCtx, result)
 		}
 	}
 
@@ -182,6 +199,33 @@ func (s *GenSource) checkLeader(ctx context.Context, changed chan<- bool) {
 	}
 }
 
+func (s *GenSource) retryUpdateSecret(ctx context.Context, bundle Bundle) {
+	defer func() {
+		s.StopUpdate <- struct{}{}
+	}()
+
+	// New exponential backoff with unlimited retries
+	bo := backoff.NewExponentialBackOff()
+	bo.MaxInterval = time.Second * 30
+	bo.MaxElapsedTime = 0
+	ticker := backoff.NewTicker(bo)
+
+	for {
+		select {
+		case <-ticker.C:
+			if err := s.updateSecret(ctx, bundle); err != nil {
+				s.Log.Error(fmt.Sprintf("attempt to update Secret %q failed: %s", certSecretName, err))
+			} else {
+				s.Log.Trace(fmt.Sprintf("updated secret %q, quitting update thread", certSecretName))
+				return
+			}
+		case <-ctx.Done():
+			s.Log.Trace("quitting update thread")
+			return
+		}
+	}
+}
+
 func (s *GenSource) updateSecret(ctx context.Context, bundle Bundle) error {
 	secret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{