diff --git a/agent-inject/agent/agent.go b/agent-inject/agent/agent.go index 8e7297da..9edc7c7c 100644 --- a/agent-inject/agent/agent.go +++ b/agent-inject/agent/agent.go @@ -26,6 +26,7 @@ const ( DefaultAgentCacheEnable = "false" DefaultAgentCacheUseAutoAuthToken = "true" DefaultAgentCacheListenerPort = "8200" + DefaultAgentUseLeaderElector = false ) // Agent is the top level structure holding all the @@ -123,8 +124,8 @@ type Agent struct { // SetSecurityContext controls whether the injected containers have a // SecurityContext set. SetSecurityContext bool - - // ExtraSecret is the Kubernetes secret to mount as a volume in the Vault agent container + + // ExtraSecret is the Kubernetes secret to mount as a volume in the Vault agent container // which can be referenced by the Agent config for secrets. Mounted at /vault/custom/ ExtraSecret string } diff --git a/deploy/injector-deployment.yaml b/deploy/injector-deployment.yaml index c6e9fd65..c0b50719 100644 --- a/deploy/injector-deployment.yaml +++ b/deploy/injector-deployment.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: vault-injector app.kubernetes.io/instance: vault spec: - replicas: 1 + replicas: 2 selector: matchLabels: app.kubernetes.io/name: vault-injector @@ -20,6 +20,38 @@ spec: spec: serviceAccountName: "vault-injector" containers: + - name: leader-elector + image: k8s.gcr.io/leader-elector:0.4 + args: + - --election=vault-agent-injector-leader + - --election-namespace=$(NAMESPACE) + - --http=0.0.0.0:4040 + - --ttl=60s + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + livenessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 - name: sidecar-injector image: "hashicorp/vault-k8s:0.6.0" imagePullPolicy: IfNotPresent @@ -42,6 +74,8 @@ spec: value: vault-agent-injector-cfg - name: AGENT_INJECT_TLS_AUTO_HOSTS value: "vault-agent-injector-svc,vault-agent-injector-svc.$(NAMESPACE),vault-agent-injector-svc.$(NAMESPACE).svc" + - name: AGENT_INJECT_USE_LEADER_ELECTOR + value: "true" args: - agent-inject - 2>&1 diff --git a/deploy/injector-leader-extras.yaml b/deploy/injector-leader-extras.yaml new file mode 100644 index 00000000..a8e7d158 --- /dev/null +++ b/deploy/injector-leader-extras.yaml @@ -0,0 +1,13 @@ +# These are created here so they can be cleaned up easily. The endpoints +# especially, since if they're left around the leader won't expire for about a +# minute. +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: vault-agent-injector-leader +--- +apiVersion: v1 +kind: Secret +metadata: + name: vault-injector-certs diff --git a/deploy/injector-rbac.yaml b/deploy/injector-rbac.yaml index 21499d09..31cf431c 100644 --- a/deploy/injector-rbac.yaml +++ b/deploy/injector-rbac.yaml @@ -38,3 +38,36 @@ subjects: - kind: ServiceAccount name: vault-injector namespace: vault +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: vault-injector-role + labels: + app.kubernetes.io/name: vault-injector + app.kubernetes.io/instance: vault +rules: +- apiGroups: [""] + resources: ["endpoints", "secrets"] + verbs: + - "create" + - "get" + - "watch" + - "list" + - "update" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vault-injector-rolebinding + labels: + app.kubernetes.io/name: vault-injector + app.kubernetes.io/instance: vault +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vault-injector-role +subjects: +- kind: ServiceAccount + name: vault-injector + namespace: vault diff --git a/deploy/kustomization.yaml b/deploy/kustomization.yaml index 066a6d68..e5a11aae 100644 --- a/deploy/kustomization.yaml +++ b/deploy/kustomization.yaml @@ -5,3 +5,4 @@ resources: - injector-mutating-webhook.yaml - injector-rbac.yaml - injector-service.yaml +- injector-leader-extras.yaml diff --git a/go.mod b/go.mod index a073f194..e96d6036 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.12 require ( github.com/armon/go-radix v1.0.0 // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/google/btree v1.0.0 // indirect github.com/hashicorp/consul v1.5.0 github.com/hashicorp/go-hclog v0.9.2 @@ -31,6 +33,7 @@ require ( k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible k8s.io/klog v1.0.0 // indirect + k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 // indirect k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d // indirect sigs.k8s.io/yaml v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 3097d400..516dd45f 100644 --- a/go.sum +++ b/go.sum @@ -65,10 +65,13 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74 h1:2MIh github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.0.0-20180919002855-2137d9196328/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -93,6 +96,8 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -222,6 +227,7 @@ github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35n github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -297,8 +303,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= github.com/oklog/run v0.0.0-20180308005104-6934b124db28/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -479,6 +487,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -487,6 +496,7 @@ gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/ory-am/dockertest.v3 v3.3.4/go.mod h1:s9mmoLkaGeAh97qygnNj4xWkiN7e1SKekYC6CovU+ek= gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -515,6 +525,8 @@ k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7v k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d h1:1P0iBJsBzxRmR+dIFnM+Iu4aLxnoa7lBqozW/0uHbT8= k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/helper/cert/notify_test.go b/helper/cert/notify_test.go index 2b220043..d5394b33 100644 --- a/helper/cert/notify_test.go +++ b/helper/cert/notify_test.go @@ -5,6 +5,8 @@ import ( "fmt" "testing" "time" + + "github.com/hashicorp/go-hclog" ) func TestNotify(t *testing.T) { @@ -64,6 +66,7 @@ func TestNotifyRace(t *testing.T) { var certSource Source = &GenSource{ Name: "Agent Inject", Hosts: []string{"some", "hosts"}, + Log: hclog.Default(), } n := NewNotify(ctx, certCh, certSource) diff --git a/helper/cert/source_gen.go b/helper/cert/source_gen.go index e5ffce81..7e344253 100644 --- a/helper/cert/source_gen.go +++ b/helper/cert/source_gen.go @@ -17,8 +17,20 @@ import ( "strings" "sync" "time" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault-k8s/leader" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + informerv1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" ) +// Name of the k8s Secret used to share the caBundle between leader and +// followers +const certSecretName = "vault-injector-certs" + // GenSource generates a self-signed CA and certificate pair. // // This generator is stateful. On the first run (last == nil to Certificate), @@ -43,6 +55,13 @@ type GenSource struct { caCert []byte caCertTemplate *x509.Certificate caSigner crypto.Signer + + K8sClient kubernetes.Interface + Namespace string + SecretsCache informerv1.SecretInformer + LeaderElector *leader.LeaderElector + + Log hclog.Logger } // Certificate implements source @@ -50,12 +69,38 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro s.mu.Lock() defer s.mu.Unlock() var result Bundle + leaderCh := make(chan bool) + + if s.LeaderElector != nil { + leaderCheck, err := s.LeaderElector.IsLeader() + if err != nil { + return result, err + } + // For followers, run different function here that reads bundle from Secret, + // and returns that in the result. That will flow through the existing + // notify channel structure, testing if it's the same cert as last, etc. + if !leaderCheck { + s.Log.Debug("Currently a follower") + return s.getBundleFromSecret() + } + s.Log.Info("Currently the leader") + + // Start a goroutine that checks for a leadership change, otherwise this + // would wait until the current certificate expires before moving on. + changeContext, cancel := context.WithCancel(ctx) + defer cancel() + go s.checkLeader(changeContext, leaderCh) + } // If we have no CA, generate it for the first time. if len(s.caCert) == 0 { if err := s.generateCA(); err != nil { return result, err } + // If we had no CA, also ensure the cert is regenerated + last = nil + + s.Log.Info("Generated CA") } // Set the CA cert @@ -79,6 +124,10 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro defer timer.Stop() select { + case <-leaderCh: + s.Log.Debug("got a leadership change, returning") + return result, fmt.Errorf("lost leadership") + case <-timer.C: // Fall through, generate cert @@ -92,11 +141,82 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro if err == nil { result.Cert = []byte(cert) result.Key = []byte(key) + + if s.LeaderElector != nil { + if err := s.updateSecret(result); err != nil { + return result, fmt.Errorf("failed to update Secret: %s", err) + } + } } return result, err } +func (s *GenSource) checkLeader(ctx context.Context, changed chan<- bool) { + for { + select { + case <-time.After(1 * time.Second): + // Check once a second for a leadership change + s.Log.Named("checkLeader").Trace("checking for leadership change") + + case <-ctx.Done(): + // Quit + return + } + + check, err := s.LeaderElector.IsLeader() + if err != nil { + s.Log.Warn("failed to check for leadership change: %s", err) + } + if !check { + s.Log.Named("checkLeader").Debug("lost the leadership, sending notification") + select { + case changed <- true: + s.Log.Named("checkLeader").Trace("sent changed <- true") + return + case <-ctx.Done(): + s.Log.Named("checkLeader").Trace("got done") + return + } + } + } +} + +func (s *GenSource) updateSecret(bundle Bundle) error { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: certSecretName, + }, + Data: map[string][]byte{ + "cert": bundle.Cert, + "key": bundle.Key, + }, + } + // Attempt updating the Secret first, and if it doesn't exist, fallback to + // create + _, err := s.K8sClient.CoreV1().Secrets(s.Namespace).Update(secret) + if errors.IsNotFound(err) { + _, err = s.K8sClient.CoreV1().Secrets(s.Namespace).Create(secret) + } + if err != nil { + return err + } + return nil +} + +func (s *GenSource) getBundleFromSecret() (Bundle, error) { + var bundle Bundle + + secret, err := s.SecretsCache.Lister().Secrets(s.Namespace).Get(certSecretName) + if err != nil { + return bundle, fmt.Errorf("failed to get secret: %s", err) + } + bundle.Cert = secret.Data["cert"] + bundle.Key = secret.Data["key"] + + return bundle, nil +} + func (s *GenSource) expiry() time.Duration { if s.Expiry > 0 { return s.Expiry diff --git a/helper/cert/source_gen_test.go b/helper/cert/source_gen_test.go index 6b7303bb..dce8f1b4 100644 --- a/helper/cert/source_gen_test.go +++ b/helper/cert/source_gen_test.go @@ -2,14 +2,23 @@ package cert import ( "context" + "encoding/json" "io/ioutil" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "testing" "time" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault-k8s/leader" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" ) // hasOpenSSL is used to determine if the openssl CLI exists for unit tests. @@ -69,6 +78,7 @@ func testGenSource() *GenSource { return &GenSource{ Name: "Test", Hosts: []string{"127.0.0.1", "localhost"}, + Log: hclog.Default(), } } @@ -116,3 +126,111 @@ func testBundleVerify(t *testing.T, bundle *Bundle) { t.Log(string(output)) require.NoError(err) } + +func TestGenSource_leader(t *testing.T) { + + if !hasOpenSSL { + t.Skip("openssl not found") + return + } + + // Generate the bundle + source := testGenSource() + + // Setup test leader service returning this host as the leader + ts := testLeaderServer(t, testGetHostname(t)) + defer ts.Close() + source.LeaderElector = leader.NewWithURL(ts.URL) + + source.Namespace = "default" + source.K8sClient = fake.NewSimpleClientset() + bundle, err := source.Certificate(context.Background(), nil) + require.NoError(t, err) + testBundleVerify(t, &bundle) + + // check that the Secret has been created + checkSecret, err := source.K8sClient.CoreV1().Secrets(source.Namespace).Get(certSecretName, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, checkSecret.Data["cert"], bundle.Cert, + "cert in the Secret should've matched what was returned from source.Certificate()", + ) + require.Equal(t, checkSecret.Data["key"], bundle.Key, + "key in the Secret should've matched what was returned from source.Certificate()", + ) +} + +func TestGenSource_follower(t *testing.T) { + + if !hasOpenSSL { + t.Skip("openssl not found") + return + } + + // Generate the bundle + source := testGenSource() + + // Setup a leader elector service that returns a different hostname, so it + // thinks we're the follower + ts := testLeaderServer(t, testGetHostname(t)+"not it") + defer ts.Close() + source.LeaderElector = leader.NewWithURL(ts.URL) + + // Setup the k8s client with a Secret for a follower to pick up + source.Namespace = "default" + secretBundle := testBundle(t) + source.K8sClient = fake.NewSimpleClientset(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: certSecretName, + Namespace: source.Namespace, + }, + Data: map[string][]byte{ + "cert": secretBundle.Cert, + "key": secretBundle.Key, + }, + }) + + // setup a Secret informer cache with the fake clientset for followers to use + factory := informers.NewSharedInformerFactoryWithOptions(source.K8sClient, 0, informers.WithNamespace(source.Namespace)) + secrets := factory.Core().V1().Secrets() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go secrets.Informer().Run(ctx.Done()) + source.SecretsCache = secrets + + bundle, err := source.Certificate(ctx, nil) + require.NoError(t, err) + + require.Equal(t, secretBundle.Cert, bundle.Cert, + "cert returned from source.Certificate() should have matched what the Secret was created with", + ) + require.Equal(t, secretBundle.Key, bundle.Key, + "key returned from source.Certificate() should have matched what the Secret was created with", + ) +} + +func testLeaderServer(t *testing.T, hostname string) *httptest.Server { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lResp := leader.LeaderResponse{ + Name: hostname, + } + body, err := json.Marshal(lResp) + if err != nil { + t.Fatalf("failed to marshal leader response: %s", err) + } + w.WriteHeader(200) + w.Write(body) + })) + return ts +} + +func testGetHostname(t *testing.T) string { + t.Helper() + + host, err := os.Hostname() + if err != nil { + t.Fatalf("failed to get hostname for test leader service: %s", err) + } + return host +} diff --git a/leader/leader.go b/leader/leader.go new file mode 100644 index 00000000..c0471da5 --- /dev/null +++ b/leader/leader.go @@ -0,0 +1,59 @@ +package leader + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "os" +) + +const defaultURL = "http://localhost:4040/" + +type LeaderElector struct { + URL string +} + +type LeaderResponse struct { + Name string `json:"name"` +} + +// New returns a LeaderElector with the default service endpoint +func New() *LeaderElector { + return &LeaderElector{ + URL: defaultURL, + } +} + +// NewWithURL returns a LeaderElector with a custom service endpoint URL +func NewWithURL(URL string) *LeaderElector { + return &LeaderElector{ + URL: URL, + } +} + +// IsLeader returns whether this host is the leader +func (le *LeaderElector) IsLeader() (bool, error) { + resp, err := http.Get(le.URL) + if err != nil { + return false, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, err + } + leaderResp := &LeaderResponse{} + err = json.Unmarshal(body, leaderResp) + if err != nil { + return false, err + } + hostname, err := os.Hostname() + if err != nil { + return false, err + } + if leaderResp.Name == hostname { + return true, nil + } + + return false, nil +} diff --git a/subcommand/injector/command.go b/subcommand/injector/command.go index ee228fce..94a81c69 100644 --- a/subcommand/injector/command.go +++ b/subcommand/injector/command.go @@ -18,11 +18,15 @@ import ( "github.com/hashicorp/go-hclog" agentInject "github.com/hashicorp/vault-k8s/agent-inject" "github.com/hashicorp/vault-k8s/helper/cert" + "github.com/hashicorp/vault-k8s/leader" "github.com/mitchellh/cli" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + informerv1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" ) type Command struct { @@ -44,6 +48,7 @@ type Command struct { flagRunAsSameUser bool // Run Vault agent as the User (uid) of the first application container flagSetSecurityContext bool // Set SecurityContext in injected containers flagTelemetryPath string // Path under which to expose metrics + flagUseLeaderElector bool // Use leader elector code flagSet *flag.FlagSet @@ -85,10 +90,41 @@ func (c *Command) Run(args []string) int { return 1 } + namespace := getNamespace() + var secrets informerv1.SecretInformer + var leaderElector *leader.LeaderElector + if c.flagUseLeaderElector { + c.UI.Info("Using leader elector logic") + factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0, informers.WithNamespace(namespace)) + secrets = factory.Core().V1().Secrets() + go secrets.Informer().Run(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), secrets.Informer().HasSynced) { + c.UI.Error("timeout syncing Secrets informer") + return 1 + } + leaderElector = leader.New() + } + + level, err := c.logLevel() + if err != nil { + c.UI.Error(fmt.Sprintf("Error setting log level: %s", err)) + return 1 + } + + logger := hclog.New(&hclog.LoggerOptions{ + Name: "handler", + Level: level, + JSONFormat: (c.flagLogFormat == "json")}) + // Determine where to source the certificates from var certSource cert.Source = &cert.GenSource{ - Name: "Agent Inject", - Hosts: strings.Split(c.flagAutoHosts, ","), + Name: "Agent Inject", + Hosts: strings.Split(c.flagAutoHosts, ","), + K8sClient: clientset, + Namespace: namespace, + SecretsCache: secrets, + LeaderElector: leaderElector, + Log: logger.Named("auto-tls"), } if c.flagCertFile != "" { certSource = &cert.DiskSource{ @@ -102,18 +138,7 @@ func (c *Command) Run(args []string) int { certCh := make(chan cert.Bundle) certNotify := cert.NewNotify(ctx, certCh, certSource) go certNotify.Run() - go c.certWatcher(ctx, certCh, clientset) - - level, err := c.logLevel() - if err != nil { - c.UI.Error(fmt.Sprintf("Error setting log level: %s", err)) - return 1 - } - - logger := hclog.New(&hclog.LoggerOptions{ - Name: "handler", - Level: level, - JSONFormat: (c.flagLogFormat == "json")}) + go c.certWatcher(ctx, certCh, clientset, logger.Named("certwatcher")) // Build the HTTP handler and server injector := agentInject.Handler{ @@ -175,6 +200,15 @@ func (c *Command) Run(args []string) int { return 0 } +func getNamespace() string { + namespace := os.Getenv("NAMESPACE") + if namespace != "" { + return namespace + } + + return "default" +} + func (c *Command) handleReady(rw http.ResponseWriter, req *http.Request) { // Always ready at this point. The main readiness check is whether // there is a TLS certificate. If we reached this point it means we @@ -191,12 +225,12 @@ func (c *Command) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) return certRaw.(*tls.Certificate), nil } -func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, clientset *kubernetes.Clientset) { +func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, clientset *kubernetes.Clientset, log hclog.Logger) { var bundle cert.Bundle for { select { case bundle = <-ch: - c.UI.Output("Updated certificate bundle received. Updating certs...") + log.Info("Updated certificate bundle received. Updating certs...") // Bundle is updated, set it up case <-time.After(1 * time.Second): @@ -212,12 +246,24 @@ func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, client crt, err := tls.X509KeyPair(bundle.Cert, bundle.Key) if err != nil { - c.UI.Error(fmt.Sprintf("Error loading TLS keypair: %s", err)) + log.Error(fmt.Sprintf("Error loading TLS keypair: %s", err)) continue } + isLeader := true + if c.flagUseLeaderElector { + // Only the leader should do the caBundle patching in k8s API + var err error + le := leader.New() + isLeader, err = le.IsLeader() + if err != nil { + log.Error(fmt.Sprintf("error checking leader: %s", err)) + continue + } + } + // If there is a MWC name set, then update the CA bundle. - if c.flagAutoName != "" && len(bundle.CACert) > 0 { + if isLeader && c.flagAutoName != "" && len(bundle.CACert) > 0 { // The CA Bundle value must be base64 encoded value := base64.StdEncoding.EncodeToString(bundle.CACert) diff --git a/subcommand/injector/flags.go b/subcommand/injector/flags.go index a975fd0c..f6eb22f2 100644 --- a/subcommand/injector/flags.go +++ b/subcommand/injector/flags.go @@ -68,6 +68,9 @@ type Specification struct { // TelemetryPath is the AGENT_INJECT_TELEMETRY_PATH environment variable. TelemetryPath string `split_words:"true"` + + // UseLeaderElector is the AGENT_INJECT_USE_LEADER_ELECTOR environment variable. + UseLeaderElector string `split_words:"true"` } func (c *Command) init() { @@ -105,6 +108,8 @@ func (c *Command) init() { fmt.Sprintf("Set SecurityContext in injected containers. Defaults to %v.", agent.DefaultAgentSetSecurityContext)) c.flagSet.StringVar(&c.flagTelemetryPath, "telemetry-path", "", "Path under which to expose metrics") + c.flagSet.BoolVar(&c.flagUseLeaderElector, "use-leader-elector", agent.DefaultAgentUseLeaderElector, + fmt.Sprintf("Use leader elector to coordinate multiple replicas when updating CA and Certs with auto-tls")) c.help = flags.Usage(help, c.flagSet) } @@ -211,5 +216,12 @@ func (c *Command) parseEnvs() error { c.flagTelemetryPath = envs.TelemetryPath } + if envs.UseLeaderElector != "" { + c.flagUseLeaderElector, err = strconv.ParseBool(envs.UseLeaderElector) + if err != nil { + return err + } + } + return nil } diff --git a/subcommand/injector/flags_test.go b/subcommand/injector/flags_test.go index 5be9e0a3..f281f650 100644 --- a/subcommand/injector/flags_test.go +++ b/subcommand/injector/flags_test.go @@ -157,6 +157,8 @@ func TestCommandEnvBools(t *testing.T) { {env: "AGENT_INJECT_RUN_AS_SAME_USER", value: false, cmdPtr: &cmd.flagRunAsSameUser}, {env: "AGENT_INJECT_SET_SECURITY_CONTEXT", value: true, cmdPtr: &cmd.flagSetSecurityContext}, {env: "AGENT_INJECT_SET_SECURITY_CONTEXT", value: false, cmdPtr: &cmd.flagSetSecurityContext}, + {env: "AGENT_INJECT_USE_LEADER_ELECTOR", value: true, cmdPtr: &cmd.flagUseLeaderElector}, + {env: "AGENT_INJECT_USE_LEADER_ELECTOR", value: false, cmdPtr: &cmd.flagUseLeaderElector}, } for _, tt := range tests {