diff --git a/pkg/server/gateway.go b/pkg/server/gateway.go index efe1718b..fc118dfe 100644 --- a/pkg/server/gateway.go +++ b/pkg/server/gateway.go @@ -18,13 +18,12 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "net" "time" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" - "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/uswitch/kiam/pkg/aws/sts" "github.com/uswitch/kiam/pkg/statsd" pb "github.com/uswitch/kiam/proto" @@ -59,28 +58,20 @@ func NewGateway(ctx context.Context, address string, caFile, certificateFile, ke retry.WithBackoff(retry.BackoffLinear(RetryInterval)), } - certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile) - if err != nil { - return nil, fmt.Errorf("error loading keypair: %v", err) - } - certPool := x509.NewCertPool() - ca, err := ioutil.ReadFile(caFile) - if err != nil { - return nil, fmt.Errorf("error reading SSL cert: %v", err) - } - if ok := certPool.AppendCertsFromPEM(ca); !ok { - return nil, fmt.Errorf("error appending certs from ca") - } - host, _, err := net.SplitHostPort(address) if err != nil { return nil, fmt.Errorf("error parsing hostname: %v", err) } + cert, caPool, err := loadCerts(certificateFile, keyFile, caFile) + if err != nil { + return nil, err + } + clientTLSMetrics.update(x509.ExtKeyUsageClientAuth, &cert, caPool) creds := credentials.NewTLS(&tls.Config{ ServerName: host, - Certificates: []tls.Certificate{certificate}, - RootCAs: certPool, + Certificates: []tls.Certificate{cert}, + RootCAs: caPool, }) dialAddress := fmt.Sprintf("dns:///%s", address) diff --git a/pkg/server/server.go b/pkg/server/server.go index 7a0642e0..d5ce1cf1 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -18,12 +18,11 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "net" "time" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" log "github.com/sirupsen/logrus" "github.com/uswitch/k8sc/official" "github.com/uswitch/kiam/pkg/aws/sts" @@ -33,7 +32,7 @@ import ( pb "github.com/uswitch/kiam/proto" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -253,25 +252,16 @@ func NewServer(config *Config) (*KiamServer, error) { server.manager = prefetch.NewManager(credentialsCache, server.pods) server.assumePolicy = Policies(NewRequestingAnnotatedRolePolicy(server.pods, arnResolver), NewNamespacePermittedRoleNamePolicy(server.namespaces, server.pods)) - certificate, err := tls.LoadX509KeyPair(config.TLS.ServerCert, config.TLS.ServerKey) + cert, caPool, err := loadCerts(config.TLS.ServerCert, config.TLS.ServerKey, config.TLS.CA) if err != nil { return nil, err } - certPool := x509.NewCertPool() - if err != nil { - return nil, err - } - ca, err := ioutil.ReadFile(config.TLS.CA) - if err != nil { - return nil, err - } - if ok := certPool.AppendCertsFromPEM(ca); !ok { - return nil, fmt.Errorf("failed to append CA cert to certPool") - } + serverTLSMetrics.update(x509.ExtKeyUsageServerAuth, &cert, caPool) + creds := credentials.NewTLS(&tls.Config{ ClientAuth: tls.RequireAndVerifyClientCert, - Certificates: []tls.Certificate{certificate}, - ClientCAs: certPool, + Certificates: []tls.Certificate{cert}, + ClientCAs: caPool, }) grpcServer := grpc.NewServer( diff --git a/pkg/server/tls.go b/pkg/server/tls.go new file mode 100644 index 00000000..031b5e02 --- /dev/null +++ b/pkg/server/tls.go @@ -0,0 +1,110 @@ +package server + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" +) + +var ( + clientTLSMetrics = newTLSMetrics("client") + serverTLSMetrics = newTLSMetrics("server") +) + +type tlsMetrics struct { + registerOnce sync.Once + verifyError prometheus.Gauge + expiration prometheus.Gauge +} + +func newTLSMetrics(subsystem string) *tlsMetrics { + return &tlsMetrics{ + verifyError: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "grpc", + Subsystem: subsystem, + Name: "tls_certificate_verify_error", + Help: "Indicates if there was an error verifying the latest gRPC " + subsystem + " TLS certificate and its expiration.", + }), + expiration: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "grpc", + Subsystem: subsystem, + Name: "tls_certificate_expiry_seconds", + Help: "Expiration time of the gRPC " + subsystem + " TLS certificate in seconds since the Unix epoch.", + }), + } +} + +func (m *tlsMetrics) Describe(ch chan<- *prometheus.Desc) { + m.verifyError.Describe(ch) + m.expiration.Describe(ch) +} + +func (m *tlsMetrics) Collect(ch chan<- prometheus.Metric) { + m.verifyError.Collect(ch) + m.expiration.Collect(ch) +} + +func (m *tlsMetrics) update(usage x509.ExtKeyUsage, cert *tls.Certificate, pool *x509.CertPool) { + m.registerOnce.Do(func() { prometheus.MustRegister(m) }) + + expiry, err := earliestExpiry(cert, pool, usage) + if err != nil { + log.Errorf("failed to verify TLS certificate: %v", err) + m.verifyError.Set(1) + } else { + m.verifyError.Set(0) + } + m.expiration.Set(float64(expiry.Unix())) +} + +func loadCerts(certFile, keyFile, caFile string) (tls.Certificate, *x509.CertPool, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("error loading TLS key pair: %v", err) + } + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS leaf cert: %v", err) + } + ca, err := ioutil.ReadFile(caFile) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("error reading TLS CAs: %v", err) + } + caPool := x509.NewCertPool() + if !caPool.AppendCertsFromPEM(ca) { + return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS CAs") + } + return cert, caPool, nil +} + +func earliestExpiry(cert *tls.Certificate, pool *x509.CertPool, usage x509.ExtKeyUsage) (time.Time, error) { + x509Cert := cert.Leaf + if x509Cert == nil { + var err error + if x509Cert, err = x509.ParseCertificate(cert.Certificate[0]); err != nil { + return time.Time{}, err + } + } + chains, err := x509Cert.Verify(x509.VerifyOptions{ + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{usage}, + }) + if err != nil { + return x509Cert.NotAfter, err + } + var t time.Time + for _, chain := range chains { + for _, cert := range chain { + if t.IsZero() || cert.NotAfter.Before(t) { + t = cert.NotAfter + } + } + } + return t, nil +}