Add metrics for TLS certificate expiration

pkg/server/gateway.go

@@ -18,13 +18,12 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"time"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
-	"github.com/grpc-ecosystem/go-grpc-prometheus"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/uswitch/kiam/pkg/aws/sts"
 	"github.com/uswitch/kiam/pkg/statsd"
 	pb "github.com/uswitch/kiam/proto"
@@ -59,28 +58,20 @@ func NewGateway(ctx context.Context, address string, caFile, certificateFile, ke
 		retry.WithBackoff(retry.BackoffLinear(RetryInterval)),
 	}
 
-	certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile)
-	if err != nil {
-		return nil, fmt.Errorf("error loading keypair: %v", err)
-	}
-	certPool := x509.NewCertPool()
-	ca, err := ioutil.ReadFile(caFile)
-	if err != nil {
-		return nil, fmt.Errorf("error reading SSL cert: %v", err)
-	}
-	if ok := certPool.AppendCertsFromPEM(ca); !ok {
-		return nil, fmt.Errorf("error appending certs from ca")
-	}
-
 	host, _, err := net.SplitHostPort(address)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing hostname: %v", err)
 	}
+	cert, caPool, err := loadCerts(certificateFile, keyFile, caFile)
+	if err != nil {
+		return nil, err
+	}
+	clientTLSMetrics.update(x509.ExtKeyUsageClientAuth, &cert, caPool)
 
 	creds := credentials.NewTLS(&tls.Config{
 		ServerName:   host,
-		Certificates: []tls.Certificate{certificate},
-		RootCAs:      certPool,
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      caPool,
 	})
 
 	dialAddress := fmt.Sprintf("dns:///%s", address)

pkg/server/server.go

@@ -18,12 +18,11 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/grpc-ecosystem/go-grpc-prometheus"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	log "github.com/sirupsen/logrus"
 	"github.com/uswitch/k8sc/official"
 	"github.com/uswitch/kiam/pkg/aws/sts"
@@ -33,7 +32,7 @@ import (
 	pb "github.com/uswitch/kiam/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -253,25 +252,16 @@ func NewServer(config *Config) (*KiamServer, error) {
 	server.manager = prefetch.NewManager(credentialsCache, server.pods)
 	server.assumePolicy = Policies(NewRequestingAnnotatedRolePolicy(server.pods, arnResolver), NewNamespacePermittedRoleNamePolicy(server.namespaces, server.pods))
 
-	certificate, err := tls.LoadX509KeyPair(config.TLS.ServerCert, config.TLS.ServerKey)
+	cert, caPool, err := loadCerts(config.TLS.ServerCert, config.TLS.ServerKey, config.TLS.CA)
 	if err != nil {
 		return nil, err
 	}
-	certPool := x509.NewCertPool()
-	if err != nil {
-		return nil, err
-	}
-	ca, err := ioutil.ReadFile(config.TLS.CA)
-	if err != nil {
-		return nil, err
-	}
-	if ok := certPool.AppendCertsFromPEM(ca); !ok {
-		return nil, fmt.Errorf("failed to append CA cert to certPool")
-	}
+	serverTLSMetrics.update(x509.ExtKeyUsageServerAuth, &cert, caPool)
+
 	creds := credentials.NewTLS(&tls.Config{
 		ClientAuth:   tls.RequireAndVerifyClientCert,
-		Certificates: []tls.Certificate{certificate},
-		ClientCAs:    certPool,
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    caPool,
 	})
 
 	grpcServer := grpc.NewServer(

pkg/server/tls.go

@@ -0,0 +1,110 @@
+package server
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	clientTLSMetrics = newTLSMetrics("client")
+	serverTLSMetrics = newTLSMetrics("server")
+)
+
+type tlsMetrics struct {
+	registerOnce sync.Once
+	verifyError  prometheus.Gauge
+	expiration   prometheus.Gauge
+}
+
+func newTLSMetrics(subsystem string) *tlsMetrics {
+	return &tlsMetrics{
+		verifyError: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "grpc",
+			Subsystem: subsystem,
+			Name:      "tls_certificate_verify_error",
+			Help:      "Indicates if there was an error verifying the latest gRPC " + subsystem + " TLS certificate and its expiration.",
+		}),
+		expiration: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "grpc",
+			Subsystem: subsystem,
+			Name:      "tls_certificate_expiry_seconds",
+			Help:      "Expiration time of the gRPC " + subsystem + " TLS certificate in seconds since the Unix epoch.",
+		}),
+	}
+}
+
+func (m *tlsMetrics) Describe(ch chan<- *prometheus.Desc) {
+	m.verifyError.Describe(ch)
+	m.expiration.Describe(ch)
+}
+
+func (m *tlsMetrics) Collect(ch chan<- prometheus.Metric) {
+	m.verifyError.Collect(ch)
+	m.expiration.Collect(ch)
+}
+
+func (m *tlsMetrics) update(usage x509.ExtKeyUsage, cert *tls.Certificate, pool *x509.CertPool) {
+	m.registerOnce.Do(func() { prometheus.MustRegister(m) })
+
+	expiry, err := earliestExpiry(cert, pool, usage)
+	if err != nil {
+		log.Errorf("failed to verify TLS certificate: %v", err)
+		m.verifyError.Set(1)
+	} else {
+		m.verifyError.Set(0)
+	}
+	m.expiration.Set(float64(expiry.Unix()))
+}
+
+func loadCerts(certFile, keyFile, caFile string) (tls.Certificate, *x509.CertPool, error) {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return tls.Certificate{}, nil, fmt.Errorf("error loading TLS key pair: %v", err)
+	}
+	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+	if err != nil {
+		return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS leaf cert: %v", err)
+	}
+	ca, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		return tls.Certificate{}, nil, fmt.Errorf("error reading TLS CAs: %v", err)
+	}
+	caPool := x509.NewCertPool()
+	if !caPool.AppendCertsFromPEM(ca) {
+		return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS CAs")
+	}
+	return cert, caPool, nil
+}
+
+func earliestExpiry(cert *tls.Certificate, pool *x509.CertPool, usage x509.ExtKeyUsage) (time.Time, error) {
+	x509Cert := cert.Leaf
+	if x509Cert == nil {
+		var err error
+		if x509Cert, err = x509.ParseCertificate(cert.Certificate[0]); err != nil {
+			return time.Time{}, err
+		}
+	}
+	chains, err := x509Cert.Verify(x509.VerifyOptions{
+		Roots:     pool,
+		KeyUsages: []x509.ExtKeyUsage{usage},
+	})
+	if err != nil {
+		return x509Cert.NotAfter, err
+	}
+	var t time.Time
+	for _, chain := range chains {
+		for _, cert := range chain {
+			if t.IsZero() || cert.NotAfter.Before(t) {
+				t = cert.NotAfter
+			}
+		}
+	}
+	return t, nil
+}