Skip to content
This repository was archived by the owner on Mar 5, 2024. It is now read-only.

Commit

Permalink
Add metrics for TLS certificate expiration (#364)
Browse files Browse the repository at this point in the history
  • Loading branch information
abursavich authored Mar 12, 2020
1 parent 88cf9ea commit aab2b6c
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 34 deletions.
25 changes: 8 additions & 17 deletions pkg/server/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"time"

grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
"github.com/grpc-ecosystem/go-grpc-prometheus"
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
"github.com/uswitch/kiam/pkg/aws/sts"
"github.com/uswitch/kiam/pkg/statsd"
pb "github.com/uswitch/kiam/proto"
Expand Down Expand Up @@ -59,28 +58,20 @@ func NewGateway(ctx context.Context, address string, caFile, certificateFile, ke
retry.WithBackoff(retry.BackoffLinear(RetryInterval)),
}

certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile)
if err != nil {
return nil, fmt.Errorf("error loading keypair: %v", err)
}
certPool := x509.NewCertPool()
ca, err := ioutil.ReadFile(caFile)
if err != nil {
return nil, fmt.Errorf("error reading SSL cert: %v", err)
}
if ok := certPool.AppendCertsFromPEM(ca); !ok {
return nil, fmt.Errorf("error appending certs from ca")
}

host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, fmt.Errorf("error parsing hostname: %v", err)
}
cert, caPool, err := loadCerts(certificateFile, keyFile, caFile)
if err != nil {
return nil, err
}
clientTLSMetrics.update(x509.ExtKeyUsageClientAuth, &cert, caPool)

creds := credentials.NewTLS(&tls.Config{
ServerName: host,
Certificates: []tls.Certificate{certificate},
RootCAs: certPool,
Certificates: []tls.Certificate{cert},
RootCAs: caPool,
})

dialAddress := fmt.Sprintf("dns:///%s", address)
Expand Down
24 changes: 7 additions & 17 deletions pkg/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,11 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"time"

"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/grpc-ecosystem/go-grpc-prometheus"
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
log "github.com/sirupsen/logrus"
"github.com/uswitch/k8sc/official"
"github.com/uswitch/kiam/pkg/aws/sts"
Expand All @@ -33,7 +32,7 @@ import (
pb "github.com/uswitch/kiam/proto"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
Expand Down Expand Up @@ -253,25 +252,16 @@ func NewServer(config *Config) (*KiamServer, error) {
server.manager = prefetch.NewManager(credentialsCache, server.pods)
server.assumePolicy = Policies(NewRequestingAnnotatedRolePolicy(server.pods, arnResolver), NewNamespacePermittedRoleNamePolicy(server.namespaces, server.pods))

certificate, err := tls.LoadX509KeyPair(config.TLS.ServerCert, config.TLS.ServerKey)
cert, caPool, err := loadCerts(config.TLS.ServerCert, config.TLS.ServerKey, config.TLS.CA)
if err != nil {
return nil, err
}
certPool := x509.NewCertPool()
if err != nil {
return nil, err
}
ca, err := ioutil.ReadFile(config.TLS.CA)
if err != nil {
return nil, err
}
if ok := certPool.AppendCertsFromPEM(ca); !ok {
return nil, fmt.Errorf("failed to append CA cert to certPool")
}
serverTLSMetrics.update(x509.ExtKeyUsageServerAuth, &cert, caPool)

creds := credentials.NewTLS(&tls.Config{
ClientAuth: tls.RequireAndVerifyClientCert,
Certificates: []tls.Certificate{certificate},
ClientCAs: certPool,
Certificates: []tls.Certificate{cert},
ClientCAs: caPool,
})

grpcServer := grpc.NewServer(
Expand Down
110 changes: 110 additions & 0 deletions pkg/server/tls.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package server

import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)

var (
clientTLSMetrics = newTLSMetrics("client")
serverTLSMetrics = newTLSMetrics("server")
)

type tlsMetrics struct {
registerOnce sync.Once
verifyError prometheus.Gauge
expiration prometheus.Gauge
}

func newTLSMetrics(subsystem string) *tlsMetrics {
return &tlsMetrics{
verifyError: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "grpc",
Subsystem: subsystem,
Name: "tls_certificate_verify_error",
Help: "Indicates if there was an error verifying the latest gRPC " + subsystem + " TLS certificate and its expiration.",
}),
expiration: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "grpc",
Subsystem: subsystem,
Name: "tls_certificate_expiry_seconds",
Help: "Expiration time of the gRPC " + subsystem + " TLS certificate in seconds since the Unix epoch.",
}),
}
}

func (m *tlsMetrics) Describe(ch chan<- *prometheus.Desc) {
m.verifyError.Describe(ch)
m.expiration.Describe(ch)
}

func (m *tlsMetrics) Collect(ch chan<- prometheus.Metric) {
m.verifyError.Collect(ch)
m.expiration.Collect(ch)
}

func (m *tlsMetrics) update(usage x509.ExtKeyUsage, cert *tls.Certificate, pool *x509.CertPool) {
m.registerOnce.Do(func() { prometheus.MustRegister(m) })

expiry, err := earliestExpiry(cert, pool, usage)
if err != nil {
log.Errorf("failed to verify TLS certificate: %v", err)
m.verifyError.Set(1)
} else {
m.verifyError.Set(0)
}
m.expiration.Set(float64(expiry.Unix()))
}

func loadCerts(certFile, keyFile, caFile string) (tls.Certificate, *x509.CertPool, error) {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return tls.Certificate{}, nil, fmt.Errorf("error loading TLS key pair: %v", err)
}
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS leaf cert: %v", err)
}
ca, err := ioutil.ReadFile(caFile)
if err != nil {
return tls.Certificate{}, nil, fmt.Errorf("error reading TLS CAs: %v", err)
}
caPool := x509.NewCertPool()
if !caPool.AppendCertsFromPEM(ca) {
return tls.Certificate{}, nil, fmt.Errorf("error parsing TLS CAs")
}
return cert, caPool, nil
}

func earliestExpiry(cert *tls.Certificate, pool *x509.CertPool, usage x509.ExtKeyUsage) (time.Time, error) {
x509Cert := cert.Leaf
if x509Cert == nil {
var err error
if x509Cert, err = x509.ParseCertificate(cert.Certificate[0]); err != nil {
return time.Time{}, err
}
}
chains, err := x509Cert.Verify(x509.VerifyOptions{
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{usage},
})
if err != nil {
return x509Cert.NotAfter, err
}
var t time.Time
for _, chain := range chains {
for _, cert := range chain {
if t.IsZero() || cert.NotAfter.Before(t) {
t = cert.NotAfter
}
}
}
return t, nil
}

0 comments on commit aab2b6c

Please sign in to comment.