Create Database CA (#9593)

Introduce Database Certificate Authority. New CA is used by Database Access to sign database certificates making them independent from Host CA. 

Co-authored-by: Marek Smoliński <[email protected]>

api/client/proto/authservice.proto

@@ -757,6 +757,11 @@ message DatabaseCSRRequest {
     bytes CSR = 1 [ (gogoproto.jsontag) = "csr" ];
     // ClusterName is the name of the cluster the request is for.
     string ClusterName = 2 [ (gogoproto.jsontag) = "cluster_name" ];
+    // SignWithDatabaseCA if set to true will use Database CA to sign the created certificate.
+    // This flag was created to enable Database CA for new proxies and don't break old one that
+    // are still using UserCA.
+    // DELETE IN 11.0.
+    bool SignWithDatabaseCA = 3 [ (gogoproto.jsontag) = "sign_with_database_ca" ];
 }
 
 // DatabaseCSRResponse contains the signed database certificate.

api/types/authority.go

@@ -402,10 +402,8 @@ func (ca *CertAuthorityV2) CheckAndSetDefaults() error {
 		return trace.Wrap(err)
 	}
 
-	switch ca.GetType() {
-	case UserCA, HostCA, JWTSigner:
-	default:
-		return trace.BadParameter("invalid CA type %q", ca.GetType())
+	if err := ca.GetType().Check(); err != nil {
+		return trace.Wrap(err)
 	}
 
 	return nil

api/types/trust.go

@@ -31,6 +31,8 @@ const (
 	HostCA CertAuthType = "host"
 	// UserCA identifies the key as a user certificate authority
 	UserCA CertAuthType = "user"
+	// DatabaseCA is a certificate authority used in database access.
+	DatabaseCA CertAuthType = "db"
 	// JWTSigner identifies type of certificate authority as JWT signer. In this
 	// case JWT is not a certificate authority because it does not issue
 	// certificates but rather is an authority that signs tokens, however it behaves
@@ -39,14 +41,17 @@ const (
 )
 
 // CertAuthTypes lists all certificate authority types.
-var CertAuthTypes = []CertAuthType{HostCA, UserCA, JWTSigner}
+var CertAuthTypes = []CertAuthType{HostCA, UserCA, DatabaseCA, JWTSigner}
 
 // Check checks if certificate authority type value is correct
 func (c CertAuthType) Check() error {
-	if c != HostCA && c != UserCA && c != JWTSigner {
-		return trace.BadParameter("'%v' authority type is not supported", c)
+	for _, caType := range CertAuthTypes {
+		if c == caType {
+			return nil
+		}
 	}
-	return nil
+
+	return trace.BadParameter("%q authority type is not supported", c)
 }
 
 // CertAuthID - id of certificate authority (it's type and domain name)

docs/pages/includes/database-access/rotation-note.mdx

@@ -1,5 +1,8 @@
 <Admonition type="note" title="Certificate Rotation">
-  Teleport signs database certificates with the host authority. As such,
-  when performing [host certificates rotation](../../setup/operations/ca-rotation.mdx),
-  the database certificates must be updated as well.
+  Teleport 9.1 introduced new database certificate authority that is only used by Database Access.
+  Older Teleport versions uses host certificate to sign Database Access certificates.
+  After upgrading to Teleport 9.1 the host certificate authority will be still used by Database Access to maintain compatibility.
+  The first [certificate rotation](../../setup/operations/ca-rotation.mdx) will rotate host and database certificates.
+  New Teleport 9.1+ installations generate database certificate authority on the first start and they are not affected
+  by the rotation procedure described above.
 </Admonition>

integration/db_integration_test.go

@@ -21,9 +21,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/gravitational/teleport"
 	apidefaults "github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
 	apievents "github.com/gravitational/teleport/api/types/events"
@@ -41,12 +43,12 @@ import (
 	"github.com/gravitational/teleport/lib/srv/db/postgres"
 	"github.com/gravitational/teleport/lib/tlsca"
 	"github.com/gravitational/teleport/lib/utils"
+	"github.com/gravitational/trace"
 
 	"github.com/google/uuid"
 	"github.com/jackc/pgconn"
 	"github.com/jonboulle/clockwork"
 	"github.com/siddontang/go-mysql/client"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson"
 )
@@ -118,6 +120,185 @@ func TestDatabaseAccessPostgresLeafCluster(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestDatabaseRotateTrustedCluster(t *testing.T) {
+	// TODO(jakule): Fix flaky test
+	t.Skip("flaky test, skip for now")
+
+	pack := setupDatabaseTest(t,
+		// set tighter rotation intervals
+		withLeafConfig(func(config *service.Config) {
+			config.PollingPeriod = 5 * time.Second
+			config.RotationConnectionInterval = 2 * time.Second
+		}),
+		withRootConfig(func(config *service.Config) {
+			config.PollingPeriod = 5 * time.Second
+			config.RotationConnectionInterval = 2 * time.Second
+		}))
+	pack.waitForLeaf(t)
+
+	var (
+		ctx             = context.Background()
+		rootCluster     = pack.root.cluster
+		authServer      = rootCluster.Process.GetAuthServer()
+		clusterRootName = rootCluster.Secrets.SiteName
+		clusterLeafName = pack.leaf.cluster.Secrets.SiteName
+	)
+
+	pw := phaseWatcher{
+		clusterRootName: clusterRootName,
+		pollingPeriod:   rootCluster.Process.Config.PollingPeriod,
+		clock:           pack.clock,
+		siteAPI:         rootCluster.GetSiteAPI(clusterLeafName),
+		certType:        types.DatabaseCA,
+	}
+
+	currentDbCA, err := pack.root.dbAuthClient.GetCertAuthority(ctx, types.CertAuthID{
+		Type:       types.DatabaseCA,
+		DomainName: clusterRootName,
+	}, false)
+	require.NoError(t, err)
+
+	rotationPhases := []string{types.RotationPhaseInit, types.RotationPhaseUpdateClients,
+		types.RotationPhaseUpdateServers, types.RotationPhaseStandby}
+
+	waitForEvent := func(process *service.TeleportProcess, event string) {
+		eventC := make(chan service.Event, 1)
+		process.WaitForEvent(context.TODO(), event, eventC)
+		select {
+		case <-eventC:
+
+		case <-time.After(20 * time.Second):
+			t.Fatalf("timeout waiting for service to broadcast event %s", event)
+		}
+	}
+
+	for _, phase := range rotationPhases {
+		errChan := make(chan error, 1)
+
+		go func() {
+			errChan <- pw.waitForPhase(phase, func() error {
+				return authServer.RotateCertAuthority(ctx, auth.RotateRequest{
+					Type:        types.DatabaseCA,
+					TargetPhase: phase,
+					Mode:        types.RotationModeManual,
+				})
+			})
+		}()
+
+		err = <-errChan
+
+		if err != nil && strings.Contains(err.Error(), "context deadline exceeded") {
+			// TODO(jakule): Workaround for CertAuthorityWatcher failing to get the correct rotation status.
+			// Query auth server directly to see if the incorrect rotation status is a rotation or watcher problem.
+			dbCA, err := pack.leaf.cluster.Process.GetAuthServer().GetCertAuthority(ctx, types.CertAuthID{
+				Type:       types.DatabaseCA,
+				DomainName: clusterRootName,
+			}, false)
+			require.NoError(t, err)
+			require.Equal(t, dbCA.GetRotation().Phase, phase)
+		} else {
+			require.NoError(t, err)
+		}
+
+		// Reload doesn't happen on Init
+		if phase == types.RotationPhaseInit {
+			continue
+		}
+
+		waitForEvent(pack.root.cluster.Process, service.TeleportReloadEvent)
+		waitForEvent(pack.leaf.cluster.Process, service.TeleportReadyEvent)
+
+		pack.waitForLeaf(t)
+	}
+
+	rotatedDbCA, err := authServer.GetCertAuthority(ctx, types.CertAuthID{
+		Type:       types.DatabaseCA,
+		DomainName: clusterRootName,
+	}, false)
+	require.NoError(t, err)
+
+	// Sanity check. Check if the CA was rotated.
+	require.NotEqual(t, currentDbCA.GetActiveKeys(), rotatedDbCA.GetActiveKeys())
+
+	// Connect to the database service in leaf cluster via root cluster.
+	dbClient, err := postgres.MakeTestClient(context.Background(), common.TestClientConfig{
+		AuthClient: pack.root.cluster.GetSiteAPI(pack.root.cluster.Secrets.SiteName),
+		AuthServer: pack.root.cluster.Process.GetAuthServer(),
+		Address:    net.JoinHostPort(Loopback, pack.root.cluster.GetPortWeb()), // Connecting via root cluster.
+		Cluster:    pack.leaf.cluster.Secrets.SiteName,
+		Username:   pack.root.user.GetName(),
+		RouteToDatabase: tlsca.RouteToDatabase{
+			ServiceName: pack.leaf.postgresService.Name,
+			Protocol:    pack.leaf.postgresService.Protocol,
+			Username:    "postgres",
+			Database:    "test",
+		},
+	})
+	require.NoError(t, err)
+
+	// Execute a query.
+	result, err := dbClient.Exec(context.Background(), "select 1").ReadAll()
+	require.NoError(t, err)
+	require.Equal(t, []*pgconn.Result{postgres.TestQueryResponse}, result)
+	require.Equal(t, uint32(1), pack.leaf.postgres.QueryCount())
+	require.Equal(t, uint32(0), pack.root.postgres.QueryCount())
+
+	// Disconnect.
+	err = dbClient.Close(context.Background())
+	require.NoError(t, err)
+}
+
+// phaseWatcher holds all arguments required by rotation watcher.
+type phaseWatcher struct {
+	clusterRootName string
+	pollingPeriod   time.Duration
+	clock           clockwork.Clock
+	siteAPI         types.Events
+	certType        types.CertAuthType
+}
+
+// waitForPhase waits until rootCluster cluster detects the rotation. fn is a rotation function that is called after
+// watcher is created.
+func (p *phaseWatcher) waitForPhase(phase string, fn func() error) error {
+	ctx, cancel := context.WithTimeout(context.Background(), p.pollingPeriod*10)
+	defer cancel()
+
+	watcher, err := services.NewCertAuthorityWatcher(ctx, services.CertAuthorityWatcherConfig{
+		ResourceWatcherConfig: services.ResourceWatcherConfig{
+			Component: teleport.ComponentProxy,
+			Clock:     p.clock,
+			Client:    p.siteAPI,
+		},
+		WatchCertTypes: []types.CertAuthType{p.certType},
+	})
+	if err != nil {
+		return err
+	}
+	defer watcher.Close()
+
+	if err := fn(); err != nil {
+		return trace.Wrap(err)
+	}
+
+	var lastPhase string
+	for i := 0; i < 10; i++ {
+		select {
+		case <-ctx.Done():
+			return trace.CompareFailed("failed to converge to phase %q, last phase %q certType: %v err: %v", phase, lastPhase, p.certType, ctx.Err())
+		case cas := <-watcher.CertAuthorityC:
+			for _, ca := range cas {
+				if ca.GetClusterName() == p.clusterRootName &&
+					ca.GetType() == p.certType &&
+					ca.GetRotation().Phase == phase {
+					return nil
+				}
+				lastPhase = ca.GetRotation().Phase
+			}
+		}
+	}
+	return trace.CompareFailed("failed to converge to phase %q, last phase %q", phase, lastPhase)
+}
+
 // TestDatabaseAccessMySQLRootCluster tests a scenario where a user connects
 // to a MySQL database running in a root cluster.
 func TestDatabaseAccessMySQLRootCluster(t *testing.T) {
@@ -1043,15 +1224,16 @@ func (p *databasePack) waitForLeaf(t *testing.T) {
 		case <-time.Tick(500 * time.Millisecond):
 			servers, err := accessPoint.GetDatabaseServers(ctx, apidefaults.Namespace)
 			if err != nil {
-				logrus.WithError(err).Debugf("Leaf cluster access point is unavailable.")
+				// Use root logger as we need a configured logger instance and the root cluster have one.
+				p.root.cluster.log.WithError(err).Debugf("Leaf cluster access point is unavailable.")
 				continue
 			}
 			if !containsDB(servers, p.leaf.mysqlService.Name) {
-				logrus.WithError(err).Debugf("Leaf db service %q is unavailable.", p.leaf.mysqlService.Name)
+				p.root.cluster.log.WithError(err).Debugf("Leaf db service %q is unavailable.", p.leaf.mysqlService.Name)
 				continue
 			}
 			if !containsDB(servers, p.leaf.postgresService.Name) {
-				logrus.WithError(err).Debugf("Leaf db service %q is unavailable.", p.leaf.postgresService.Name)
+				p.root.cluster.log.WithError(err).Debugf("Leaf db service %q is unavailable.", p.leaf.postgresService.Name)
 				continue
 			}
 			return

integration/helpers.go

@@ -325,10 +325,24 @@ func (s *InstanceSecrets) GetCAs() ([]types.CertAuthority, error) {
 		return nil, trace.Wrap(err)
 	}
 
-	return []types.CertAuthority{
-		hostCA,
-		userCA,
-	}, nil
+	dbCA, err := types.NewCertAuthority(types.CertAuthoritySpecV2{
+		Type:        types.DatabaseCA,
+		ClusterName: s.SiteName,
+		ActiveKeys: types.CAKeySet{
+			TLS: []*types.TLSKeyPair{{
+				Key:     s.PrivKey,
+				KeyType: types.PrivateKeyType_RAW,
+				Cert:    s.TLSCACert,
+			}},
+		},
+		Roles:      []string{},
+		SigningAlg: types.CertAuthoritySpecV2_RSA_SHA2_512,
+	})
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	return []types.CertAuthority{hostCA, userCA, dbCA}, nil
 }
 
 func (s *InstanceSecrets) AllowedLogins() []string {
@@ -370,7 +384,7 @@ func (s *InstanceSecrets) GetIdentity() *auth.Identity {
 	return i
 }
 
-// GetSiteAPI() is a helper which returns an API endpoint to a site with
+// GetSiteAPI is a helper which returns an API endpoint to a site with
 // a given name. i endpoint implements HTTP-over-SSH access to the
 // site's auth server.
 func (i *TeleInstance) GetSiteAPI(siteName string) auth.ClientI {