feat: update vault secrets from config (#3080)

* feat: update vault secrets from config * chore: create or update vault secrets * chore: move vault to pkg
supabase · Jan 31, 2025 · 722c814 · 722c814
1 parent 5017dcb
commit 722c814
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 24 deletions.
diff --git a/internal/db/push/push.go b/internal/db/push/push.go
@@ -14,6 +14,7 @@ import (
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/internal/utils/flags"
 	"github.com/supabase/cli/pkg/migration"
+	"github.com/supabase/cli/pkg/vault"
 )
 
 func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
@@ -82,6 +83,9 @@ func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles,
 			} else if !shouldPush {
 				return errors.New(context.Canceled)
 			}
+			if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil {
+				return err
+			}
 			if err := migration.ApplyMigrations(ctx, pending, conn, afero.NewIOFS(fsys)); err != nil {
 				return err
 			}

diff --git a/internal/db/reset/reset.go b/internal/db/reset/reset.go
@@ -23,11 +23,11 @@ import (
 	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/gen/keys"
 	"github.com/supabase/cli/internal/migration/apply"
-	"github.com/supabase/cli/internal/migration/list"
 	"github.com/supabase/cli/internal/migration/repair"
 	"github.com/supabase/cli/internal/seed/buckets"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/migration"
+	"github.com/supabase/cli/pkg/vault"
 )
 
 func Run(ctx context.Context, version string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
@@ -242,22 +242,10 @@ func resetRemote(ctx context.Context, version string, config pgconn.Config, fsys
 	if err := migration.DropUserSchemas(ctx, conn); err != nil {
 		return err
 	}
-	migrations, err := list.LoadPartialMigrations(version, fsys)
-	if err != nil {
-		return err
-	}
-	if err := migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys)); err != nil {
+	if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil {
 		return err
 	}
-	if !utils.Config.Db.Seed.Enabled {
-		// Skip because --no-seed flag is set
-		return nil
-	}
-	seeds, err := migration.GetPendingSeeds(ctx, utils.Config.Db.Seed.SqlPaths, conn, afero.NewIOFS(fsys))
-	if err != nil {
-		return err
-	}
-	return migration.SeedData(ctx, seeds, conn, afero.NewIOFS(fsys))
+	return apply.MigrateAndSeed(ctx, version, conn, fsys)
 }
 
 func LikeEscapeSchema(schemas []string) (result []string) {

diff --git a/internal/db/start/start.go b/internal/db/start/start.go
@@ -25,6 +25,7 @@ import (
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/internal/utils/flags"
 	"github.com/supabase/cli/pkg/migration"
+	"github.com/supabase/cli/pkg/vault"
 )
 
 var (
@@ -372,6 +373,10 @@ func SetupDatabase(ctx context.Context, conn *pgx.Conn, host string, w io.Writer
 	if err := initSchema(ctx, conn, host, w); err != nil {
 		return err
 	}
+	// Create vault secrets first so roles.sql can reference them
+	if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil {
+		return err
+	}
 	err := migration.SeedGlobals(ctx, []string{utils.CustomRolesPath}, conn, afero.NewIOFS(fsys))
 	if errors.Is(err, os.ErrNotExist) {
 		return nil

diff --git a/internal/migration/up/up.go b/internal/migration/up/up.go
@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/afero"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/migration"
+	"github.com/supabase/cli/pkg/vault"
 )
 
 func Run(ctx context.Context, includeAll bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
@@ -23,6 +24,9 @@ func Run(ctx context.Context, includeAll bool, config pgconn.Config, fsys afero.
 	if err != nil {
 		return err
 	}
+	if err := vault.UpsertVaultSecrets(ctx, utils.Config.Db.Vault, conn); err != nil {
+		return err
+	}
 	return migration.ApplyMigrations(ctx, pending, conn, afero.NewIOFS(fsys))
 }
 

diff --git a/pkg/config/db.go b/pkg/config/db.go
@@ -51,15 +51,16 @@ type (
 	}
 
 	db struct {
-		Image        string   `toml:"-"`
-		Port         uint16   `toml:"port"`
-		ShadowPort   uint16   `toml:"shadow_port"`
-		MajorVersion uint     `toml:"major_version"`
-		Password     string   `toml:"-"`
-		RootKey      string   `toml:"-" mapstructure:"root_key"`
-		Pooler       pooler   `toml:"pooler"`
-		Seed         seed     `toml:"seed"`
-		Settings     settings `toml:"settings"`
+		Image        string            `toml:"-"`
+		Port         uint16            `toml:"port"`
+		ShadowPort   uint16            `toml:"shadow_port"`
+		MajorVersion uint              `toml:"major_version"`
+		Password     string            `toml:"-"`
+		RootKey      string            `toml:"-" mapstructure:"root_key"`
+		Pooler       pooler            `toml:"pooler"`
+		Seed         seed              `toml:"seed"`
+		Settings     settings          `toml:"settings"`
+		Vault        map[string]Secret `toml:"vault"`
 	}
 
 	seed struct {

diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml
@@ -42,6 +42,9 @@ default_pool_size = 20
 # Maximum number of client connections allowed.
 max_client_conn = 100
 
+# [db.vault]
+# secret_key = "env(SECRET_VALUE)"
+
 [db.seed]
 # If enabled, seeds the database after migrations during a db reset.
 enabled = true

diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml
@@ -42,6 +42,9 @@ default_pool_size = 20
 # Maximum number of client connections allowed.
 max_client_conn = 100
 
+[db.vault]
+test_key = "test_value"
+
 [db.seed]
 # If enabled, seeds the database after migrations during a db reset.
 enabled = true

diff --git a/pkg/vault/batch.go b/pkg/vault/batch.go
@@ -0,0 +1,60 @@
+package vault
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/go-errors/errors"
+	"github.com/jackc/pgx/v4"
+	"github.com/supabase/cli/pkg/config"
+	"github.com/supabase/cli/pkg/pgxv5"
+)
+
+const (
+	CREATE_VAULT_KV = "SELECT vault.create_secret($1, $2)"
+	READ_VAULT_KV   = "SELECT id, name FROM vault.secrets WHERE name = ANY($1)"
+	UPDATE_VAULT_KV = "SELECT vault.update_secret($1, $2)"
+)
+
+type VaultTable struct {
+	Id   string
+	Name string
+}
+
+func UpsertVaultSecrets(ctx context.Context, secrets map[string]config.Secret, conn *pgx.Conn) error {
+	var keys []string
+	toInsert := map[string]string{}
+	for k, v := range secrets {
+		if len(v.SHA256) > 0 {
+			keys = append(keys, k)
+			toInsert[k] = v.Value
+		}
+	}
+	if len(keys) == 0 {
+		return nil
+	}
+	fmt.Fprintln(os.Stderr, "Updating vault secrets...")
+	rows, err := conn.Query(ctx, READ_VAULT_KV, keys)
+	if err != nil {
+		return errors.Errorf("failed to read vault: %w", err)
+	}
+	toUpdate, err := pgxv5.CollectRows[VaultTable](rows)
+	if err != nil {
+		return err
+	}
+	batch := pgx.Batch{}
+	for _, r := range toUpdate {
+		secret := secrets[r.Name]
+		batch.Queue(UPDATE_VAULT_KV, r.Id, secret.Value)
+		delete(toInsert, r.Name)
+	}
+	// Remaining secrets should be created
+	for k, v := range toInsert {
+		batch.Queue(CREATE_VAULT_KV, v, k)
+	}
+	if err := conn.SendBatch(ctx, &batch).Close(); err != nil {
+		return errors.Errorf("failed to update vault: %w", err)
+	}
+	return nil
+}