Merge pull request #8343 from dolthub/zachmu/schema-commit

support for schemas in various version control operations

go/cmd/dolt/commands/cvcmds/verify_constraints.go

@@ -30,7 +30,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/merge"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
-	"github.com/dolthub/dolt/go/libraries/utils/set"
 	"github.com/dolthub/dolt/go/store/types"
 )
 
@@ -85,7 +84,12 @@ func (cmd VerifyConstraintsCmd) Exec(ctx context.Context, commandStr string, arg
 			return commands.HandleVErrAndExitCode(errhand.BuildDError("Unable to read table names.").AddCause(err).Build(), nil)
 		}
 	}
-	tableSet := set.NewStrSet(tableNames)
+	tableSet := doltdb.NewTableNameSet(nil)
+
+	// TODO: schema names
+	for _, tableName := range tableNames {
+		tableSet.Add(doltdb.TableName{Name: tableName})
+	}
 
 	comparingRoot, err := dEnv.HeadRoot(ctx)
 	if err != nil {

go/libraries/doltcore/diff/database_schema_deltas.go

@@ -0,0 +1,106 @@
+// Copyright 2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package diff
+
+import (
+	"context"
+
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/utils/set"
+)
+
+// DatabaseSchemaDelta represents a change in the set of database schemas between two roots
+type DatabaseSchemaDelta struct {
+	FromName string
+	ToName   string
+}
+
+func (d DatabaseSchemaDelta) IsAdd() bool {
+	return d.FromName == "" && d.ToName != ""
+}
+
+func (d DatabaseSchemaDelta) IsDrop() bool {
+	return d.FromName != "" && d.ToName == ""
+}
+
+func (d DatabaseSchemaDelta) CurName() string {
+	if d.ToName != "" {
+		return d.ToName
+	}
+	return d.FromName
+}
+
+// GetDatabaseSchemaDeltas returns a list of DatabaseSchemaDelta objects representing the changes in database schemas
+func GetDatabaseSchemaDeltas(ctx context.Context, fromRoot, toRoot doltdb.RootValue) ([]DatabaseSchemaDelta, error) {
+	fromNames, err := getDatabaseSchemaNames(ctx, fromRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	toNames, err := getDatabaseSchemaNames(ctx, toRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	// short circuit for common case where there are no schemas (dolt)
+	if fromNames.Size() == 0 && toNames.Size() == 0 {
+		return nil, nil
+	}
+
+	// generate a diff for each schema name that's present in one root but not the other
+	var deltas []DatabaseSchemaDelta
+	fromNames.Iterate(func(name string) (cont bool) {
+		if !toNames.Contains(name) {
+			deltas = append(deltas, DatabaseSchemaDelta{FromName: name})
+		}
+		return true
+	})
+
+	toNames.Iterate(func(name string) (cont bool) {
+		if !fromNames.Contains(name) {
+			deltas = append(deltas, DatabaseSchemaDelta{ToName: name})
+		}
+		return true
+	})
+
+	return deltas, nil
+}
+
+// GetStagedUnstagedDatabaseSchemaDeltas represents staged and unstaged changes as DatabaseSchemaDelta slices.
+func GetStagedUnstagedDatabaseSchemaDeltas(ctx context.Context, roots doltdb.Roots) (staged, unstaged []DatabaseSchemaDelta, err error) {
+	staged, err = GetDatabaseSchemaDeltas(ctx, roots.Head, roots.Staged)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	unstaged, err = GetDatabaseSchemaDeltas(ctx, roots.Staged, roots.Working)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return staged, unstaged, nil
+}
+
+func getDatabaseSchemaNames(ctx context.Context, root doltdb.RootValue) (*set.StrSet, error) {
+	dbSchemaNames := set.NewEmptyStrSet()
+	dbSchemas, err := root.GetDatabaseSchemas(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, dbSchema := range dbSchemas {
+		dbSchemaNames.Add(dbSchema.Name)
+	}
+	return dbSchemaNames, nil
+}

go/libraries/doltcore/doltdb/foreign_key_coll.go

@@ -490,7 +490,7 @@ OuterLoop:
 // and any keys in the collection are unresolved. A "dirty resolution" is performed, which matches the column names to
 // tags, and then a standard tag comparison is performed. If a table or column is not in the map, then the foreign key
 // is ignored.
-func (fkc *ForeignKeyCollection) GetMatchingKey(fk ForeignKey, allSchemas map[string]schema.Schema, matchUnresolvedKeyToResolvedKey bool) (ForeignKey, bool) {
+func (fkc *ForeignKeyCollection) GetMatchingKey(fk ForeignKey, allSchemas map[TableName]schema.Schema, matchUnresolvedKeyToResolvedKey bool) (ForeignKey, bool) {
 	if !fk.IsResolved() {
 		// The given foreign key is unresolved, so we only look for matches on unresolved keys
 	OuterLoopUnresolved:
@@ -543,11 +543,13 @@ OuterLoopResolved:
 				len(fk.ReferencedTableColumns) != len(existingFk.UnresolvedFKDetails.ReferencedTableColumns) {
 				continue
 			}
-			tblSch, ok := allSchemas[existingFk.TableName]
+			// TODO: schema name
+			tblSch, ok := allSchemas[TableName{Name: existingFk.TableName}]
 			if !ok {
 				continue
 			}
-			refTblSch, ok := allSchemas[existingFk.ReferencedTableName]
+			// TODO: schema name
+			refTblSch, ok := allSchemas[TableName{Name: existingFk.ReferencedTableName}]
 			if !ok {
 				continue
 			}

go/libraries/doltcore/doltdb/root_val.go

@@ -455,11 +455,10 @@ func GetExistingColumns(
 	return existingCols, nil
 }
 
-func GetAllSchemas(ctx context.Context, root RootValue) (map[string]schema.Schema, error) {
-	m := make(map[string]schema.Schema)
+func GetAllSchemas(ctx context.Context, root RootValue) (map[TableName]schema.Schema, error) {
+	m := make(map[TableName]schema.Schema)
 	err := root.IterTables(ctx, func(name TableName, table *Table, sch schema.Schema) (stop bool, err error) {
-		// TODO: schema name
-		m[name.Name] = sch
+		m[name] = sch
 		return false, nil
 	})
 

go/libraries/doltcore/doltdb/table.go

@@ -256,7 +256,7 @@ func (t *Table) clearConflicts(ctx context.Context) (*Table, error) {
 }
 
 // GetConflictSchemas returns the merge conflict schemas for this table.
-func (t *Table) GetConflictSchemas(ctx context.Context, tblName string) (base, sch, mergeSch schema.Schema, err error) {
+func (t *Table) GetConflictSchemas(ctx context.Context, tblName TableName) (base, sch, mergeSch schema.Schema, err error) {
 	if t.Format() == types.Format_DOLT {
 		return t.getProllyConflictSchemas(ctx, tblName)
 	}
@@ -267,7 +267,7 @@ func (t *Table) GetConflictSchemas(ctx context.Context, tblName string) (base, s
 // The conflict schema is implicitly determined based on the first conflict in the artifacts table.
 // For now, we will enforce that all conflicts in the artifacts table must have the same schema set (base, ours, theirs).
 // In the future, we may be able to display conflicts in a way that allows different conflict schemas to coexist.
-func (t *Table) getProllyConflictSchemas(ctx context.Context, tblName string) (base, sch, mergeSch schema.Schema, err error) {
+func (t *Table) getProllyConflictSchemas(ctx context.Context, tblName TableName) (base, sch, mergeSch schema.Schema, err error) {
 	arts, err := t.GetArtifacts(ctx)
 	if err != nil {
 		return nil, nil, nil, err
@@ -331,12 +331,12 @@ func (t *Table) getProllyConflictSchemas(ctx context.Context, tblName string) (b
 	return baseSch, ourSch, theirSch, nil
 }
 
-func tableFromRootIsh(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, h hash.Hash, tblName string) (*Table, bool, error) {
+func tableFromRootIsh(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, h hash.Hash, tblName TableName) (*Table, bool, error) {
 	rv, err := LoadRootValueFromRootIshAddr(ctx, vrw, ns, h)
 	if err != nil {
 		return nil, false, err
 	}
-	tbl, ok, err := rv.GetTable(ctx, TableName{Name: tblName})
+	tbl, ok, err := rv.GetTable(ctx, tblName)
 	if err != nil {
 		return nil, false, err
 	}

go/libraries/doltcore/doltdb/workingset.go

@@ -137,8 +137,8 @@ type MergeState struct {
 type SchemaConflict struct {
 	ToSch, FromSch    schema.Schema
 	ToFks, FromFks    []ForeignKey
-	ToParentSchemas   map[string]schema.Schema
-	FromParentSchemas map[string]schema.Schema
+	ToParentSchemas   map[TableName]schema.Schema
+	FromParentSchemas map[TableName]schema.Schema
 	toTbl, fromTbl    *Table
 }
 

go/libraries/doltcore/env/actions/commit.go

@@ -47,21 +47,26 @@ func GetCommitStaged(
 		return nil, datas.ErrEmptyCommitMessage
 	}
 
-	staged, notStaged, err := diff.GetStagedUnstagedTableDeltas(ctx, roots)
+	stagedTables, notStaged, err := diff.GetStagedUnstagedTableDeltas(ctx, roots)
 	if err != nil {
 		return nil, err
 	}
 
 	var stagedTblNames []doltdb.TableName
-	for _, td := range staged {
+	for _, td := range stagedTables {
 		n := td.ToName
 		if td.IsDrop() {
 			n = td.FromName
 		}
 		stagedTblNames = append(stagedTblNames, n)
 	}
 
-	isEmpty := len(staged) == 0
+	stagedSchemas, _, err := diff.GetStagedUnstagedDatabaseSchemaDeltas(ctx, roots)
+	if err != nil {
+		return nil, err
+	}
+
+	isEmpty := len(stagedTables) == 0 && len(stagedSchemas) == 0
 	allowEmpty := ws.MergeActive() || props.AllowEmpty || props.Amend
 
 	if isEmpty && props.SkipEmpty {

go/libraries/doltcore/env/actions/reset.go

@@ -25,7 +25,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
-	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 	"github.com/dolthub/dolt/go/store/datas"
 )
@@ -81,7 +80,7 @@ func resetHardTables(ctx *sql.Context, dbData env.DbData, cSpecStr string, roots
 		return nil, doltdb.Roots{}, err
 	}
 	for _, name := range staged {
-		delete(untracked, name)
+		delete(untracked, doltdb.TableName{Name: name})
 	}
 
 	newWkRoot := roots.Head
@@ -102,15 +101,15 @@ func resetHardTables(ctx *sql.Context, dbData env.DbData, cSpecStr string, roots
 	}
 
 	for name := range untracked {
-		tname, tbl, exists, err := resolve.Table(ctx, roots.Working, name)
+		tbl, exists, err := roots.Working.GetTable(ctx, name)
 		if err != nil {
 			return nil, doltdb.Roots{}, err
 		}
 		if !exists {
 			return nil, doltdb.Roots{}, fmt.Errorf("untracked table %s does not exist in working set", name)
 		}
 
-		newWkRoot, err = newWkRoot.PutTable(ctx, tname, tbl)
+		newWkRoot, err = newWkRoot.PutTable(ctx, name, tbl)
 		if err != nil {
 			return nil, doltdb.Roots{}, fmt.Errorf("failed to write table back to database: %s", err)
 		}
@@ -334,11 +333,11 @@ func CleanUntracked(ctx context.Context, roots doltdb.Roots, tables []string, dr
 
 // mapColumnTags takes a map from table name to schema.Schema and generates
 // a map from column tags to table names (see RootValue.GetAllSchemas).
-func mapColumnTags(tables map[string]schema.Schema) (m map[uint64]string) {
+func mapColumnTags(tables map[doltdb.TableName]schema.Schema) (m map[uint64]string) {
 	m = make(map[uint64]string, len(tables))
 	for tbl, sch := range tables {
 		for _, tag := range sch.GetAllCols().Tags {
-			m[tag] = tbl
+			m[tag] = tbl.Name
 		}
 	}
 	return

go/libraries/doltcore/env/actions/staged.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/diff"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 )
 
 func StageTables(ctx context.Context, roots doltdb.Roots, tbls []doltdb.TableName, filterIgnoredTables bool) (doltdb.Roots, error) {
@@ -45,7 +46,58 @@ func StageAllTables(ctx context.Context, roots doltdb.Roots, filterIgnoredTables
 		return doltdb.Roots{}, err
 	}
 
-	return StageTables(ctx, roots, tbls, filterIgnoredTables)
+	roots, err = StageTables(ctx, roots, tbls, filterIgnoredTables)
+	if err != nil {
+		return doltdb.Roots{}, err
+	}
+
+	roots, err = StageAllSchemas(ctx, roots)
+	if err != nil {
+		return doltdb.Roots{}, err
+	}
+
+	return roots, nil
+}
+
+func StageAllSchemas(ctx context.Context, roots doltdb.Roots) (doltdb.Roots, error) {
+	newStaged, err := MoveAllSchemasBetweenRoots(ctx, roots.Working, roots.Staged)
+	if err != nil {
+		return doltdb.Roots{}, err
+	}
+
+	roots.Staged = newStaged
+	return roots, nil
+}
+
+// MoveAllSchemasBetweenRoots copies all schemas from the src RootValue to the dest RootValue.
+func MoveAllSchemasBetweenRoots(ctx context.Context, src, dest doltdb.RootValue) (doltdb.RootValue, error) {
+	srcSchemaNames, err := getDatabaseSchemaNames(ctx, src)
+	if err != nil {
+		return nil, err
+	}
+
+	if srcSchemaNames.Size() == 0 {
+		return dest, nil
+	}
+
+	destSchemaNames, err := getDatabaseSchemaNames(ctx, dest)
+	if err != nil {
+		return nil, err
+	}
+
+	srcSchemaNames.Iterate(func(schemaName string) (cont bool) {
+		if !destSchemaNames.Contains(schemaName) {
+			dest, err = dest.CreateDatabaseSchema(ctx, schema.DatabaseSchema{
+				Name: schemaName,
+			})
+			if err != nil {
+				return false
+			}
+		}
+		return true
+	})
+
+	return dest, nil
 }
 
 func StageDatabase(ctx context.Context, roots doltdb.Roots) (doltdb.Roots, error) {