From 45eeb19d7d952531c3102add91cad61f1d02a410 Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 16:45:45 -0400 Subject: [PATCH 1/7] add fix and timestamp default --- create.go | 14 ++------- dialect.go | 20 +++++++++++++ fix.go | 45 +++++++++++++++++++++++++++++ fix_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++ goose.go | 5 ++++ migrate.go | 38 +++++++++++++++++++++++++ 6 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 fix.go create mode 100644 fix_test.go diff --git a/create.go b/create.go index b86307652..6761dec32 100644 --- a/create.go +++ b/create.go @@ -6,22 +6,12 @@ import ( "os" "path/filepath" "text/template" + "time" ) // Create writes a new blank migration file. func CreateWithTemplate(db *sql.DB, dir string, migrationTemplate *template.Template, name, migrationType string) error { - migrations, err := CollectMigrations(dir, minVersion, maxVersion) - if err != nil { - return err - } - - // Initial version. - version := "00001" - - if last, err := migrations.Last(); err == nil { - version = fmt.Sprintf("%05v", last.Version+1) - } - + version := time.Now().Format(timestampFormat) filename := fmt.Sprintf("%v_%v.%v", version, name, migrationType) fpath := filepath.Join(dir, filename) diff --git a/dialect.go b/dialect.go index 488f5e040..edc2c35e7 100644 --- a/dialect.go +++ b/dialect.go @@ -61,6 +61,10 @@ func (pg PostgresDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (pg PostgresDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (pg PostgresDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -91,6 +95,10 @@ func (m MySQLDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m MySQLDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m MySQLDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -120,6 +128,10 @@ func (m Sqlite3Dialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m Sqlite3Dialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m Sqlite3Dialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -150,6 +162,10 @@ func (rs RedshiftDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (rs RedshiftDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (rs RedshiftDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -180,6 +196,10 @@ func (m TiDBDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m TiDBDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m TiDBDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { diff --git a/fix.go b/fix.go new file mode 100644 index 000000000..abb9d19a1 --- /dev/null +++ b/fix.go @@ -0,0 +1,45 @@ +package goose + +import ( + "database/sql" + "fmt" + "os" + "strings" +) + +func Fix(db *sql.DB, dir string) error { + migrations, err := CollectMigrations(dir, minVersion, maxVersion) + if err != nil { + return err + } + + // split into timestamped and versioned migrations + tsMigrations, err := migrations.Timestamped() + if err != nil { + return err + } + + vMigrations, err := migrations.Versioned() + if err != nil { + return err + } + // Initial version. + version := int64(1) + if last, err := vMigrations.Last(); err == nil { + version = last.Version + 1 + } + + // fix filenames by replacing timestamps with sequential versions + for _, tsm := range tsMigrations { + oldPath := tsm.Source + newPath := strings.Replace(oldPath, fmt.Sprintf("%d", tsm.Version), fmt.Sprintf("%05v", version), 1) + + if err := os.Rename(oldPath, newPath); err != nil { + return err + } + + version++ + } + + return nil +} diff --git a/fix_test.go b/fix_test.go new file mode 100644 index 000000000..e2f9fc9ff --- /dev/null +++ b/fix_test.go @@ -0,0 +1,81 @@ +package goose + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestFix(t *testing.T) { + dir, err := ioutil.TempDir("", "tmptest") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) // clean up + defer os.Remove("goose") // clean up + + commands := []string{ + "go build -i -o goose ./cmd/goose", + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files are in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } + + // add more migrations and then fix it + commands = []string{ + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err = ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files still in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } +} diff --git a/goose.go b/goose.go index 51ca6ecf4..1c492b0d8 100644 --- a/goose.go +++ b/goose.go @@ -11,6 +11,7 @@ var ( duplicateCheckOnce sync.Once minVersion = int64(0) maxVersion = int64((1 << 63) - 1) + timestampFormat = "20060102150405" ) // Run runs a goose command. @@ -64,6 +65,10 @@ func Run(command string, db *sql.DB, dir string, args ...string) error { if err := DownTo(db, dir, version); err != nil { return err } + case "fix": + if err := Fix(db, dir); err != nil { + return err + } case "redo": if err := Redo(db, dir); err != nil { return err diff --git a/migrate.go b/migrate.go index 4774af022..f998b27b4 100644 --- a/migrate.go +++ b/migrate.go @@ -8,6 +8,7 @@ import ( "path/filepath" "runtime" "sort" + "time" ) var ( @@ -76,6 +77,43 @@ func (ms Migrations) Last() (*Migration, error) { return ms[len(ms)-1], nil } +// Versioned gets versioned migrations. +func (ms Migrations) Versioned() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + + if versionTime.Before(time.Unix(0, 0)) || err != nil { + migrations = append(migrations, m) + } + } + + return migrations, nil +} + +// Timestamped gets the timestamped migrations. +func (ms Migrations) Timestamped() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + if err != nil { + // probably not a timestamp + continue + } + + if versionTime.After(time.Unix(0, 0)) { + migrations = append(migrations, m) + } + } + return migrations, nil +} + func (ms Migrations) String() string { str := "" for _, m := range ms { From 286883adf54fb4d6e749d95a4596bebfe44d191d Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 16:51:07 -0400 Subject: [PATCH 2/7] add db fix as well --- dialect.go | 1 + fix.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dialect.go b/dialect.go index edc2c35e7..1f21aa6a3 100644 --- a/dialect.go +++ b/dialect.go @@ -10,6 +10,7 @@ import ( type SQLDialect interface { createVersionTableSQL() string // sql string to create the db version table insertVersionSQL() string // sql string to insert the initial version table row + updateVersionSQL() string // sql string to update version dbVersionQuery(db *sql.DB) (*sql.Rows, error) } diff --git a/fix.go b/fix.go index abb9d19a1..caa9ff8e6 100644 --- a/fix.go +++ b/fix.go @@ -29,6 +29,12 @@ func Fix(db *sql.DB, dir string) error { version = last.Version + 1 } + // fix db table as well + tx, err := db.Begin() + if err != nil { + log.Fatal("db.Begin: ", err) + } + // fix filenames by replacing timestamps with sequential versions for _, tsm := range tsMigrations { oldPath := tsm.Source @@ -38,8 +44,13 @@ func Fix(db *sql.DB, dir string) error { return err } + if _, err := tx.Exec(GetDialect().updateVersionSQL(), version, tsm.Version); err != nil { + tx.Rollback() + return err + } + version++ } - return nil + return tx.Commit() } From 22d96449ec7edf128453482eec3a30a5bf7817ca Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 17:02:57 -0400 Subject: [PATCH 3/7] remove db fix, unexport stuff --- cmd/goose/main.go | 4 ++-- fix.go | 20 ++++---------------- goose.go | 2 +- migrate.go | 4 ++-- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index dcc59101b..87b9506ea 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -26,8 +26,8 @@ func main() { args := flags.Args() - if len(args) > 1 && args[0] == "create" { - if err := goose.Run("create", nil, *dir, args[1:]...); err != nil { + if len(args) > 1 && (args[0] == "create" || args[0] == "fix") { + if err := goose.Run(args[0], nil, *dir, args[1:]...); err != nil { log.Fatalf("goose run: %v", err) } return diff --git a/fix.go b/fix.go index caa9ff8e6..34f570556 100644 --- a/fix.go +++ b/fix.go @@ -1,25 +1,24 @@ package goose import ( - "database/sql" "fmt" "os" "strings" ) -func Fix(db *sql.DB, dir string) error { +func Fix(dir string) error { migrations, err := CollectMigrations(dir, minVersion, maxVersion) if err != nil { return err } // split into timestamped and versioned migrations - tsMigrations, err := migrations.Timestamped() + tsMigrations, err := migrations.timestamped() if err != nil { return err } - vMigrations, err := migrations.Versioned() + vMigrations, err := migrations.versioned() if err != nil { return err } @@ -29,12 +28,6 @@ func Fix(db *sql.DB, dir string) error { version = last.Version + 1 } - // fix db table as well - tx, err := db.Begin() - if err != nil { - log.Fatal("db.Begin: ", err) - } - // fix filenames by replacing timestamps with sequential versions for _, tsm := range tsMigrations { oldPath := tsm.Source @@ -44,13 +37,8 @@ func Fix(db *sql.DB, dir string) error { return err } - if _, err := tx.Exec(GetDialect().updateVersionSQL(), version, tsm.Version); err != nil { - tx.Rollback() - return err - } - version++ } - return tx.Commit() + return nil } diff --git a/goose.go b/goose.go index 1c492b0d8..ff4b5e84c 100644 --- a/goose.go +++ b/goose.go @@ -66,7 +66,7 @@ func Run(command string, db *sql.DB, dir string, args ...string) error { return err } case "fix": - if err := Fix(db, dir); err != nil { + if err := Fix(dir); err != nil { return err } case "redo": diff --git a/migrate.go b/migrate.go index f998b27b4..552944278 100644 --- a/migrate.go +++ b/migrate.go @@ -78,7 +78,7 @@ func (ms Migrations) Last() (*Migration, error) { } // Versioned gets versioned migrations. -func (ms Migrations) Versioned() (Migrations, error) { +func (ms Migrations) versioned() (Migrations, error) { var migrations Migrations // assume that the user will never have more than 19700101000000 migrations @@ -95,7 +95,7 @@ func (ms Migrations) Versioned() (Migrations, error) { } // Timestamped gets the timestamped migrations. -func (ms Migrations) Timestamped() (Migrations, error) { +func (ms Migrations) timestamped() (Migrations, error) { var migrations Migrations // assume that the user will never have more than 19700101000000 migrations From c6c49ff9953c8f541d027cb679c7b357a5da770c Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 17:08:25 -0400 Subject: [PATCH 4/7] remove yolo --- fix_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fix_test.go b/fix_test.go index e2f9fc9ff..263ebf7bc 100644 --- a/fix_test.go +++ b/fix_test.go @@ -21,10 +21,10 @@ func TestFix(t *testing.T) { commands := []string{ "go build -i -o goose ./cmd/goose", - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create create_table", dir), + fmt.Sprintf("./goose -dir=%s create add_users", dir), + fmt.Sprintf("./goose -dir=%s create add_indices", dir), + fmt.Sprintf("./goose -dir=%s create update_users", dir), fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), } @@ -52,8 +52,8 @@ func TestFix(t *testing.T) { // add more migrations and then fix it commands = []string{ - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create remove_column", dir), + fmt.Sprintf("./goose -dir=%s create create_books_table", dir), fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), } From 3c2c9d9076bc96c4ad57b3ccc69fa623e5ff4372 Mon Sep 17 00:00:00 2001 From: 1vn Date: Wed, 31 Oct 2018 12:27:15 -0400 Subject: [PATCH 5/7] add output --- cmd/goose/main.go | 12 ++++++++++-- fix.go | 2 ++ fix_test.go | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index 87b9506ea..cfb9043ea 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -26,8 +26,16 @@ func main() { args := flags.Args() - if len(args) > 1 && (args[0] == "create" || args[0] == "fix") { - if err := goose.Run(args[0], nil, *dir, args[1:]...); err != nil { + if len(args) > 1 && args[0] == "create" { + if err := goose.Run("create", nil, *dir, args[1:]...); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { log.Fatalf("goose run: %v", err) } return diff --git a/fix.go b/fix.go index 34f570556..dcd801000 100644 --- a/fix.go +++ b/fix.go @@ -3,6 +3,7 @@ package goose import ( "fmt" "os" + "path/filepath" "strings" ) @@ -37,6 +38,7 @@ func Fix(dir string) error { return err } + log.Printf("RENAMED %s => %s", filepath.Base(oldPath), filepath.Base(newPath)) version++ } diff --git a/fix_test.go b/fix_test.go index 263ebf7bc..956070c91 100644 --- a/fix_test.go +++ b/fix_test.go @@ -25,7 +25,7 @@ func TestFix(t *testing.T) { fmt.Sprintf("./goose -dir=%s create add_users", dir), fmt.Sprintf("./goose -dir=%s create add_indices", dir), fmt.Sprintf("./goose -dir=%s create update_users", dir), - fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), } for _, cmd := range commands { @@ -54,7 +54,7 @@ func TestFix(t *testing.T) { commands = []string{ fmt.Sprintf("./goose -dir=%s create remove_column", dir), fmt.Sprintf("./goose -dir=%s create create_books_table", dir), - fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), } for _, cmd := range commands { From a4a53bdf99114be929c0a80e08997c0824c7129f Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 6 Nov 2018 14:14:09 -0500 Subject: [PATCH 6/7] add to example as well --- examples/go-migrations/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/go-migrations/main.go b/examples/go-migrations/main.go index 64226ea3a..1f1ea5a2e 100644 --- a/examples/go-migrations/main.go +++ b/examples/go-migrations/main.go @@ -33,6 +33,14 @@ func main() { return } + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + if len(args) < 3 { flags.Usage() return From 90f0cf504b284bfa2b294586e631a4fde3cf3179 Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 6 Nov 2018 16:54:31 -0500 Subject: [PATCH 7/7] update documentation --- README.md | 15 +++++++++------ cmd/goose/main.go | 3 ++- examples/go-migrations/main.go | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index efedf5ba0..e60d33c79 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,7 @@ Goose is a database migration tool. Manage your database schema by creating incr - goose pkg doesn't register any SQL drivers anymore, thus no driver `panic()` conflict within your codebase! - goose pkg doesn't have any vendor dependencies anymore -- We encourage using sequential versioning of migration files - (rather than timestamps-based versioning) to prevent version - mismatch and migration colissions +- We use timestamped migrations by default but recommend a hybrid approach of using timestamps in the development process and sequential versions in production. # Install @@ -51,7 +49,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp Options: -dir string @@ -74,14 +72,14 @@ Examples: Create a new SQL migration. $ goose create add_some_column sql - $ Created new file: 00001_add_some_column.sql + $ Created new file: 20170506082420_add_some_column.sql Edit the newly created file to define the behavior of your migration. You can also create a Go migration, if you then invoke it with [your own goose binary](#go-migrations): $ goose create fetch_user_data go - $ Created new file: 00002_fetch_user_data.go + $ Created new file: 20170506082421_fetch_user_data.go ## up @@ -241,6 +239,11 @@ func Down(tx *sql.Tx) error { } ``` +# Hybrid Versioning +We strongly recommend adopting a hybrid versioning approach, using both timestamps and sequential numbers. Migrations created during the development process are timestamped and sequential versions are ran on production. We believe this method will prevent the problem of conflicting versions when writing software in a team environment. + +To help you adopt this approach, `create` will use the current timestamp as the migration version. When you're ready to deploy your migrations in a production environment, we also provide a helpful `fix` command to convert your migrations into sequential order, while preserving the timestamp ordering. We recommend running `fix` in the CI pipeline, and only when the migrations are ready for production. + ## License Licensed under [MIT License](./LICENSE) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index cfb9043ea..15b4caf6f 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -125,6 +125,7 @@ Commands: reset Roll back all migrations status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` ) diff --git a/examples/go-migrations/main.go b/examples/go-migrations/main.go index 1f1ea5a2e..bb323afd1 100644 --- a/examples/go-migrations/main.go +++ b/examples/go-migrations/main.go @@ -125,6 +125,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` )