From f25b6f1ffa0f1fe936292c87033d1e3f51bbe0c1 Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Thu, 30 Jan 2025 09:42:56 -0800 Subject: [PATCH 1/2] Cherry-pick 770dcf0914e1bf8d3931f1d6b2e4cf7892955367 with conflicts --- go/os2/file.go | 50 +++++ .../backup/vtctlbackup/backup_utils.go | 12 ++ go/test/endtoend/cluster/vttablet_process.go | 60 ++++++ go/vt/mysqlctl/backupengine.go | 3 +- go/vt/mysqlctl/blackbox/utils.go | 197 ++++++++++++++++++ go/vt/mysqlctl/builtinbackupengine.go | 5 +- go/vt/mysqlctl/filebackupstorage/file.go | 12 +- go/vt/mysqlctl/mysqld.go | 15 +- 8 files changed, 343 insertions(+), 11 deletions(-) create mode 100644 go/os2/file.go create mode 100644 go/vt/mysqlctl/blackbox/utils.go diff --git a/go/os2/file.go b/go/os2/file.go new file mode 100644 index 00000000000..7c284ed1fc1 --- /dev/null +++ b/go/os2/file.go @@ -0,0 +1,50 @@ +/* +Copyright 2025 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package os2 + +import ( + "io/fs" + "os" +) + +const ( + // PermFile is a FileMode for regular files without world permission bits. + PermFile fs.FileMode = 0660 + // PermDirectory is a FileMode for directories without world permission bits. + PermDirectory fs.FileMode = 0770 +) + +// Create is identical to os.Create except uses 0660 permission +// rather than 0666, to exclude world read/write bit. +func Create(name string) (*os.File, error) { + return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, PermFile) +} + +// WriteFile is identical to os.WriteFile except permission of 0660 is used. +func WriteFile(name string, data []byte) error { + return os.WriteFile(name, data, PermFile) +} + +// Mkdir is identical to os.Mkdir except permission of 0770 is used. +func Mkdir(path string) error { + return os.Mkdir(path, PermDirectory) +} + +// MkdirAll is identical to os.MkdirAll except permission of 0770 is used. +func MkdirAll(path string) error { + return os.MkdirAll(path, PermDirectory) +} diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 392b38f8956..6df477e4c54 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -409,6 +409,18 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int, cDe return vterrors.Errorf(vtrpc.Code_UNKNOWN, "test failure: %s", test.name) } } + + t.Run("check for files created with global permissions", func(t *testing.T) { + t.Logf("Confirming that none of the MySQL data directories that we've created have files with global permissions") + for _, ks := range localCluster.Keyspaces { + for _, shard := range ks.Shards { + for _, tablet := range shard.Vttablets { + tablet.VttabletProcess.ConfirmDataDirHasNoGlobalPerms(t) + } + } + } + }) + return nil } diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index d4f0e3f1963..fa1045cffc6 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "os/exec" @@ -35,6 +36,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" @@ -702,6 +705,63 @@ func (vttablet *VttabletProcess) IsShutdown() bool { return vttablet.proc == nil } +// ConfirmDataDirHasNoGlobalPerms confirms that no files in the tablet's data directory +// have any global/world/other permissions enabled. +func (vttablet *VttabletProcess) ConfirmDataDirHasNoGlobalPerms(t *testing.T) { + datadir := vttablet.Directory + if _, err := os.Stat(datadir); errors.Is(err, os.ErrNotExist) { + t.Logf("Data directory %s no longer exists, skipping permissions check", datadir) + return + } + + var allowedFiles = []string{ + // These are intentionally created with the world/other read bit set by mysqld itself + // during the --initialize[-insecure] step. + // See: https://dev.mysql.com/doc/mysql-security-excerpt/en/creating-ssl-rsa-files-using-mysql.html + // "On Unix and Unix-like systems, the file access mode is 644 for certificate files + // (that is, world readable) and 600 for key files (that is, accessible only by the + // account that runs the server)." + path.Join("data", "ca.pem"), + path.Join("data", "client-cert.pem"), + path.Join("data", "public_key.pem"), + path.Join("data", "server-cert.pem"), + // The domain socket must have global perms for anyone to use it. + "mysql.sock", + // These files are created by xtrabackup. + path.Join("tmp", "xtrabackup_checkpoints"), + path.Join("tmp", "xtrabackup_info"), + } + + var matches []string + fsys := os.DirFS(datadir) + err := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, _ error) error { + // first check if the file should be skipped + for _, name := range allowedFiles { + if strings.HasSuffix(p, name) { + return nil + } + } + + info, err := d.Info() + if err != nil { + return err + } + + // check if any global bit is on the filemode + if info.Mode()&0007 != 0 { + matches = append(matches, fmt.Sprintf( + "%s (%s)", + path.Join(datadir, p), + info.Mode(), + )) + } + return nil + }) + + require.NoError(t, err, "Error walking directory") + require.Empty(t, matches, "Found files with global permissions: %s\n", strings.Join(matches, "\n")) +} + // VttabletProcessInstance returns a VttabletProcess handle for vttablet process // configured with the given Config. // The process must be manually started by calling setup() diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index c483aff3d78..b3490f7b837 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -30,6 +30,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/os2" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstats" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -705,7 +706,7 @@ func createStateFile(cnf *Mycnf) error { // rename func to openStateFile // change to return a *File fname := filepath.Join(cnf.TabletDir(), RestoreState) - fd, err := os.Create(fname) + fd, err := os2.Create(fname) if err != nil { return fmt.Errorf("unable to create file: %v", err) } diff --git a/go/vt/mysqlctl/blackbox/utils.go b/go/vt/mysqlctl/blackbox/utils.go new file mode 100644 index 00000000000..ff7236a41b8 --- /dev/null +++ b/go/vt/mysqlctl/blackbox/utils.go @@ -0,0 +1,197 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blackbox + +import ( + "context" + "fmt" + "os" + "path" + "slices" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/capabilities" + "vitess.io/vitess/go/os2" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl" + "vitess.io/vitess/go/vt/mysqlctl/backupstats" + "vitess.io/vitess/go/vt/mysqlctl/filebackupstorage" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/proto/vttime" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" +) + +type StatSummary struct { + DestinationCloseStats int + DestinationOpenStats int + DestinationWriteStats int + SourceCloseStats int + SourceOpenStats int + SourceReadStats int +} + +func GetStats(stats *backupstats.FakeStats) StatSummary { + var ss StatSummary + + for _, sr := range stats.ScopeReturns { + switch sr.ScopeV[backupstats.ScopeOperation] { + case "Destination:Close": + ss.DestinationCloseStats += len(sr.TimedIncrementCalls) + case "Destination:Open": + ss.DestinationOpenStats += len(sr.TimedIncrementCalls) + case "Destination:Write": + if len(sr.TimedIncrementBytesCalls) > 0 { + ss.DestinationWriteStats++ + } + case "Source:Close": + ss.SourceCloseStats += len(sr.TimedIncrementCalls) + case "Source:Open": + ss.SourceOpenStats += len(sr.TimedIncrementCalls) + case "Source:Read": + if len(sr.TimedIncrementBytesCalls) > 0 { + ss.SourceReadStats++ + } + } + } + return ss +} + +func AssertLogs(t *testing.T, expectedLogs []string, logger *logutil.MemoryLogger) { + for _, log := range expectedLogs { + require.Truef(t, slices.ContainsFunc(logger.LogEvents(), func(event *logutilpb.Event) bool { + return event.GetValue() == log + }), "%s is missing from the logs", log) + } +} + +func SetupCluster(ctx context.Context, t *testing.T, dirs, filesPerDir int) (backupRoot string, keyspace string, shard string, ts *topo.Server) { + // Set up local backup directory + id := fmt.Sprintf("%d", time.Now().UnixNano()) + backupRoot = fmt.Sprintf("testdata/builtinbackup_test_%s", id) + filebackupstorage.FileBackupStorageRoot = backupRoot + require.NoError(t, createBackupDir(backupRoot, "innodb", "log", "datadir")) + dataDir := path.Join(backupRoot, "datadir") + // Add some files under data directory to force backup to execute semaphore acquire inside + // backupFiles() method (https://github.com/vitessio/vitess/blob/main/go/vt/mysqlctl/builtinbackupengine.go#L483). + for dirI := range dirs { + dirName := "test" + strconv.Itoa(dirI+1) + require.NoError(t, createBackupDir(dataDir, dirName)) + require.NoError(t, createBackupFiles(path.Join(dataDir, dirName), filesPerDir, "ibd")) + } + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(backupRoot)) + }) + + needIt, err := NeedInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + fpath := path.Join("log", mysql.DynamicRedoLogSubdir) + if err := createBackupDir(backupRoot, fpath); err != nil { + require.Failf(t, err.Error(), "failed to create directory: %s", fpath) + } + } + + // Set up topo + keyspace, shard = "mykeyspace", "-" + ts = memorytopo.NewServer(ctx, "cell1") + t.Cleanup(func() { + ts.Close() + }) + + require.NoError(t, ts.CreateKeyspace(ctx, keyspace, &topodata.Keyspace{})) + require.NoError(t, ts.CreateShard(ctx, keyspace, shard)) + + tablet := topo.NewTablet(100, "cell1", "mykeyspace-00-80-0100") + tablet.Keyspace = keyspace + tablet.Shard = shard + + require.NoError(t, ts.CreateTablet(ctx, tablet)) + + _, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { + si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"} + + now := time.Now() + si.PrimaryTermStartTime = &vttime.Time{Seconds: int64(now.Second()), Nanoseconds: int32(now.Nanosecond())} + + return nil + }) + require.NoError(t, err) + return backupRoot, keyspace, shard, ts +} + +// NeedInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory. +// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the +// (/. by default) called "#innodb_redo". See: +// +// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity +func NeedInnoDBRedoLogSubdir() (needIt bool, err error) { + mysqldVersionStr, err := mysqlctl.GetVersionString() + if err != nil { + return needIt, err + } + _, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr) + if err != nil { + return needIt, err + } + versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch) + capableOf := mysql.ServerVersionCapableOf(versionStr) + if capableOf == nil { + return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr) + } + return capableOf(capabilities.DynamicRedoLogCapacityFlavorCapability) +} + +const MysqlShutdownTimeout = 1 * time.Minute + +func SetBuiltinBackupMysqldDeadline(t time.Duration) time.Duration { + old := mysqlctl.BuiltinBackupMysqldTimeout + mysqlctl.BuiltinBackupMysqldTimeout = t + + return old +} + +func createBackupDir(root string, dirs ...string) error { + for _, dir := range dirs { + if err := os.MkdirAll(path.Join(root, dir), 0755); err != nil { + return err + } + } + + return nil +} + +func createBackupFiles(root string, fileCount int, ext string) error { + for i := 0; i < fileCount; i++ { + f, err := os2.Create(path.Join(root, fmt.Sprintf("%d.%s", i, ext))) + if err != nil { + return err + } + if _, err := f.Write([]byte("hello, world!")); err != nil { + return err + } + defer f.Close() + } + + return nil +} diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 77c0ac43f00..acef8395c9f 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -39,6 +39,7 @@ import ( "vitess.io/vitess/go/ioutil" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/os2" "vitess.io/vitess/go/protoutil" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" @@ -199,10 +200,10 @@ func (fe *FileEntry) open(cnf *Mycnf, readOnly bool) (*os.File, error) { } } else { dir := path.Dir(name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { + if err := os2.MkdirAll(dir); err != nil { return nil, vterrors.Wrapf(err, "cannot create destination directory %v", dir) } - if fd, err = os.Create(name); err != nil { + if fd, err = os2.Create(name); err != nil { return nil, vterrors.Wrapf(err, "cannot create destination file %v", name) } } diff --git a/go/vt/mysqlctl/filebackupstorage/file.go b/go/vt/mysqlctl/filebackupstorage/file.go index 99148d9169b..cbba1424d8a 100644 --- a/go/vt/mysqlctl/filebackupstorage/file.go +++ b/go/vt/mysqlctl/filebackupstorage/file.go @@ -27,6 +27,12 @@ import ( "github.com/spf13/pflag" +<<<<<<< HEAD +======= + "vitess.io/vitess/go/os2" + "vitess.io/vitess/go/vt/mysqlctl/errors" + +>>>>>>> 770dcf0914 (Replace uses of os.Create with os2.Create within backup/restore workflows (#17648)) "vitess.io/vitess/go/ioutil" "vitess.io/vitess/go/vt/concurrency" stats "vitess.io/vitess/go/vt/mysqlctl/backupstats" @@ -110,7 +116,7 @@ func (fbh *FileBackupHandle) AddFile(ctx context.Context, filename string, files return nil, fmt.Errorf("AddFile cannot be called on read-only backup") } p := path.Join(FileBackupStorageRoot, fbh.dir, fbh.name, filename) - f, err := os.Create(p) + f, err := os2.Create(p) if err != nil { return nil, err } @@ -186,13 +192,13 @@ func (fbs *FileBackupStorage) ListBackups(ctx context.Context, dir string) ([]ba func (fbs *FileBackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) { // Make sure the directory exists. p := path.Join(FileBackupStorageRoot, dir) - if err := os.MkdirAll(p, os.ModePerm); err != nil { + if err := os2.MkdirAll(p); err != nil { return nil, err } // Create the subdirectory for this named backup. p = path.Join(p, name) - if err := os.Mkdir(p, os.ModePerm); err != nil { + if err := os2.Mkdir(p); err != nil { return nil, err } diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 952c0987c82..fa242d561a8 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -46,6 +46,11 @@ import ( "vitess.io/vitess/config" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/sqlerror" +<<<<<<< HEAD +======= + "vitess.io/vitess/go/os2" + "vitess.io/vitess/go/osutil" +>>>>>>> 770dcf0914 (Replace uses of os.Create with os2.Create within backup/restore workflows (#17648)) "vitess.io/vitess/go/protoutil" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs" @@ -898,7 +903,7 @@ func (mysqld *Mysqld) initConfig(cnf *Mycnf, outFile string) error { return err } - return os.WriteFile(outFile, []byte(configData), 0o664) + return os2.WriteFile(outFile, []byte(configData)) } func (mysqld *Mysqld) getMycnfTemplate() string { @@ -1048,7 +1053,7 @@ func (mysqld *Mysqld) ReinitConfig(ctx context.Context, cnf *Mycnf) error { func (mysqld *Mysqld) createDirs(cnf *Mycnf) error { tabletDir := cnf.TabletDir() log.Infof("creating directory %s", tabletDir) - if err := os.MkdirAll(tabletDir, os.ModePerm); err != nil { + if err := os2.MkdirAll(tabletDir); err != nil { return err } for _, dir := range TopLevelDirs() { @@ -1058,7 +1063,7 @@ func (mysqld *Mysqld) createDirs(cnf *Mycnf) error { } for _, dir := range cnf.directoryList() { log.Infof("creating directory %s", dir) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { + if err := os2.MkdirAll(dir); err != nil { return err } // FIXME(msolomon) validate permissions? @@ -1082,14 +1087,14 @@ func (mysqld *Mysqld) createTopDir(cnf *Mycnf, dir string) error { if os.IsNotExist(err) { topdir := path.Join(tabletDir, dir) log.Infof("creating directory %s", topdir) - return os.MkdirAll(topdir, os.ModePerm) + return os2.MkdirAll(topdir) } return err } linkto := path.Join(target, vtname) source := path.Join(tabletDir, dir) log.Infof("creating directory %s", linkto) - err = os.MkdirAll(linkto, os.ModePerm) + err = os2.MkdirAll(linkto) if err != nil { return err } From 730976ce38015dd5bf922869df98d83de17fc2f6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 30 Jan 2025 13:43:14 -0500 Subject: [PATCH 2/2] Fix conflicts Signed-off-by: Matt Lord --- go/vt/mysqlctl/blackbox/utils.go | 197 ----------------------- go/vt/mysqlctl/filebackupstorage/file.go | 7 +- go/vt/mysqlctl/mysqld.go | 4 - 3 files changed, 1 insertion(+), 207 deletions(-) delete mode 100644 go/vt/mysqlctl/blackbox/utils.go diff --git a/go/vt/mysqlctl/blackbox/utils.go b/go/vt/mysqlctl/blackbox/utils.go deleted file mode 100644 index ff7236a41b8..00000000000 --- a/go/vt/mysqlctl/blackbox/utils.go +++ /dev/null @@ -1,197 +0,0 @@ -/* -Copyright 2024 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package blackbox - -import ( - "context" - "fmt" - "os" - "path" - "slices" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/mysql/capabilities" - "vitess.io/vitess/go/os2" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/mysqlctl" - "vitess.io/vitess/go/vt/mysqlctl/backupstats" - "vitess.io/vitess/go/vt/mysqlctl/filebackupstorage" - logutilpb "vitess.io/vitess/go/vt/proto/logutil" - "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/proto/vttime" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/memorytopo" -) - -type StatSummary struct { - DestinationCloseStats int - DestinationOpenStats int - DestinationWriteStats int - SourceCloseStats int - SourceOpenStats int - SourceReadStats int -} - -func GetStats(stats *backupstats.FakeStats) StatSummary { - var ss StatSummary - - for _, sr := range stats.ScopeReturns { - switch sr.ScopeV[backupstats.ScopeOperation] { - case "Destination:Close": - ss.DestinationCloseStats += len(sr.TimedIncrementCalls) - case "Destination:Open": - ss.DestinationOpenStats += len(sr.TimedIncrementCalls) - case "Destination:Write": - if len(sr.TimedIncrementBytesCalls) > 0 { - ss.DestinationWriteStats++ - } - case "Source:Close": - ss.SourceCloseStats += len(sr.TimedIncrementCalls) - case "Source:Open": - ss.SourceOpenStats += len(sr.TimedIncrementCalls) - case "Source:Read": - if len(sr.TimedIncrementBytesCalls) > 0 { - ss.SourceReadStats++ - } - } - } - return ss -} - -func AssertLogs(t *testing.T, expectedLogs []string, logger *logutil.MemoryLogger) { - for _, log := range expectedLogs { - require.Truef(t, slices.ContainsFunc(logger.LogEvents(), func(event *logutilpb.Event) bool { - return event.GetValue() == log - }), "%s is missing from the logs", log) - } -} - -func SetupCluster(ctx context.Context, t *testing.T, dirs, filesPerDir int) (backupRoot string, keyspace string, shard string, ts *topo.Server) { - // Set up local backup directory - id := fmt.Sprintf("%d", time.Now().UnixNano()) - backupRoot = fmt.Sprintf("testdata/builtinbackup_test_%s", id) - filebackupstorage.FileBackupStorageRoot = backupRoot - require.NoError(t, createBackupDir(backupRoot, "innodb", "log", "datadir")) - dataDir := path.Join(backupRoot, "datadir") - // Add some files under data directory to force backup to execute semaphore acquire inside - // backupFiles() method (https://github.com/vitessio/vitess/blob/main/go/vt/mysqlctl/builtinbackupengine.go#L483). - for dirI := range dirs { - dirName := "test" + strconv.Itoa(dirI+1) - require.NoError(t, createBackupDir(dataDir, dirName)) - require.NoError(t, createBackupFiles(path.Join(dataDir, dirName), filesPerDir, "ibd")) - } - t.Cleanup(func() { - require.NoError(t, os.RemoveAll(backupRoot)) - }) - - needIt, err := NeedInnoDBRedoLogSubdir() - require.NoError(t, err) - if needIt { - fpath := path.Join("log", mysql.DynamicRedoLogSubdir) - if err := createBackupDir(backupRoot, fpath); err != nil { - require.Failf(t, err.Error(), "failed to create directory: %s", fpath) - } - } - - // Set up topo - keyspace, shard = "mykeyspace", "-" - ts = memorytopo.NewServer(ctx, "cell1") - t.Cleanup(func() { - ts.Close() - }) - - require.NoError(t, ts.CreateKeyspace(ctx, keyspace, &topodata.Keyspace{})) - require.NoError(t, ts.CreateShard(ctx, keyspace, shard)) - - tablet := topo.NewTablet(100, "cell1", "mykeyspace-00-80-0100") - tablet.Keyspace = keyspace - tablet.Shard = shard - - require.NoError(t, ts.CreateTablet(ctx, tablet)) - - _, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { - si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"} - - now := time.Now() - si.PrimaryTermStartTime = &vttime.Time{Seconds: int64(now.Second()), Nanoseconds: int32(now.Nanosecond())} - - return nil - }) - require.NoError(t, err) - return backupRoot, keyspace, shard, ts -} - -// NeedInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory. -// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the -// (/. by default) called "#innodb_redo". See: -// -// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity -func NeedInnoDBRedoLogSubdir() (needIt bool, err error) { - mysqldVersionStr, err := mysqlctl.GetVersionString() - if err != nil { - return needIt, err - } - _, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr) - if err != nil { - return needIt, err - } - versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch) - capableOf := mysql.ServerVersionCapableOf(versionStr) - if capableOf == nil { - return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr) - } - return capableOf(capabilities.DynamicRedoLogCapacityFlavorCapability) -} - -const MysqlShutdownTimeout = 1 * time.Minute - -func SetBuiltinBackupMysqldDeadline(t time.Duration) time.Duration { - old := mysqlctl.BuiltinBackupMysqldTimeout - mysqlctl.BuiltinBackupMysqldTimeout = t - - return old -} - -func createBackupDir(root string, dirs ...string) error { - for _, dir := range dirs { - if err := os.MkdirAll(path.Join(root, dir), 0755); err != nil { - return err - } - } - - return nil -} - -func createBackupFiles(root string, fileCount int, ext string) error { - for i := 0; i < fileCount; i++ { - f, err := os2.Create(path.Join(root, fmt.Sprintf("%d.%s", i, ext))) - if err != nil { - return err - } - if _, err := f.Write([]byte("hello, world!")); err != nil { - return err - } - defer f.Close() - } - - return nil -} diff --git a/go/vt/mysqlctl/filebackupstorage/file.go b/go/vt/mysqlctl/filebackupstorage/file.go index cbba1424d8a..9c3e8165e2c 100644 --- a/go/vt/mysqlctl/filebackupstorage/file.go +++ b/go/vt/mysqlctl/filebackupstorage/file.go @@ -27,13 +27,8 @@ import ( "github.com/spf13/pflag" -<<<<<<< HEAD -======= - "vitess.io/vitess/go/os2" - "vitess.io/vitess/go/vt/mysqlctl/errors" - ->>>>>>> 770dcf0914 (Replace uses of os.Create with os2.Create within backup/restore workflows (#17648)) "vitess.io/vitess/go/ioutil" + "vitess.io/vitess/go/os2" "vitess.io/vitess/go/vt/concurrency" stats "vitess.io/vitess/go/vt/mysqlctl/backupstats" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index fa242d561a8..48a17f0cfc3 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -46,11 +46,7 @@ import ( "vitess.io/vitess/config" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/sqlerror" -<<<<<<< HEAD -======= "vitess.io/vitess/go/os2" - "vitess.io/vitess/go/osutil" ->>>>>>> 770dcf0914 (Replace uses of os.Create with os2.Create within backup/restore workflows (#17648)) "vitess.io/vitess/go/protoutil" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs"