Cherry-pick 770dcf0 with conflicts

go/os2/file.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package os2
+
+import (
+	"io/fs"
+	"os"
+)
+
+const (
+	// PermFile is a FileMode for regular files without world permission bits.
+	PermFile fs.FileMode = 0660
+	// PermDirectory is a FileMode for directories without world permission bits.
+	PermDirectory fs.FileMode = 0770
+)
+
+// Create is identical to os.Create except uses 0660 permission
+// rather than 0666, to exclude world read/write bit.
+func Create(name string) (*os.File, error) {
+	return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, PermFile)
+}
+
+// WriteFile is identical to os.WriteFile except permission of 0660 is used.
+func WriteFile(name string, data []byte) error {
+	return os.WriteFile(name, data, PermFile)
+}
+
+// Mkdir is identical to os.Mkdir except permission of 0770 is used.
+func Mkdir(path string) error {
+	return os.Mkdir(path, PermDirectory)
+}
+
+// MkdirAll is identical to os.MkdirAll except permission of 0770 is used.
+func MkdirAll(path string) error {
+	return os.MkdirAll(path, PermDirectory)
+}

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -425,6 +425,18 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int, cDe
 			return vterrors.Errorf(vtrpc.Code_UNKNOWN, "test failure: %s", test.name)
 		}
 	}
+
+	t.Run("check for files created with global permissions", func(t *testing.T) {
+		t.Logf("Confirming that none of the MySQL data directories that we've created have files with global permissions")
+		for _, ks := range localCluster.Keyspaces {
+			for _, shard := range ks.Shards {
+				for _, tablet := range shard.Vttablets {
+					tablet.VttabletProcess.ConfirmDataDirHasNoGlobalPerms(t)
+				}
+			}
+		}
+	})
+
 	return nil
 }
 

go/test/endtoend/cluster/vttablet_process.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"os"
 	"os/exec"
@@ -35,6 +36,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"vitess.io/vitess/go/constants/sidecar"
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/sqltypes"
@@ -703,6 +706,63 @@ func (vttablet *VttabletProcess) IsShutdown() bool {
 	return vttablet.proc == nil
 }
 
+// ConfirmDataDirHasNoGlobalPerms confirms that no files in the tablet's data directory
+// have any global/world/other permissions enabled.
+func (vttablet *VttabletProcess) ConfirmDataDirHasNoGlobalPerms(t *testing.T) {
+	datadir := vttablet.Directory
+	if _, err := os.Stat(datadir); errors.Is(err, os.ErrNotExist) {
+		t.Logf("Data directory %s no longer exists, skipping permissions check", datadir)
+		return
+	}
+
+	var allowedFiles = []string{
+		// These are intentionally created with the world/other read bit set by mysqld itself
+		// during the --initialize[-insecure] step.
+		// See: https://dev.mysql.com/doc/mysql-security-excerpt/en/creating-ssl-rsa-files-using-mysql.html
+		// "On Unix and Unix-like systems, the file access mode is 644 for certificate files
+		// (that is, world readable) and 600 for key files (that is, accessible only by the
+		// account that runs the server)."
+		path.Join("data", "ca.pem"),
+		path.Join("data", "client-cert.pem"),
+		path.Join("data", "public_key.pem"),
+		path.Join("data", "server-cert.pem"),
+		// The domain socket must have global perms for anyone to use it.
+		"mysql.sock",
+		// These files are created by xtrabackup.
+		path.Join("tmp", "xtrabackup_checkpoints"),
+		path.Join("tmp", "xtrabackup_info"),
+	}
+
+	var matches []string
+	fsys := os.DirFS(datadir)
+	err := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, _ error) error {
+		// first check if the file should be skipped
+		for _, name := range allowedFiles {
+			if strings.HasSuffix(p, name) {
+				return nil
+			}
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		// check if any global bit is on the filemode
+		if info.Mode()&0007 != 0 {
+			matches = append(matches, fmt.Sprintf(
+				"%s (%s)",
+				path.Join(datadir, p),
+				info.Mode(),
+			))
+		}
+		return nil
+	})
+
+	require.NoError(t, err, "Error walking directory")
+	require.Empty(t, matches, "Found files with global permissions: %s\n", strings.Join(matches, "\n"))
+}
+
 // VttabletProcessInstance returns a VttabletProcess handle for vttablet process
 // configured with the given Config.
 // The process must be manually started by calling setup()

go/vt/mysqlctl/backupengine.go

@@ -31,6 +31,7 @@ import (
 
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/mysql/replication"
+	"vitess.io/vitess/go/os2"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/mysqlctl/backupstats"
 	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
@@ -723,7 +724,7 @@ func createStateFile(cnf *Mycnf) error {
 	// rename func to openStateFile
 	// change to return a *File
 	fname := filepath.Join(cnf.TabletDir(), RestoreState)
-	fd, err := os.Create(fname)
+	fd, err := os2.Create(fname)
 	if err != nil {
 		return fmt.Errorf("unable to create file: %v", err)
 	}

go/vt/mysqlctl/blackbox/utils.go

@@ -0,0 +1,197 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package blackbox
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/mysql/capabilities"
+	"vitess.io/vitess/go/os2"
+	"vitess.io/vitess/go/vt/logutil"
+	"vitess.io/vitess/go/vt/mysqlctl"
+	"vitess.io/vitess/go/vt/mysqlctl/backupstats"
+	"vitess.io/vitess/go/vt/mysqlctl/filebackupstorage"
+	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	"vitess.io/vitess/go/vt/proto/topodata"
+	"vitess.io/vitess/go/vt/proto/vttime"
+	"vitess.io/vitess/go/vt/topo"
+	"vitess.io/vitess/go/vt/topo/memorytopo"
+)
+
+type StatSummary struct {
+	DestinationCloseStats int
+	DestinationOpenStats  int
+	DestinationWriteStats int
+	SourceCloseStats      int
+	SourceOpenStats       int
+	SourceReadStats       int
+}
+
+func GetStats(stats *backupstats.FakeStats) StatSummary {
+	var ss StatSummary
+
+	for _, sr := range stats.ScopeReturns {
+		switch sr.ScopeV[backupstats.ScopeOperation] {
+		case "Destination:Close":
+			ss.DestinationCloseStats += len(sr.TimedIncrementCalls)
+		case "Destination:Open":
+			ss.DestinationOpenStats += len(sr.TimedIncrementCalls)
+		case "Destination:Write":
+			if len(sr.TimedIncrementBytesCalls) > 0 {
+				ss.DestinationWriteStats++
+			}
+		case "Source:Close":
+			ss.SourceCloseStats += len(sr.TimedIncrementCalls)
+		case "Source:Open":
+			ss.SourceOpenStats += len(sr.TimedIncrementCalls)
+		case "Source:Read":
+			if len(sr.TimedIncrementBytesCalls) > 0 {
+				ss.SourceReadStats++
+			}
+		}
+	}
+	return ss
+}
+
+func AssertLogs(t *testing.T, expectedLogs []string, logger *logutil.MemoryLogger) {
+	for _, log := range expectedLogs {
+		require.Truef(t, slices.ContainsFunc(logger.LogEvents(), func(event *logutilpb.Event) bool {
+			return event.GetValue() == log
+		}), "%s is missing from the logs", log)
+	}
+}
+
+func SetupCluster(ctx context.Context, t *testing.T, dirs, filesPerDir int) (backupRoot string, keyspace string, shard string, ts *topo.Server) {
+	// Set up local backup directory
+	id := fmt.Sprintf("%d", time.Now().UnixNano())
+	backupRoot = fmt.Sprintf("testdata/builtinbackup_test_%s", id)
+	filebackupstorage.FileBackupStorageRoot = backupRoot
+	require.NoError(t, createBackupDir(backupRoot, "innodb", "log", "datadir"))
+	dataDir := path.Join(backupRoot, "datadir")
+	// Add some files under data directory to force backup to execute semaphore acquire inside
+	// backupFiles() method (https://github.com/vitessio/vitess/blob/main/go/vt/mysqlctl/builtinbackupengine.go#L483).
+	for dirI := range dirs {
+		dirName := "test" + strconv.Itoa(dirI+1)
+		require.NoError(t, createBackupDir(dataDir, dirName))
+		require.NoError(t, createBackupFiles(path.Join(dataDir, dirName), filesPerDir, "ibd"))
+	}
+	t.Cleanup(func() {
+		require.NoError(t, os.RemoveAll(backupRoot))
+	})
+
+	needIt, err := NeedInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		fpath := path.Join("log", mysql.DynamicRedoLogSubdir)
+		if err := createBackupDir(backupRoot, fpath); err != nil {
+			require.Failf(t, err.Error(), "failed to create directory: %s", fpath)
+		}
+	}
+
+	// Set up topo
+	keyspace, shard = "mykeyspace", "-"
+	ts = memorytopo.NewServer(ctx, "cell1")
+	t.Cleanup(func() {
+		ts.Close()
+	})
+
+	require.NoError(t, ts.CreateKeyspace(ctx, keyspace, &topodata.Keyspace{}))
+	require.NoError(t, ts.CreateShard(ctx, keyspace, shard))
+
+	tablet := topo.NewTablet(100, "cell1", "mykeyspace-00-80-0100")
+	tablet.Keyspace = keyspace
+	tablet.Shard = shard
+
+	require.NoError(t, ts.CreateTablet(ctx, tablet))
+
+	_, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error {
+		si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"}
+
+		now := time.Now()
+		si.PrimaryTermStartTime = &vttime.Time{Seconds: int64(now.Second()), Nanoseconds: int32(now.Nanosecond())}
+
+		return nil
+	})
+	require.NoError(t, err)
+	return backupRoot, keyspace, shard, ts
+}
+
+// NeedInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory.
+// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the
+// <innodb_log_group_home_dir> (<datadir>/. by default) called "#innodb_redo". See:
+//
+//	https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity
+func NeedInnoDBRedoLogSubdir() (needIt bool, err error) {
+	mysqldVersionStr, err := mysqlctl.GetVersionString()
+	if err != nil {
+		return needIt, err
+	}
+	_, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr)
+	if err != nil {
+		return needIt, err
+	}
+	versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch)
+	capableOf := mysql.ServerVersionCapableOf(versionStr)
+	if capableOf == nil {
+		return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr)
+	}
+	return capableOf(capabilities.DynamicRedoLogCapacityFlavorCapability)
+}
+
+const MysqlShutdownTimeout = 1 * time.Minute
+
+func SetBuiltinBackupMysqldDeadline(t time.Duration) time.Duration {
+	old := mysqlctl.BuiltinBackupMysqldTimeout
+	mysqlctl.BuiltinBackupMysqldTimeout = t
+
+	return old
+}
+
+func createBackupDir(root string, dirs ...string) error {
+	for _, dir := range dirs {
+		if err := os.MkdirAll(path.Join(root, dir), 0755); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func createBackupFiles(root string, fileCount int, ext string) error {
+	for i := 0; i < fileCount; i++ {
+		f, err := os2.Create(path.Join(root, fmt.Sprintf("%d.%s", i, ext)))
+		if err != nil {
+			return err
+		}
+		if _, err := f.Write([]byte("hello, world!")); err != nil {
+			return err
+		}
+		defer f.Close()
+	}
+
+	return nil
+}

go/vt/mysqlctl/builtinbackupengine.go

@@ -39,6 +39,7 @@ import (
 	"vitess.io/vitess/go/ioutil"
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/mysql/replication"
+	"vitess.io/vitess/go/os2"
 	"vitess.io/vitess/go/protoutil"
 	"vitess.io/vitess/go/vt/concurrency"
 	"vitess.io/vitess/go/vt/log"
@@ -198,10 +199,10 @@ func (fe *FileEntry) open(cnf *Mycnf, readOnly bool) (*os.File, error) {
 		}
 	} else {
 		dir := path.Dir(name)
-		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		if err := os2.MkdirAll(dir); err != nil {
 			return nil, vterrors.Wrapf(err, "cannot create destination directory %v", dir)
 		}
-		if fd, err = os.Create(name); err != nil {
+		if fd, err = os2.Create(name); err != nil {
 			return nil, vterrors.Wrapf(err, "cannot create destination file %v", name)
 		}
 	}