feat: enable configuration of some directory modes for dfdaemon (#2340)

Co-authored-by: Gaius <[email protected]>

client/config/peerhost.go

@@ -52,12 +52,15 @@ type DaemonOption struct {
 	GCInterval util.Duration `mapstructure:"gcInterval" yaml:"gcInterval"`
 	Metrics    string        `mapstructure:"metrics" yaml:"metrics"`
 
-	WorkHome    string `mapstructure:"workHome" yaml:"workHome"`
-	CacheDir    string `mapstructure:"cacheDir" yaml:"cacheDir"`
-	LogDir      string `mapstructure:"logDir" yaml:"logDir"`
-	PluginDir   string `mapstructure:"pluginDir" yaml:"pluginDir"`
-	DataDir     string `mapstructure:"dataDir" yaml:"dataDir"`
-	KeepStorage bool   `mapstructure:"keepStorage" yaml:"keepStorage"`
+	WorkHome     string `mapstructure:"workHome" yaml:"workHome"`
+	WorkHomeMode uint32 `mapstructure:"workHomeMode" yaml:"workHomeMode"`
+	CacheDir     string `mapstructure:"cacheDir" yaml:"cacheDir"`
+	CacheDirMode uint32 `mapstructure:"cacheDirMode" yaml:"cacheDirMode"`
+	LogDir       string `mapstructure:"logDir" yaml:"logDir"`
+	PluginDir    string `mapstructure:"pluginDir" yaml:"pluginDir"`
+	DataDir      string `mapstructure:"dataDir" yaml:"dataDir"`
+	DataDirMode  uint32 `mapstructure:"dataDirMode" yaml:"dataDirMode"`
+	KeepStorage  bool   `mapstructure:"keepStorage" yaml:"keepStorage"`
 
 	Security      GlobalSecurityOption `mapstructure:"security" yaml:"security"`
 	Scheduler     SchedulerOption      `mapstructure:"scheduler" yaml:"scheduler"`

client/config/peerhost_test.go

@@ -247,13 +247,16 @@ func TestPeerHostOption_Load(t *testing.T) {
 		GCInterval: util.Duration{
 			Duration: 60000000000,
 		},
-		Metrics:     ":8000",
-		WorkHome:    "/tmp/dragonfly/dfdaemon/",
-		CacheDir:    "/var/cache/dragonfly/",
-		LogDir:      "/var/log/dragonfly/",
-		PluginDir:   "/tmp/dragonfly/dfdaemon/plugins/",
-		DataDir:     "/var/lib/dragonfly/",
-		KeepStorage: false,
+		Metrics:      ":8000",
+		WorkHome:     "/tmp/dragonfly/dfdaemon/",
+		WorkHomeMode: 0755,
+		CacheDir:     "/var/cache/dragonfly/",
+		CacheDirMode: 0700,
+		LogDir:       "/var/log/dragonfly/",
+		PluginDir:    "/tmp/dragonfly/dfdaemon/plugins/",
+		DataDir:      "/var/lib/dragonfly/",
+		DataDirMode:  0700,
+		KeepStorage:  false,
 		Scheduler: SchedulerOption{
 			Manager: ManagerOption{
 				Enable: false,

client/config/testdata/config/daemon.yaml

@@ -7,10 +7,13 @@ metrics: ":8000"
 aliveTime: 0s
 gcInterval: 1m0s
 workHome: /tmp/dragonfly/dfdaemon/
+workHomeMode: 0755
 cacheDir: /var/cache/dragonfly/
+cacheDirMode: 0700
 logDir: /var/log/dragonfly/
 pluginDir: /tmp/dragonfly/dfdaemon/plugins/
 dataDir: /var/lib/dragonfly/
+dataDirMode: 0700
 keepStorage: false
 scheduler:
   manager:

client/daemon/daemon.go

@@ -239,8 +239,9 @@ func New(opt *config.DaemonOption, d dfpath.Dfpath) (Daemon, error) {
 			logger.Infof("step 4: leave task %s/%s state ok", request.TaskID, request.PeerID)
 		}
 	}
+	dirMode := os.FileMode(opt.DataDirMode)
 	storageManager, err := storage.NewStorageManager(opt.Storage.StoreStrategy, &opt.Storage,
-		gcCallback, storage.WithGCInterval(opt.GCInterval.Duration))
+		gcCallback, dirMode, storage.WithGCInterval(opt.GCInterval.Duration))
 	if err != nil {
 		return nil, err
 	}

client/daemon/peer/peertask_manager_test.go

@@ -269,7 +269,7 @@ func setupPeerTaskManagerComponents(ctrl *gomock.Controller, opt componentsOptio
 			TaskExpireTime: util.Duration{
 				Duration: -1 * time.Second,
 			},
-		}, func(request storage.CommonTaskRequest) {})
+		}, func(request storage.CommonTaskRequest) {}, os.FileMode(0755))
 	return sched, storageManager
 }
 

client/daemon/peer/peertask_stream_backsource_partial_test.go

@@ -225,7 +225,7 @@ func setupBackSourcePartialComponents(ctrl *gomock.Controller, testBytes []byte,
 			TaskExpireTime: util.Duration{
 				Duration: -1 * time.Second,
 			},
-		}, func(request storage.CommonTaskRequest) {})
+		}, func(request storage.CommonTaskRequest) {}, os.FileMode(0755))
 	return sched, storageManager
 }
 

client/daemon/peer/piece_manager_test.go

@@ -75,7 +75,7 @@ func TestPieceManager_DownloadSource(t *testing.T) {
 			TaskExpireTime: clientutil.Duration{
 				Duration: -1 * time.Second,
 			},
-		}, func(request storage.CommonTaskRequest) {})
+		}, func(request storage.CommonTaskRequest) {}, os.FileMode(0755))
 
 	hash := md5.New()
 	hash.Write(testBytes)

client/daemon/peer/traffic_shaper_test.go

@@ -231,7 +231,7 @@ func trafficShaperSetupPeerTaskManagerComponents(ctrl *gomock.Controller, opt tr
 			TaskExpireTime: util.Duration{
 				Duration: -1 * time.Second,
 			},
-		}, func(request storage.CommonTaskRequest) {})
+		}, func(request storage.CommonTaskRequest) {}, os.FileMode(0755))
 	return sched, storageManager
 }
 

client/daemon/storage/const.go

@@ -26,7 +26,7 @@ const (
 	taskMetadata = "metadata"
 
 	defaultFileMode      = os.FileMode(0644)
-	defaultDirectoryMode = os.FileMode(0755)
+	defaultDirectoryMode = os.FileMode(0755) // used unless overridden in config
 )
 
 var (

client/daemon/storage/local_storage_test.go

@@ -142,7 +142,7 @@ func TestLocalTaskStore_PutAndGetPiece(t *testing.T) {
 						Duration: time.Minute,
 					},
 				}, func(request CommonTaskRequest) {
-				})
+				}, defaultDirectoryMode)
 			assert.Nil(err)
 
 			_, err = tc.create(sm.(*storageManager), taskID, peerID)

client/daemon/storage/storage_manager.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -136,6 +137,7 @@ type storageManager struct {
 	dataPathStat       *syscall.Stat_t
 	gcCallback         func(CommonTaskRequest)
 	gcInterval         time.Duration
+	dataDirMode        fs.FileMode
 
 	indexRWMutex       sync.RWMutex
 	indexTask2PeerTask map[string][]*localTaskStore // key: task id, value: slice of localTaskStore
@@ -149,7 +151,12 @@ var _ Manager = (*storageManager)(nil)
 
 type GCCallback func(request CommonTaskRequest)
 
-func NewStorageManager(storeStrategy config.StoreStrategy, opt *config.StorageOption, gcCallback GCCallback, moreOpts ...func(*storageManager) error) (Manager, error) {
+func NewStorageManager(storeStrategy config.StoreStrategy, opt *config.StorageOption, gcCallback GCCallback, dirMode fs.FileMode, moreOpts ...func(*storageManager) error) (Manager, error) {
+	dataDirMode := defaultDirectoryMode
+	// If dirMode isn't in config, use default
+	if dirMode != os.FileMode(0) {
+		dataDirMode = defaultDirectoryMode
+	}
 	if !path.IsAbs(opt.DataPath) {
 		abs, err := filepath.Abs(opt.DataPath)
 		if err != nil {
@@ -159,7 +166,7 @@ func NewStorageManager(storeStrategy config.StoreStrategy, opt *config.StorageOp
 	}
 	stat, err := os.Stat(opt.DataPath)
 	if os.IsNotExist(err) {
-		if err := os.MkdirAll(opt.DataPath, defaultDirectoryMode); err != nil {
+		if err := os.MkdirAll(opt.DataPath, dataDirMode); err != nil {
 			return nil, err
 		}
 		stat, err = os.Stat(opt.DataPath)
@@ -182,6 +189,7 @@ func NewStorageManager(storeStrategy config.StoreStrategy, opt *config.StorageOp
 		dataPathStat:          stat.Sys().(*syscall.Stat_t),
 		gcCallback:            gcCallback,
 		gcInterval:            time.Minute,
+		dataDirMode:           dataDirMode,
 		indexTask2PeerTask:    map[string][]*localTaskStore{},
 		subIndexTask2PeerTask: map[string][]*localSubTaskStore{},
 	}
@@ -409,7 +417,14 @@ func (s *storageManager) CreateTask(req *RegisterTaskRequest) (TaskStorageDriver
 
 		SugaredLoggerOnWith: logger.With("task", req.TaskID, "peer", req.PeerID, "component", "localTaskStore"),
 	}
-	if err := os.MkdirAll(t.dataDir, defaultDirectoryMode); err != nil && !os.IsExist(err) {
+
+	dataDirMode := defaultDirectoryMode
+	// If dirMode isn't in config, use default
+	if s.dataDirMode != os.FileMode(0) {
+		dataDirMode = s.dataDirMode
+	}
+
+	if err := os.MkdirAll(t.dataDir, dataDirMode); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
 	t.touch()

cmd/dfget/cmd/daemon.go

@@ -103,10 +103,16 @@ func initDaemonDfpath(cfg *config.DaemonOption) (dfpath.Dfpath, error) {
 	if cfg.WorkHome != "" {
 		options = append(options, dfpath.WithWorkHome(cfg.WorkHome))
 	}
+	if os.FileMode(cfg.WorkHomeMode) != os.FileMode(0) {
+		options = append(options, dfpath.WithWorkHomeMode(os.FileMode(cfg.WorkHomeMode)))
+	}
 
 	if cfg.CacheDir != "" {
 		options = append(options, dfpath.WithCacheDir(cfg.CacheDir))
 	}
+	if os.FileMode(cfg.CacheDirMode) != os.FileMode(0) {
+		options = append(options, dfpath.WithCacheDirMode(os.FileMode(cfg.CacheDirMode)))
+	}
 
 	if cfg.LogDir != "" {
 		options = append(options, dfpath.WithLogDir(cfg.LogDir))
@@ -126,6 +132,12 @@ func initDaemonDfpath(cfg *config.DaemonOption) (dfpath.Dfpath, error) {
 	}
 	options = append(options, dfpath.WithDataDir(dataDir))
 
+	dataDirMode := dfpath.DefaultDataDirMode
+	if os.FileMode(cfg.DataDirMode) != os.FileMode(0) {
+		dataDirMode = os.FileMode(cfg.DataDirMode)
+	}
+	options = append(options, dfpath.WithDataDirMode(dataDirMode))
+
 	return dfpath.New(options...)
 }
 

manager/searcher/testdata/plugin/searcher.go

@@ -19,8 +19,9 @@ package main
 import (
 	"context"
 
-	"d7y.io/dragonfly/v2/manager/models"
 	"go.uber.org/zap"
+
+	"d7y.io/dragonfly/v2/manager/models"
 )
 
 type searcher struct{}

pkg/dfpath/dfpath.go

@@ -31,9 +31,12 @@ import (
 // Dfpath is the interface used for init project path.
 type Dfpath interface {
 	WorkHome() string
+	WorkHomeMode() fs.FileMode
 	CacheDir() string
+	CacheDirMode() fs.FileMode
 	LogDir() string
 	DataDir() string
+	DataDirMode() fs.FileMode
 	PluginDir() string
 	DaemonSockPath() string
 	DaemonLockPath() string
@@ -43,9 +46,12 @@ type Dfpath interface {
 // Dfpath provides init project path function.
 type dfpath struct {
 	workHome       string
+	workHomeMode   fs.FileMode
 	cacheDir       string
+	cacheDirMode   fs.FileMode
 	logDir         string
 	dataDir        string
+	dataDirMode    fs.FileMode
 	pluginDir      string
 	daemonSockPath string
 	daemonLockPath string
@@ -69,13 +75,27 @@ func WithWorkHome(dir string) Option {
 	}
 }
 
+// WithWorkHomeMode sets the workHome directory mode
+func WithWorkHomeMode(mode fs.FileMode) Option {
+	return func(d *dfpath) {
+		d.workHomeMode = mode
+	}
+}
+
 // WithCacheDir set the cache directory.
 func WithCacheDir(dir string) Option {
 	return func(d *dfpath) {
 		d.cacheDir = dir
 	}
 }
 
+// WithCacheDirMode sets the cacheDir mode
+func WithCacheDirMode(mode fs.FileMode) Option {
+	return func(d *dfpath) {
+		d.cacheDirMode = mode
+	}
+}
+
 // WithLogDir set the log directory.
 func WithLogDir(dir string) Option {
 	return func(d *dfpath) {
@@ -90,6 +110,13 @@ func WithDataDir(dir string) Option {
 	}
 }
 
+// WithDataDirMode sets the dataDir mode
+func WithDataDirMode(mode fs.FileMode) Option {
+	return func(d *dfpath) {
+		d.dataDirMode = mode
+	}
+}
+
 // WithPluginDir set plugin directory.
 func WithPluginDir(dir string) Option {
 	return func(d *dfpath) {
@@ -109,9 +136,11 @@ func New(options ...Option) (Dfpath, error) {
 	cache.Do(func() {
 		d := &dfpath{
 			workHome:       DefaultWorkHome,
+			workHomeMode:   DefaultWorkHomeMode,
 			logDir:         DefaultLogDir,
 			pluginDir:      DefaultPluginDir,
 			cacheDir:       DefaultCacheDir,
+			cacheDirMode:   DefaultCacheDirMode,
 			daemonSockPath: DefaultDownloadUnixSocketPath,
 		}
 
@@ -124,7 +153,7 @@ func New(options ...Option) (Dfpath, error) {
 		d.dfgetLockPath = filepath.Join(d.workHome, "dfget.lock")
 
 		// Create workhome directory.
-		if err := os.MkdirAll(d.workHome, fs.FileMode(0755)); err != nil {
+		if err := os.MkdirAll(d.workHome, d.workHomeMode); err != nil {
 			cache.err = multierror.Append(cache.err, err)
 		}
 
@@ -144,13 +173,13 @@ func New(options ...Option) (Dfpath, error) {
 		}
 
 		// Create cache directory.
-		if err := os.MkdirAll(d.cacheDir, fs.FileMode(0755)); err != nil {
+		if err := os.MkdirAll(d.cacheDir, d.cacheDirMode); err != nil {
 			cache.err = multierror.Append(cache.err, err)
 		}
 
 		// Create data directory.
 		if d.dataDir != "" {
-			if err := os.MkdirAll(d.dataDir, fs.FileMode(0755)); err != nil {
+			if err := os.MkdirAll(d.dataDir, d.dataDirMode); err != nil {
 				cache.err = multierror.Append(cache.err, err)
 			}
 		}
@@ -170,10 +199,18 @@ func (d *dfpath) WorkHome() string {
 	return d.workHome
 }
 
+func (d *dfpath) WorkHomeMode() fs.FileMode {
+	return d.workHomeMode
+}
+
 func (d *dfpath) CacheDir() string {
 	return d.cacheDir
 }
 
+func (d *dfpath) CacheDirMode() fs.FileMode {
+	return d.cacheDirMode
+}
+
 func (d *dfpath) LogDir() string {
 	return d.logDir
 }
@@ -182,6 +219,10 @@ func (d *dfpath) DataDir() string {
 	return d.dataDir
 }
 
+func (d *dfpath) DataDirMode() fs.FileMode {
+	return d.dataDirMode
+}
+
 func (d *dfpath) PluginDir() string {
 	return d.pluginDir
 }

pkg/dfpath/dfpath_darwin.go

@@ -25,9 +25,12 @@ import (
 )
 
 var DefaultWorkHome = filepath.Join(user.HomeDir(), ".dragonfly")
+var DefaultWorkHomeMode = os.FileMode(0755)
 var DefaultCacheDir = filepath.Join(DefaultWorkHome, "cache")
+var DefaultCacheDirMode = os.FileMode(0755)
 var DefaultConfigDir = filepath.Join(DefaultWorkHome, "config")
 var DefaultLogDir = filepath.Join(DefaultWorkHome, "logs")
 var DefaultDataDir = filepath.Join(DefaultWorkHome, "data")
+var DefaultDataDirMode = os.FileMode(0755)
 var DefaultPluginDir = filepath.Join(DefaultWorkHome, "plugins")
 var DefaultDownloadUnixSocketPath = filepath.Join(DefaultWorkHome, "dfdaemon.sock")

pkg/dfpath/dfpath_linux.go

@@ -18,10 +18,15 @@
 
 package dfpath
 
+import "os"
+
 var DefaultWorkHome = "/usr/local/dragonfly"
+var DefaultWorkHomeMode = os.FileMode(0755)
 var DefaultCacheDir = "/var/cache/dragonfly"
+var DefaultCacheDirMode = os.FileMode(0755)
 var DefaultConfigDir = "/etc/dragonfly"
 var DefaultLogDir = "/var/log/dragonfly"
 var DefaultDataDir = "/var/lib/dragonfly"
+var DefaultDataDirMode = os.FileMode(0755)
 var DefaultPluginDir = "/usr/local/dragonfly/plugins"
 var DefaultDownloadUnixSocketPath = "/var/run/dfdaemon.sock"