From f5989b99facbfa18c1b5e1df9da2d4fe85583265 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Mon, 7 Apr 2025 18:03:09 +0300 Subject: [PATCH 01/15] Add MFS command line options, extend existing mount functions for MFS, set defaults. --- cmd/ipfs/kubo/daemon.go | 10 ++++++++- config/init.go | 1 + config/mounts.go | 1 + core/commands/mount_unix.go | 10 ++++++++- core/core.go | 1 + fuse/mfs/mfs_unix.go | 34 +++++++++++++++++++++++++++++++ fuse/mfs/mount_unix.go | 21 +++++++++++++++++++ fuse/node/mount_test.go | 4 +++- fuse/node/mount_unix.go | 36 ++++++++++++++++++++++++++------- test/3nodetest/bootstrap/config | 3 ++- test/3nodetest/client/config | 3 ++- test/3nodetest/server/config | 3 ++- 12 files changed, 114 insertions(+), 13 deletions(-) create mode 100644 fuse/mfs/mfs_unix.go create mode 100644 fuse/mfs/mount_unix.go diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 7ee5953070c..2c30ac110f3 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -56,6 +56,7 @@ const ( initProfileOptionKwd = "init-profile" ipfsMountKwd = "mount-ipfs" ipnsMountKwd = "mount-ipns" + mfsMountKwd = "mount-mfs" migrateKwd = "migrate" mountKwd = "mount" offlineKwd = "offline" // global option @@ -173,6 +174,7 @@ Headers. cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"), cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), + cmds.StringOption(mfsMountKwd, "Path to the mountpoint for MFS (if using --mount). Defaults to config setting."), cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow RPC API access to unlisted hashes"), cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"), cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"), @@ -1058,17 +1060,23 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { nsdir = cfg.Mounts.IPNS } + mfdir, found := req.Options[mfsMountKwd].(string) + if !found { + mfdir = cfg.Mounts.MFS + } + node, err := cctx.ConstructNode() if err != nil { return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) } - err = nodeMount.Mount(node, fsdir, nsdir) + err = nodeMount.Mount(node, fsdir, nsdir, mfdir) if err != nil { return err } fmt.Printf("IPFS mounted at: %s\n", fsdir) fmt.Printf("IPNS mounted at: %s\n", nsdir) + fmt.Printf("MFS mounted at: %s\n", mfdir) return nil } diff --git a/config/init.go b/config/init.go index a0351bd8b18..33871d23d02 100644 --- a/config/init.go +++ b/config/init.go @@ -58,6 +58,7 @@ func InitWithIdentity(identity Identity) (*Config, error) { Mounts: Mounts{ IPFS: "/ipfs", IPNS: "/ipns", + MFS: "/mfs", }, Ipns: Ipns{ diff --git a/config/mounts.go b/config/mounts.go index dfdd1e5bf6c..571316cf386 100644 --- a/config/mounts.go +++ b/config/mounts.go @@ -4,5 +4,6 @@ package config type Mounts struct { IPFS string IPNS string + MFS string FuseAllowOther bool } diff --git a/core/commands/mount_unix.go b/core/commands/mount_unix.go index 52a1b843b80..79705a1b7fd 100644 --- a/core/commands/mount_unix.go +++ b/core/commands/mount_unix.go @@ -18,6 +18,7 @@ import ( const ( mountIPFSPathOptionName = "ipfs-path" mountIPNSPathOptionName = "ipns-path" + mountMFSPathOptionName = "mfs-path" ) var MountCmd = &cmds.Command{ @@ -81,6 +82,7 @@ baz Options: []cmds.Option{ cmds.StringOption(mountIPFSPathOptionName, "f", "The path where IPFS should be mounted."), cmds.StringOption(mountIPNSPathOptionName, "n", "The path where IPNS should be mounted."), + cmds.StringOption(mountMFSPathOptionName, "m", "The path where MFS should be mounted."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfg, err := env.(*oldcmds.Context).GetConfig() @@ -109,7 +111,12 @@ baz nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare! } - err = nodeMount.Mount(nd, fsdir, nsdir) + mfdir, found := req.Options[mountMFSPathOptionName].(string) + if !found { + nsdir = cfg.Mounts.MFS + } + + err = nodeMount.Mount(nd, fsdir, nsdir, mfdir) if err != nil { return err } @@ -124,6 +131,7 @@ baz cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, mounts *config.Mounts) error { fmt.Fprintf(w, "IPFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPFS)) fmt.Fprintf(w, "IPNS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPNS)) + fmt.Fprintf(w, "MFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.MFS)) return nil }), diff --git a/core/core.go b/core/core.go index 54c98752762..3440895e72b 100644 --- a/core/core.go +++ b/core/core.go @@ -134,6 +134,7 @@ type IpfsNode struct { type Mounts struct { Ipfs mount.Mount Ipns mount.Mount + Mfs mount.Mount } // Close calls Close() on the App object diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go new file mode 100644 index 00000000000..4a57ee540e9 --- /dev/null +++ b/fuse/mfs/mfs_unix.go @@ -0,0 +1,34 @@ +//go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse +// +build linux darwin freebsd netbsd openbsd +// +build !nofuse + +package mfs + +import ( + "context" + + "bazil.org/fuse" + "bazil.org/fuse/fs" + + "github.com/ipfs/kubo/core" +) + +type FileSystem struct { + root Root +} + +func (fs FileSystem) Root() (fs.Node, error) { + return fs.root, nil +} + +type Root struct{} + +func (root Root) Attr(ctx context.Context, attr *fuse.Attr) error { + return nil +} + +func NewFileSystem(*core.IpfsNode) fs.FS { + return FileSystem{ + root: Root{}, + } +} diff --git a/fuse/mfs/mount_unix.go b/fuse/mfs/mount_unix.go new file mode 100644 index 00000000000..7fe72e8df17 --- /dev/null +++ b/fuse/mfs/mount_unix.go @@ -0,0 +1,21 @@ +//go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse +// +build linux darwin freebsd netbsd openbsd +// +build !nofuse + +package mfs + +import ( + core "github.com/ipfs/kubo/core" + mount "github.com/ipfs/kubo/fuse/mount" +) + +// Mount mounts MFS at a given location, and returns a mount.Mount instance. +func Mount(ipfs *core.IpfsNode, mountpoint string) (mount.Mount, error) { + cfg, err := ipfs.Repo.Config() + if err != nil { + return nil, err + } + allowOther := cfg.Mounts.FuseAllowOther + fsys := NewFileSystem(ipfs) + return mount.NewMount(ipfs.Process, fsys, mountpoint, allowOther) +} diff --git a/fuse/node/mount_test.go b/fuse/node/mount_test.go index 178fddcf665..1947f759ffa 100644 --- a/fuse/node/mount_test.go +++ b/fuse/node/mount_test.go @@ -56,10 +56,12 @@ func TestExternalUnmount(t *testing.T) { ipfsDir := dir + "/ipfs" ipnsDir := dir + "/ipns" + mfsDir := dir + "/mfs" mkdir(t, ipfsDir) mkdir(t, ipnsDir) + mkdir(t, mfsDir) - err = Mount(node, ipfsDir, ipnsDir) + err = Mount(node, ipfsDir, ipnsDir, mfsDir) if err != nil { if strings.Contains(err.Error(), "unable to check fuse version") || err == fuse.ErrOSXFUSENotFound { t.Skip(err) diff --git a/fuse/node/mount_unix.go b/fuse/node/mount_unix.go index a5a2a371688..45dffba1b94 100644 --- a/fuse/node/mount_unix.go +++ b/fuse/node/mount_unix.go @@ -11,6 +11,7 @@ import ( core "github.com/ipfs/kubo/core" ipns "github.com/ipfs/kubo/fuse/ipns" + mfs "github.com/ipfs/kubo/fuse/mfs" mount "github.com/ipfs/kubo/fuse/mount" rofs "github.com/ipfs/kubo/fuse/readonly" @@ -31,7 +32,7 @@ var platformFuseChecks = func(*core.IpfsNode) error { return nil } -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { // check if we already have live mounts. // if the user said "Mount", then there must be something wrong. // so, close them and try again. @@ -43,15 +44,19 @@ func Mount(node *core.IpfsNode, fsdir, nsdir string) error { // best effort _ = node.Mounts.Ipns.Unmount() } + if node.Mounts.Mfs != nil && node.Mounts.Mfs.IsActive() { + // best effort + _ = node.Mounts.Mfs.Unmount() + } if err := platformFuseChecks(node); err != nil { return err } - return doMount(node, fsdir, nsdir) + return doMount(node, fsdir, nsdir, mfdir) } -func doMount(node *core.IpfsNode, fsdir, nsdir string) error { +func doMount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { fmtFuseErr := func(err error, mountpoint string) error { s := err.Error() if strings.Contains(s, fuseNoDirectory) { @@ -67,8 +72,8 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { } // this sync stuff is so that both can be mounted simultaneously. - var fsmount, nsmount mount.Mount - var err1, err2 error + var fsmount, nsmount, mfmount mount.Mount + var err1, err2, err3 error var wg sync.WaitGroup @@ -86,6 +91,12 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { }() } + wg.Add(1) + go func() { + defer wg.Done() + mfmount, err3 = mfs.Mount(node, mfdir) + }() + wg.Wait() if err1 != nil { @@ -96,22 +107,33 @@ func doMount(node *core.IpfsNode, fsdir, nsdir string) error { log.Errorf("error mounting IPNS %s for IPFS %s: %s", nsdir, fsdir, err2) } - if err1 != nil || err2 != nil { + if err3 != nil { + log.Errorf("error mounting MFS %s: %s", mfdir, err3) + } + + if err1 != nil || err2 != nil || err3 != nil { if fsmount != nil { _ = fsmount.Unmount() } if nsmount != nil { _ = nsmount.Unmount() } + if mfmount != nil { + _ = mfmount.Unmount() + } if err1 != nil { return fmtFuseErr(err1, fsdir) } - return fmtFuseErr(err2, nsdir) + if err2 != nil { + return fmtFuseErr(err2, nsdir) + } + return fmtFuseErr(err3, mfdir) } // setup node state, so that it can be canceled node.Mounts.Ipfs = fsmount node.Mounts.Ipns = nsmount + node.Mounts.Mfs = mfmount return nil } diff --git a/test/3nodetest/bootstrap/config b/test/3nodetest/bootstrap/config index ac441a19f16..e22f25e909b 100644 --- a/test/3nodetest/bootstrap/config +++ b/test/3nodetest/bootstrap/config @@ -15,7 +15,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "Current": "0.1.7", diff --git a/test/3nodetest/client/config b/test/3nodetest/client/config index 86ef0668d72..fa8f923d5c3 100644 --- a/test/3nodetest/client/config +++ b/test/3nodetest/client/config @@ -17,7 +17,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", diff --git a/test/3nodetest/server/config b/test/3nodetest/server/config index fb16a6d7a87..1e9db2a6332 100644 --- a/test/3nodetest/server/config +++ b/test/3nodetest/server/config @@ -17,7 +17,8 @@ }, "Mounts": { "IPFS": "/ipfs", - "IPNS": "/ipns" + "IPNS": "/ipns", + "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", From 47aa0f097351dc4fb823797a482a4721372054e0 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Tue, 8 Apr 2025 15:47:33 +0300 Subject: [PATCH 02/15] Directory listing and file stat. --- fuse/mfs/mfs_unix.go | 105 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 4a57ee540e9..8e9911a872f 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -6,29 +6,118 @@ package mfs import ( "context" + "os" + "strings" + "syscall" "bazil.org/fuse" "bazil.org/fuse/fs" + "github.com/ipfs/boxo/mfs" "github.com/ipfs/kubo/core" + "github.com/spaolacci/murmur3" ) +// FUSE filesystem mounted at /mfs. type FileSystem struct { - root Root + root Dir } -func (fs FileSystem) Root() (fs.Node, error) { - return fs.root, nil +// Get filesystem root. +func (fs *FileSystem) Root() (fs.Node, error) { + return &fs.root, nil } -type Root struct{} +// Inode numbers generated with murmur3 of the file path. +func GetInode(path string) uint64 { + return uint64(murmur3.Sum32([]byte(path))) +} + +// FUSE Adapter for MFS directories. +type Dir struct { + mfsDir *mfs.Directory +} + +// Directory attributes. +func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { + attr.Mode = os.FileMode(os.ModeDir | 0755) + return nil +} -func (root Root) Attr(ctx context.Context, attr *fuse.Attr) error { +// Access files in a directory. +func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { + mfsNode, err := dir.mfsDir.Child(req.Name) + if err != nil { + return nil, syscall.Errno(syscall.ENOENT) + } + switch mfsNode.Type() { + case mfs.TDir: + mfsDir := mfsNode.(*mfs.Directory) + result := Dir{ + mfsDir: mfsDir, + } + return &result, nil + case mfs.TFile: + mfsFile := mfsNode.(*mfs.File) + result := File{ + mfsFile: mfsFile, + } + return &result, nil + } + + return nil, syscall.Errno(syscall.ENOENT) +} + +// List MFS directory (ls). +func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + var res []fuse.Dirent + nodes, err := dir.mfsDir.List(ctx) + if err != nil { + return nil, err + } + + for _, node := range nodes { + res = append(res, fuse.Dirent{ + Inode: GetInode(strings.Join([]string{dir.mfsDir.Path(), node.Name}, "/")), + Type: fuse.DT_File, + Name: node.Name, + }) + } + return res, nil +} + +// FUSE adapter for MFS files. +type File struct { + mfsFile *mfs.File +} + +// File attributes. +func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { + size, err := file.mfsFile.Size() + if err != nil { + return err + } + attr.Size = uint64(size) + + mtime, err := file.mfsFile.ModTime() + if err != nil { + return err + } + attr.Mtime = mtime + + mode, err := file.mfsFile.Mode() + if err != nil { + return err + } + attr.Mode = mode return nil } -func NewFileSystem(*core.IpfsNode) fs.FS { - return FileSystem{ - root: Root{}, +// Create new filesystem. +func NewFileSystem(ipfs *core.IpfsNode) fs.FS { + return &FileSystem{ + root: Dir{ + mfsDir: ipfs.FilesRoot.GetDirectory(), + }, } } From 6fee6d53757f7efce2112ce25dc83fedabbd6d97 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Thu, 10 Apr 2025 15:49:31 +0300 Subject: [PATCH 03/15] Add a read-only MFS view. --- fuse/mfs/mfs_unix.go | 74 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 8e9911a872f..fac9791148a 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -6,8 +6,9 @@ package mfs import ( "context" + "io" "os" - "strings" + "sync" "syscall" "bazil.org/fuse" @@ -15,7 +16,6 @@ import ( "github.com/ipfs/boxo/mfs" "github.com/ipfs/kubo/core" - "github.com/spaolacci/murmur3" ) // FUSE filesystem mounted at /mfs. @@ -28,11 +28,6 @@ func (fs *FileSystem) Root() (fs.Node, error) { return &fs.root, nil } -// Inode numbers generated with murmur3 of the file path. -func GetInode(path string) uint64 { - return uint64(murmur3.Sum32([]byte(path))) -} - // FUSE Adapter for MFS directories. type Dir struct { mfsDir *mfs.Directory @@ -41,6 +36,8 @@ type Dir struct { // Directory attributes. func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Mode = os.FileMode(os.ModeDir | 0755) + attr.Size = 4096 + attr.Blocks = 8 return nil } @@ -52,15 +49,13 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. } switch mfsNode.Type() { case mfs.TDir: - mfsDir := mfsNode.(*mfs.Directory) result := Dir{ - mfsDir: mfsDir, + mfsDir: mfsNode.(*mfs.Directory), } return &result, nil case mfs.TFile: - mfsFile := mfsNode.(*mfs.File) result := File{ - mfsFile: mfsFile, + mfsFile: mfsNode.(*mfs.File), } return &result, nil } @@ -78,9 +73,8 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { for _, node := range nodes { res = append(res, fuse.Dirent{ - Inode: GetInode(strings.Join([]string{dir.mfsDir.Path(), node.Name}, "/")), - Type: fuse.DT_File, - Name: node.Name, + Type: fuse.DT_File, + Name: node.Name, }) } return res, nil @@ -98,6 +92,11 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { return err } attr.Size = uint64(size) + if size%512 == 0 { + attr.Blocks = uint64(size / 512) + } else { + attr.Blocks = uint64(size/512 + 1) + } mtime, err := file.mfsFile.ModTime() if err != nil { @@ -113,6 +112,53 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { return nil } +// Open an MFS file. +func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + accessMode := req.Flags & fuse.OpenAccessModeMask + flags := mfs.Flags{ + Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, + Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, + Sync: req.Flags|fuse.OpenSync > 0, + } + fd, err := file.mfsFile.Open(flags) + if err != nil { + return nil, err + } + return &FileHandler{ + mfsFD: fd, + }, nil +} + +// Wrapper for MFS's file descriptor that conforms to the FUSE fs.Handler +// interface. +type FileHandler struct { + mfsFD mfs.FileDescriptor + mu sync.Mutex +} + +// Read a opened MFS file. +func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) + if err != nil { + return err + } + + buf := make([]byte, req.Size) + l, err := fh.mfsFD.Read(buf) + + resp.Data = buf[:l] + + switch err { + case nil, io.EOF, io.ErrUnexpectedEOF: + return nil + default: + return err + } +} + // Create new filesystem. func NewFileSystem(ipfs *core.IpfsNode) fs.FS { return &FileSystem{ From 5e0469619a270b6678193eb318e0bc73d2d67494 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Thu, 10 Apr 2025 16:41:35 +0300 Subject: [PATCH 04/15] Add mkdir and interface checks. --- fuse/mfs/mfs_unix.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index fac9791148a..bc1f065bd27 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -80,6 +80,17 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { return res, nil } +// Mkdir in MFS. +func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { + mfsDir, err := dir.mfsDir.Mkdir(req.Name) + if err != nil { + return nil, err + } + return &Dir{ + mfsDir: mfsDir, + }, nil +} + // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File @@ -167,3 +178,27 @@ func NewFileSystem(ipfs *core.IpfsNode) fs.FS { }, } } + +// Check that our structs implement all the interfaces we want. +type mfDir interface { + fs.Node + fs.HandleReadDirAller + fs.NodeRequestLookuper + fs.NodeMkdirer +} + +var _ mfDir = (*Dir)(nil) + +type mfFile interface { + fs.Node + fs.NodeOpener +} + +var _ mfFile = (*File)(nil) + +type mfHandler interface { + fs.Handle + fs.HandleReader +} + +var _ mfHandler = (*FileHandler)(nil) From cda7cf78fe9018ad472952d52398aaf81380c3c2 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Thu, 10 Apr 2025 17:32:50 +0300 Subject: [PATCH 05/15] Add remove and rename functionality. --- fuse/mfs/mfs_unix.go | 57 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index bc1f065bd27..6a8cdf4b316 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -31,9 +31,10 @@ func (fs *FileSystem) Root() (fs.Node, error) { // FUSE Adapter for MFS directories. type Dir struct { mfsDir *mfs.Directory + mu sync.Mutex } -// Directory attributes. +// Directory attributes (stat). func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Mode = os.FileMode(os.ModeDir | 0755) attr.Size = 4096 @@ -63,7 +64,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. return nil, syscall.Errno(syscall.ENOENT) } -// List MFS directory (ls). +// List (ls) MFS directory. func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { var res []fuse.Dirent nodes, err := dir.mfsDir.List(ctx) @@ -80,7 +81,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { return res, nil } -// Mkdir in MFS. +// Mkdir (mkdir) in MFS. func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { mfsDir, err := dir.mfsDir.Mkdir(req.Name) if err != nil { @@ -91,6 +92,54 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err }, nil } +// Remove (rm/rmdir) an MFS file. +func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + // Check for empty directory. + if req.Dir { + targetNode, err := dir.mfsDir.Child(req.Name) + if err != nil { + return err + } + target := targetNode.(*mfs.Directory) + + children, err := target.ListNames(ctx) + if err != nil { + return err + } + if len(children) > 0 { + return os.ErrExist + } + } + err := dir.mfsDir.Unlink(req.Name) + if err != nil { + return err + } + return dir.mfsDir.Flush() +} + +// Move (mv) an MFS file. +func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { + dir.mu.Lock() + defer dir.mu.Unlock() + + file, err := dir.mfsDir.Child(req.OldName) + if err != nil { + return err + } + node, err := file.GetNode() + if err != nil { + return err + } + targetDir := newDir.(*Dir) + + err = targetDir.mfsDir.AddChild(req.NewName, node) + if err != nil { + return err + } + + return dir.mfsDir.Unlink(req.OldName) +} + // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File @@ -185,6 +234,8 @@ type mfDir interface { fs.HandleReadDirAller fs.NodeRequestLookuper fs.NodeMkdirer + fs.NodeRenamer + fs.NodeRemover } var _ mfDir = (*Dir)(nil) From 5b9deba775ec05d6c811242c1646df349dfdd3e1 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Fri, 11 Apr 2025 12:19:22 +0300 Subject: [PATCH 06/15] Implement all required write interfaces. --- fuse/mfs/mfs_unix.go | 123 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 6a8cdf4b316..f3ce7feb31a 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -14,6 +14,8 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" + dag "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/mfs" "github.com/ipfs/kubo/core" ) @@ -31,7 +33,7 @@ func (fs *FileSystem) Root() (fs.Node, error) { // FUSE Adapter for MFS directories. type Dir struct { mfsDir *mfs.Directory - mu sync.Mutex + mu sync.RWMutex } // Directory attributes (stat). @@ -44,6 +46,9 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { // Access files in a directory. func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { + dir.mu.RLock() + defer dir.mu.RUnlock() + mfsNode, err := dir.mfsDir.Child(req.Name) if err != nil { return nil, syscall.Errno(syscall.ENOENT) @@ -66,6 +71,9 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. // List (ls) MFS directory. func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + dir.mu.RLock() + defer dir.mu.RUnlock() + var res []fuse.Dirent nodes, err := dir.mfsDir.List(ctx) if err != nil { @@ -83,6 +91,9 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { // Mkdir (mkdir) in MFS. func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { + dir.mu.Lock() + defer dir.mu.Unlock() + mfsDir, err := dir.mfsDir.Mkdir(req.Name) if err != nil { return nil, err @@ -94,6 +105,9 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err // Remove (rm/rmdir) an MFS file. func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + dir.mu.Lock() + defer dir.mu.Unlock() + // Check for empty directory. if req.Dir { targetNode, err := dir.mfsDir.Child(req.Name) @@ -140,6 +154,53 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.N return dir.mfsDir.Unlink(req.OldName) } +// Create (touch) an MFS file. +func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + dir.mu.Lock() + defer dir.mu.Unlock() + + node := dag.NodeWithData(ft.FilePBData(nil, 0)) + if err := node.SetCidBuilder(dir.mfsDir.GetCidBuilder()); err != nil { + return nil, nil, err + } + + if err := dir.mfsDir.AddChild(req.Name, node); err != nil { + return nil, nil, err + } + + if err := dir.mfsDir.Flush(); err != nil { + return nil, nil, err + } + + mfsNode, err := dir.mfsDir.Child(req.Name) + if err != nil { + return nil, nil, err + } + mfsFile := mfsNode.(*mfs.File) + + file := File{ + mfsFile: mfsFile, + } + + // Read access flags and create a handler. + accessMode := req.Flags & fuse.OpenAccessModeMask + flags := mfs.Flags{ + Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, + Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, + Sync: req.Flags|fuse.OpenSync > 0, + } + + fd, err := mfsFile.Open(flags) + if err != nil { + return nil, nil, err + } + handler := FileHandler{ + mfsFD: fd, + } + + return &file, &handler, nil +} + // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File @@ -189,6 +250,11 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op }, nil } +// Sync the file's contents to MFS. +func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { + return file.mfsFile.Sync() +} + // Wrapper for MFS's file descriptor that conforms to the FUSE fs.Handler // interface. type FileHandler struct { @@ -198,16 +264,18 @@ type FileHandler struct { // Read a opened MFS file. func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - fh.mu.Lock() - defer fh.mu.Unlock() + buf := make([]byte, req.Size) - _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) - if err != nil { - return err - } + l, err := func() (int, error) { + fh.mu.Lock() + defer fh.mu.Unlock() - buf := make([]byte, req.Size) - l, err := fh.mfsFD.Read(buf) + _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) + if err != nil { + return 0, err + } + return fh.mfsFD.Read(buf) + }() resp.Data = buf[:l] @@ -219,6 +287,38 @@ func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fu } } +// Write writes to an opened MFS file. +func (fh *FileHandler) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + l, err := func() (int, error) { + fh.mu.Lock() + defer fh.mu.Unlock() + + return fh.mfsFD.WriteAt(req.Data, req.Offset) + }() + if err != nil { + return err + } + resp.Size = l + + return nil +} + +// Flushes the file's buffer. +func (fh *FileHandler) Flush(ctx context.Context, req *fuse.FlushRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + return fh.mfsFD.Flush() +} + +// Closes the file. +func (fh *FileHandler) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + + return fh.mfsFD.Close() +} + // Create new filesystem. func NewFileSystem(ipfs *core.IpfsNode) fs.FS { return &FileSystem{ @@ -236,6 +336,7 @@ type mfDir interface { fs.NodeMkdirer fs.NodeRenamer fs.NodeRemover + fs.NodeCreater } var _ mfDir = (*Dir)(nil) @@ -243,6 +344,7 @@ var _ mfDir = (*Dir)(nil) type mfFile interface { fs.Node fs.NodeOpener + fs.NodeFsyncer } var _ mfFile = (*File)(nil) @@ -250,6 +352,9 @@ var _ mfFile = (*File)(nil) type mfHandler interface { fs.Handle fs.HandleReader + fs.HandleWriter + fs.HandleFlusher + fs.HandleReleaser } var _ mfHandler = (*FileHandler)(nil) From 6e1a4550229b74c0cbfa6be30868477326fffc5d Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Fri, 11 Apr 2025 13:45:37 +0300 Subject: [PATCH 07/15] Adjust mount functions for other architechtures. --- fuse/node/mount_nofuse.go | 2 +- fuse/node/mount_notsupp.go | 2 +- fuse/node/mount_windows.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fuse/node/mount_nofuse.go b/fuse/node/mount_nofuse.go index e6f512f8eef..bc93bbc0d8b 100644 --- a/fuse/node/mount_nofuse.go +++ b/fuse/node/mount_nofuse.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { return errors.New("not compiled in") } diff --git a/fuse/node/mount_notsupp.go b/fuse/node/mount_notsupp.go index e9762a3e4bd..565a66d45df 100644 --- a/fuse/node/mount_notsupp.go +++ b/fuse/node/mount_notsupp.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { return errors.New("FUSE not supported on OpenBSD or NetBSD. See #5334 (https://github.com/ipfs/kubo/issues/5334).") } diff --git a/fuse/node/mount_windows.go b/fuse/node/mount_windows.go index 33393f99a90..7a474523e20 100644 --- a/fuse/node/mount_windows.go +++ b/fuse/node/mount_windows.go @@ -4,7 +4,7 @@ import ( "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { // TODO // currently a no-op, but we don't want to return an error return nil From d1b4cc123f6072ca8d2eb6ee996368e59d4c3b2d Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Fri, 11 Apr 2025 16:09:52 +0300 Subject: [PATCH 08/15] Merge branch 'master' into feat/10710-mfs-fuse-mount --- config/internal.go | 1 + core/corehttp/webui.go | 3 +- core/node/bitswap.go | 9 +- core/node/core.go | 1 + core/node/provider.go | 88 +++++++++++++------ docs/changelogs/v0.35.md | 34 +++++-- docs/config.md | 27 ++++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- .../t0119-prometheus-data/prometheus_metrics | 1 + test/sharness/t0165-keystore-data/README.md | 2 +- 15 files changed, 130 insertions(+), 54 deletions(-) diff --git a/config/internal.go b/config/internal.go index f43746534a3..fe7e8432085 100644 --- a/config/internal.go +++ b/config/internal.go @@ -14,5 +14,6 @@ type InternalBitswap struct { EngineTaskWorkerCount OptionalInteger MaxOutstandingBytesPerPeer OptionalInteger ProviderSearchDelay OptionalDuration + ProviderSearchMaxResults OptionalInteger WantHaveReplaceSize OptionalInteger } diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index 32d76a49deb..387a5b9ca72 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // WebUI version confirmed to work with this Kubo version -const WebUIPath = "/ipfs/bafybeibpaa5kqrj4gkemiswbwndjqiryl65cks64ypwtyerxixu56gnvvm" // v4.6.0 +const WebUIPath = "/ipfs/bafybeibfd5kbebqqruouji6ct5qku3tay273g7mt24mmrfzrsfeewaal5y" // v4.7.0 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeibpaa5kqrj4gkemiswbwndjqiryl65cks64ypwtyerxixu56gnvvm", // v4.6.0 "/ipfs/bafybeiata4qg7xjtwgor6r5dw63jjxyouenyromrrb4lrewxrlvav7gzgi", // v4.5.0 "/ipfs/bafybeigp3zm7cqoiciqk5anlheenqjsgovp7j7zq6hah4nu6iugdgb4nby", // v4.4.2 "/ipfs/bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4", // v4.4.1 diff --git a/core/node/bitswap.go b/core/node/bitswap.go index 4bcc9706604..2408fe3715e 100644 --- a/core/node/bitswap.go +++ b/core/node/bitswap.go @@ -28,6 +28,7 @@ const ( DefaultEngineTaskWorkerCount = 8 DefaultMaxOutstandingBytesPerPeer = 1 << 20 DefaultProviderSearchDelay = 1000 * time.Millisecond + DefaultMaxProviders = 10 // matching BitswapClientDefaultMaxProviders from https://github.com/ipfs/boxo/blob/v0.29.1/bitswap/internal/defaults/defaults.go#L15 DefaultWantHaveReplaceSize = 1024 ) @@ -79,11 +80,13 @@ func Bitswap(provide bool) interface{} { var provider routing.ContentDiscovery if provide { - // We need to hardcode the default because it is an - // internal setting in boxo. + var maxProviders int = DefaultMaxProviders + if in.Cfg.Internal.Bitswap != nil { + maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders)) + } pqm, err := rpqm.New(bitswapNetwork, in.Rt, - rpqm.WithMaxProviders(10), + rpqm.WithMaxProviders(maxProviders), rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...), ) if err != nil { diff --git a/core/node/core.go b/core/node/core.go index 1c372b642c3..cb34399394d 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -115,6 +115,7 @@ func FetcherConfig(bs blockservice.BlockService) FetchersOut { // path resolution should not fetch new blocks via exchange. offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore())) offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs) + offlineIpldFetcher.SkipNotFound = true // carries onto the UnixFSFetcher below offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify) diff --git a/core/node/provider.go b/core/node/provider.go index 51b12862b7a..4638aad4dc5 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -7,8 +7,10 @@ import ( "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/fetcher" + "github.com/ipfs/boxo/mfs" pin "github.com/ipfs/boxo/pinning/pinner" provider "github.com/ipfs/boxo/provider" + "github.com/ipfs/go-cid" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" "go.uber.org/fx" @@ -136,14 +138,8 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro var keyProvider fx.Option switch reprovideStrategy { - case "all", "": - keyProvider = fx.Provide(newProvidingStrategy(false, false)) - case "roots": - keyProvider = fx.Provide(newProvidingStrategy(true, true)) - case "pinned": - keyProvider = fx.Provide(newProvidingStrategy(true, false)) - case "flat": - keyProvider = fx.Provide(provider.NewBlockstoreProvider) + case "all", "", "roots", "pinned", "mfs", "pinned+mfs", "flat": + keyProvider = fx.Provide(newProvidingStrategy(reprovideStrategy)) default: return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy)) } @@ -159,32 +155,68 @@ func OfflineProviders() fx.Option { return fx.Provide(provider.NewNoopProvider) } -func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} { +func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFunc { + return func(ctx context.Context) (<-chan cid.Cid, error) { + err := mfsRoot.FlushMemFree(ctx) + if err != nil { + return nil, fmt.Errorf("error flushing mfs, cannot provide MFS: %w", err) + } + rootNode, err := mfsRoot.GetDirectory().GetNode() + if err != nil { + return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err) + } + + kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher) + return kcf(ctx) + } + +} + +func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc { + return func(ctx context.Context) (<-chan cid.Cid, error) { + rootNode, err := mfsRoot.GetDirectory().GetNode() + if err != nil { + return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err) + } + ch := make(chan cid.Cid, 1) + ch <- rootNode.Cid() + close(ch) + return ch, nil + } +} + +func newProvidingStrategy(strategy string) interface{} { type input struct { fx.In - Pinner pin.Pinner - Blockstore blockstore.Blockstore - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + Pinner pin.Pinner + Blockstore blockstore.Blockstore + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixFSFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` + MFSRoot *mfs.Root } return func(in input) provider.KeyChanFunc { - // Pinner-related CIDs will be buffered in memory to avoid - // deadlocking the pinner when the providing process is slow. - - if onlyRoots { - return provider.NewBufferedProvider( - provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher), + switch strategy { + case "roots": + return provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)) + case "pinned": + return provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)) + case "pinned+mfs": + return provider.NewPrioritizedProvider( + provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)), + mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher), ) - } - - if onlyPinned { - return provider.NewBufferedProvider( - provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher), + case "mfs": + return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher) + case "flat": + return provider.NewBlockstoreProvider(in.Blockstore) + default: // "all", "" + return provider.NewPrioritizedProvider( + provider.NewPrioritizedProvider( + provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)), + mfsRootProvider(in.MFSRoot), + ), + provider.NewBlockstoreProvider(in.Blockstore), ) } - - return provider.NewPrioritizedProvider( - provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)), - provider.NewBlockstoreProvider(in.Blockstore), - ) } } diff --git a/docs/changelogs/v0.35.md b/docs/changelogs/v0.35.md index d1432a1f41c..9c29d43c992 100644 --- a/docs/changelogs/v0.35.md +++ b/docs/changelogs/v0.35.md @@ -10,6 +10,10 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [Overview](#overview) - [πŸ”¦ Highlights](#-highlights) + - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs) + - [Additional new configuration options](#additional-new-configuration-options) + - [Grid view in WebUI](#grid-view-in-webui) + - [Expose MFS as FUSE mount point](#expose-mfs-as-fuse-mount-point) - [πŸ“¦οΈ Important dependency updates](#-important-dependency-updates) - [πŸ“ Changelog](#-changelog) - [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) @@ -18,15 +22,35 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. ### πŸ”¦ Highlights -##### `Routing.IgnoreProviders` +#### Dedicated `Reprovider.Strategy` for MFS -This new option allows ignoring specific peer IDs when returned by the content -routing system as providers of content. See the -[documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) -for for information. +The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme). + +Previously, the `pinned` reprovider strategy required manual pin management: each dataset update meant pinning the new version and unpinning the old one. Now, new strategiesβ€”`mfs` and `pinned+mfs`β€”let users limit announcements to data explicitly placed in MFS. This simplifies updating datasets and announcing only the latest version to the Amino DHT. + +Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS alone to manage updates and announcements, eliminating the need for manual pinning and unpinning. We hope this makes it easier to publish just the data that matters to you. + +See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details. + +#### Additional new configuration options + +- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md##internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones. +- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content. + +#### Grid view in WebUI + +The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen: + +> ![image](https://github.com/user-attachments/assets/80dcf0d0-8103-426f-ae91-416fb25d32b6) + +#### Expose MFS as a FUSE mount point + +The MFS root is now available as a read/write FUSE mount point at `/mfs`. This filesystem is mounted in the same way as `/ipfs` and `/ipns` when running `ipfs mount` or `ipfs daemon --mount`. #### πŸ“¦οΈ Important dependency updates +- update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0) + ### πŸ“ Changelog ### πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors diff --git a/docs/config.md b/docs/config.md index 6056510b8a3..367fffb83c7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,7 +79,8 @@ config file at runtime. - [`Internal.Bitswap.EngineBlockstoreWorkerCount`](#internalbitswapengineblockstoreworkercount) - [`Internal.Bitswap.EngineTaskWorkerCount`](#internalbitswapenginetaskworkercount) - [`Internal.Bitswap.MaxOutstandingBytesPerPeer`](#internalbitswapmaxoutstandingbytesperpeer) - - [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay) + - [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay) + - [`Internal.Bitswap.ProviderSearchMaxResults`](#internalbitswapprovidersearchmaxresults) - [`Internal.UnixFSShardingSizeThreshold`](#internalunixfsshardingsizethreshold) - [`Ipns`](#ipns) - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod) @@ -119,7 +120,7 @@ config file at runtime. - [`Routing.Type`](#routingtype) - [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient) - [`Routing.LoopbackAddressesOnLanDHT`](#routingloopbackaddressesonlandht) - - [`Routing.IgnoreProviders`](#routingignoreproviders) + - [`Routing.IgnoreProviders`](#routingignoreproviders) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - [`Routing.Routers: Parameters`](#routingrouters-parameters) @@ -1181,7 +1182,7 @@ deteriorate the quality provided to less aggressively-wanting peers. Type: `optionalInteger` (byte count, `null` means default which is 1MB) -### `Internal.Bitswap.ProviderSearchDelay` +#### `Internal.Bitswap.ProviderSearchDelay` This parameter determines how long to wait before looking for providers outside of bitswap. Other routing systems like the Amino DHT are able to provide results in less than a second, so lowering @@ -1189,6 +1190,13 @@ this number will allow faster peers lookups in some cases. Type: `optionalDuration` (`null` means default which is 1s) +#### `Internal.Bitswap.ProviderSearchMaxResults` + +Maximum number of providers bitswap client should aim at before it stops searching for new ones. +Setting to 0 means unlimited. + +Type: `optionalInteger` (`null` means default which is 10) + ### `Internal.UnixFSShardingSizeThreshold` The sharding threshold used internally to decide whether a UnixFS directory should be sharded or not. @@ -1587,10 +1595,10 @@ Type: `optionalDuration` (unset for the default) Tells reprovider what should be announced. Valid strategies are: - `"all"` - announce all CIDs of stored blocks - - Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore -- `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks) + - Order: root blocks of direct and recursive pins and MFS root are announced first, then the rest of blockstore +- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks) - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins -- `"roots"` - only announce the root block of explicitly pinned CIDs +- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`) - **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks. It makes sense only for use cases where the entire DAG is fetched in full, and a graceful resume does not have to be guaranteed: the lack of child @@ -1598,10 +1606,15 @@ Tells reprovider what should be announced. Valid strategies are: providers for the missing block in the middle of a file, unless the peer happens to already be connected to a provider and ask for child CID over bitswap. +- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`) + - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced. +- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies. + - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache. + - Order: first `pinned` and then the locally available part of `mfs`. - `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything. > [!IMPORTANT] -> Reproviding larger pinsets using the `all`, `pinned`, or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT. +> Reproviding larger pinsets using the `all`, `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT. > This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle. > The `flat` strategy can be used to lower memory requirements, but only recommended if memory utilization is too high, prioritization of pins is not necessary, and it is acceptable to announce every block cached in the local repository. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 38b3e64e9a1..bfaa528c9e7 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.1 + github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f8fc2a88567..eb2997529af 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 1abbe78123d..8975eb21054 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.1 + github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 91d34dbbdb0..e926a459635 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 3f6a1e39bcd..0cf72f9ca8a 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -116,7 +116,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.1 // indirect + github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-datastore v0.8.2 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 757ab2b7c52..4fef97504e1 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw= +github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= diff --git a/test/sharness/t0119-prometheus-data/prometheus_metrics b/test/sharness/t0119-prometheus-data/prometheus_metrics index 60475da439f..07cd44d8f5c 100644 --- a/test/sharness/t0119-prometheus-data/prometheus_metrics +++ b/test/sharness/t0119-prometheus-data/prometheus_metrics @@ -1,3 +1,4 @@ +exchange_bitswap_requests_in_flight exchange_bitswap_response_bytes_bucket exchange_bitswap_response_bytes_count exchange_bitswap_response_bytes_sum diff --git a/test/sharness/t0165-keystore-data/README.md b/test/sharness/t0165-keystore-data/README.md index 4c0a68b5169..298b7708e95 100644 --- a/test/sharness/t0165-keystore-data/README.md +++ b/test/sharness/t0165-keystore-data/README.md @@ -8,7 +8,7 @@ openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 > openssl_rsa.pem ``` secp key used in the 'restrict import key' test. -From: https://www.openssl.org/docs/man1.1.1/man1/openssl-genpkey.html +From: https://docs.openssl.org/1.1.1/man1/genpkey/ ```bash openssl genpkey -genparam -algorithm EC -out ecp.pem \ -pkeyopt ec_paramgen_curve:secp384r1 \ From d4cdb93ddd6c24e4575c43fd89cb86594db34185 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Fri, 11 Apr 2025 16:12:21 +0300 Subject: [PATCH 09/15] Write a basic read/write test. --- fuse/mfs/mfs_test.go | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 fuse/mfs/mfs_test.go diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go new file mode 100644 index 00000000000..01a18f4a4af --- /dev/null +++ b/fuse/mfs/mfs_test.go @@ -0,0 +1,73 @@ +//go:build !nofuse && !openbsd && !netbsd && !plan9 +// +build !nofuse,!openbsd,!netbsd,!plan9 + +package mfs + +import ( + "context" + "os" + "strings" + "testing" + + "bazil.org/fuse" + "bazil.org/fuse/fs" + "bazil.org/fuse/fs/fstestutil" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/core/node" + "github.com/libp2p/go-libp2p-testing/ci" +) + +func setUp(t *testing.T) (fs.FS, *fstestutil.Mount) { + if ci.NoFuse() { + t.Skip("Skipping FUSE tests") + } + + node, err := core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + fs := NewFileSystem(node) + mnt, err := fstestutil.MountedT(t, fs, nil) + if err == fuse.ErrOSXFUSENotFound { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + + return fs, mnt +} + +func TestReadWrite(t *testing.T) { + _, mnt := setUp(t) + defer mnt.Close() + + t.Run("write", func(t *testing.T) { + f, err := os.Create(mnt.Dir + "/testrw") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write([]byte("test read/write")) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + f, err := os.Open(mnt.Dir + "/testrw") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + buf := make([]byte, 15) + l, err := f.Read(buf) + if err != nil { + t.Fatal(err) + } + if strings.Compare("test read/write", string(buf[:l])) != 0 { + t.Fatal("read and write not equal") + } + }) +} From 8bba26e595f640ff63fd6ed797211b4cad7cc162 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Mon, 14 Apr 2025 10:39:30 +0300 Subject: [PATCH 10/15] Write more basic tests, add a mutex to the file object, fix modtime. --- fuse/mfs/mfs_test.go | 166 +++++++++++++++++++++++++++++++++++++++---- fuse/mfs/mfs_unix.go | 29 ++++++-- 2 files changed, 178 insertions(+), 17 deletions(-) diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go index 01a18f4a4af..ab83b1090f3 100644 --- a/fuse/mfs/mfs_test.go +++ b/fuse/mfs/mfs_test.go @@ -4,10 +4,13 @@ package mfs import ( + "bytes" "context" + "crypto/rand" + iofs "io/fs" "os" - "strings" "testing" + "time" "bazil.org/fuse" "bazil.org/fuse/fs" @@ -17,16 +20,21 @@ import ( "github.com/libp2p/go-libp2p-testing/ci" ) -func setUp(t *testing.T) (fs.FS, *fstestutil.Mount) { +// Create an Ipfs.Node, a filesystem and a mount point. +func setUp(t *testing.T, ipfs *core.IpfsNode) (fs.FS, *fstestutil.Mount) { if ci.NoFuse() { t.Skip("Skipping FUSE tests") } - node, err := core.NewNode(context.Background(), &node.BuildCfg{}) - if err != nil { - t.Fatal(err) + if ipfs == nil { + var err error + ipfs, err = core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } } - fs := NewFileSystem(node) + + fs := NewFileSystem(ipfs) mnt, err := fstestutil.MountedT(t, fs, nil) if err == fuse.ErrOSXFUSENotFound { t.Skip(err) @@ -38,36 +46,170 @@ func setUp(t *testing.T) (fs.FS, *fstestutil.Mount) { return fs, mnt } +// Test reading and writing a file. func TestReadWrite(t *testing.T) { - _, mnt := setUp(t) + _, mnt := setUp(t, nil) defer mnt.Close() + path := mnt.Dir + "/testrw" + content := make([]byte, 1024) + _, err := rand.Read(content) + if err != nil { + t.Fatal(err) + } + t.Run("write", func(t *testing.T) { - f, err := os.Create(mnt.Dir + "/testrw") + f, err := os.Create(path) if err != nil { t.Fatal(err) } defer f.Close() - _, err = f.Write([]byte("test read/write")) + _, err = f.Write(content) if err != nil { t.Fatal(err) } }) t.Run("read", func(t *testing.T) { - f, err := os.Open(mnt.Dir + "/testrw") + f, err := os.Open(path) if err != nil { t.Fatal(err) } defer f.Close() - buf := make([]byte, 15) + buf := make([]byte, 1024) l, err := f.Read(buf) if err != nil { t.Fatal(err) } - if strings.Compare("test read/write", string(buf[:l])) != 0 { + if bytes.Equal(content, buf[:l]) != true { t.Fatal("read and write not equal") } }) } + +// Test creating a directory. +func TestMkdir(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + path := mnt.Dir + "/foo/bar/baz/qux/quux" + + t.Run("write", func(t *testing.T) { + err := os.MkdirAll(path, iofs.ModeDir) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + stat, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if !stat.IsDir() { + t.Fatal("not dir") + } + }) +} + +// Test file persistence across mounts. +func TestPersistence(t *testing.T) { + ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + + content := make([]byte, 1024) + _, err = rand.Read(content) + if err != nil { + t.Fatal(err) + } + + t.Run("write", func(t *testing.T) { + _, mnt := setUp(t, ipfs) + defer mnt.Close() + path := mnt.Dir + "/testpersistence" + + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + _, mnt := setUp(t, ipfs) + defer mnt.Close() + path := mnt.Dir + "/testpersistence" + + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + buf := make([]byte, 1024) + l, err := f.Read(buf) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(content, buf[:l]) != true { + t.Fatal("read and write not equal") + } + }) +} + +// Test getting the file attributes. +func TestAttr(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + path := mnt.Dir + "/testattr" + content := make([]byte, 1024) + _, err := rand.Read(content) + if err != nil { + t.Fatal(err) + } + + t.Run("write", func(t *testing.T) { + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + }) + t.Run("read", func(t *testing.T) { + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + if fi.IsDir() { + t.Fatal("file is a directory") + } + + if fi.ModTime().After(time.Now()) { + t.Fatal("future modtime") + } + if time.Since(fi.ModTime()) > time.Second { + t.Fatal("past modtime") + } + + if fi.Name() != "testattr" { + t.Fatal("invalid filename") + } + + if fi.Size() != 1024 { + t.Fatal("invalid size") + } + }) +} diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index f3ce7feb31a..1ce8c56097e 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -10,6 +10,7 @@ import ( "os" "sync" "syscall" + "time" "bazil.org/fuse" "bazil.org/fuse/fs" @@ -176,6 +177,8 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. if err != nil { return nil, nil, err } + mfsNode.SetModTime(time.Now()) + mfsFile := mfsNode.(*mfs.File) file := File{ @@ -204,10 +207,14 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File + mu sync.RWMutex } // File attributes. func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { + file.mu.RLock() + defer file.mu.RUnlock() + size, err := file.mfsFile.Size() if err != nil { return err @@ -235,6 +242,9 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { // Open an MFS file. func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + file.mu.Lock() + defer file.mu.Unlock() + accessMode := req.Flags & fuse.OpenAccessModeMask flags := mfs.Flags{ Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, @@ -245,6 +255,14 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op if err != nil { return nil, err } + + if flags.Write { + err := file.mfsFile.SetModTime(time.Now()) + if err != nil { + return nil, err + } + } + return &FileHandler{ mfsFD: fd, }, nil @@ -252,6 +270,9 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op // Sync the file's contents to MFS. func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { + file.mu.Lock() + defer file.mu.Unlock() + return file.mfsFile.Sync() } @@ -289,12 +310,10 @@ func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fu // Write writes to an opened MFS file. func (fh *FileHandler) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { - l, err := func() (int, error) { - fh.mu.Lock() - defer fh.mu.Unlock() + fh.mu.Lock() + defer fh.mu.Unlock() - return fh.mfsFD.WriteAt(req.Data, req.Offset) - }() + l, err := fh.mfsFD.WriteAt(req.Data, req.Offset) if err != nil { return err } From d7342aa5f10a177036e81aed031694812079f08f Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Mon, 14 Apr 2025 12:34:55 +0300 Subject: [PATCH 11/15] Add a concurrency test, remove mutexes from file and directory structures. --- fuse/mfs/mfs_test.go | 92 +++++++++++++++++++++++++++++++++++++++++--- fuse/mfs/mfs_unix.go | 54 ++++++-------------------- 2 files changed, 98 insertions(+), 48 deletions(-) diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go index ab83b1090f3..479c471acf0 100644 --- a/fuse/mfs/mfs_test.go +++ b/fuse/mfs/mfs_test.go @@ -7,8 +7,10 @@ import ( "bytes" "context" "crypto/rand" + "errors" iofs "io/fs" "os" + "strconv" "testing" "time" @@ -52,7 +54,7 @@ func TestReadWrite(t *testing.T) { defer mnt.Close() path := mnt.Dir + "/testrw" - content := make([]byte, 1024) + content := make([]byte, 8196) _, err := rand.Read(content) if err != nil { t.Fatal(err) @@ -77,7 +79,7 @@ func TestReadWrite(t *testing.T) { } defer f.Close() - buf := make([]byte, 1024) + buf := make([]byte, 8196) l, err := f.Read(buf) if err != nil { t.Fatal(err) @@ -119,7 +121,7 @@ func TestPersistence(t *testing.T) { t.Fatal(err) } - content := make([]byte, 1024) + content := make([]byte, 8196) _, err = rand.Read(content) if err != nil { t.Fatal(err) @@ -152,7 +154,7 @@ func TestPersistence(t *testing.T) { } defer f.Close() - buf := make([]byte, 1024) + buf := make([]byte, 8196) l, err := f.Read(buf) if err != nil { t.Fatal(err) @@ -169,7 +171,7 @@ func TestAttr(t *testing.T) { defer mnt.Close() path := mnt.Dir + "/testattr" - content := make([]byte, 1024) + content := make([]byte, 8196) _, err := rand.Read(content) if err != nil { t.Fatal(err) @@ -208,8 +210,86 @@ func TestAttr(t *testing.T) { t.Fatal("invalid filename") } - if fi.Size() != 1024 { + if fi.Size() != 8196 { t.Fatal("invalid size") } }) } + +// Test concurrent access to the filesystem. +func TestConcurrentRW(t *testing.T) { + _, mnt := setUp(t, nil) + defer mnt.Close() + + files := 5 + fileWorkers := 5 + + path := mnt.Dir + "/testconcurrent" + content := make([][]byte, files) + + for i := range content { + content[i] = make([]byte, 8196) + _, err := rand.Read(content[i]) + if err != nil { + t.Fatal(err) + } + } + + t.Run("write", func(t *testing.T) { + errs := make(chan (error), 1) + for i := 0; i < files; i++ { + go func() { + var err error + defer func() { errs <- err }() + + f, err := os.Create(path + strconv.Itoa(i)) + if err != nil { + return + } + defer f.Close() + + _, err = f.Write(content[i]) + if err != nil { + return + } + }() + } + for i := 0; i < files; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } + }) + t.Run("read", func(t *testing.T) { + errs := make(chan (error), 1) + for i := 0; i < files*fileWorkers; i++ { + go func() { + var err error + defer func() { errs <- err }() + + f, err := os.Open(path + strconv.Itoa(i/fileWorkers)) + if err != nil { + return + } + defer f.Close() + + buf := make([]byte, 8196) + l, err := f.Read(buf) + if err != nil { + return + } + if bytes.Equal(content[i/fileWorkers], buf[:l]) != true { + err = errors.New("read and write not equal") + return + } + }() + } + for i := 0; i < files; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } + }) +} diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 1ce8c56097e..1e68e89a4e3 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -34,7 +34,6 @@ func (fs *FileSystem) Root() (fs.Node, error) { // FUSE Adapter for MFS directories. type Dir struct { mfsDir *mfs.Directory - mu sync.RWMutex } // Directory attributes (stat). @@ -47,9 +46,6 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { // Access files in a directory. func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { - dir.mu.RLock() - defer dir.mu.RUnlock() - mfsNode, err := dir.mfsDir.Child(req.Name) if err != nil { return nil, syscall.Errno(syscall.ENOENT) @@ -72,9 +68,6 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. // List (ls) MFS directory. func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - dir.mu.RLock() - defer dir.mu.RUnlock() - var res []fuse.Dirent nodes, err := dir.mfsDir.List(ctx) if err != nil { @@ -92,9 +85,6 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { // Mkdir (mkdir) in MFS. func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { - dir.mu.Lock() - defer dir.mu.Unlock() - mfsDir, err := dir.mfsDir.Mkdir(req.Name) if err != nil { return nil, err @@ -106,9 +96,6 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err // Remove (rm/rmdir) an MFS file. func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { - dir.mu.Lock() - defer dir.mu.Unlock() - // Check for empty directory. if req.Dir { targetNode, err := dir.mfsDir.Child(req.Name) @@ -134,9 +121,6 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { // Move (mv) an MFS file. func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { - dir.mu.Lock() - defer dir.mu.Unlock() - file, err := dir.mfsDir.Child(req.OldName) if err != nil { return err @@ -157,9 +141,6 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.N // Create (touch) an MFS file. func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { - dir.mu.Lock() - defer dir.mu.Unlock() - node := dag.NodeWithData(ft.FilePBData(nil, 0)) if err := node.SetCidBuilder(dir.mfsDir.GetCidBuilder()); err != nil { return nil, nil, err @@ -177,7 +158,9 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. if err != nil { return nil, nil, err } - mfsNode.SetModTime(time.Now()) + if err := mfsNode.SetModTime(time.Now()); err != nil { + return nil, nil, err + } mfsFile := mfsNode.(*mfs.File) @@ -207,14 +190,10 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File - mu sync.RWMutex } // File attributes. func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { - file.mu.RLock() - defer file.mu.RUnlock() - size, err := file.mfsFile.Size() if err != nil { return err @@ -242,9 +221,6 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { // Open an MFS file. func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { - file.mu.Lock() - defer file.mu.Unlock() - accessMode := req.Flags & fuse.OpenAccessModeMask flags := mfs.Flags{ Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, @@ -257,8 +233,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op } if flags.Write { - err := file.mfsFile.SetModTime(time.Now()) - if err != nil { + if err := file.mfsFile.SetModTime(time.Now()); err != nil { return nil, err } } @@ -270,9 +245,6 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op // Sync the file's contents to MFS. func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { - file.mu.Lock() - defer file.mu.Unlock() - return file.mfsFile.Sync() } @@ -285,18 +257,16 @@ type FileHandler struct { // Read a opened MFS file. func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - buf := make([]byte, req.Size) + fh.mu.Lock() + defer fh.mu.Unlock() - l, err := func() (int, error) { - fh.mu.Lock() - defer fh.mu.Unlock() + _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) + if err != nil { + return err + } - _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) - if err != nil { - return 0, err - } - return fh.mfsFD.Read(buf) - }() + buf := make([]byte, req.Size) + l, err := fh.mfsFD.Read(buf) resp.Data = buf[:l] From c64cfff9b32b08c856dae63d6e1bb935a5851b49 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Tue, 22 Apr 2025 14:58:21 +0300 Subject: [PATCH 12/15] Refactor naming(mfdir -> mfsdir) and add documentation. --- cmd/ipfs/kubo/daemon.go | 8 ++++---- core/commands/mount_unix.go | 4 ++-- docs/config.md | 8 ++++++++ docs/experimental-features.md | 2 +- docs/fuse.md | 9 ++++++--- fuse/mfs/mfs_unix.go | 12 ++++++------ fuse/node/mount_nofuse.go | 2 +- fuse/node/mount_notsupp.go | 2 +- fuse/node/mount_unix.go | 12 ++++++------ fuse/node/mount_windows.go | 2 +- 10 files changed, 36 insertions(+), 25 deletions(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 2c30ac110f3..7f7a1adcc1b 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -1060,9 +1060,9 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { nsdir = cfg.Mounts.IPNS } - mfdir, found := req.Options[mfsMountKwd].(string) + mfsdir, found := req.Options[mfsMountKwd].(string) if !found { - mfdir = cfg.Mounts.MFS + mfsdir = cfg.Mounts.MFS } node, err := cctx.ConstructNode() @@ -1070,13 +1070,13 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) } - err = nodeMount.Mount(node, fsdir, nsdir, mfdir) + err = nodeMount.Mount(node, fsdir, nsdir, mfsdir) if err != nil { return err } fmt.Printf("IPFS mounted at: %s\n", fsdir) fmt.Printf("IPNS mounted at: %s\n", nsdir) - fmt.Printf("MFS mounted at: %s\n", mfdir) + fmt.Printf("MFS mounted at: %s\n", mfsdir) return nil } diff --git a/core/commands/mount_unix.go b/core/commands/mount_unix.go index 79705a1b7fd..42342d43c20 100644 --- a/core/commands/mount_unix.go +++ b/core/commands/mount_unix.go @@ -111,12 +111,12 @@ baz nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare! } - mfdir, found := req.Options[mountMFSPathOptionName].(string) + mfsdir, found := req.Options[mountMFSPathOptionName].(string) if !found { nsdir = cfg.Mounts.MFS } - err = nodeMount.Mount(nd, fsdir, nsdir, mfdir) + err = nodeMount.Mount(nd, fsdir, nsdir, mfsdir) if err != nil { return err } diff --git a/docs/config.md b/docs/config.md index 367fffb83c7..43852f2ab5b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1315,6 +1315,14 @@ Default: `/ipns` Type: `string` (filesystem path) +### `Mounts.MFS` + +Mountpoint for `/mfs/`. + +Default: `/mfs` + +Type: `string` (filesystem path) + ### `Mounts.FuseAllowOther` Sets the 'FUSE allow other'-option on the mount point. diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 3fccdad197f..9aecfa13734 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -404,7 +404,7 @@ We also support the use of protocol names of the form /x/$NAME/http where $NAME ## FUSE -FUSE makes it possible to mount `/ipfs` and `/ipns` namespaces in your OS, +FUSE makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS using a subset of filesystem abstractions. It is considered EXPERIMENTAL due to limited (and buggy) support on some platforms. diff --git a/docs/fuse.md b/docs/fuse.md index 7744a0d457b..6f2b8ca8642 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -2,7 +2,7 @@ **EXPERIMENTAL:** FUSE support is limited, YMMV. -Kubo makes it possible to mount `/ipfs` and `/ipns` namespaces in your OS, +Kubo makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS. ## Install FUSE @@ -50,18 +50,20 @@ speak with us, or if you figure something new out, please add to this document! ## Prepare mountpoints -By default ipfs uses `/ipfs` and `/ipns` directories for mounting, this can be -changed in config. You will have to create the `/ipfs` and `/ipns` directories +By default ipfs uses `/ipfs`, `/ipns` and `/mfs` directories for mounting, this can be +changed in config. You will have to create the `/ipfs`, `/ipns` and `/mfs` directories explicitly. Note that modifying root requires sudo permissions. ```sh # make the directories sudo mkdir /ipfs sudo mkdir /ipns +sudo mkdir /mfs # chown them so ipfs can use them without root permissions sudo chown /ipfs sudo chown /ipns +sudo chown /mfs ``` Depending on whether you are using OSX or Linux, follow the proceeding instructions. @@ -145,6 +147,7 @@ set for user running `ipfs mount` command. ``` sudo umount /ipfs sudo umount /ipns +sudo umount /mfs ``` #### Mounting fails with "error mounting: could not resolve name" diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 1e68e89a4e3..1d4671ad8e8 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -318,7 +318,7 @@ func NewFileSystem(ipfs *core.IpfsNode) fs.FS { } // Check that our structs implement all the interfaces we want. -type mfDir interface { +type mfsDir interface { fs.Node fs.HandleReadDirAller fs.NodeRequestLookuper @@ -328,17 +328,17 @@ type mfDir interface { fs.NodeCreater } -var _ mfDir = (*Dir)(nil) +var _ mfsDir = (*Dir)(nil) -type mfFile interface { +type mfsFile interface { fs.Node fs.NodeOpener fs.NodeFsyncer } -var _ mfFile = (*File)(nil) +var _ mfsFile = (*File)(nil) -type mfHandler interface { +type mfsHandler interface { fs.Handle fs.HandleReader fs.HandleWriter @@ -346,4 +346,4 @@ type mfHandler interface { fs.HandleReleaser } -var _ mfHandler = (*FileHandler)(nil) +var _ mfsHandler = (*FileHandler)(nil) diff --git a/fuse/node/mount_nofuse.go b/fuse/node/mount_nofuse.go index bc93bbc0d8b..7423cb24d34 100644 --- a/fuse/node/mount_nofuse.go +++ b/fuse/node/mount_nofuse.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("not compiled in") } diff --git a/fuse/node/mount_notsupp.go b/fuse/node/mount_notsupp.go index 565a66d45df..79ac0e79165 100644 --- a/fuse/node/mount_notsupp.go +++ b/fuse/node/mount_notsupp.go @@ -9,6 +9,6 @@ import ( core "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("FUSE not supported on OpenBSD or NetBSD. See #5334 (https://github.com/ipfs/kubo/issues/5334).") } diff --git a/fuse/node/mount_unix.go b/fuse/node/mount_unix.go index 45dffba1b94..9846d7a42db 100644 --- a/fuse/node/mount_unix.go +++ b/fuse/node/mount_unix.go @@ -32,7 +32,7 @@ var platformFuseChecks = func(*core.IpfsNode) error { return nil } -func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // check if we already have live mounts. // if the user said "Mount", then there must be something wrong. // so, close them and try again. @@ -53,10 +53,10 @@ func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { return err } - return doMount(node, fsdir, nsdir, mfdir) + return doMount(node, fsdir, nsdir, mfsdir) } -func doMount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { +func doMount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { fmtFuseErr := func(err error, mountpoint string) error { s := err.Error() if strings.Contains(s, fuseNoDirectory) { @@ -94,7 +94,7 @@ func doMount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { wg.Add(1) go func() { defer wg.Done() - mfmount, err3 = mfs.Mount(node, mfdir) + mfmount, err3 = mfs.Mount(node, mfsdir) }() wg.Wait() @@ -108,7 +108,7 @@ func doMount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { } if err3 != nil { - log.Errorf("error mounting MFS %s: %s", mfdir, err3) + log.Errorf("error mounting MFS %s: %s", mfsdir, err3) } if err1 != nil || err2 != nil || err3 != nil { @@ -128,7 +128,7 @@ func doMount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { if err2 != nil { return fmtFuseErr(err2, nsdir) } - return fmtFuseErr(err3, mfdir) + return fmtFuseErr(err3, mfsdir) } // setup node state, so that it can be canceled diff --git a/fuse/node/mount_windows.go b/fuse/node/mount_windows.go index 7a474523e20..42e6bc10b9b 100644 --- a/fuse/node/mount_windows.go +++ b/fuse/node/mount_windows.go @@ -4,7 +4,7 @@ import ( "github.com/ipfs/kubo/core" ) -func Mount(node *core.IpfsNode, fsdir, nsdir, mfdir string) error { +func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // TODO // currently a no-op, but we don't want to return an error return nil From 7e57b858d886fbcd742070da67d28212e09f705c Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Tue, 22 Apr 2025 15:46:54 +0300 Subject: [PATCH 13/15] Add CID retrieval through ipfs_cid xattr. --- fuse/mfs/mfs_test.go | 37 +++++++++++++++++++++++++++++++++++++ fuse/mfs/mfs_unix.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go index 479c471acf0..08799a9fedb 100644 --- a/fuse/mfs/mfs_test.go +++ b/fuse/mfs/mfs_test.go @@ -10,6 +10,7 @@ import ( "errors" iofs "io/fs" "os" + "slices" "strconv" "testing" "time" @@ -293,3 +294,39 @@ func TestConcurrentRW(t *testing.T) { } }) } + +// Test ipfs_cid extended attribute +func TestMFSRootXattr(t *testing.T) { + ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) + if err != nil { + t.Fatal(err) + } + + fs, mnt := setUp(t, ipfs) + defer mnt.Close() + + node, err := fs.Root() + if err != nil { + t.Fatal(err) + } + + root := node.(*Dir) + + req := fuse.GetxattrRequest{ + Name: "ipfs_cid", + } + res := fuse.GetxattrResponse{} + err = root.Getxattr(context.Background(), &req, &res) + if err != nil { + t.Fatal(err) + } + + ipldNode, err := ipfs.FilesRoot.GetDirectory().GetNode() + if err != nil { + t.Fatal(err) + } + + if slices.Compare(res.Xattr, []byte(ipldNode.Cid().String())) != 0 { + t.Fatal("xattr cid not equal to mfs root cid") + } +} diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 1d4671ad8e8..12753f8d634 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -187,6 +187,21 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. return &file, &handler, nil } +// Get dir xattr. +func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + switch req.Name { + case "ipfs_cid": + node, err := dir.mfsDir.GetNode() + if err != nil { + return err + } + resp.Xattr = []byte(node.Cid().String()) + return nil + default: + return fuse.ErrNoXattr + } +} + // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File @@ -248,6 +263,21 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { return file.mfsFile.Sync() } +// Get file xattr. +func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + switch req.Name { + case "ipfs_cid": + node, err := file.mfsFile.GetNode() + if err != nil { + return err + } + resp.Xattr = []byte(node.Cid().String()) + return nil + default: + return fuse.ErrNoXattr + } +} + // Wrapper for MFS's file descriptor that conforms to the FUSE fs.Handler // interface. type FileHandler struct { @@ -320,6 +350,7 @@ func NewFileSystem(ipfs *core.IpfsNode) fs.FS { // Check that our structs implement all the interfaces we want. type mfsDir interface { fs.Node + fs.NodeGetxattrer fs.HandleReadDirAller fs.NodeRequestLookuper fs.NodeMkdirer @@ -332,6 +363,7 @@ var _ mfsDir = (*Dir)(nil) type mfsFile interface { fs.Node + fs.NodeGetxattrer fs.NodeOpener fs.NodeFsyncer } From 642487239eaa4f72230e6b81b17031b9654e029b Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Mon, 28 Apr 2025 15:47:36 +0300 Subject: [PATCH 14/15] Add docs, add xattr listing, fix bugs for mv and stat, refactor. --- docs/fuse.md | 19 +++++++++++ fuse/mfs/mfs_test.go | 18 +++++++--- fuse/mfs/mfs_unix.go | 81 +++++++++++++++++++++++++++++++------------- 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/docs/fuse.md b/docs/fuse.md index 6f2b8ca8642..67e1a06e008 100644 --- a/docs/fuse.md +++ b/docs/fuse.md @@ -107,6 +107,25 @@ ipfs config --json Mounts.FuseAllowOther true ipfs daemon --mount ``` +## MFS mountpoint + +Since kubo release v0.35.0, it supports mounting the MFS(Mutable File System) +root as a FUSE filesystem at `/mfs`, which enables you to manipulate +content-addressed data like regular files. The CID of a file/directory is +retrievable via the `ipfs_cid` extended attribute. + +```sh +getfattr -n ipfs_cid /mfs/welcome-to-IPFS.jpg +getfattr: Removing leading '/' from absolute path names +# file: mfs/welcome-to-IPFS.jpg +ipfs_cid="QmaeXDdwpUeKQcMy7d5SFBfVB4y7LtREbhm5KizawPsBSH" +``` + +Please note that the operations supported by the MFS FUSE mountpoint are +limited. Since the MFS wasn't designed to store file attributes like ownership +information, permissions and creation date, some applications like `vim` and +`sed` may misbehave due to missing functionality. + ## Troubleshooting #### `Permission denied` or `fusermount: user has no write access to mountpoint` error in Linux diff --git a/fuse/mfs/mfs_test.go b/fuse/mfs/mfs_test.go index 08799a9fedb..cedbe996723 100644 --- a/fuse/mfs/mfs_test.go +++ b/fuse/mfs/mfs_test.go @@ -312,11 +312,21 @@ func TestMFSRootXattr(t *testing.T) { root := node.(*Dir) - req := fuse.GetxattrRequest{ + listReq := fuse.ListxattrRequest{} + listRes := fuse.ListxattrResponse{} + err = root.Listxattr(context.Background(), &listReq, &listRes) + if err != nil { + t.Fatal(err) + } + if slices.Compare(listRes.Xattr, []byte("ipfs_cid\x00")) != 0 { + t.Fatal("list xattr returns invalid value") + } + + getReq := fuse.GetxattrRequest{ Name: "ipfs_cid", } - res := fuse.GetxattrResponse{} - err = root.Getxattr(context.Background(), &req, &res) + getRes := fuse.GetxattrResponse{} + err = root.Getxattr(context.Background(), &getReq, &getRes) if err != nil { t.Fatal(err) } @@ -326,7 +336,7 @@ func TestMFSRootXattr(t *testing.T) { t.Fatal(err) } - if slices.Compare(res.Xattr, []byte(ipldNode.Cid().String())) != 0 { + if slices.Compare(getRes.Xattr, []byte(ipldNode.Cid().String())) != 0 { t.Fatal("xattr cid not equal to mfs root cid") } } diff --git a/fuse/mfs/mfs_unix.go b/fuse/mfs/mfs_unix.go index 12753f8d634..91cad257d92 100644 --- a/fuse/mfs/mfs_unix.go +++ b/fuse/mfs/mfs_unix.go @@ -21,6 +21,14 @@ import ( "github.com/ipfs/kubo/core" ) +const ( + ipfsCIDXattr = "ipfs_cid" + mfsDirMode = os.ModeDir | 0755 + mfsFileMode = 0644 + blockSize = 512 + dirSize = 8 +) + // FUSE filesystem mounted at /mfs. type FileSystem struct { root Dir @@ -38,18 +46,23 @@ type Dir struct { // Directory attributes (stat). func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Mode = os.FileMode(os.ModeDir | 0755) - attr.Size = 4096 - attr.Blocks = 8 + attr.Mode = mfsDirMode + attr.Size = dirSize * blockSize + attr.Blocks = dirSize return nil } // Access files in a directory. func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { mfsNode, err := dir.mfsDir.Child(req.Name) - if err != nil { + switch err { + case os.ErrNotExist: return nil, syscall.Errno(syscall.ENOENT) + case nil: + default: + return nil, err } + switch mfsNode.Type() { case mfs.TDir: result := Dir{ @@ -75,8 +88,12 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { } for _, node := range nodes { + nodeType := fuse.DT_File + if node.Type == 1 { + nodeType = fuse.DT_Dir + } res = append(res, fuse.Dirent{ - Type: fuse.DT_File, + Type: nodeType, Name: node.Name, }) } @@ -131,12 +148,23 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.N } targetDir := newDir.(*Dir) + // Remove file if exists + err = targetDir.mfsDir.Unlink(req.NewName) + if err != nil && err != os.ErrNotExist { + return err + } + err = targetDir.mfsDir.AddChild(req.NewName, node) if err != nil { return err } - return dir.mfsDir.Unlink(req.OldName) + err = dir.mfsDir.Unlink(req.OldName) + if err != nil { + return err + } + + return dir.mfsDir.Flush() } // Create (touch) an MFS file. @@ -187,10 +215,16 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse. return &file, &handler, nil } +// List dir xattr. +func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { + resp.Append(ipfsCIDXattr) + return nil +} + // Get dir xattr. func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { switch req.Name { - case "ipfs_cid": + case ipfsCIDXattr: node, err := dir.mfsDir.GetNode() if err != nil { return err @@ -209,28 +243,19 @@ type File struct { // File attributes. func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { - size, err := file.mfsFile.Size() - if err != nil { - return err - } + size, _ := file.mfsFile.Size() + attr.Size = uint64(size) - if size%512 == 0 { - attr.Blocks = uint64(size / 512) + if size%blockSize == 0 { + attr.Blocks = uint64(size / blockSize) } else { - attr.Blocks = uint64(size/512 + 1) + attr.Blocks = uint64(size/blockSize + 1) } - mtime, err := file.mfsFile.ModTime() - if err != nil { - return err - } + mtime, _ := file.mfsFile.ModTime() attr.Mtime = mtime - mode, err := file.mfsFile.Mode() - if err != nil { - return err - } - attr.Mode = mode + attr.Mode = mfsFileMode return nil } @@ -263,10 +288,16 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { return file.mfsFile.Sync() } +// List file xattr. +func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { + resp.Append(ipfsCIDXattr) + return nil +} + // Get file xattr. func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { switch req.Name { - case "ipfs_cid": + case ipfsCIDXattr: node, err := file.mfsFile.GetNode() if err != nil { return err @@ -351,6 +382,7 @@ func NewFileSystem(ipfs *core.IpfsNode) fs.FS { type mfsDir interface { fs.Node fs.NodeGetxattrer + fs.NodeListxattrer fs.HandleReadDirAller fs.NodeRequestLookuper fs.NodeMkdirer @@ -364,6 +396,7 @@ var _ mfsDir = (*Dir)(nil) type mfsFile interface { fs.Node fs.NodeGetxattrer + fs.NodeListxattrer fs.NodeOpener fs.NodeFsyncer } From 3f885952b164a19819ac567e53945878ded6da98 Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Wed, 30 Apr 2025 17:45:44 +0300 Subject: [PATCH 15/15] Add some sharness tests and sharness support for MFS. --- .github/workflows/sharness.yml | 2 +- test/sharness/lib/test-lib.sh | 7 +++-- test/sharness/t0030-mount.sh | 42 +++++++++++++++++++++++--- test/sharness/t0270-filestore.sh | 2 +- test/sharness/t0271-filestore-utils.sh | 2 +- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index 6e452010f74..effef1cb923 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -46,7 +46,7 @@ jobs: env: TEST_DOCKER: 1 TEST_PLUGIN: 0 - TEST_FUSE: 0 + TEST_FUSE: 1 TEST_VERBOSE: 1 TEST_JUNIT: 1 TEST_EXPENSIVE: 1 diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index e5714d6223b..63863ff2f8f 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -206,9 +206,10 @@ test_init_ipfs() { ' test_expect_success "prepare config -- mounting" ' - mkdir mountdir ipfs ipns && + mkdir mountdir ipfs ipns mfs && test_config_set Mounts.IPFS "$(pwd)/ipfs" && - test_config_set Mounts.IPNS "$(pwd)/ipns" || + test_config_set Mounts.IPNS "$(pwd)/ipns" && + test_config_set Mounts.MFS "$(pwd)/mfs" || test_fsh cat "\"$IPFS_PATH/config\"" ' @@ -300,12 +301,14 @@ test_mount_ipfs() { test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && + do_umount "$(pwd)/mfs" || true && ipfs mount >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && + echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' diff --git a/test/sharness/t0030-mount.sh b/test/sharness/t0030-mount.sh index 0c0983d0c41..f7fbec70744 100755 --- a/test/sharness/t0030-mount.sh +++ b/test/sharness/t0030-mount.sh @@ -37,7 +37,7 @@ test_expect_success "'ipfs mount' output looks good" ' ' test_expect_success "setup and publish default IPNS value" ' - mkdir "$(pwd)/ipfs" "$(pwd)/ipns" && + mkdir "$(pwd)/ipfs" "$(pwd)/ipns" "$(pwd)/mfs" && ipfsi 0 name publish QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn ' @@ -46,12 +46,14 @@ test_expect_success "setup and publish default IPNS value" ' test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && - ipfsi 0 mount -f "$(pwd)/ipfs" -n "$(pwd)/ipns" >actual + do_umount "$(pwd)/mfs" || true && + ipfsi 0 mount -f "$(pwd)/ipfs" -n "$(pwd)/ipns" -m "$(pwd)/mfs" >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && + echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' @@ -67,13 +69,45 @@ test_expect_success FUSE "can resolve ipns names" ' test_cmp expected actual ' +test_expect_success FUSE "create mfs file" ' + touch mfs/testfile && + ipfs files ls | grep testfile +' + +test_expect_success FUSE "create mfs dir" ' + mkdir mfs/testdir && + ipfs files ls | grep testdir +' + +test_expect_success FUSE "read mfs file from fuse" ' + echo content | ipfs files write -e /testfile && + cat mfs/testfile | grep content +' + +test_expect_success FUSE "test file xattr" ' + echo content > mfs/testfile && + getfattr -n ipfs_cid mfs/testfile +' + +test_expect_success FUSE "test file removal" ' + touch mfs/testfile && + rm mfs/testfile +' + +test_expect_success FUSE "test nested dirs" ' + mkdir -p mfs/foo/bar/baz/qux && + echo content > mfs/foo/bar/baz/qux/quux && + ipfs files stat /foo/bar/baz/qux/quux +' + test_expect_success "mount directories cannot be removed while active" ' - test_must_fail rmdir ipfs ipns 2>/dev/null + test_must_fail rmdir ipfs ipns mfs 2>/dev/null ' test_expect_success "unmount directories" ' do_umount "$(pwd)/ipfs" && - do_umount "$(pwd)/ipns" + do_umount "$(pwd)/ipns" && + do_umount "$(pwd)/mfs" ' test_expect_success "mount directories can be removed after shutdown" ' diff --git a/test/sharness/t0270-filestore.sh b/test/sharness/t0270-filestore.sh index f2f63b0de80..fc377c2d264 100755 --- a/test/sharness/t0270-filestore.sh +++ b/test/sharness/t0270-filestore.sh @@ -63,7 +63,7 @@ test_filestore_adds() { init_ipfs_filestore() { test_expect_success "clean up old node" ' - rm -rf "$IPFS_PATH" mountdir ipfs ipns + rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs diff --git a/test/sharness/t0271-filestore-utils.sh b/test/sharness/t0271-filestore-utils.sh index e7c11646cda..5fd33565932 100755 --- a/test/sharness/t0271-filestore-utils.sh +++ b/test/sharness/t0271-filestore-utils.sh @@ -10,7 +10,7 @@ test_description="Test out the filestore nocopy functionality" test_init_filestore() { test_expect_success "clean up old node" ' - rm -rf "$IPFS_PATH" mountdir ipfs ipns + rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs