Skip to content

Commit

Permalink
preliminary support for unixfs Mode and ModTime
Browse files Browse the repository at this point in the history 
This commit introduces initial Mode and ModTime support
for single filesystem files and webfiles.

The ipfs add options --preserve-mode and --preserve-mtime are
used to store the original mode and last modified time of the
file being added, the options --mode, --mtime and --mtime-nsecs
are used to store custom values.

A custom value of 0 is a no-op.

The preserve flags and custom options are mutually exclusive,
if both are provided the custom options take precedence.

Majority of of the code was from #7754 written by kstuart

Replaces #7753

Closes #6920
  • Loading branch information
@gammazero
gammazero committed Aug 13, 2024
1 parent a339e6e commit 095c18e
Show file tree
Hide file tree
Showing 17 changed files with 884 additions and 65 deletions.
58 changes: 49 additions & 9 deletions client/rpc/apifile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"time"

"github.com/ipfs/boxo/files"
unixfs "github.com/ipfs/boxo/ipld/unixfs"
Expand All @@ -24,9 +26,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)
}

var stat struct {
Hash string
Type string
Size int64 // unixfs size
Hash string
Type string
Size int64 // unixfs size
Mode os.FileMode
Mtime int64
MtimeNsecs int
}
err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat)
if err != nil {
Expand All @@ -35,9 +40,9 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)

switch stat.Type {
case "file":
return api.getFile(ctx, p, stat.Size)
return api.getFile(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
case "directory":
return api.getDir(ctx, p, stat.Size)
return api.getDir(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
default:
return nil, fmt.Errorf("unsupported file type '%s'", stat.Type)
}
Expand All @@ -49,6 +54,9 @@ type apiFile struct {
size int64
path path.Path

mode os.FileMode
mtime time.Time

r *Response
at int64
}
Expand Down Expand Up @@ -128,17 +136,31 @@ func (f *apiFile) Close() error {
return nil
}

func (f *apiFile) Mode() os.FileMode {
return f.mode
}

func (f *apiFile) ModTime() time.Time {
return f.mtime
}

func (f *apiFile) Size() (int64, error) {
return f.size, nil
}

func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) {
func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) {
f := &apiFile{
ctx: ctx,
core: api.core(),
size: size,
path: p,
}
if mode != 0 {
f.mode = os.FileMode(mode)
}
if mtime != 0 {
f.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC()
}

return f, f.reset()
}
Expand Down Expand Up @@ -195,13 +217,13 @@ func (it *apiIter) Next() bool {

switch it.cur.Type {
case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory:
it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size))
it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
if err != nil {
it.err = err
return false
}
case unixfs.TFile:
it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size))
it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
if err != nil {
it.err = err
return false
Expand All @@ -223,13 +245,24 @@ type apiDir struct {
size int64
path path.Path

mode os.FileMode
mtime time.Time

dec *json.Decoder
}

func (d *apiDir) Close() error {
return nil
}

func (d *apiDir) Mode() os.FileMode {
return d.mode
}

func (d *apiDir) ModTime() time.Time {
return d.mtime
}

func (d *apiDir) Size() (int64, error) {
return d.size, nil
}
Expand All @@ -242,7 +275,7 @@ func (d *apiDir) Entries() files.DirIterator {
}
}

func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) {
func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) {
resp, err := api.core().Request("ls", p.String()).
Option("resolve-size", true).
Option("stream", true).Send(ctx)
Expand All @@ -262,6 +295,13 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (file
dec: json.NewDecoder(resp.Output),
}

if mode != 0 {
d.mode = os.FileMode(mode)
}
if mtime != 0 {
d.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC()
}

return d, nil
}

Expand Down
18 changes: 15 additions & 3 deletions client/rpc/unixfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"os"

"github.com/ipfs/boxo/files"
unixfs "github.com/ipfs/boxo/ipld/unixfs"
Expand All @@ -22,6 +23,10 @@ type addEvent struct {
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`

Mode os.FileMode `json:",omitempty"`
Mtime int64 `json:",omitempty"`
MtimeNsecs int `json:",omitempty"`
}

type UnixfsAPI HttpApi
Expand Down Expand Up @@ -94,9 +99,12 @@ loop:

if options.Events != nil {
ifevt := &iface.AddEvent{
Name: out.Name,
Size: out.Size,
Bytes: out.Bytes,
Name: out.Name,
Size: out.Size,
Bytes: out.Bytes,
Mode: out.Mode,
Mtime: out.Mtime,
MtimeNsecs: out.MtimeNsecs,
}

if out.Hash != "" {
Expand Down Expand Up @@ -129,6 +137,10 @@ type lsLink struct {
Size uint64
Type unixfs_pb.Data_DataType
Target string

Mode os.FileMode
Mtime int64
MtimeNsecs int
}

type lsObject struct {
Expand Down
93 changes: 83 additions & 10 deletions core/commands/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"io"
"os"
gopath "path"
"strconv"
"strings"
"time"

"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"
Expand All @@ -25,11 +27,31 @@ import (
// ErrDepthLimitExceeded indicates that the max depth has been exceeded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

type TimeParts struct {
t *time.Time
}

func (t TimeParts) MarshalJSON() ([]byte, error) {
return t.t.MarshalJSON()
}

// UnmarshalJSON implements the json.Unmarshaler interface.
// The time is expected to be a quoted string in RFC 3339 format.
func (t *TimeParts) UnmarshalJSON(data []byte) (err error) {
// Fractional seconds are handled implicitly by Parse.
tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data))
*t = TimeParts{&tt}
return
}

type AddEvent struct {
Name string
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`
Name string
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`
Mode string `json:",omitempty"`
Mtime int64 `json:",omitempty"`
MtimeNsecs int `json:",omitempty"`
}

const (
Expand All @@ -50,6 +72,12 @@ const (
inlineOptionName = "inline"
inlineLimitOptionName = "inline-limit"
toFilesOptionName = "to-files"

preserveModeOptionName = "preserve-mode"
preserveMtimeOptionName = "preserve-mtime"
modeOptionName = "mode"
mtimeOptionName = "mtime"
mtimeNsecsOptionName = "mtime-nsecs"
)

const adderOutChanSize = 8
Expand Down Expand Up @@ -166,6 +194,12 @@ See 'dag export' and 'dag import' for more information.
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),

cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries"),
cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries"),
cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries"),
cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch)"),
cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),
},
PreRun: func(req *cmds.Request, env cmds.Environment) error {
quiet, _ := req.Options[quietOptionName].(bool)
Expand Down Expand Up @@ -217,6 +251,11 @@ See 'dag export' and 'dag import' for more information.
inline, _ := req.Options[inlineOptionName].(bool)
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
preserveMode, _ := req.Options[preserveModeOptionName].(bool)
preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool)
mode, _ := req.Options[modeOptionName].(uint)
mtime, _ := req.Options[mtimeOptionName].(int64)
mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)

if chunker == "" {
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
Expand Down Expand Up @@ -272,6 +311,19 @@ See 'dag export' and 'dag import' for more information.

options.Unixfs.Progress(progress),
options.Unixfs.Silent(silent),

options.Unixfs.PreserveMode(preserveMode),
options.Unixfs.PreserveMtime(preserveMtime),
}

if mode != 0 {
opts = append(opts, options.Unixfs.Mode(os.FileMode(mode)))
}

if mtime != 0 {
opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs)))
} else if mtimeNsecs != 0 {
fmt.Println("option", mtimeNsecsOptionName, "ignored as no valid", mtimeOptionName, "value provided")
}

if cidVerSet {
Expand Down Expand Up @@ -383,12 +435,33 @@ See 'dag export' and 'dag import' for more information.
output.Name = gopath.Join(addit.Name(), output.Name)
}

if err := res.Emit(&AddEvent{
Name: output.Name,
Hash: h,
Bytes: output.Bytes,
Size: output.Size,
}); err != nil {
output.Mode = addit.Node().Mode()
if ts := addit.Node().ModTime(); !ts.IsZero() {
output.Mtime = addit.Node().ModTime().Unix()
output.MtimeNsecs = addit.Node().ModTime().Nanosecond()
}

addEvent := AddEvent{
Name: output.Name,
Hash: h,
Bytes: output.Bytes,
Size: output.Size,
Mtime: output.Mtime,
MtimeNsecs: output.MtimeNsecs,
}

if output.Mode != 0 {
addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8)
}

if output.Mtime > 0 {
addEvent.Mtime = output.Mtime
if output.MtimeNsecs > 0 {
addEvent.MtimeNsecs = output.MtimeNsecs
}
}

if err := res.Emit(&addEvent); err != nil {
return err
}
}
Expand Down
2 changes: 2 additions & 0 deletions core/commands/commands_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ func TestCommands(t *testing.T) {
"/files/rm",
"/files/stat",
"/files/write",
"/files/chmod",
"/files/touch",
"/filestore",
"/filestore/dups",
"/filestore/ls",
Expand Down
Loading

0 comments on commit 095c18e

Please sign in to comment.