From 795d1ea5400eb3bb18d8b248e8494251ea8db359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 15:00:51 +0200 Subject: [PATCH 01/23] coreapi unixfs: use fileAdder directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 50 ++++++++++++++++++++++++ core/coreapi/interface/unixfs.go | 4 +- core/coreapi/name_test.go | 3 +- core/coreapi/pin_test.go | 9 +++-- core/coreapi/unixfs.go | 40 ++++++++++++++++--- core/coreapi/unixfs_test.go | 5 ++- 6 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 core/coreapi/interface/options/unixfs.go diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go new file mode 100644 index 00000000000..8dc9806a776 --- /dev/null +++ b/core/coreapi/interface/options/unixfs.go @@ -0,0 +1,50 @@ +package options + +import ( + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" +) + +type UnixfsAddSettings struct { + CidVersion int + MhType uint64 + + InlineLimit int +} + +type UnixfsAddOption func(*UnixfsAddSettings) error + +func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { + options := &UnixfsAddSettings{ + CidVersion: -1, + MhType: mh.SHA2_256, + + InlineLimit: 0, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type unixfsOpts struct{} + +var Unixfs unixfsOpts + +func (unixfsOpts) CidVersion(version int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.CidVersion = version + return nil + } +} + +func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.MhType = mhtype + return nil + } +} diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 4a3aff6fc53..10febd9faa7 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -4,13 +4,15 @@ import ( "context" "io" + options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) // UnixfsAPI is the basic interface to immutable files in IPFS type UnixfsAPI interface { // Add imports the data from the reader into merkledag file - Add(context.Context, io.Reader) (ResolvedPath, error) + Add(context.Context, io.ReadCloser, ...options.UnixfsAddOption) (ResolvedPath, error) // Cat returns a reader for the file Cat(context.Context, Path) (Reader, error) diff --git a/core/coreapi/name_test.go b/core/coreapi/name_test.go index d226664842d..650d487b4e8 100644 --- a/core/coreapi/name_test.go +++ b/core/coreapi/name_test.go @@ -3,6 +3,7 @@ package coreapi_test import ( "context" "io" + "io/ioutil" "math/rand" "testing" "time" @@ -16,7 +17,7 @@ import ( var rnd = rand.New(rand.NewSource(0x62796532303137)) func addTestObject(ctx context.Context, api coreiface.CoreAPI) (coreiface.Path, error) { - return api.Unixfs().Add(ctx, &io.LimitedReader{R: rnd, N: 4092}) + return api.Unixfs().Add(ctx, ioutil.NopCloser(&io.LimitedReader{R: rnd, N: 4092})) } func TestBasicPublishResolve(t *testing.T) { diff --git a/core/coreapi/pin_test.go b/core/coreapi/pin_test.go index 9bbf16c9cfa..fbae2280291 100644 --- a/core/coreapi/pin_test.go +++ b/core/coreapi/pin_test.go @@ -2,6 +2,7 @@ package coreapi_test import ( "context" + "io/ioutil" "strings" "testing" @@ -15,7 +16,7 @@ func TestPinAdd(t *testing.T) { t.Error(err) } - p, err := api.Unixfs().Add(ctx, strings.NewReader("foo")) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) if err != nil { t.Error(err) } @@ -33,7 +34,7 @@ func TestPinSimple(t *testing.T) { t.Error(err) } - p, err := api.Unixfs().Add(ctx, strings.NewReader("foo")) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) if err != nil { t.Error(err) } @@ -82,12 +83,12 @@ func TestPinRecursive(t *testing.T) { t.Error(err) } - p0, err := api.Unixfs().Add(ctx, strings.NewReader("foo")) + p0, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) if err != nil { t.Error(err) } - p1, err := api.Unixfs().Add(ctx, strings.NewReader("bar")) + p1, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("bar"))) if err != nil { t.Error(err) } diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 1a2479ab995..27962202e23 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -5,10 +5,12 @@ import ( "io" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" + options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" coreunix "github.com/ipfs/go-ipfs/core/coreunix" - uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) @@ -16,16 +18,44 @@ type UnixfsAPI CoreAPI // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. -func (api *UnixfsAPI) Add(ctx context.Context, r io.Reader) (coreiface.ResolvedPath, error) { - k, err := coreunix.AddWithContext(ctx, api.node, r) +func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) { + _, err := options.UnixfsAddOptions(opts...) + if err != nil { + return nil, err + } + + outChan := make(chan interface{}, 1) + + fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG) if err != nil { return nil, err } - c, err := cid.Decode(k) + + fileAdder.Out = outChan + + err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil)) if err != nil { return nil, err } - return coreiface.IpfsPath(c), nil + + if _, err = fileAdder.Finalize(); err != nil { + return nil, err + } + + for { + select { + case r := <-outChan: + output := r.(*coreunix.AddedObject) + if output.Hash != "" { + c, err := cid.Parse(output.Hash) + if err != nil { + return nil, err + } + + return coreiface.IpfsPath(c), err + } + } + } } // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index bc78448f551..e7c6d153778 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "fmt" "io" + "io/ioutil" "math" "strings" "testing" @@ -133,7 +134,7 @@ func TestAdd(t *testing.T) { } str := strings.NewReader(helloStr) - p, err := api.Unixfs().Add(ctx, str) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str)) if err != nil { t.Error(err) } @@ -165,7 +166,7 @@ func TestAddEmptyFile(t *testing.T) { } str := strings.NewReader("") - p, err := api.Unixfs().Add(ctx, str) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str)) if err != nil { t.Error(err) } From c68ab5620a8a1026c2f817c5831ca671b24ca70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 15:59:53 +0200 Subject: [PATCH 02/23] coreapi unixfs: cid prefix options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 8 +++- core/coreapi/interface/unixfs.go | 1 + core/coreapi/unixfs.go | 52 ++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 8dc9806a776..ffed75577ea 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -8,7 +8,9 @@ type UnixfsAddSettings struct { CidVersion int MhType uint64 - InlineLimit int + InlineLimit int + RawLeaves bool + RawLeavesSet bool } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -18,7 +20,9 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { CidVersion: -1, MhType: mh.SHA2_256, - InlineLimit: 0, + InlineLimit: 0, + RawLeaves: false, + RawLeavesSet: false, } for _, opt := range opts { diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 10febd9faa7..acc3b960ce3 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -10,6 +10,7 @@ import ( ) // UnixfsAPI is the basic interface to immutable files in IPFS +// NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { // Add imports the data from the reader into merkledag file Add(context.Context, io.ReadCloser, ...options.UnixfsAddOption) (ResolvedPath, error) diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 27962202e23..afa66f0aeab 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -2,15 +2,19 @@ package coreapi import ( "context" + "errors" + "fmt" "io" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" - options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - coreunix "github.com/ipfs/go-ipfs/core/coreunix" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreunix" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" + dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) @@ -19,11 +23,42 @@ type UnixfsAPI CoreAPI // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) { - _, err := options.UnixfsAddOptions(opts...) + settings, err := options.UnixfsAddOptions(opts...) if err != nil { return nil, err } + // TODO: move to options + // (hash != "sha2-256") -> CIDv1 + if settings.MhType != mh.SHA2_256 { + switch settings.CidVersion { + case 0: + return nil, errors.New("CIDv0 only supports sha2-256") + case 1, -1: + settings.CidVersion = 1 + default: + return nil, fmt.Errorf("unknown CID version: %d", settings.CidVersion) + } + } else { + if settings.CidVersion < 0 { + // Default to CIDv0 + settings.CidVersion = 0 + } + } + + // cidV1 -> raw blocks (by default) + if settings.CidVersion > 0 && !settings.RawLeavesSet { + settings.RawLeaves = true + } + + prefix, err := dag.PrefixForCidVersion(settings.CidVersion) + if err != nil { + return nil, err + } + + prefix.MhType = settings.MhType + prefix.MhLength = -1 + outChan := make(chan interface{}, 1) fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG) @@ -32,6 +67,17 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. } fileAdder.Out = outChan + //fileAdder.Chunker = chunker + //fileAdder.Progress = progress + //fileAdder.Hidden = hidden + //fileAdder.Trickle = trickle + //fileAdder.Wrap = wrap + //fileAdder.Pin = dopin + fileAdder.Silent = false + fileAdder.RawLeaves = settings.RawLeaves + //fileAdder.NoCopy = nocopy + //fileAdder.Name = pathName + fileAdder.CidBuilder = prefix err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil)) if err != nil { From 06a4218a0324beb4addff61ffa737a22294b17e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 16:00:28 +0200 Subject: [PATCH 03/23] coreapi unixfs: better add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/unixfs_test.go | 157 +++++++++++++++++------------------- 1 file changed, 76 insertions(+), 81 deletions(-) diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index e7c6d153778..f8f63b909dd 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -11,15 +11,16 @@ import ( "strings" "testing" - core "github.com/ipfs/go-ipfs/core" - coreapi "github.com/ipfs/go-ipfs/core/coreapi" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/coreapi" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" - options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - coreunix "github.com/ipfs/go-ipfs/core/coreunix" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreunix" mock "github.com/ipfs/go-ipfs/core/mock" - keystore "github.com/ipfs/go-ipfs/keystore" - repo "github.com/ipfs/go-ipfs/repo" + "github.com/ipfs/go-ipfs/keystore" + "github.com/ipfs/go-ipfs/repo" + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" ci "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto" cbor "gx/ipfs/QmSywXfm2v4Qkp4DcFqo8eehj49dJK3bdUnaLVxrdFLMQn/go-ipld-cbor" unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" @@ -39,7 +40,6 @@ var hello = "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" var helloStr = "hello, world!" // `echo -n | ipfs add` -var emptyFile = "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" func makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]*core.IpfsNode, []coreiface.CoreAPI, error) { mn := mocknet.New(ctx) @@ -133,84 +133,79 @@ func TestAdd(t *testing.T) { t.Error(err) } - str := strings.NewReader(helloStr) - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str)) - if err != nil { - t.Error(err) - } - - if p.String() != hello { - t.Fatalf("expected path %s, got: %s", hello, p) - } - - r, err := api.Unixfs().Cat(ctx, p) - if err != nil { - t.Fatal(err) - } - buf := make([]byte, len(helloStr)) - _, err = io.ReadFull(r, buf) - if err != nil { - t.Error(err) - } - - if string(buf) != helloStr { - t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err) - } -} - -func TestAddEmptyFile(t *testing.T) { - ctx := context.Background() - _, api, err := makeAPI(ctx) - if err != nil { - t.Error(err) - } - - str := strings.NewReader("") - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str)) - if err != nil { - t.Error(err) - } - - if p.String() != emptyFile { - t.Fatalf("expected path %s, got: %s", hello, p) - } -} - -func TestCatBasic(t *testing.T) { - ctx := context.Background() - node, api, err := makeAPI(ctx) - if err != nil { - t.Fatal(err) - } - - hr := strings.NewReader(helloStr) - p, err := coreunix.Add(node, hr) - if err != nil { - t.Fatal(err) + cases := []struct { + name string + data string + path string + err string + opts []options.UnixfsAddOption + }{ + { + name: "simpleAdd", + data: helloStr, + path: hello, + opts: []options.UnixfsAddOption{}, + }, + { + name: "addEmpty", + data: "", + path: "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH", + }, + { + name: "addCidV1", + data: helloStr, + path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, + }, + { + name: "addCidSha3", + data: helloStr, + path: "/ipfs/zb2wwnYtXBxpndNABjtYxWAPt3cwWNRnc11iT63fvkYV78iRb", + opts: []options.UnixfsAddOption{options.Unixfs.Hash(mh.SHA3_256)}, + }, + { + name: "addCidSha3Cid0", + data: helloStr, + err: "CIDv0 only supports sha2-256", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, + }, } - p = "/ipfs/" + p - if p != hello { - t.Fatalf("expected CID %s, got: %s", hello, p) - } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + str := strings.NewReader(testCase.data) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), testCase.opts...) + if testCase.err != "" { + if err == nil { + t.Fatalf("expected an error: %s", testCase.err) + } + if err.Error() != testCase.err { + t.Fatalf("expected an error: '%s' != '%s'", err.Error(), testCase.err) + } + return + } + if err != nil { + t.Error(err) + } - helloPath, err := coreiface.ParsePath(hello) - if err != nil { - t.Fatal(err) - } + if p.String() != testCase.path { + t.Fatalf("expected path %s, got: %s", hello, p) + } - r, err := api.Unixfs().Cat(ctx, helloPath) - if err != nil { - t.Fatal(err) - } + r, err := api.Unixfs().Cat(ctx, p) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, len(testCase.data)) + _, err = io.ReadFull(r, buf) + if err != nil { + t.Error(err) + } - buf := make([]byte, len(helloStr)) - _, err = io.ReadFull(r, buf) - if err != nil { - t.Error(err) - } - if string(buf) != helloStr { - t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err) + if string(buf) != testCase.data { + t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err) + } + }) } } @@ -226,7 +221,7 @@ func TestCatEmptyFile(t *testing.T) { t.Fatal(err) } - emptyFilePath, err := coreiface.ParsePath(emptyFile) + emptyFilePath, err := coreiface.ParsePath("/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH") if err != nil { t.Fatal(err) } From 041e55d7066c78f6d209460548c4eb4b858c6f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 16:19:32 +0200 Subject: [PATCH 04/23] coreapi unixfs: options for RawLeaves / Inline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 15 +++++++++++++++ core/coreapi/unixfs.go | 10 +++++++++- core/coreapi/unixfs_test.go | 22 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index ffed75577ea..3c46ed086e6 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -52,3 +52,18 @@ func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { return nil } } + +func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.RawLeaves = enable + settings.RawLeavesSet = true + return nil + } +} + +func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.InlineLimit = limit + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index afa66f0aeab..19e4fc90c07 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -12,7 +12,8 @@ import ( cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" - files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" + "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" @@ -79,6 +80,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. //fileAdder.Name = pathName fileAdder.CidBuilder = prefix + if settings.InlineLimit > 0 { + fileAdder.CidBuilder = cidutil.InlineBuilder{ + Builder: fileAdder.CidBuilder, + Limit: settings.InlineLimit, + } + } + err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil)) if err != nil { return nil, err diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index f8f63b909dd..c5d483a080f 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -140,6 +140,7 @@ func TestAdd(t *testing.T) { err string opts []options.UnixfsAddOption }{ + // Simple cases { name: "simpleAdd", data: helloStr, @@ -151,12 +152,20 @@ func TestAdd(t *testing.T) { data: "", path: "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH", }, + // CIDv1 version / rawLeaves { name: "addCidV1", data: helloStr, path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, }, + { + name: "addCidV1NoLeaves", + data: helloStr, + path: "/ipfs/zdj7WY4GbN8NDbTW1dfCShAQNVovams2xhq9hVCx5vXcjvT8g", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(false)}, + }, + // Non sha256 hash vs CID { name: "addCidSha3", data: helloStr, @@ -169,6 +178,19 @@ func TestAdd(t *testing.T) { err: "CIDv0 only supports sha2-256", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, }, + // Inline + { + name: "addInline", + data: helloStr, + path: "/ipfs/zaYomJdLndMku8P9LHngHB5w2CQ7NenLbv", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32)}, + }, + { //TODO: after coreapi add is used in `ipfs add`, consider making this default for inline + name: "addInlineRaw", + data: helloStr, + path: "/ipfs/zj7Gr8AcBreqGEfrnR5kPFe", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.RawLeaves(true)}, + }, } for _, testCase := range cases { From 6337e69ff7e00b31c7cfe101161d652ac27c2502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 16:40:31 +0200 Subject: [PATCH 05/23] coreapi unixfs: layout/chunker options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 27 ++++++++++++++++++++++++ core/coreapi/unixfs.go | 12 +++++++++-- core/coreapi/unixfs_test.go | 15 ++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 3c46ed086e6..fe41af9a801 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -4,6 +4,13 @@ import ( mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" ) +type Layout int + +const ( + BalancedLayout Layout = iota + TrickleLeyout +) + type UnixfsAddSettings struct { CidVersion int MhType uint64 @@ -11,6 +18,9 @@ type UnixfsAddSettings struct { InlineLimit int RawLeaves bool RawLeavesSet bool + + Chunker string + Layout Layout } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -23,6 +33,9 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { InlineLimit: 0, RawLeaves: false, RawLeavesSet: false, + + Chunker: "size-262144", + Layout: BalancedLayout, } for _, opt := range opts { @@ -67,3 +80,17 @@ func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return nil } } + +func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Chunker = chunker + return nil + } +} + +func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Layout = layout + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 19e4fc90c07..1133964cb5e 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -68,10 +68,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. } fileAdder.Out = outChan - //fileAdder.Chunker = chunker + fileAdder.Chunker = settings.Chunker //fileAdder.Progress = progress //fileAdder.Hidden = hidden - //fileAdder.Trickle = trickle //fileAdder.Wrap = wrap //fileAdder.Pin = dopin fileAdder.Silent = false @@ -80,6 +79,15 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. //fileAdder.Name = pathName fileAdder.CidBuilder = prefix + switch settings.Layout { + case options.BalancedLayout: + // Default + case options.TrickleLeyout: + fileAdder.Trickle = true + default: + return nil, fmt.Errorf("unknown layout: %d", settings.Layout) + } + if settings.InlineLimit > 0 { fileAdder.CidBuilder = cidutil.InlineBuilder{ Builder: fileAdder.CidBuilder, diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index c5d483a080f..bcad984a5ce 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -191,6 +191,19 @@ func TestAdd(t *testing.T) { path: "/ipfs/zj7Gr8AcBreqGEfrnR5kPFe", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.RawLeaves(true)}, }, + // Chunker / Layout + { + name: "addChunks", + data: strings.Repeat("aoeuidhtns", 200), + path: "/ipfs/QmRo11d4QJrST47aaiGVJYwPhoNA4ihRpJ5WaxBWjWDwbX", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4")}, + }, + { + name: "addChunksTrickle", + data: strings.Repeat("aoeuidhtns", 200), + path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLeyout)}, + }, } for _, testCase := range cases { @@ -211,7 +224,7 @@ func TestAdd(t *testing.T) { } if p.String() != testCase.path { - t.Fatalf("expected path %s, got: %s", hello, p) + t.Errorf("expected path %s, got: %s", testCase.path, p) } r, err := api.Unixfs().Cat(ctx, p) From c056e5a8dee4f7e36c2a077fbd2fc49f938ed541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 17:02:03 +0200 Subject: [PATCH 06/23] coreapi unixfs: Simpler output handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/unixfs.go | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 1133964cb5e..1ea3f222a67 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -10,11 +10,10 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreunix" - cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + uio "gx/ipfs/QmPL8bYtbACcSFFiSr4s2du7Na382NxRADR8hC7D9FkEA2/go-unixfs/io" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" - uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) @@ -60,20 +59,17 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. prefix.MhType = settings.MhType prefix.MhLength = -1 - outChan := make(chan interface{}, 1) - fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG) if err != nil { return nil, err } - fileAdder.Out = outChan fileAdder.Chunker = settings.Chunker //fileAdder.Progress = progress //fileAdder.Hidden = hidden //fileAdder.Wrap = wrap //fileAdder.Pin = dopin - fileAdder.Silent = false + fileAdder.Silent = true fileAdder.RawLeaves = settings.RawLeaves //fileAdder.NoCopy = nocopy //fileAdder.Name = pathName @@ -100,24 +96,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. return nil, err } - if _, err = fileAdder.Finalize(); err != nil { + nd, err := fileAdder.Finalize() + if err != nil { return nil, err } - for { - select { - case r := <-outChan: - output := r.(*coreunix.AddedObject) - if output.Hash != "" { - c, err := cid.Parse(output.Hash) - if err != nil { - return nil, err - } - - return coreiface.IpfsPath(c), err - } - } - } + return coreiface.IpfsPath(nd.Cid()), err + } // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. From 49946c69c4f3e12faa2d8c68116c9f863f666e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 23:05:22 +0200 Subject: [PATCH 07/23] coreapi unixfs: pin/local/hash-only options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 29 ++++++++++++ core/coreapi/unixfs.go | 57 ++++++++++++++++++++++-- core/coreapi/unixfs_test.go | 56 +++++++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index fe41af9a801..6012ce77b23 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -21,6 +21,10 @@ type UnixfsAddSettings struct { Chunker string Layout Layout + + Pin bool + OnlyHash bool + Local bool } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -36,6 +40,10 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { Chunker: "size-262144", Layout: BalancedLayout, + + Pin: false, + OnlyHash: false, + Local: false, } for _, opt := range opts { @@ -94,3 +102,24 @@ func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { return nil } } + +func (unixfsOpts) Pin(pin bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Pin = pin + return nil + } +} + +func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.OnlyHash = hashOnly + return nil + } +} + +func (unixfsOpts) Local(local bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Local = local + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 1ea3f222a67..0d10b025235 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -4,18 +4,25 @@ import ( "context" "errors" "fmt" + "github.com/ipfs/go-ipfs/core" "io" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreunix" - uio "gx/ipfs/QmPL8bYtbACcSFFiSr4s2du7Na382NxRADR8hC7D9FkEA2/go-unixfs/io" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" + offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" + uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" + mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs" dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" + dagtest "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag/test" + blockservice "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" + bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore" ) type UnixfsAPI CoreAPI @@ -59,7 +66,33 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. prefix.MhType = settings.MhType prefix.MhLength = -1 - fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG) + n := api.node + if settings.OnlyHash { + nilnode, err := core.NewNode(ctx, &core.BuildCfg{ + //TODO: need this to be true or all files + // hashed will be stored in memory! + NilRepo: true, + }) + if err != nil { + return nil, err + } + n = nilnode + } + + addblockstore := n.Blockstore + //if !(fscache || nocopy) { + addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker) + //} + + exch := n.Exchange + if settings.Local { + exch = offline.Exchange(addblockstore) + } + + bserv := blockservice.New(addblockstore, exch) // hash security 001 + dserv := dag.NewDAGService(bserv) + + fileAdder, err := coreunix.NewAdder(ctx, n.Pinning, n.Blockstore, dserv) if err != nil { return nil, err } @@ -68,7 +101,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. //fileAdder.Progress = progress //fileAdder.Hidden = hidden //fileAdder.Wrap = wrap - //fileAdder.Pin = dopin + fileAdder.Pin = settings.Pin fileAdder.Silent = true fileAdder.RawLeaves = settings.RawLeaves //fileAdder.NoCopy = nocopy @@ -91,6 +124,19 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. } } + if settings.OnlyHash { + md := dagtest.Mock() + emptyDirNode := ft.EmptyDirNode() + // Use the same prefix for the "empty" MFS root as for the file adder. + emptyDirNode.SetCidBuilder(fileAdder.CidBuilder) + mr, err := mfs.NewRoot(ctx, md, emptyDirNode, nil) + if err != nil { + return nil, err + } + + fileAdder.SetMfsRoot(mr) + } + err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil)) if err != nil { return nil, err @@ -101,8 +147,11 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. return nil, err } - return coreiface.IpfsPath(nd.Cid()), err + if settings.Pin { + err = fileAdder.PinRoot() + } + return coreiface.IpfsPath(nd.Cid()), err } // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index bcad984a5ce..2c39f7709c3 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -204,6 +204,13 @@ func TestAdd(t *testing.T) { path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLeyout)}, }, + // Local + { + name: "addLocal", // better cases in sharness + data: helloStr, + path: hello, + opts: []options.UnixfsAddOption{options.Unixfs.Local(true)}, + }, } for _, testCase := range cases { @@ -244,6 +251,55 @@ func TestAdd(t *testing.T) { } } +func TestAddPinned(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + str := strings.NewReader(helloStr) + _, err = api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.Pin(true)) + if err != nil { + t.Error(err) + } + + pins, err := api.Pin().Ls(ctx) + if len(pins) != 1 { + t.Fatalf("expected 1 pin, got %d", len(pins)) + } + + if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { + t.Fatalf("got unexpected pin: %s", pins[0].Path().String()) + } +} + +func TestAddHashOnly(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + str := strings.NewReader(helloStr) + p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.HashOnly(true)) + if err != nil { + t.Error(err) + } + + if p.String() != hello { + t.Errorf("unxepected path: %s", p.String()) + } + + _, err = api.Block().Get(ctx, p) + if err == nil { + t.Fatal("expected an error") + } + if err.Error() != "blockservice: key not found" { + t.Errorf("unxepected error: %s", err.Error()) + } +} + func TestCatEmptyFile(t *testing.T) { ctx := context.Background() node, api, err := makeAPI(ctx) From 679d60993d7b6782d5e68d20bc73aa3d2fbbff0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 23:15:07 +0200 Subject: [PATCH 08/23] coreapi unixfs: cleanup options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 41 ++++++++++++++++++++++-- core/coreapi/unixfs.go | 35 +------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 6012ce77b23..6abfd9622ab 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -1,7 +1,12 @@ package options import ( + "errors" + "fmt" + + cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" + dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" ) type Layout int @@ -29,7 +34,7 @@ type UnixfsAddSettings struct { type UnixfsAddOption func(*UnixfsAddSettings) error -func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { +func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) { options := &UnixfsAddSettings{ CidVersion: -1, MhType: mh.SHA2_256, @@ -49,11 +54,41 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) { for _, opt := range opts { err := opt(options) if err != nil { - return nil, err + return nil, cid.Prefix{}, err + } + } + + // (hash != "sha2-256") -> CIDv1 + if options.MhType != mh.SHA2_256 { + switch options.CidVersion { + case 0: + return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256") + case 1, -1: + options.CidVersion = 1 + default: + return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion) + } + } else { + if options.CidVersion < 0 { + // Default to CIDv0 + options.CidVersion = 0 } } - return options, nil + // cidV1 -> raw blocks (by default) + if options.CidVersion > 0 && !options.RawLeavesSet { + options.RawLeaves = true + } + + prefix, err := dag.PrefixForCidVersion(options.CidVersion) + if err != nil { + return nil, cid.Prefix{}, err + } + + prefix.MhType = options.MhType + prefix.MhLength = -1 + + return options, prefix, nil } type unixfsOpts struct{} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 0d10b025235..399e2688885 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -2,7 +2,6 @@ package coreapi import ( "context" - "errors" "fmt" "github.com/ipfs/go-ipfs/core" "io" @@ -11,7 +10,6 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreunix" - mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" @@ -30,42 +28,11 @@ type UnixfsAPI CoreAPI // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) { - settings, err := options.UnixfsAddOptions(opts...) + settings, prefix, err := options.UnixfsAddOptions(opts...) if err != nil { return nil, err } - // TODO: move to options - // (hash != "sha2-256") -> CIDv1 - if settings.MhType != mh.SHA2_256 { - switch settings.CidVersion { - case 0: - return nil, errors.New("CIDv0 only supports sha2-256") - case 1, -1: - settings.CidVersion = 1 - default: - return nil, fmt.Errorf("unknown CID version: %d", settings.CidVersion) - } - } else { - if settings.CidVersion < 0 { - // Default to CIDv0 - settings.CidVersion = 0 - } - } - - // cidV1 -> raw blocks (by default) - if settings.CidVersion > 0 && !settings.RawLeavesSet { - settings.RawLeaves = true - } - - prefix, err := dag.PrefixForCidVersion(settings.CidVersion) - if err != nil { - return nil, err - } - - prefix.MhType = settings.MhType - prefix.MhLength = -1 - n := api.node if settings.OnlyHash { nilnode, err := core.NewNode(ctx, &core.BuildCfg{ From 9887a05e43906db7a0d9fc9ae05f9ed0f74083f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Sep 2018 23:44:49 +0200 Subject: [PATCH 09/23] coreapi unixfs: docs on options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 29 +++++++++++++++++++++++- core/coreapi/unixfs.go | 2 +- core/coreapi/unixfs_test.go | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 6abfd9622ab..9b003e1aff9 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -13,7 +13,7 @@ type Layout int const ( BalancedLayout Layout = iota - TrickleLeyout + TrickleLayout ) type UnixfsAddSettings struct { @@ -95,6 +95,8 @@ type unixfsOpts struct{} var Unixfs unixfsOpts +// CidVersion specifies which CID version to use. Defaults to 0 unless an option +// that depends on CIDv1 is passed. func (unixfsOpts) CidVersion(version int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.CidVersion = version @@ -102,6 +104,9 @@ func (unixfsOpts) CidVersion(version int) UnixfsAddOption { } } +// Hash function to use. Implies CIDv1 if not set to sha2-256 (default). +// +// Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.MhType = mhtype @@ -109,6 +114,8 @@ func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { } } +// RawLeaves specifies whether to use raw blocks for leaves (data nodes with no +// links) instead of wrapping them with unixfs structures. func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.RawLeaves = enable @@ -117,6 +124,11 @@ func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { } } +// InlineLimit sets the amount of bytes below which blocks will be encoded +// directly into CID instead of being stored and addressed by it's hash +// +// Note that while there is no hard limit on the number of bytes here, it should +// be kept at something reasonably low like 32b (default for 'ipfs add') func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.InlineLimit = limit @@ -124,6 +136,11 @@ func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { } } +// Chunker specifies settings for the chunking algorithm to use. +// +// Default: size-262144, formats: +// size-[bytes] - Simple chunker splitting data into blocks of n bytes +// rabin-[min]-[avg]-[max] - Rabin chunker func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Chunker = chunker @@ -131,6 +148,10 @@ func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { } } +// Layout tells the adder how to balance data between leaves. +// options.BalancedLayout is the default, it's optimized for static seekable +// files. +// options.TrickleLayout is optimized for streaming data, func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Layout = layout @@ -138,6 +159,7 @@ func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { } } +// Pin tells the adder to pin the file root recursively after adding func (unixfsOpts) Pin(pin bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Pin = pin @@ -145,6 +167,8 @@ func (unixfsOpts) Pin(pin bool) UnixfsAddOption { } } +// HashOnly will make the adder calculate data hash without storing it in the +// blockstore or announcing it to the network func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.OnlyHash = hashOnly @@ -152,6 +176,9 @@ func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { } } +// Local will add the data to blockstore without announcing it to the network +// +// Note that this doesn't prevent other nodes from getting this data func (unixfsOpts) Local(local bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Local = local diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 399e2688885..d6800ff77bd 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -78,7 +78,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. switch settings.Layout { case options.BalancedLayout: // Default - case options.TrickleLeyout: + case options.TrickleLayout: fileAdder.Trickle = true default: return nil, fmt.Errorf("unknown layout: %d", settings.Layout) diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 2c39f7709c3..30b7ccf3507 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -202,7 +202,7 @@ func TestAdd(t *testing.T) { name: "addChunksTrickle", data: strings.Repeat("aoeuidhtns", 200), path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", - opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLeyout)}, + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLayout)}, }, // Local { From 374fb00cc36c7c094d037188e4c082d33c5bfd63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 2 Oct 2018 09:42:50 +0200 Subject: [PATCH 10/23] coreapi unixfs: separate option to enable inlining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 25 ++++++++++++++++++++---- core/coreapi/unixfs.go | 2 +- core/coreapi/unixfs_test.go | 21 ++++++++++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 9b003e1aff9..df6f4fc7112 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -20,6 +20,7 @@ type UnixfsAddSettings struct { CidVersion int MhType uint64 + Inline bool InlineLimit int RawLeaves bool RawLeavesSet bool @@ -39,7 +40,8 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, CidVersion: -1, MhType: mh.SHA2_256, - InlineLimit: 0, + Inline: false, + InlineLimit: 32, RawLeaves: false, RawLeavesSet: false, @@ -124,11 +126,26 @@ func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { } } +// Inline tells the adder to inline small blocks into CIDs +func (unixfsOpts) Inline(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Inline = enable + return nil + } +} + // InlineLimit sets the amount of bytes below which blocks will be encoded -// directly into CID instead of being stored and addressed by it's hash +// directly into CID instead of being stored and addressed by it's hash. +// Specifying this option won't enable block inlining. For that use `Inline` +// option. Default: 32 bytes +// +// Note that while there is no hard limit on the number of bytes, it should +// be kept at a reasonably low value, like 64 bytes if you intend to display +// these hashes. Larger values like 256 bytes will work fine, but may affect +// de-duplication of smaller blocks. // -// Note that while there is no hard limit on the number of bytes here, it should -// be kept at something reasonably low like 32b (default for 'ipfs add') +// Setting this value too high may cause various problems, such as render some +// blocks unfetchable func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.InlineLimit = limit diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index d6800ff77bd..31910afd320 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -84,7 +84,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. return nil, fmt.Errorf("unknown layout: %d", settings.Layout) } - if settings.InlineLimit > 0 { + if settings.Inline { fileAdder.CidBuilder = cidutil.InlineBuilder{ Builder: fileAdder.CidBuilder, Limit: settings.InlineLimit, diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 30b7ccf3507..207470a8144 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -40,6 +40,7 @@ var hello = "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" var helloStr = "hello, world!" // `echo -n | ipfs add` +var emptyFile = "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" func makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]*core.IpfsNode, []coreiface.CoreAPI, error) { mn := mocknet.New(ctx) @@ -150,7 +151,7 @@ func TestAdd(t *testing.T) { { name: "addEmpty", data: "", - path: "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH", + path: emptyFile, }, // CIDv1 version / rawLeaves { @@ -183,13 +184,25 @@ func TestAdd(t *testing.T) { name: "addInline", data: helloStr, path: "/ipfs/zaYomJdLndMku8P9LHngHB5w2CQ7NenLbv", - opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32)}, + opts: []options.UnixfsAddOption{options.Unixfs.Inline(true)}, + }, + { + name: "addInlineLimit", + data: helloStr, + path: "/ipfs/zaYomJdLndMku8P9LHngHB5w2CQ7NenLbv", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true)}, + }, + { + name: "addInlineZero", + data: "", + path: "/ipfs/z2yYDV", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(0), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, { //TODO: after coreapi add is used in `ipfs add`, consider making this default for inline name: "addInlineRaw", data: helloStr, path: "/ipfs/zj7Gr8AcBreqGEfrnR5kPFe", - opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.RawLeaves(true)}, + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, // Chunker / Layout { @@ -312,7 +325,7 @@ func TestCatEmptyFile(t *testing.T) { t.Fatal(err) } - emptyFilePath, err := coreiface.ParsePath("/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH") + emptyFilePath, err := coreiface.ParsePath(emptyFile) if err != nil { t.Fatal(err) } From 675106d75968612049c5aba28060cc6c65303b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 02:08:57 +0200 Subject: [PATCH 11/23] coreapi unixfs: rebase fixes for fileAdder.AddAllAndPin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 2 +- core/coreapi/unixfs.go | 16 +++------------- core/coreunix/add.go | 16 ++++++++-------- core/coreunix/add_test.go | 4 ++-- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 40e170cdb71..d7d6b33a267 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -302,7 +302,7 @@ You can now check what blocks have been created by: var err error defer func() { errCh <- err }() defer close(outChan) - err = fileAdder.AddAllAndPin(req.Files) + _, err = fileAdder.AddAllAndPin(req.Files) }() err = res.Emit(outChan) diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 31910afd320..03bc6f73c0d 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -68,7 +68,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. //fileAdder.Progress = progress //fileAdder.Hidden = hidden //fileAdder.Wrap = wrap - fileAdder.Pin = settings.Pin + fileAdder.Pin = settings.Pin && !settings.OnlyHash fileAdder.Silent = true fileAdder.RawLeaves = settings.RawLeaves //fileAdder.NoCopy = nocopy @@ -104,21 +104,11 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. fileAdder.SetMfsRoot(mr) } - err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil)) + nd, err := fileAdder.AddAllAndPin(files.NewReaderFile("", "", r, nil)) if err != nil { return nil, err } - - nd, err := fileAdder.Finalize() - if err != nil { - return nil, err - } - - if settings.Pin { - err = fileAdder.PinRoot() - } - - return coreiface.IpfsPath(nd.Cid()), err + return coreiface.IpfsPath(nd.Cid()), nil } // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 3792db9430a..55be2cd55da 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -399,7 +399,7 @@ func (adder *Adder) addNode(node ipld.Node, path string) error { } // AddAllAndPin adds the given request's files and pin them. -func (adder *Adder) AddAllAndPin(file files.File) error { +func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) { if adder.Pin { adder.unlocker = adder.blockstore.PinLock() } @@ -420,30 +420,30 @@ func (adder *Adder) AddAllAndPin(file files.File) error { // Finished the list of files. break } else if err != nil { - return err + return nil, err } if err := adder.addFile(file); err != nil { - return err + return nil, err } } break default: if err := adder.addFile(file); err != nil { - return err + return nil, err } break } // copy intermediary nodes from editor to our actual dagservice - _, err := adder.Finalize() + nd, err := adder.Finalize() if err != nil { - return err + return nil, err } if !adder.Pin { - return nil + return nd, nil } - return adder.PinRoot() + return nd, adder.PinRoot() } func (adder *Adder) addFile(file files.File) error { diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index bba5d7ce8e7..5185c4f4d8f 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -85,7 +85,7 @@ func TestAddGCLive(t *testing.T) { go func() { defer close(addDone) defer close(out) - err := adder.AddAllAndPin(slf) + _, err := adder.AddAllAndPin(slf) if err != nil { t.Fatal(err) @@ -191,7 +191,7 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) { go func() { defer close(adder.Out) - err = adder.AddAllAndPin(file) + _, err = adder.AddAllAndPin(file) if err != nil { t.Fatal(err) } From e6bc92342503edba0b7d71cfafa1747fa1cc6816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 15:05:46 +0200 Subject: [PATCH 12/23] coreapi unixfs: multi file support in unixfs coreapi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/unixfs.go | 4 +-- core/coreapi/name_test.go | 3 +- core/coreapi/pin_test.go | 9 +++--- core/coreapi/unixfs.go | 7 ++--- core/coreapi/unixfs_test.go | 49 ++++++++++++++++++-------------- core/corehttp/gateway_handler.go | 4 ++- 6 files changed, 41 insertions(+), 35 deletions(-) diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index acc3b960ce3..92168503eca 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -2,10 +2,10 @@ package iface import ( "context" - "io" options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) @@ -13,7 +13,7 @@ import ( // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { // Add imports the data from the reader into merkledag file - Add(context.Context, io.ReadCloser, ...options.UnixfsAddOption) (ResolvedPath, error) + Add(context.Context, files.File, ...options.UnixfsAddOption) (ResolvedPath, error) // Cat returns a reader for the file Cat(context.Context, Path) (Reader, error) diff --git a/core/coreapi/name_test.go b/core/coreapi/name_test.go index 650d487b4e8..a245b710c4a 100644 --- a/core/coreapi/name_test.go +++ b/core/coreapi/name_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" ipath "gx/ipfs/QmcjwUb36Z16NJkvDX6ccXPqsFswo6AsRXynyXcLLCphV2/go-path" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" @@ -17,7 +18,7 @@ import ( var rnd = rand.New(rand.NewSource(0x62796532303137)) func addTestObject(ctx context.Context, api coreiface.CoreAPI) (coreiface.Path, error) { - return api.Unixfs().Add(ctx, ioutil.NopCloser(&io.LimitedReader{R: rnd, N: 4092})) + return api.Unixfs().Add(ctx, files.NewReaderFile("", "", ioutil.NopCloser(&io.LimitedReader{R: rnd, N: 4092}), nil)) } func TestBasicPublishResolve(t *testing.T) { diff --git a/core/coreapi/pin_test.go b/core/coreapi/pin_test.go index fbae2280291..a9a7547c3db 100644 --- a/core/coreapi/pin_test.go +++ b/core/coreapi/pin_test.go @@ -2,7 +2,6 @@ package coreapi_test import ( "context" - "io/ioutil" "strings" "testing" @@ -16,7 +15,7 @@ func TestPinAdd(t *testing.T) { t.Error(err) } - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) + p, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Error(err) } @@ -34,7 +33,7 @@ func TestPinSimple(t *testing.T) { t.Error(err) } - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) + p, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Error(err) } @@ -83,12 +82,12 @@ func TestPinRecursive(t *testing.T) { t.Error(err) } - p0, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("foo"))) + p0, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Error(err) } - p1, err := api.Unixfs().Add(ctx, ioutil.NopCloser(strings.NewReader("bar"))) + p1, err := api.Unixfs().Add(ctx, strFile("bar")()) if err != nil { t.Error(err) } diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 03bc6f73c0d..9ba1c29c3cb 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/ipfs/go-ipfs/core" - "io" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -12,7 +11,7 @@ import ( cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline" - "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs" @@ -27,7 +26,7 @@ type UnixfsAPI CoreAPI // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. -func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) { +func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) { settings, prefix, err := options.UnixfsAddOptions(opts...) if err != nil { return nil, err @@ -104,7 +103,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options. fileAdder.SetMfsRoot(mr) } - nd, err := fileAdder.AddAllAndPin(files.NewReaderFile("", "", r, nil)) + nd, err := fileAdder.AddAllAndPin(files) if err != nil { return nil, err } diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 207470a8144..f1fd90203b3 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -22,6 +22,7 @@ import ( mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" ci "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" cbor "gx/ipfs/QmSywXfm2v4Qkp4DcFqo8eehj49dJK3bdUnaLVxrdFLMQn/go-ipld-cbor" unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" datastore "gx/ipfs/QmUyz7JTJzgegC6tiJrfby3mPhzcdswVtG4x58TQ6pq8jV/go-datastore" @@ -127,6 +128,12 @@ func makeAPI(ctx context.Context) (*core.IpfsNode, coreiface.CoreAPI, error) { return nd[0], api[0], nil } +func strFile(data string) func() files.File { + return func() files.File { + return files.NewReaderFile("", "", ioutil.NopCloser(strings.NewReader(data)), nil) + } +} + func TestAdd(t *testing.T) { ctx := context.Background() _, api, err := makeAPI(ctx) @@ -136,7 +143,7 @@ func TestAdd(t *testing.T) { cases := []struct { name string - data string + data func() files.File path string err string opts []options.UnixfsAddOption @@ -144,83 +151,83 @@ func TestAdd(t *testing.T) { // Simple cases { name: "simpleAdd", - data: helloStr, + data: strFile(helloStr), path: hello, opts: []options.UnixfsAddOption{}, }, { name: "addEmpty", - data: "", + data: strFile(""), path: emptyFile, }, // CIDv1 version / rawLeaves { name: "addCidV1", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, }, { name: "addCidV1NoLeaves", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zdj7WY4GbN8NDbTW1dfCShAQNVovams2xhq9hVCx5vXcjvT8g", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(false)}, }, // Non sha256 hash vs CID { name: "addCidSha3", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zb2wwnYtXBxpndNABjtYxWAPt3cwWNRnc11iT63fvkYV78iRb", opts: []options.UnixfsAddOption{options.Unixfs.Hash(mh.SHA3_256)}, }, { name: "addCidSha3Cid0", - data: helloStr, + data: strFile(helloStr), err: "CIDv0 only supports sha2-256", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, }, // Inline { name: "addInline", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zaYomJdLndMku8P9LHngHB5w2CQ7NenLbv", opts: []options.UnixfsAddOption{options.Unixfs.Inline(true)}, }, { name: "addInlineLimit", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zaYomJdLndMku8P9LHngHB5w2CQ7NenLbv", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true)}, }, { name: "addInlineZero", - data: "", + data: strFile(""), path: "/ipfs/z2yYDV", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(0), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, { //TODO: after coreapi add is used in `ipfs add`, consider making this default for inline name: "addInlineRaw", - data: helloStr, + data: strFile(helloStr), path: "/ipfs/zj7Gr8AcBreqGEfrnR5kPFe", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, // Chunker / Layout { name: "addChunks", - data: strings.Repeat("aoeuidhtns", 200), + data: strFile(strings.Repeat("aoeuidhtns", 200)), path: "/ipfs/QmRo11d4QJrST47aaiGVJYwPhoNA4ihRpJ5WaxBWjWDwbX", opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4")}, }, { name: "addChunksTrickle", - data: strings.Repeat("aoeuidhtns", 200), + data: strFile(strings.Repeat("aoeuidhtns", 200)), path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLayout)}, }, // Local { name: "addLocal", // better cases in sharness - data: helloStr, + data: strFile(helloStr), path: hello, opts: []options.UnixfsAddOption{options.Unixfs.Local(true)}, }, @@ -228,8 +235,7 @@ func TestAdd(t *testing.T) { for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { - str := strings.NewReader(testCase.data) - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), testCase.opts...) + p, err := api.Unixfs().Add(ctx, testCase.data(), testCase.opts...) if testCase.err != "" { if err == nil { t.Fatalf("expected an error: %s", testCase.err) @@ -247,7 +253,7 @@ func TestAdd(t *testing.T) { t.Errorf("expected path %s, got: %s", testCase.path, p) } - r, err := api.Unixfs().Cat(ctx, p) + /*r, err := api.Unixfs().Cat(ctx, p) if err != nil { t.Fatal(err) } @@ -259,7 +265,8 @@ func TestAdd(t *testing.T) { if string(buf) != testCase.data { t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err) - } + }*/ + }) } } @@ -271,8 +278,7 @@ func TestAddPinned(t *testing.T) { t.Error(err) } - str := strings.NewReader(helloStr) - _, err = api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.Pin(true)) + _, err = api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.Pin(true)) if err != nil { t.Error(err) } @@ -294,8 +300,7 @@ func TestAddHashOnly(t *testing.T) { t.Error(err) } - str := strings.NewReader(helloStr) - p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.HashOnly(true)) + p, err := api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.HashOnly(true)) if err != nil { t.Error(err) } diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 9e0b8e1ec86..e5e2ae78ac3 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "net/url" "os" @@ -25,6 +26,7 @@ import ( humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" chunker "gx/ipfs/QmULKgr55cSWR8Kiwy3cVRcAiGVnR6EVSaB7hJcWS4138p/go-ipfs-chunker" routing "gx/ipfs/QmVBnJDKhtFXTRVjXKinqpwGu8t1DyNqPKan2iGX8PR8xG/go-libp2p-routing" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" @@ -398,7 +400,7 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, nam } func (i *gatewayHandler) postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { - p, err := i.api.Unixfs().Add(ctx, r.Body) + p, err := i.api.Unixfs().Add(ctx, files.NewReaderFile("", "", ioutil.NopCloser(r.Body), nil)) if err != nil { internalWebError(w, err) return From 8dda69575a1c20fd9ed7e458a6cc0f44ebdd9bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 17:21:07 +0200 Subject: [PATCH 13/23] coreapi unixfs: unixfs.Get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/unixfs.go | 8 ++ core/coreapi/unixfile.go | 199 +++++++++++++++++++++++++++++++ core/coreapi/unixfs.go | 9 ++ 3 files changed, 216 insertions(+) create mode 100644 core/coreapi/unixfile.go diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 92168503eca..69e73182261 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -13,8 +13,16 @@ import ( // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { // Add imports the data from the reader into merkledag file + // + // TODO: a long useful comment on how to use this for many different scenarios Add(context.Context, files.File, ...options.UnixfsAddOption) (ResolvedPath, error) + // Get returns a read-only handle to a file tree referenced by a path + // + // Note that some implementations of this API may apply the specified context + // to operations performed on the returned file + Get(context.Context, Path) (files.File, error) + // Cat returns a reader for the file Cat(context.Context, Path) (Reader, error) diff --git a/core/coreapi/unixfile.go b/core/coreapi/unixfile.go new file mode 100644 index 00000000000..95335075416 --- /dev/null +++ b/core/coreapi/unixfile.go @@ -0,0 +1,199 @@ +package coreapi + +import ( + "bytes" + "context" + "errors" + "io" + "io/ioutil" + "os" + gopath "path" + "time" + + files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" + uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io" + dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" + ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" +) + +// Number to file to prefetch in directories +// TODO: should we allow setting this via context hint? +const prefetchFiles = 4 + +// TODO: this probably belongs in go-unixfs (and could probably replace a chunk of it's interface in the long run) + +type sizeInfo struct { + size int64 + name string + modTime time.Time +} + +func (s *sizeInfo) Name() string { + return s.name +} + +func (s *sizeInfo) Size() int64 { + return s.size +} + +func (s *sizeInfo) Mode() os.FileMode { + return 0444 // all read +} + +func (s *sizeInfo) ModTime() time.Time { + return s.modTime +} + +func (s *sizeInfo) IsDir() bool { + return false +} + +func (s *sizeInfo) Sys() interface{} { + return nil +} + +type ufsDirectory struct { + ctx context.Context + dserv ipld.DAGService + + files chan *ipld.Link + + name string + path string +} + +func (d *ufsDirectory) Close() error { + return files.ErrNotReader +} + +func (d *ufsDirectory) Read(_ []byte) (int, error) { + return 0, files.ErrNotReader +} + +func (d *ufsDirectory) FileName() string { + return d.name +} + +func (d *ufsDirectory) FullPath() string { + return d.path +} + +func (d *ufsDirectory) IsDirectory() bool { + return true +} + +func (d *ufsDirectory) NextFile() (files.File, error) { + l, ok := <-d.files + if !ok { + return nil, io.EOF + } + + nd, err := l.GetNode(d.ctx, d.dserv) + if err != nil { + return nil, err + } + + return newUnixfsFile(d.ctx, d.dserv, nd, l.Name, d) +} + +type ufsFile struct { + uio.DagReader + + name string + path string +} + +func (f *ufsFile) IsDirectory() bool { + return false +} + +func (f *ufsFile) NextFile() (files.File, error) { + return nil, files.ErrNotDirectory +} + +func (f *ufsFile) FileName() string { + return f.name +} + +func (f *ufsFile) FullPath() string { + return f.path +} + +func (f *ufsFile) Size() (int64, error) { + return int64(f.DagReader.Size()), nil +} + +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, path string) (files.File, error) { + dir, err := uio.NewDirectoryFromNode(dserv, nd) + if err != nil { + return nil, err + } + + fileCh := make(chan *ipld.Link, prefetchFiles) + go func() { + dir.ForEachLink(ctx, func(link *ipld.Link) error { + select { + case fileCh <- link: + case <-ctx.Done(): + return ctx.Err() + } + return nil + }) + + close(fileCh) + }() + + return &ufsDirectory{ + ctx: ctx, + dserv: dserv, + + files: fileCh, + + name: name, + path: path, + }, nil +} + +func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, name string, parent files.File) (files.File, error) { + path := name + if parent != nil { + path = gopath.Join(parent.FullPath(), name) + } + + switch dn := nd.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.RawData()) + if err != nil { + return nil, err + } + if fsn.IsDir() { + return newUnixfsDir(ctx, dserv, nd, name, path) + } + + case *dag.RawNode: + + r := ioutil.NopCloser(bytes.NewReader(dn.RawData())) + fi := &sizeInfo{ + size: int64(len(dn.RawData())), + } + + return files.NewReaderFile("", "", r, fi), nil + default: + return nil, errors.New("unknown node type") + } + + dr, err := uio.NewDagReader(ctx, nd, dserv) + if err != nil { + return nil, err + } + + return &ufsFile{ + DagReader: dr, + + name: name, + path: path, + }, nil +} + +var _ os.FileInfo = &sizeInfo{} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 9ba1c29c3cb..80f59d278c3 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -110,6 +110,15 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options return coreiface.IpfsPath(nd.Cid()), nil } +func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.File, error) { + nd, err := api.core().ResolveNode(ctx, p) + if err != nil { + return nil, err + } + + return newUnixfsFile(ctx, api.node.DAG, nd, "", nil) +} + // Cat returns the data contained by an IPFS or IPNS object(s) at path `p`. func (api *UnixfsAPI) Cat(ctx context.Context, p coreiface.Path) (coreiface.Reader, error) { dget := api.node.DAG // TODO: use a session here once routing perf issues are resolved From c6f8e7e2a6d92dd16dc5376cbc930dea68731cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 22:06:35 +0200 Subject: [PATCH 14/23] coreapi unixfs: test recursive adding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/unixfile.go | 3 +- core/coreapi/unixfs_test.go | 106 ++++++++++++++++++++++++++++++------ core/coreunix/add.go | 4 +- 3 files changed, 92 insertions(+), 21 deletions(-) diff --git a/core/coreapi/unixfile.go b/core/coreapi/unixfile.go index 95335075416..dcec250cbb3 100644 --- a/core/coreapi/unixfile.go +++ b/core/coreapi/unixfile.go @@ -163,7 +163,7 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, nam switch dn := nd.(type) { case *dag.ProtoNode: - fsn, err := ft.FSNodeFromBytes(nd.RawData()) + fsn, err := ft.FSNodeFromBytes(dn.Data()) if err != nil { return nil, err } @@ -172,7 +172,6 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, nam } case *dag.RawNode: - r := ioutil.NopCloser(bytes.NewReader(dn.RawData())) fi := &sizeInfo{ size: int64(len(dn.RawData())), diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index f1fd90203b3..c446032c0c6 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -142,11 +142,12 @@ func TestAdd(t *testing.T) { } cases := []struct { - name string - data func() files.File - path string - err string - opts []options.UnixfsAddOption + name string + data func() files.File + path string + err string + recursive bool + opts []options.UnixfsAddOption }{ // Simple cases { @@ -231,11 +232,45 @@ func TestAdd(t *testing.T) { path: hello, opts: []options.UnixfsAddOption{options.Unixfs.Local(true)}, }, + // multi file + { + name: "simpleDir", + data: func() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + }, + recursive: true, + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + }, + { + name: "twoLevelDir", + data: func() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewSliceFile("t/abc", "t/abc", []files.File{ + files.NewReaderFile("t/abc/def", "t/abc/def", ioutil.NopCloser(strings.NewReader("world")), nil), + }), + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + }, + recursive: true, + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + }, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { - p, err := api.Unixfs().Add(ctx, testCase.data(), testCase.opts...) + + data := testCase.data() + if testCase.recursive { + data = files.NewSliceFile("", "", []files.File{ + data, + }) + } + + p, err := api.Unixfs().Add(ctx, data, testCase.opts...) if testCase.err != "" { if err == nil { t.Fatalf("expected an error: %s", testCase.err) @@ -246,27 +281,64 @@ func TestAdd(t *testing.T) { return } if err != nil { - t.Error(err) + t.Fatal(err) } if p.String() != testCase.path { t.Errorf("expected path %s, got: %s", testCase.path, p) } - /*r, err := api.Unixfs().Cat(ctx, p) - if err != nil { - t.Fatal(err) + var cmpFile func(orig files.File, got files.File) + cmpFile = func(orig files.File, got files.File) { + if orig.IsDirectory() != got.IsDirectory() { + t.Fatal("file type mismatch") + } + + if !orig.IsDirectory() { + defer orig.Close() + defer got.Close() + + do, err := ioutil.ReadAll(orig) + if err != nil { + t.Fatal(err) + } + + dg, err := ioutil.ReadAll(got) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(do, dg) { + t.Fatal("data not equal") + } + + return + } + + for { + fo, err := orig.NextFile() + fg, err2 := got.NextFile() + + if err != nil { + if err == io.EOF && err2 == io.EOF { + break + } + t.Fatal(err) + } + if err2 != nil { + t.Fatal(err) + } + + cmpFile(fo, fg) + } } - buf := make([]byte, len(testCase.data)) - _, err = io.ReadFull(r, buf) + + f, err := api.Unixfs().Get(ctx, p) if err != nil { - t.Error(err) + t.Fatal(err) } - if string(buf) != testCase.data { - t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err) - }*/ - + cmpFile(testCase.data(), f) }) } } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 55be2cd55da..d5a152f57f7 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -415,14 +415,14 @@ func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) { // single files.File f is treated as a directory, affecting hidden file // semantics. for { - file, err := file.NextFile() + f, err := file.NextFile() if err == io.EOF { // Finished the list of files. break } else if err != nil { return nil, err } - if err := adder.addFile(file); err != nil { + if err := adder.addFile(f); err != nil { return nil, err } } From 18fec6eb5f4ec73654baa26903c34e38c0866bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 22:30:45 +0200 Subject: [PATCH 15/23] coreapi unixfs: wrap option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 13 ++++ core/coreapi/unixfs.go | 2 +- core/coreapi/unixfs_test.go | 77 +++++++++++++++++++----- 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index df6f4fc7112..abbea968193 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -31,6 +31,8 @@ type UnixfsAddSettings struct { Pin bool OnlyHash bool Local bool + + Wrap bool } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -51,6 +53,8 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Pin: false, OnlyHash: false, Local: false, + + Wrap: false, } for _, opt := range opts { @@ -202,3 +206,12 @@ func (unixfsOpts) Local(local bool) UnixfsAddOption { return nil } } + +// Wrap tells the adder to wrap the added file structure with an additional +// directory. +func (unixfsOpts) Wrap(wrap bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Wrap = wrap + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 80f59d278c3..650ecda036d 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -66,7 +66,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options fileAdder.Chunker = settings.Chunker //fileAdder.Progress = progress //fileAdder.Hidden = hidden - //fileAdder.Wrap = wrap + fileAdder.Wrap = settings.Wrap fileAdder.Pin = settings.Pin && !settings.OnlyHash fileAdder.Silent = true fileAdder.RawLeaves = settings.RawLeaves diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index c446032c0c6..30b1e571e63 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -134,6 +134,18 @@ func strFile(data string) func() files.File { } } +func twoLevelDir() func() files.File { + return func() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewSliceFile("t/abc", "t/abc", []files.File{ + files.NewReaderFile("t/abc/def", "t/abc/def", ioutil.NopCloser(strings.NewReader("world")), nil), + }), + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + } +} + func TestAdd(t *testing.T) { ctx := context.Background() _, api, err := makeAPI(ctx) @@ -142,12 +154,16 @@ func TestAdd(t *testing.T) { } cases := []struct { - name string - data func() files.File - path string - err string + name string + data func() files.File + + path string + err string + recursive bool - opts []options.UnixfsAddOption + wrapped bool + + opts []options.UnixfsAddOption }{ // Simple cases { @@ -232,6 +248,12 @@ func TestAdd(t *testing.T) { path: hello, opts: []options.UnixfsAddOption{options.Unixfs.Local(true)}, }, + { + name: "hashOnly", // test (non)fetchability + data: strFile(helloStr), + path: hello, + opts: []options.UnixfsAddOption{options.Unixfs.HashOnly(true)}, + }, // multi file { name: "simpleDir", @@ -245,18 +267,36 @@ func TestAdd(t *testing.T) { path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", }, { - name: "twoLevelDir", + name: "twoLevelDir", + data: twoLevelDir(), + recursive: true, + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + }, + // wrapped + { + name: "addWrapped", + path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE", data: func() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewSliceFile("t/abc", "t/abc", []files.File{ - files.NewReaderFile("t/abc/def", "t/abc/def", ioutil.NopCloser(strings.NewReader("world")), nil), - }), - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), - }) + return files.NewReaderFile("foo", "foo", ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, + wrapped: true, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + }, + { + name: "twoLevelDirWrapped", + data: twoLevelDir(), recursive: true, - path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + wrapped: true, + path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + }, + { + name: "twoLevelInlineHash", + data: twoLevelDir(), + recursive: true, + wrapped: true, + path: "/ipfs/zBunoruKoyCHKkALNSWxDvj4L7yuQnMgQ4hUa9j1Z64tVcDEcu6Zdetyu7eeFCxMPfxb7YJvHeFHoFoHMkBUQf6vfdhmi", + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true), options.Unixfs.Hash(mh.SHA3)}, }, } @@ -338,7 +378,14 @@ func TestAdd(t *testing.T) { t.Fatal(err) } - cmpFile(testCase.data(), f) + orig := testCase.data() + if testCase.wrapped { + orig = files.NewSliceFile("", "", []files.File{ + orig, + }) + } + + cmpFile(orig, f) }) } } From 3e754d64fbd9770a1e09af3dfb492f599b7fe21f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 22:49:42 +0200 Subject: [PATCH 16/23] coreapi unixfs: hidden opiton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 14 ++++- core/coreapi/unixfs.go | 2 +- core/coreapi/unixfs_test.go | 67 ++++++++++++++++++++---- 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index abbea968193..7d7af5b818e 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -32,7 +32,8 @@ type UnixfsAddSettings struct { OnlyHash bool Local bool - Wrap bool + Wrap bool + Hidden bool } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -54,7 +55,8 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, OnlyHash: false, Local: false, - Wrap: false, + Wrap: false, + Hidden: false, } for _, opt := range opts { @@ -215,3 +217,11 @@ func (unixfsOpts) Wrap(wrap bool) UnixfsAddOption { return nil } } + +// Hidden enables adding of hidden files (files prefixed with '.') +func (unixfsOpts) Hidden(hidden bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Hidden = hidden + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 650ecda036d..7917d18f2c2 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -65,7 +65,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options fileAdder.Chunker = settings.Chunker //fileAdder.Progress = progress - //fileAdder.Hidden = hidden + fileAdder.Hidden = settings.Hidden fileAdder.Wrap = settings.Wrap fileAdder.Pin = settings.Pin && !settings.OnlyHash fileAdder.Silent = true diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 30b1e571e63..14577eae0a9 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -146,6 +146,12 @@ func twoLevelDir() func() files.File { } } +func wrapped(f files.File) files.File { + return files.NewSliceFile("", "", []files.File{ + f, + }) +} + func TestAdd(t *testing.T) { ctx := context.Background() _, api, err := makeAPI(ctx) @@ -154,14 +160,14 @@ func TestAdd(t *testing.T) { } cases := []struct { - name string - data func() files.File + name string + data func() files.File + expect func(files.File) files.File path string err string recursive bool - wrapped bool opts []options.UnixfsAddOption }{ @@ -279,14 +285,14 @@ func TestAdd(t *testing.T) { data: func() files.File { return files.NewReaderFile("foo", "foo", ioutil.NopCloser(strings.NewReader(helloStr)), nil) }, - wrapped: true, - opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + expect: wrapped, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, { name: "twoLevelDirWrapped", data: twoLevelDir(), recursive: true, - wrapped: true, + expect: wrapped, path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, @@ -294,10 +300,51 @@ func TestAdd(t *testing.T) { name: "twoLevelInlineHash", data: twoLevelDir(), recursive: true, - wrapped: true, + expect: wrapped, path: "/ipfs/zBunoruKoyCHKkALNSWxDvj4L7yuQnMgQ4hUa9j1Z64tVcDEcu6Zdetyu7eeFCxMPfxb7YJvHeFHoFoHMkBUQf6vfdhmi", opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true), options.Unixfs.Hash(mh.SHA3)}, }, + // hidden + { + name: "hiddenFiles", + data: func() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewReaderFile("t/.bar", "t/.bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + }, + recursive: true, + path: "/ipfs/QmehGvpf2hY196MzDFmjL8Wy27S4jbgGDUAhBJyvXAwr3g", + opts: []options.UnixfsAddOption{options.Unixfs.Hidden(true)}, + }, + { + name: "hiddenFileAlwaysAdded", + data: func() files.File { + return files.NewReaderFile(".foo", ".foo", ioutil.NopCloser(strings.NewReader(helloStr)), nil) + }, + recursive: true, + path: hello, + }, + { + name: "hiddenFilesNotAdded", + data: func() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewReaderFile("t/.bar", "t/.bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + }, + expect: func(files.File) files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) + }, + recursive: true, + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + opts: []options.UnixfsAddOption{options.Unixfs.Hidden(false)}, + }, } for _, testCase := range cases { @@ -379,10 +426,8 @@ func TestAdd(t *testing.T) { } orig := testCase.data() - if testCase.wrapped { - orig = files.NewSliceFile("", "", []files.File{ - orig, - }) + if testCase.expect != nil { + orig = testCase.expect(orig) } cmpFile(orig, f) From f9cf84965dc88eaed9fc09d92e36f72d214e2994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Oct 2018 23:17:18 +0200 Subject: [PATCH 17/23] coreapi unixfs: stdin-name option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 19 +++++++++++++---- core/coreapi/unixfs.go | 2 +- core/coreapi/unixfs_test.go | 27 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 7d7af5b818e..90ad53e9ea3 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -32,8 +32,9 @@ type UnixfsAddSettings struct { OnlyHash bool Local bool - Wrap bool - Hidden bool + Wrap bool + Hidden bool + StdinName string } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -55,8 +56,9 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, OnlyHash: false, Local: false, - Wrap: false, - Hidden: false, + Wrap: false, + Hidden: false, + StdinName: "", } for _, opt := range opts { @@ -225,3 +227,12 @@ func (unixfsOpts) Hidden(hidden bool) UnixfsAddOption { return nil } } + +// StdinName is the name set for files which don specify FilePath as +// os.Stdin.Name() +func (unixfsOpts) StdinName(name string) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.StdinName = name + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 7917d18f2c2..fc971bf6e06 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -71,7 +71,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options fileAdder.Silent = true fileAdder.RawLeaves = settings.RawLeaves //fileAdder.NoCopy = nocopy - //fileAdder.Name = pathName + fileAdder.Name = settings.StdinName fileAdder.CidBuilder = prefix switch settings.Layout { diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 14577eae0a9..ba92e0484d5 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -8,6 +8,7 @@ import ( "io" "io/ioutil" "math" + "os" "strings" "testing" @@ -288,6 +289,32 @@ func TestAdd(t *testing.T) { expect: wrapped, opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, + { + name: "stdinWrapped", + path: "/ipfs/QmU3r81oZycjHS9oaSHw37ootMFuFUw1DvMLKXPsezdtqU", + data: func() files.File { + return files.NewReaderFile("", os.Stdin.Name(), ioutil.NopCloser(strings.NewReader(helloStr)), nil) + }, + expect: func(files.File) files.File { + return files.NewSliceFile("", "", []files.File{ + files.NewReaderFile("QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", ioutil.NopCloser(strings.NewReader(helloStr)), nil), + }) + }, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + }, + { + name: "stdinNamed", + path: "/ipfs/QmQ6cGBmb3ZbdrQW1MRm1RJnYnaxCqfssz7CrTa9NEhQyS", + data: func() files.File { + return files.NewReaderFile("", os.Stdin.Name(), ioutil.NopCloser(strings.NewReader(helloStr)), nil) + }, + expect: func(files.File) files.File { + return files.NewSliceFile("", "", []files.File{ + files.NewReaderFile("test", "test", ioutil.NopCloser(strings.NewReader(helloStr)), nil), + }) + }, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")}, + }, { name: "twoLevelDirWrapped", data: twoLevelDir(), From a81ec29baa59d7c62622eec8636ec3e49b34b906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Oct 2018 01:00:26 +0200 Subject: [PATCH 18/23] coreapi unixfs: progress events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 5 +- core/commands/tar.go | 8 +- core/coreapi/interface/options/unixfs.go | 35 ++++++ core/coreapi/interface/unixfs.go | 9 ++ core/coreapi/unixfs.go | 7 +- core/coreapi/unixfs_test.go | 139 ++++++++++++++++++++--- core/coreunix/add.go | 28 ++--- core/coreunix/add_test.go | 12 +- 8 files changed, 199 insertions(+), 44 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index d7d6b33a267..df811af542f 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -8,6 +8,7 @@ import ( core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreunix" filestore "github.com/ipfs/go-ipfs/filestore" ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" @@ -371,7 +372,7 @@ You can now check what blocks have been created by: break LOOP } - output := out.(*coreunix.AddedObject) + output := out.(*coreiface.AddEvent) if len(output.Hash) > 0 { lastHash = output.Hash if quieter { @@ -451,5 +452,5 @@ You can now check what blocks have been created by: } }, }, - Type: coreunix.AddedObject{}, + Type: coreiface.AddEvent{}, } diff --git a/core/commands/tar.go b/core/commands/tar.go index a2fa3da20cc..670679032f7 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -7,7 +7,7 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - "github.com/ipfs/go-ipfs/core/coreunix" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" tar "github.com/ipfs/go-ipfs/tar" dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" path "gx/ipfs/QmcjwUb36Z16NJkvDX6ccXPqsFswo6AsRXynyXcLLCphV2/go-path" @@ -60,12 +60,12 @@ represent it. c := node.Cid() fi.FileName() - res.SetOutput(&coreunix.AddedObject{ + res.SetOutput(&coreiface.AddEvent{ Name: fi.FileName(), Hash: c.String(), }) }, - Type: coreunix.AddedObject{}, + Type: coreiface.AddEvent{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { v, err := unwrapOutput(res.Output()) @@ -73,7 +73,7 @@ represent it. return nil, err } - o, ok := v.(*coreunix.AddedObject) + o, ok := v.(*coreiface.AddEvent) if !ok { return nil, e.TypeErr(o, v) } diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 90ad53e9ea3..da99b42f662 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -35,6 +35,10 @@ type UnixfsAddSettings struct { Wrap bool Hidden bool StdinName string + + Events chan<- interface{} + Silent bool + Progress bool } type UnixfsAddOption func(*UnixfsAddSettings) error @@ -59,6 +63,10 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Wrap: false, Hidden: false, StdinName: "", + + Events: nil, + Silent: false, + Progress: false, } for _, opt := range opts { @@ -236,3 +244,30 @@ func (unixfsOpts) StdinName(name string) UnixfsAddOption { return nil } } + +// Events specifies channel which will be used to report events about ongoing +// Add operation. +// +// Note that if this channel blocks it may slowdown the adder +func (unixfsOpts) Events(sink chan<- interface{}) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Events = sink + return nil + } +} + +// Silent reduces event output +func (unixfsOpts) Silent(silent bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Silent = silent + return nil + } +} + +// Progress tells the adder whether to enable progress events +func (unixfsOpts) Progress(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Progress = enable + return nil + } +} diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 69e73182261..c622e210e58 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -9,6 +9,14 @@ import ( ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) +// TODO: ideas on making this more coreapi-ish without breaking the http API? +type AddEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + // UnixfsAPI is the basic interface to immutable files in IPFS // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { @@ -24,6 +32,7 @@ type UnixfsAPI interface { Get(context.Context, Path) (files.File, error) // Cat returns a reader for the file + // TODO: Remove in favour of Get (if we use Get on a file we still have reader directly, so..) Cat(context.Context, Path) (Reader, error) // Ls returns the list of links in a directory diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index fc971bf6e06..4bd60d159e1 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -64,11 +64,14 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options } fileAdder.Chunker = settings.Chunker - //fileAdder.Progress = progress + if settings.Events != nil { + fileAdder.Out = settings.Events + fileAdder.Progress = settings.Progress + } fileAdder.Hidden = settings.Hidden fileAdder.Wrap = settings.Wrap fileAdder.Pin = settings.Pin && !settings.OnlyHash - fileAdder.Silent = true + fileAdder.Silent = settings.Silent fileAdder.RawLeaves = settings.RawLeaves //fileAdder.NoCopy = nocopy fileAdder.Name = settings.StdinName diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index ba92e0484d5..eb02b3e5182 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -9,7 +9,9 @@ import ( "io/ioutil" "math" "os" + "strconv" "strings" + "sync" "testing" "github.com/ipfs/go-ipfs/core" @@ -147,6 +149,13 @@ func twoLevelDir() func() files.File { } } +func flatDir() files.File { + return files.NewSliceFile("t", "t", []files.File{ + files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), + files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), + }) +} + func wrapped(f files.File) files.File { return files.NewSliceFile("", "", []files.File{ f, @@ -170,6 +179,8 @@ func TestAdd(t *testing.T) { recursive bool + events []coreiface.AddEvent + opts []options.UnixfsAddOption }{ // Simple cases @@ -263,13 +274,8 @@ func TestAdd(t *testing.T) { }, // multi file { - name: "simpleDir", - data: func() files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), - }) - }, + name: "simpleDir", + data: flatDir, recursive: true, path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", }, @@ -300,7 +306,7 @@ func TestAdd(t *testing.T) { files.NewReaderFile("QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", ioutil.NopCloser(strings.NewReader(helloStr)), nil), }) }, - opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)}, }, { name: "stdinNamed", @@ -313,7 +319,7 @@ func TestAdd(t *testing.T) { files.NewReaderFile("test", "test", ioutil.NopCloser(strings.NewReader(helloStr)), nil), }) }, - opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")}, + opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")}, }, { name: "twoLevelDirWrapped", @@ -363,19 +369,71 @@ func TestAdd(t *testing.T) { }) }, expect: func(files.File) files.File { - return files.NewSliceFile("t", "t", []files.File{ - files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil), - files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil), - }) + return flatDir() }, recursive: true, path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", opts: []options.UnixfsAddOption{options.Unixfs.Hidden(false)}, }, + // Events / Progress + { + name: "simpleAddEvent", + data: strFile(helloStr), + path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", + events: []coreiface.AddEvent{ + {Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Hash: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Size: strconv.Itoa(len(helloStr))}, + }, + opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)}, + }, + { + name: "silentAddEvent", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, + {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, + }, + recursive: true, + opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, + }, + { + name: "dirAddEvents", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "t/abc/def", Hash: "QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk", Size: "13"}, + {Name: "t/bar", Hash: "QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY", Size: "14"}, + {Name: "t/foo", Hash: "QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ", Size: "14"}, + {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, + {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, + }, + recursive: true, + }, + { + name: "progress1M", + data: func() files.File { + r := bytes.NewReader(bytes.Repeat([]byte{0}, 1000000)) + return files.NewReaderFile("", "", ioutil.NopCloser(r), nil) + }, + path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", + events: []coreiface.AddEvent{ + {Name: "", Bytes: 262144}, + {Name: "", Bytes: 524288}, + {Name: "", Bytes: 786432}, + {Name: "", Bytes: 1000000}, + {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Size: "1000256"}, + }, + recursive: true, + opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, + }, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // recursive logic data := testCase.data() if testCase.recursive { @@ -384,7 +442,58 @@ func TestAdd(t *testing.T) { }) } - p, err := api.Unixfs().Add(ctx, data, testCase.opts...) + // handle events if relevant to test case + + opts := testCase.opts + eventOut := make(chan interface{}) + var evtWg sync.WaitGroup + if len(testCase.events) > 0 { + opts = append(opts, options.Unixfs.Events(eventOut)) + evtWg.Add(1) + + go func() { + defer evtWg.Done() + expected := testCase.events + + for evt := range eventOut { + event, ok := evt.(*coreiface.AddEvent) + if !ok { + t.Fatal("unexpected event type") + } + + if len(expected) < 1 { + t.Fatal("got more events than expected") + } + + if expected[0].Size != event.Size { + t.Errorf("Event.Size didn't match, %s != %s", expected[0].Size, event.Size) + } + + if expected[0].Name != event.Name { + t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) + } + + if expected[0].Hash != event.Hash { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Hash, event.Hash) + } + if expected[0].Bytes != event.Bytes { + t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes) + } + + expected = expected[1:] + } + + if len(expected) > 0 { + t.Fatalf("%d event(s) didn't arrive", len(expected)) + } + }() + } + + // Add! + + p, err := api.Unixfs().Add(ctx, data, opts...) + close(eventOut) + evtWg.Wait() if testCase.err != "" { if err == nil { t.Fatalf("expected an error: %s", testCase.err) @@ -402,6 +511,8 @@ func TestAdd(t *testing.T) { t.Errorf("expected path %s, got: %s", testCase.path, p) } + // compare file structure with Unixfs().Get + var cmpFile func(orig files.File, got files.File) cmpFile = func(orig files.File, got files.File) { if orig.IsDirectory() != got.IsDirectory() { diff --git a/core/coreunix/add.go b/core/coreunix/add.go index d5a152f57f7..84324c7ec41 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -11,19 +11,20 @@ import ( "strconv" core "github.com/ipfs/go-ipfs/core" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/pin" - unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" - balanced "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/balanced" - ihelper "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/helpers" - trickle "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/trickle" - dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" posinfo "gx/ipfs/QmPG32VXR5jmpo9q8R9FNdR4Ae97Ky9CiZE6SctJLUB79H/go-ipfs-posinfo" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" + balanced "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/balanced" + ihelper "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/helpers" + trickle "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/trickle" chunker "gx/ipfs/QmULKgr55cSWR8Kiwy3cVRcAiGVnR6EVSaB7hJcWS4138p/go-ipfs-chunker" logging "gx/ipfs/QmZChCsSt8DctjceaL56Eibc29CVQq4dGKRXC5JRZ6Ppae/go-log" mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs" + dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore" ) @@ -46,13 +47,6 @@ type Object struct { Size string } -type AddedObject struct { - Name string - Hash string `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` -} - // NewAdder Returns a new Adder used for a file add operation. func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds ipld.DAGService) (*Adder, error) { return &Adder{ @@ -75,7 +69,7 @@ type Adder struct { pinning pin.Pinner blockstore bstore.GCBlockstore dagService ipld.DAGService - Out chan interface{} + Out chan<- interface{} Progress bool Hidden bool Pin bool @@ -570,7 +564,7 @@ func (adder *Adder) maybePauseForGC() error { } // outputDagnode sends dagnode info over the output channel -func outputDagnode(out chan interface{}, name string, dn ipld.Node) error { +func outputDagnode(out chan<- interface{}, name string, dn ipld.Node) error { if out == nil { return nil } @@ -580,7 +574,7 @@ func outputDagnode(out chan interface{}, name string, dn ipld.Node) error { return err } - out <- &AddedObject{ + out <- &coreiface.AddEvent{ Hash: o.Hash, Name: name, Size: o.Size, @@ -615,7 +609,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) { type progressReader struct { file files.File - out chan interface{} + out chan<- interface{} bytes int64 lastProgress int64 } @@ -626,7 +620,7 @@ func (i *progressReader) Read(p []byte) (int, error) { i.bytes += int64(n) if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF { i.lastProgress = i.bytes - i.out <- &AddedObject{ + i.out <- &coreiface.AddEvent{ Name: i.file.FileName(), Bytes: i.bytes, } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 5185c4f4d8f..4af1803b35a 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/ipfs/go-ipfs/core" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/pin/gc" "github.com/ipfs/go-ipfs/repo" @@ -96,7 +97,7 @@ func TestAddGCLive(t *testing.T) { addedHashes := make(map[string]struct{}) select { case o := <-out: - addedHashes[o.(*AddedObject).Hash] = struct{}{} + addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{} case <-addDone: t.Fatal("add shouldnt complete yet") } @@ -124,7 +125,7 @@ func TestAddGCLive(t *testing.T) { // receive next object from adder o := <-out - addedHashes[o.(*AddedObject).Hash] = struct{}{} + addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{} <-gcstarted @@ -140,7 +141,7 @@ func TestAddGCLive(t *testing.T) { var last cid.Cid for a := range out { // wait for it to finish - c, err := cid.Decode(a.(*AddedObject).Hash) + c, err := cid.Decode(a.(*coreiface.AddEvent).Hash) if err != nil { t.Fatal(err) } @@ -178,7 +179,8 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) { if err != nil { t.Fatal(err) } - adder.Out = make(chan interface{}) + out := make(chan interface{}) + adder.Out = out adder.Progress = true adder.RawLeaves = rawLeaves adder.NoCopy = true @@ -196,7 +198,7 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) { t.Fatal(err) } }() - for range adder.Out { + for range out { } exp := 0 From d51ce96f55b84ea83599487f2a87b43998ac3902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Oct 2018 01:24:57 +0200 Subject: [PATCH 19/23] coreapi unixfs: filestore opts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 35 ++++++++++++++++++++++++ core/coreapi/unixfs.go | 19 ++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index da99b42f662..810e3c6e841 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -31,6 +31,8 @@ type UnixfsAddSettings struct { Pin bool OnlyHash bool Local bool + FsCache bool + NoCopy bool Wrap bool Hidden bool @@ -59,6 +61,8 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Pin: false, OnlyHash: false, Local: false, + FsCache: false, + NoCopy: false, Wrap: false, Hidden: false, @@ -76,6 +80,17 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, } } + // nocopy -> rawblocks + if options.NoCopy && !options.RawLeaves { + // fixed? + if options.RawLeavesSet { + return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") + } + + // No, satisfy mandatory constraint. + options.RawLeaves = true + } + // (hash != "sha2-256") -> CIDv1 if options.MhType != mh.SHA2_256 { switch options.CidVersion { @@ -271,3 +286,23 @@ func (unixfsOpts) Progress(enable bool) UnixfsAddOption { return nil } } + +// FsCache tells the adder to check the filestore for pre-existing blocks +// +// Experimental +func (unixfsOpts) FsCache(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.FsCache = enable + return nil + } +} + +// NoCopy tells the adder to add the files using filestore. Implies RawLeaves. +// +// Experimental +func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.NoCopy = enable + return nil + } +} diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 4bd60d159e1..5950985886a 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/filestore" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -33,6 +34,16 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options } n := api.node + + cfg, err := n.Repo.Config() + if err != nil { + return nil, err + } + + if settings.NoCopy && !cfg.Experimental.FilestoreEnabled { + return nil, filestore.ErrFilestoreNotEnabled + } + if settings.OnlyHash { nilnode, err := core.NewNode(ctx, &core.BuildCfg{ //TODO: need this to be true or all files @@ -46,9 +57,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options } addblockstore := n.Blockstore - //if !(fscache || nocopy) { - addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker) - //} + if !(settings.FsCache || settings.NoCopy) { + addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker) + } exch := n.Exchange if settings.Local { @@ -73,7 +84,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options fileAdder.Pin = settings.Pin && !settings.OnlyHash fileAdder.Silent = settings.Silent fileAdder.RawLeaves = settings.RawLeaves - //fileAdder.NoCopy = nocopy + fileAdder.NoCopy = settings.NoCopy fileAdder.Name = settings.StdinName fileAdder.CidBuilder = prefix From 69eb015b9f043fc6d1d3a276f1677e02961d468c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Oct 2018 01:57:25 +0200 Subject: [PATCH 20/23] ipfs add uses CoreAPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 156 ++++++++--------------------------------- core/coreapi/unixfs.go | 8 +++ 2 files changed, 39 insertions(+), 125 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index df811af542f..05484e9ec07 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,25 +6,15 @@ import ( "os" "strings" - core "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreunix" - filestore "github.com/ipfs/go-ipfs/filestore" - ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs" - dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag" - dagtest "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag/test" - blockservice "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice" + options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb" - cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" - offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline" cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" cmds "gx/ipfs/QmXTmUCBtDUrzDYVzASogLiNph7EBuYqEgPL7QoHNMzUnz/go-ipfs-cmds" - mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs" - bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore" ) // ErrDepthLimitExceeded indicates that the max depth has been exceeded. @@ -149,23 +139,11 @@ You can now check what blocks have been created by: return nil }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - n, err := cmdenv.GetNode(env) + api, err := cmdenv.GetApi(env) if err != nil { return err } - cfg, err := n.Repo.Config() - if err != nil { - return err - } - // check if repo will exceed storage limit if added - // TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated) - // TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon - //if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil { - // res.SetError(err, cmdkit.ErrNormal) - // return - //} - progress, _ := req.Options[progressOptionName].(bool) trickle, _ := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) @@ -182,131 +160,59 @@ You can now check what blocks have been created by: inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) pathName, _ := req.Options[stdinPathName].(string) - - // The arguments are subject to the following constraints. - // - // nocopy -> filestoreEnabled - // nocopy -> rawblocks - // (hash != sha2-256) -> cidv1 - - // NOTE: 'rawblocks -> cidv1' is missing. Legacy reasons. - - // nocopy -> filestoreEnabled - if nocopy && !cfg.Experimental.FilestoreEnabled { - return cmdkit.Errorf(cmdkit.ErrClient, filestore.ErrFilestoreNotEnabled.Error()) - } - - // nocopy -> rawblocks - if nocopy && !rawblks { - // fixed? - if rbset { - return fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") - } - // No, satisfy mandatory constraint. - rawblks = true - } - - // (hash != "sha2-256") -> CIDv1 - if hashFunStr != "sha2-256" && cidVer == 0 { - if cidVerSet { - return cmdkit.Errorf(cmdkit.ErrClient, "CIDv0 only supports sha2-256") - } - cidVer = 1 - } - - // cidV1 -> raw blocks (by default) - if cidVer > 0 && !rbset { - rawblks = true - } - - prefix, err := dag.PrefixForCidVersion(cidVer) - if err != nil { - return err - } + local, _ := req.Options["local"].(bool) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)) } - prefix.MhType = hashFunCode - prefix.MhLength = -1 - - if hash { - nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ - //TODO: need this to be true or all files - // hashed will be stored in memory! - NilRepo: true, - }) - if err != nil { - return err - } - n = nilnode - } + events := make(chan interface{}, adderOutChanSize) - addblockstore := n.Blockstore - if !(fscache || nocopy) { - addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker) - } + opts := []options.UnixfsAddOption{ + options.Unixfs.Hash(hashFunCode), - exch := n.Exchange - local, _ := req.Options["local"].(bool) - if local { - exch = offline.Exchange(addblockstore) - } + options.Unixfs.Inline(inline), + options.Unixfs.InlineLimit(inlineLimit), - bserv := blockservice.New(addblockstore, exch) // hash security 001 - dserv := dag.NewDAGService(bserv) + options.Unixfs.Chunker(chunker), - outChan := make(chan interface{}, adderOutChanSize) + options.Unixfs.Pin(dopin), + options.Unixfs.HashOnly(hash), + options.Unixfs.Local(local), + options.Unixfs.FsCache(fscache), + options.Unixfs.Nocopy(nocopy), - fileAdder, err := coreunix.NewAdder(req.Context, n.Pinning, n.Blockstore, dserv) - if err != nil { - return err + options.Unixfs.Wrap(wrap), + options.Unixfs.Hidden(hidden), + options.Unixfs.StdinName(pathName), + + options.Unixfs.Progress(progress), + options.Unixfs.Silent(silent), + options.Unixfs.Events(events), } - fileAdder.Out = outChan - fileAdder.Chunker = chunker - fileAdder.Progress = progress - fileAdder.Hidden = hidden - fileAdder.Trickle = trickle - fileAdder.Wrap = wrap - fileAdder.Pin = dopin && !hash - fileAdder.Silent = silent - fileAdder.RawLeaves = rawblks - fileAdder.NoCopy = nocopy - fileAdder.Name = pathName - fileAdder.CidBuilder = prefix - - if inline { - fileAdder.CidBuilder = cidutil.InlineBuilder{ - Builder: fileAdder.CidBuilder, - Limit: inlineLimit, - } + if cidVerSet { + opts = append(opts, options.Unixfs.CidVersion(cidVer)) } - if hash { - md := dagtest.Mock() - emptyDirNode := ft.EmptyDirNode() - // Use the same prefix for the "empty" MFS root as for the file adder. - emptyDirNode.SetCidBuilder(fileAdder.CidBuilder) - mr, err := mfs.NewRoot(req.Context, md, emptyDirNode, nil) - if err != nil { - return err - } + if rbset { + opts = append(opts, options.Unixfs.RawLeaves(rawblks)) + } - fileAdder.SetMfsRoot(mr) + if trickle { + opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } errCh := make(chan error) go func() { var err error defer func() { errCh <- err }() - defer close(outChan) - _, err = fileAdder.AddAllAndPin(req.Files) + defer close(events) + _, err = api.Unixfs().Add(req.Context, req.Files, opts...) }() - err = res.Emit(outChan) + err = res.Emit(events) if err != nil { return err } diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 5950985886a..53a3ea4fb39 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -40,6 +40,14 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options return nil, err } + // check if repo will exceed storage limit if added + // TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated) + // TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon + //if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil { + // res.SetError(err, cmdkit.ErrNormal) + // return + //} + if settings.NoCopy && !cfg.Experimental.FilestoreEnabled { return nil, filestore.ErrFilestoreNotEnabled } From 051c33087ba52461d1708da1e18c06bab57af4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Oct 2018 04:07:22 +0200 Subject: [PATCH 21/23] coreapi unixfile: simplify RawNode case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/unixfile.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/coreapi/unixfile.go b/core/coreapi/unixfile.go index dcec250cbb3..f100bbffb89 100644 --- a/core/coreapi/unixfile.go +++ b/core/coreapi/unixfile.go @@ -1,11 +1,9 @@ package coreapi import ( - "bytes" "context" "errors" "io" - "io/ioutil" "os" gopath "path" "time" @@ -172,12 +170,6 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, nam } case *dag.RawNode: - r := ioutil.NopCloser(bytes.NewReader(dn.RawData())) - fi := &sizeInfo{ - size: int64(len(dn.RawData())), - } - - return files.NewReaderFile("", "", r, fi), nil default: return nil, errors.New("unknown node type") } From 4b1f733864491216df4641b8f88c4b02fc083384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Oct 2018 20:39:43 +0200 Subject: [PATCH 22/23] coreapi unixfs: fix inline doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/options/unixfs.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 810e3c6e841..9249af89539 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -170,13 +170,10 @@ func (unixfsOpts) Inline(enable bool) UnixfsAddOption { // Specifying this option won't enable block inlining. For that use `Inline` // option. Default: 32 bytes // -// Note that while there is no hard limit on the number of bytes, it should -// be kept at a reasonably low value, like 64 bytes if you intend to display -// these hashes. Larger values like 256 bytes will work fine, but may affect -// de-duplication of smaller blocks. -// -// Setting this value too high may cause various problems, such as render some -// blocks unfetchable +// Note that while there is no hard limit on the number of bytes, it should be +// kept at a reasonably low value, such as 64 and no more than 1k. Setting this +// value too high may cause various problems, such as render some +// blocks unfetchable. func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.InlineLimit = limit From 35e8d6fba1f20a38cb7b04069f7d140bdc4daa81 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Oct 2018 21:12:53 -0700 Subject: [PATCH 23/23] update unixfs inline option comment to give us room to change things (addressing CR) License: MIT Signed-off-by: Steven Allen --- core/coreapi/interface/options/unixfs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 9249af89539..d486981c3a6 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -171,9 +171,8 @@ func (unixfsOpts) Inline(enable bool) UnixfsAddOption { // option. Default: 32 bytes // // Note that while there is no hard limit on the number of bytes, it should be -// kept at a reasonably low value, such as 64 and no more than 1k. Setting this -// value too high may cause various problems, such as render some -// blocks unfetchable. +// kept at a reasonably low value, such as 64; implementations may choose to +// reject anything larger. func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.InlineLimit = limit