From e71dce5dfba3616e5ade4abf0844925389497a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 18 Oct 2017 20:10:22 +0200 Subject: [PATCH 1/4] gateway: fix seeker can't seek on specific files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/corehttp/gateway_handler.go | 13 ++++++++++++- test/sharness/t0110-gateway-data/foo.block | 2 ++ test/sharness/t0110-gateway-data/foofoo.block | Bin 0 -> 82 bytes test/sharness/t0110-gateway.sh | 15 +++++++++++++-- unixfs/io/pbdagreader.go | 11 ++++++++++- 5 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 test/sharness/t0110-gateway-data/foo.block create mode 100644 test/sharness/t0110-gateway-data/foofoo.block diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index f02e874070b..33616940451 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -268,7 +268,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr if !dir { name := gopath.Base(urlPath) - http.ServeContent(w, r, name, modtime, dr) + i.serverFile(w, r, name, modtime, dr) return } @@ -372,6 +372,17 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr } } +func (i *gatewayHandler) serverFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) { + http.ServeContent(w, req, name, modtime, content) + //TODO: check for errors in ServeContent.. somehow + + // If http.ServeContent can't figure out content size it won't write it to the + // responseWriter, Content-Length not being set is a good indicator of this + if req.Method != "HEAD" && w.Header().Get("Content-Length") == "" { + io.Copy(w, content) + } +} + func (i *gatewayHandler) postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { p, err := i.api.Unixfs().Add(ctx, r.Body) if err != nil { diff --git a/test/sharness/t0110-gateway-data/foo.block b/test/sharness/t0110-gateway-data/foo.block new file mode 100644 index 00000000000..39c7ef60b82 --- /dev/null +++ b/test/sharness/t0110-gateway-data/foo.block @@ -0,0 +1,2 @@ + + foo \ No newline at end of file diff --git a/test/sharness/t0110-gateway-data/foofoo.block b/test/sharness/t0110-gateway-data/foofoo.block new file mode 100644 index 0000000000000000000000000000000000000000..9e5177b183ca963f4b15a2e8ee981f55bded1bcd GIT binary patch literal 82 zcmd;L;b4+r6H?()5>hxn>F@iqhke#C%;{!*ou>UDv3KXa&^K4qTVK9O7y5BOl{i%Z Ds+T9Z literal 0 HcmV?d00001 diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh index 6041b39c980..b64cb799f76 100755 --- a/test/sharness/t0110-gateway.sh +++ b/test/sharness/t0110-gateway.sh @@ -124,7 +124,7 @@ test_expect_success "HEAD 'index.html' has no content" ' # test ipfs readonly api test_curl_gateway_api() { - curl -sfo actual "http://127.0.0.1:$port/api/v0/$1" + curl -sfo actual "http://127.0.0.1:$port/api/v0/$1" } test_expect_success "get IPFS directory file through readonly API succeeds" ' @@ -140,7 +140,7 @@ test_expect_success "refs IPFS directory file through readonly API succeeds" ' ' test_expect_success "test gateway api is sanitized" ' -for cmd in "add" "block/put" "bootstrap" "config" "dht" "diag" "dns" "get" "id" "mount" "name/publish" "object/put" "object/new" "object/patch" "pin" "ping" "refs/local" "repo" "resolve" "stats" "swarm" "file" "update" "version" "bitswap"; do + for cmd in "add" "block/put" "bootstrap" "config" "dht" "diag" "dns" "get" "id" "mount" "name/publish" "object/put" "object/new" "object/patch" "pin" "ping" "refs/local" "repo" "resolve" "stats" "swarm" "file" "update" "version" "bitswap"; do test_curl_resp_http_code "http://127.0.0.1:$port/api/v0/$cmd" "HTTP/1.1 404 Not Found" done ' @@ -155,6 +155,17 @@ test_expect_success "try fetching it from gateway" ' test_cmp rfile ffile ' +test_expect_success "Add compact blocks" ' + ipfs block put ../t0110-gateway-data/foo.block && + FOO2_HASH=$(ipfs block put ../t0110-gateway-data/foofoo.block) && + printf "foofoo" > expected +' + +test_expect_success "GET compact blocks succeeds" ' + curl -o actual "http://127.0.0.1:$port/ipfs/$FOO2_HASH" && + test_cmp expected actual +' + test_kill_ipfs_daemon test_done diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 0b75fd916c0..7ba63649ff2 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -243,7 +243,16 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) { return dr.Seek(noffset, io.SeekStart) case io.SeekEnd: noffset := int64(dr.pbdata.GetFilesize()) - offset - return dr.Seek(noffset, io.SeekStart) + n, err := dr.Seek(noffset, io.SeekStart) + + // Return negative number if we can't figure out the file size. Using io.EOF + // for this seems to be good(-enough) solution as it's only returned by + // precalcNextBuf when we step out of file range. + // This is needed for gateway to function properly + if err == io.EOF && *dr.pbdata.Type == ftpb.Data_File { + return -1, nil + } + return n, err default: return 0, errors.New("invalid whence") } From 3fbb77153b676c773dd12821cf46be1ed351f65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 18 Oct 2017 22:25:36 +0200 Subject: [PATCH 2/4] gateway: custom seeker for files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/corehttp/gateway_handler.go | 24 ++++++++++++++++++++++++ unixfs/io/pbdagreader.go | 7 +++++++ 2 files changed, 31 insertions(+) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 33616940451..0a7e657de81 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -372,7 +372,31 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr } } +type sizeReadSeeker interface { + Size() uint64 + + io.ReadSeeker +} + +type sizeSeeker struct { + sizeReadSeeker +} + +func (s *sizeSeeker) Seek(offset int64, whence int) (int64, error) { + if whence == io.SeekEnd && offset == 0 { + return int64(s.Size()), nil + } + + return s.sizeReadSeeker.Seek(offset, whence) +} + func (i *gatewayHandler) serverFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) { + if sp, ok := content.(sizeReadSeeker); ok { + content = &sizeSeeker{ + sizeReadSeeker: sp, + } + } + http.ServeContent(w, req, name, modtime, content) //TODO: check for errors in ServeContent.. somehow diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 7ba63649ff2..dcd383460d0 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -188,6 +188,9 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) { if offset < 0 { return -1, errors.New("Invalid offset") } + if offset == dr.offset { + return offset, nil + } // Grab cached protobuf object (solely to make code look cleaner) pb := dr.pbdata @@ -239,6 +242,10 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) { return offset, nil case io.SeekCurrent: // TODO: be smarter here + if offset == 0 { + return dr.offset, nil + } + noffset := dr.offset + offset return dr.Seek(noffset, io.SeekStart) case io.SeekEnd: From 913f9641e4518ffa2609e09429254019385222af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 18 Oct 2017 22:26:03 +0200 Subject: [PATCH 3/4] merkledag: keep key order in dedupeKeys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/corehttp/gateway_handler.go | 2 +- merkledag/merkledag.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 0a7e657de81..19b0de41c41 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -373,7 +373,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr } type sizeReadSeeker interface { - Size() uint64 + Size() uint64 io.ReadSeeker } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 92cb5fa866f..bcf0e84a1c5 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -307,11 +307,14 @@ func GetNodes(ctx context.Context, ds DAGService, keys []*cid.Cid) []NodeGetter // Remove duplicates from a list of keys func dedupeKeys(cids []*cid.Cid) []*cid.Cid { + out := make([]*cid.Cid, 0, len(cids)) set := cid.NewSet() for _, c := range cids { - set.Add(c) + if set.Visit(c) { + out = append(out, c) + } } - return set.Keys() + return out } func newNodePromise(ctx context.Context) NodeGetter { From cbccd847aba20203da8fc3232712ce150936f38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Oct 2017 13:36:37 +0200 Subject: [PATCH 4/4] gateway: apply review to serveFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/corehttp/gateway_handler.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 19b0de41c41..6e19f94ec08 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -268,7 +268,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr if !dir { name := gopath.Base(urlPath) - i.serverFile(w, r, name, modtime, dr) + i.serveFile(w, r, name, modtime, dr) return } @@ -390,7 +390,7 @@ func (s *sizeSeeker) Seek(offset int64, whence int) (int64, error) { return s.sizeReadSeeker.Seek(offset, whence) } -func (i *gatewayHandler) serverFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) { +func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) { if sp, ok := content.(sizeReadSeeker); ok { content = &sizeSeeker{ sizeReadSeeker: sp, @@ -398,13 +398,6 @@ func (i *gatewayHandler) serverFile(w http.ResponseWriter, req *http.Request, na } http.ServeContent(w, req, name, modtime, content) - //TODO: check for errors in ServeContent.. somehow - - // If http.ServeContent can't figure out content size it won't write it to the - // responseWriter, Content-Length not being set is a good indicator of this - if req.Method != "HEAD" && w.Header().Get("Content-Length") == "" { - io.Copy(w, content) - } } func (i *gatewayHandler) postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {