Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

make the tar writer handle sharded ipfs directories #4873

Merged
merged 5 commits into from
Mar 25, 2018
Merged

make the tar writer handle sharded ipfs directories #4873

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
234ad18
make the tar writer handle sharded ipfs directories
Stebalien Mar 23, 2018
dd94858
fix error when encountering a sharded directory when expecting a file
Stebalien Mar 23, 2018
6895fc6
return a nice error when listing sharded directories
Stebalien Mar 23, 2018
0dc9842
test ipfs get on sharded directory
Stebalien Mar 23, 2018
71a0a5c
appease go lint
Stebalien Mar 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/commands/unixfs/ls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ possible, please use 'ipfs ls' instead.
switch t {
case unixfspb.Data_File:
break
case unixfspb.Data_HAMTShard:
// We need a streaming ls API for this.
res.SetError(fmt.Errorf("cannot list large directories yet"), cmdkit.ErrNormal)
return
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be okay to have this without a streaming ls. It's just gonna take its time to run, just like dir-listings on the gateway for sharded directories.

The streaming ls is an optimization (an important one nevertheless).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fine by me (although let's fix that in a different PR #4874).

case unixfspb.Data_Directory:
links := make([]LsLink, len(merkleNode.Links()))
output.Objects[hash].Links = links
Expand Down
6 changes: 5 additions & 1 deletion test/sharness/t0260-sharding-flag.sh → test/sharness/t0260-sharding.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# MIT Licensed; see the LICENSE file in this repository.
#

test_description="Test global enable sharding flag"
test_description="Test directory sharding"

. lib/test-lib.sh

Expand All @@ -23,6 +23,10 @@ test_add_large_dir() {
echo "$exphash" > sharddir_exp &&
test_cmp sharddir_exp sharddir_out
'
test_expect_success "ipfs get on very large directory succeeds" '
ipfs get -o testdata-out "$exphash" &&
test_cmp testdata testdata-out
'
}

test_init_ipfs
Expand Down
21 changes: 10 additions & 11 deletions unixfs/archive/tar/writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,22 @@ func NewWriter(ctx context.Context, dag ipld.DAGService, archive bool, compressi
}

func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error {
dir, err := uio.NewDirectoryFromNode(w.Dag, nd)
if err != nil {
return err
}
if err := writeDirHeader(w.TarW, fpath); err != nil {
return err
}

for i, ng := range ipld.GetDAG(w.ctx, w.Dag, nd) {
child, err := ng.Get(w.ctx)
return dir.ForEachLink(w.ctx, func(l *ipld.Link) error {
child, err := w.Dag.Get(w.ctx, l.Cid)
if err != nil {
return err
}

npath := path.Join(fpath, nd.Links()[i].Name)
if err := w.WriteNode(child, npath); err != nil {
return err
}
}

return nil
npath := path.Join(fpath, l.Name)
return w.WriteNode(child, npath)
})
}

func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error {
Expand Down Expand Up @@ -83,7 +82,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error {
switch pb.GetType() {
case upb.Data_Metadata:
fallthrough
case upb.Data_Directory:
case upb.Data_Directory, upb.Data_HAMTShard:
return w.writeDir(nd, fpath)
case upb.Data_Raw:
fallthrough
Expand Down
2 changes: 1 addition & 1 deletion unixfs/io/pbdagreader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error {
}

switch pb.GetType() {
case ftpb.Data_Directory:
case ftpb.Data_Directory, ftpb.Data_HAMTShard:
// A directory should not exist within a file
return ft.ErrInvalidDirLocation
case ftpb.Data_File:
Expand Down