core/commands/unixfs: Add 'ipfs unixfs ls ...'

This is similar to 'ipfs ls ...', but it:

* Lists file sizes that match the content size:

    $ ipfs unixfs ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4
    QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a File 1947624  busybox
    $ ipfs cat /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4/busybox | wc -c
    1947624

  'ipfs ls ...', on the other hand, is using the Merkle-descendant
  size, which also includes fanout links and the typing information
  unixfs objects store in their Data:

    $ ipfs ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4
    QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a 1948128 busybox

* Includes a separate column that explicitly lists the entry type
  (e.g. File, Directory, ...).  'ipfs ls ...' hints at this by
  appending a trailing slash to directory names.

    $ ipfs unixfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02
    QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 Directory 1948183 bin
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn Directory 4       dev
    QmUz1Z5jnQEjwr78fiMk5babwjJBDmhN5sx5HvPiTGGGjM Directory 1207    etc
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn Directory 4       proc
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn Directory 4       run
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn Directory 4       sys
    $ ipfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02
    QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 1948183 bin/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       dev/
    QmUz1Z5jnQEjwr78fiMk5babwjJBDmhN5sx5HvPiTGGGjM 1207    etc/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       proc/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       run/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       sys/

  We probably want to handle this consistently between the two UIs, so
  I can update 'ipfs ls' if the explicit syntax seems better.

License: MIT
Signed-off-by: W. Trevor King <[email protected]>

core/commands/root.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs"
 	evlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
 )
 
@@ -35,6 +36,7 @@ DATA STRUCTURE COMMANDS
 
     block         Interact with raw blocks in the datastore
     object        Interact with raw dag nodes
+    unixfs        Interact with Unix filesystem objects
 
 ADVANCED COMMANDS
 
@@ -102,6 +104,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"stats":     StatsCmd,
 	"swarm":     SwarmCmd,
 	"tour":      tourCmd,
+	"unixfs":    unixfs.UnixFSCmd,
 	"update":    UpdateCmd,
 	"version":   VersionCmd,
 	"bitswap":   BitswapCmd,

core/commands/unixfs/ls.go

@@ -0,0 +1,138 @@
+package unixfs
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
+	cmds "github.com/ipfs/go-ipfs/commands"
+	core "github.com/ipfs/go-ipfs/core"
+	path "github.com/ipfs/go-ipfs/path"
+	unixfs "github.com/ipfs/go-ipfs/unixfs"
+	unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
+)
+
+type LsLink struct {
+	Name, Hash string
+	Size       uint64
+	Type       unixfspb.Data_DataType
+}
+
+type LsObject struct {
+	Argument string
+	Links    []LsLink
+}
+
+type LsOutput struct {
+	Objects []*LsObject
+}
+
+var LsCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "List directory contents for Unix-filesystem objects",
+		ShortDescription: `
+Retrieves the object named by <ipfs-or-ipns-path> and displays the
+contents with the following format:
+
+  <hash> <type> <size> <name>
+
+For files, the child size is the total size of the file contents.  For
+directories, the child size is the IPFS link size.
+`,
+	},
+
+	Arguments: []cmds.Argument{
+		cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to list links from").EnableStdin(),
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption("headers", "", "Print table headers (Hash, Size, Name)"),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		node, err := req.Context().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		// get options early -> exit early in case of error
+		if _, _, err := req.Option("headers").Bool(); err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		paths := req.Arguments()
+
+		output := make([]*LsObject, len(paths))
+		for i, fpath := range paths {
+			dagnode, err := core.Resolve(req.Context().Context, node, path.Path(fpath))
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			output[i] = &LsObject{
+				Argument: paths[i],
+				Links:    make([]LsLink, len(dagnode.Links)),
+			}
+			for j, link := range dagnode.Links {
+				ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+				defer cancel()
+				link.Node, err = link.GetNode(ctx, node.DAG)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+				d, err := unixfs.FromBytes(link.Node.Data)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+				lsLink := LsLink{
+					Name: link.Name,
+					Hash: link.Hash.B58String(),
+					Type: d.GetType(),
+				}
+				if lsLink.Type == unixfspb.Data_File {
+					lsLink.Size = d.GetFilesize()
+				} else {
+					lsLink.Size = link.Size
+				}
+				output[i].Links[j] = lsLink
+			}
+		}
+
+		res.SetOutput(&LsOutput{Objects: output})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+
+			headers, _, _ := res.Request().Option("headers").Bool()
+			output := res.Output().(*LsOutput)
+			buf := new(bytes.Buffer)
+			w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0)
+			for _, object := range output.Objects {
+				if len(output.Objects) > 1 {
+					fmt.Fprintf(w, "%s:\n", object.Argument)
+				}
+				if headers {
+					fmt.Fprintln(w, "Hash\tType\tSize\tName")
+				}
+				for _, link := range object.Links {
+					fmt.Fprintf(w, "%s\t%s\t%v\t%s\n",
+						link.Hash, link.Type.String(), link.Size, link.Name)
+				}
+				if len(output.Objects) > 1 {
+					fmt.Fprintln(w)
+				}
+			}
+			w.Flush()
+
+			return buf, nil
+		},
+	},
+	Type: LsOutput{},
+}

core/commands/unixfs/unixfs.go

@@ -0,0 +1,21 @@
+package unixfs
+
+import cmds "github.com/ipfs/go-ipfs/commands"
+
+var UnixFSCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Interact with ipfs objects representing Unix filesystems",
+		ShortDescription: `
+'ipfs unixfs' provides a familar interface to filesystems represtented
+by IPFS objects that hides IPFS-implementation details like layout
+objects (e.g. fanout and chunking).
+`,
+		Synopsis: `
+ipfs unixfs ls <path>...  - List directory contents for <path>...
+`,
+	},
+
+	Subcommands: map[string]*cmds.Command{
+		"ls": LsCmd,
+	},
+}

test/sharness/t0200-unixfs-ls.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+#
+# Copyright (c) 2014 Christian Couder
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test unixfs ls command"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+test_ls_cmd() {
+
+	test_expect_success "'ipfs add -r testData' succeeds" '
+		mkdir -p testData testData/d1 testData/d2 &&
+		echo "test" >testData/f1 &&
+		echo "data" >testData/f2 &&
+		echo "hello" >testData/d1/a &&
+		random 128 42 >testData/d1/128 &&
+		echo "world" >testData/d2/a &&
+		random 1024 42 >testData/d2/1024 &&
+		ipfs add -r testData >actual_add
+	'
+
+	test_expect_success "'ipfs add' output looks good" '
+		cat <<-\EOF >expected_add &&
+			added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128
+			added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a
+			added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1
+			added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024
+			added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a
+			added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2
+			added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1
+			added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2
+			added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData
+		EOF
+		test_cmp expected_add actual_add
+	'
+
+	test_expect_success "'ipfs unixfs ls <three dir hashes>' succeeds" '
+		ipfs unixfs ls QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls
+	'
+
+	test_expect_success "'ipfs unixfs ls <three dir hashes>' output looks good" '
+		cat <<-\EOF >expected_ls &&
+			QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
+			QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss Directory 246  d1
+			QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy Directory 1143 d2
+			QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH File      5    f1
+			QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M File      5    f2
+
+			QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
+			QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd File 1024 1024
+			QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL File 6    a
+
+			QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
+			QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe File 128 128
+			QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN File 6   a
+
+		EOF
+		test_cmp expected_ls actual_ls
+	'
+
+	test_expect_success "'ipfs unixfs ls --headers <three dir hashes>' succeeds" '
+		ipfs unixfs ls --headers QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls_headers
+	'
+
+	test_expect_success "'ipfs unixfs ls --headers <three dir hashes>' output looks good" '
+		cat <<-\EOF >expected_ls_headers &&
+			QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
+			Hash                                           Type      Size Name
+			QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss Directory 246  d1
+			QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy Directory 1143 d2
+			QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH File      5    f1
+			QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M File      5    f2
+
+			QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
+			Hash                                           Type Size Name
+			QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd File 1024 1024
+			QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL File 6    a
+
+			QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
+			Hash                                           Type Size Name
+			QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe File 128  128
+			QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN File 6    a
+
+		EOF
+		test_cmp expected_ls_headers actual_ls_headers
+	'
+}
+
+# should work offline
+test_ls_cmd
+
+# should work online
+test_launch_ipfs_daemon
+test_ls_cmd
+test_kill_ipfs_daemon
+
+test_done