From 460c2119e4ccbe0d08e64a04be4445c78a72c851 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 25 Sep 2019 18:41:32 -0700 Subject: [PATCH 1/2] daemon: support unix domain sockets for the API/gateway All the work was client-side. Unix domain socket multiaddrs already worked server-side. fixes #4218 --- cmd/ipfs/daemon.go | 8 +++- cmd/ipfs/main.go | 30 +++++++++++--- docs/config.md | 16 ++++++++ go.mod | 2 +- go.sum | 8 ++-- test/dependencies/pollEndpoint/main.go | 50 ++++-------------------- test/sharness/lib/test-lib.sh | 2 +- test/sharness/t0060-daemon.sh | 2 +- test/sharness/t0067-unix-api.sh | 30 ++++++++++++++ test/sharness/t0111-gateway-writeable.sh | 2 +- 10 files changed, 92 insertions(+), 58 deletions(-) create mode 100755 test/sharness/t0067-unix-api.sh diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 06b629cf308..26d1a72c4da 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -416,7 +416,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // Setting to 1 lets us multiply it with other stats to add the version labels ipfsInfoMetric.With(prometheus.Labels{ "version": version.CurrentVersionNumber, - "commit": version.CurrentCommit, + "commit": version.CurrentCommit, }).Set(1) // initialize metrics collector @@ -492,7 +492,11 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error for _, listener := range listeners { // we might have listened to /tcp/0 - lets see what we are listing on fmt.Printf("API server listening on %s\n", listener.Multiaddr()) - fmt.Printf("WebUI: http://%s/webui\n", listener.Addr()) + // Browsers require TCP. + switch listener.Addr().Network() { + case "tcp", "tcp4", "tcp6": + fmt.Printf("WebUI: http://%s/webui\n", listener.Addr()) + } } // by default, we don't let you load arbitrary ipfs objects through the api, diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index 8ae51e72adf..ccb914ef9fb 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "math/rand" + "net" + "net/http" "os" "runtime/pprof" "strings" @@ -22,7 +24,7 @@ import ( "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/go-ipfs-cmds/cli" - "github.com/ipfs/go-ipfs-cmds/http" + cmdhttp "github.com/ipfs/go-ipfs-cmds/http" "github.com/ipfs/go-ipfs-config" u "github.com/ipfs/go-ipfs-util" logging "github.com/ipfs/go-log" @@ -249,23 +251,39 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) { if err != nil { return nil, err } - _, host, err := manet.DialArgs(apiAddr) + network, host, err := manet.DialArgs(apiAddr) if err != nil { return nil, err } // Construct the executor. - opts := []http.ClientOpt{ - http.ClientWithAPIPrefix(corehttp.APIPath), + opts := []cmdhttp.ClientOpt{ + cmdhttp.ClientWithAPIPrefix(corehttp.APIPath), } // Fallback on a local executor if we (a) have a repo and (b) aren't // forcing a daemon. if !daemonRequested && fsrepo.IsInitialized(cctx.ConfigRoot) { - opts = append(opts, http.ClientWithFallback(exe)) + opts = append(opts, cmdhttp.ClientWithFallback(exe)) + } + + switch network { + case "tcp", "tcp4", "tcp6": + case "unix": + path := host + host = "unix" + opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", path) + }, + }, + })) + default: + return nil, fmt.Errorf("unsupported API address: %s", apiAddr) } - return http.NewClient(host, opts...), nil + return cmdhttp.NewClient(host, opts...), nil } // commandDetails returns a command's details for the command given by |path|. diff --git a/docs/config.md b/docs/config.md index 883a650b038..42bcf58d0e5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -77,16 +77,32 @@ Contains information about various listener addresses to be used by this node. - `API` Multiaddr or array of multiaddrs describing the address to serve the local HTTP API on. +Supported Transports: + +* tcp/ip{4,6} - `/ipN/.../tcp/...` +* unix - `/unix/path/to/socket` + Default: `/ip4/127.0.0.1/tcp/5001` - `Gateway` Multiaddr or array of multiaddrs describing the address to serve the local gateway on. +Supported Transports: + +* tcp/ip{4,6} - `/ipN/.../tcp/...` +* unix - `/unix/path/to/socket` + Default: `/ip4/127.0.0.1/tcp/8080` - `Swarm` Array of multiaddrs describing which addresses to listen on for p2p swarm connections. +Supported Transports: + +* tcp/ip{4,6} - `/ipN/.../tcp/...` +* websocket - `/ipN/.../tcp/.../ws` +* quic - `/ipN/.../udp/.../quic` + Default: ```json [ diff --git a/go.mod b/go.mod index 6582be08fbb..636c62adcda 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-ipfs-blockstore v0.1.0 github.com/ipfs/go-ipfs-chunker v0.0.1 - github.com/ipfs/go-ipfs-cmds v0.1.0 + github.com/ipfs/go-ipfs-cmds v0.1.1 github.com/ipfs/go-ipfs-config v0.0.11 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 diff --git a/go.sum b/go.sum index 41e5960ff9f..c999c92f0ac 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= -github.com/ipfs/go-ipfs-cmds v0.1.0 h1:0CEde9EcxByej8+L6d1PST57J4ambRPyCTjLG5Ymou8= -github.com/ipfs/go-ipfs-cmds v0.1.0/go.mod h1:TiK4e7/V31tuEb8YWDF8lN3qrnDH+BS7ZqWIeYJlAs8= +github.com/ipfs/go-ipfs-cmds v0.1.1 h1:H9/BLf5rcsULHMj/x8gC0e5o+raYhqk1OQsfzbGMNM4= +github.com/ipfs/go-ipfs-cmds v0.1.1/go.mod h1:k1zMXcOLtljA9iAnZHddbH69yVm5+weRL0snmMD/rK0= github.com/ipfs/go-ipfs-config v0.0.11 h1:5/4nas2CQXiKr2/MLxU24GDGTBvtstQIQezuk7ltOQQ= github.com/ipfs/go-ipfs-config v0.0.11/go.mod h1:wveA8UT5ywN26oKStByzmz1CO6cXwLKKM6Jn/Hfw08I= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -596,8 +596,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= -github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/test/dependencies/pollEndpoint/main.go b/test/dependencies/pollEndpoint/main.go index 67a5e2be0d5..7eadcd3f702 100644 --- a/test/dependencies/pollEndpoint/main.go +++ b/test/dependencies/pollEndpoint/main.go @@ -3,10 +3,6 @@ package main import ( "flag" - "fmt" - "io/ioutil" - "net/http" - "net/url" "os" "time" @@ -16,11 +12,10 @@ import ( ) var ( - host = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on") - endpoint = flag.String("ep", "/version", "which http endpoint path to hit") - tries = flag.Int("tries", 10, "how many tries to make before failing") - timeout = flag.Duration("tout", time.Second, "how long to wait between attempts") - verbose = flag.Bool("v", false, "verbose logging") + host = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on") + tries = flag.Int("tries", 10, "how many tries to make before failing") + timeout = flag.Duration("tout", time.Second, "how long to wait between attempts") + verbose = flag.Bool("v", false, "verbose logging") ) var log = logging.Logger("pollEndpoint") @@ -33,37 +28,23 @@ func main() { if err != nil { log.Fatal("NewMultiaddr() failed: ", err) } - p := addr.Protocols() - if len(p) < 2 { - log.Fatal("need two protocols in host flag (/ip/tcp): ", addr) - } - _, host, err := manet.DialArgs(addr) - if err != nil { - log.Fatal("manet.DialArgs() failed: ", err) - } if *verbose { // lower log level logging.SetDebugLogging() } - // construct url to dial - var u url.URL - u.Scheme = "http" - u.Host = host - u.Path = *endpoint - // show what we got start := time.Now() - log.Debugf("starting at %s, tries: %d, timeout: %s, url: %s", start, *tries, *timeout, u) + log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr) for *tries > 0 { - - err := checkOK(http.Get(u.String())) + c, err := manet.Dial(addr) if err == nil { log.Debugf("ok - endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start)) + c.Close() os.Exit(0) } - log.Debug("get failed: ", err) + log.Debug("connect failed: ", err) time.Sleep(*timeout) *tries-- } @@ -71,18 +52,3 @@ func main() { log.Error("failed.") os.Exit(1) } - -func checkOK(resp *http.Response, err error) error { - if err == nil { // request worked - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - return nil - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - fmt.Fprintf(os.Stderr, "pollEndpoint: ioutil.ReadAll() Error: %s", err) - } - return fmt.Errorf("response not OK. %d %s %q", resp.StatusCode, resp.Status, string(body)) - } - return err -} diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index c16118898a9..f803c31eb78 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -230,7 +230,7 @@ test_launch_ipfs_daemon() { # we say the daemon is ready when the API server is ready. test_expect_success "'ipfs daemon' is ready" ' - pollEndpoint -ep=/version -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || + pollEndpoint -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' } diff --git a/test/sharness/t0060-daemon.sh b/test/sharness/t0060-daemon.sh index a6cdc962469..3256355dcc3 100755 --- a/test/sharness/t0060-daemon.sh +++ b/test/sharness/t0060-daemon.sh @@ -170,7 +170,7 @@ test_expect_success "'ipfs daemon' should be able to run with a pipe attached to ' test_expect_success "daemon with pipe eventually becomes live" ' - pollEndpoint -host='$API_MADDR' -ep=/version -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr && + pollEndpoint -host='$API_MADDR' -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr && test_kill_repeat_10_sec $DAEMON_PID || test_fsh cat stdin_daemon_out || test_fsh cat stdin_daemon_err || test_fsh cat stdin_poll_apiout || test_fsh cat stdin_poll_apierr ' diff --git a/test/sharness/t0067-unix-api.sh b/test/sharness/t0067-unix-api.sh new file mode 100755 index 00000000000..4f1e34ca4af --- /dev/null +++ b/test/sharness/t0067-unix-api.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test unix API transport" + +. lib/test-lib.sh + +test_init_ipfs + +# We can't use the trash dir as the full name must be longer less than 108 bytes +# long (because that's the max unix domain socket path length). +SOCKDIR="$(mktemp -d "${TMPDIR:-/tmp}/unix-api-sharness.XXXXXX")" + +test_expect_success "configure" ' + peerid=$(ipfs config Identity.PeerID) && + ipfs config Addresses.API "/unix/$SOCKDIR/sock" +' + +test_launch_ipfs_daemon + +test_expect_success "client works" ' + printf "$peerid" >expected && + ipfs --api="/unix/$SOCKDIR/sock" id -f="" >actual && + test_cmp expected actual +' + +test_kill_ipfs_daemon +test_done diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index 423151c6a36..f63b49bcf4b 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -30,7 +30,7 @@ test_launch_ipfs_daemon port=$GWAY_PORT test_expect_success "ipfs daemon up" ' - pollEndpoint -host $GWAY_MADDR -ep=/version -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || + pollEndpoint -host $GWAY_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' From 8e346a993e1bdee0b49dba4f4557a4d9d7705b9f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 30 Sep 2019 18:47:55 -0700 Subject: [PATCH 2/2] test(sharness): fix tests for commands lib update We no longer spit out the annoying http error. --- test/sharness/t0040-add-and-cat.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 0d7c98f7169..edd3e120714 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -494,7 +494,6 @@ test_add_cat_expensive() { } test_add_named_pipe() { - err_prefix=$1 test_expect_success "useful error message when adding a named pipe" ' mkfifo named-pipe && test_expect_code 1 ipfs add named-pipe 2>actual && @@ -510,7 +509,7 @@ test_add_named_pipe() { mkfifo named-pipe-dir/named-pipe && STAT=$(generic_stat named-pipe-dir/named-pipe) && test_expect_code 1 ipfs add -r named-pipe-dir 2>actual && - printf "Error:$err_prefix unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && + printf "Error: unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && rm named-pipe-dir/named-pipe && rmdir named-pipe-dir && test_cmp expected actual @@ -796,7 +795,7 @@ test_add_cat_expensive "--cid-version=1" "bafybeidkj5ecbhrqmzrcee2rw7qwsx24z3364 # encoded with the blake2b-256 hash funtion test_add_cat_expensive '--hash=blake2b-256' "bafykbzaceb26fnq5hz5iopzamcb4yqykya5x6a4nvzdmcyuu4rj2akzs3z7r6" -test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&progress=true&recursive=true&stream-channels=true:" +test_add_named_pipe test_add_pwd_is_symlink @@ -827,7 +826,7 @@ test_expect_success "ipfs cat file fails" ' test_must_fail ipfs cat $(cat oh_hash) ' -test_add_named_pipe "" +test_add_named_pipe test_add_pwd_is_symlink