From c0ab433186e60a54deeae59bd2e295dde524f559 Mon Sep 17 00:00:00 2001 From: Max Chechel Date: Fri, 11 Jan 2019 00:39:45 +0300 Subject: [PATCH 1/3] Fixed and cleaned up TestIpfsStressRead License: MIT Signed-off-by: Max Chechel --- fuse/readonly/ipfs_test.go | 53 ++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index 0c245649836..c8afaf0d674 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -4,13 +4,14 @@ package readonly import ( "bytes" - "errors" + "context" "fmt" "io/ioutil" "math/rand" "os" "path" - "sync" + "strconv" + "strings" "testing" core "github.com/ipfs/go-ipfs/core" @@ -162,49 +163,45 @@ func TestIpfsStressRead(t *testing.T) { paths = append(paths, npaths...) } - // Now read a bunch, concurrently - wg := sync.WaitGroup{} - errs := make(chan error) + t.Parallel() for s := 0; s < 4; s++ { - wg.Add(1) - go func() { - defer wg.Done() - + t.Run(strconv.Itoa(s), func(t *testing.T) { for i := 0; i < 2000; i++ { - item, _ := iface.ParsePath(paths[rand.Intn(len(paths))]) - fname := path.Join(mnt.Dir, item.String()) + item, err := iface.ParsePath(paths[rand.Intn(len(paths))]) + if err != nil { + t.Fatal(err) + } + + relpath := strings.Replace(item.String(), "/ipfs/", "/", 1) + fname := path.Join(mnt.Dir, relpath) + rbuf, err := ioutil.ReadFile(fname) if err != nil { - errs <- err + t.Fatal(err) } - read, err := api.Unixfs().Get(nd.Context(), item) + //nd.Context() is never closed which leads to + //hitting 8128 goroutine limit in go test -race mode + ctx, cancelFunc := context.WithCancel(context.Background()) + + read, err := api.Unixfs().Get(ctx, item) if err != nil { - errs <- err + t.Fatal(err) } data, err := ioutil.ReadAll(read.(files.File)) if err != nil { - errs <- err + t.Fatal(err) } + cancelFunc() + if !bytes.Equal(rbuf, data) { - errs <- errors.New("incorrect read") + t.Fatal("incorrect read") } } - }() - } - - go func() { - wg.Wait() - close(errs) - }() - - for err := range errs { - if err != nil { - t.Fatal(err) - } + }) } } From 498474f35842111cd676008513cd87ecff7856eb Mon Sep 17 00:00:00 2001 From: Max Chechel Date: Fri, 11 Jan 2019 01:42:25 +0300 Subject: [PATCH 2/3] fixed TestIpfsStressRead on Windows License: MIT Signed-off-by: Max Chechel --- fuse/readonly/ipfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index c8afaf0d674..f7d5fb2e0b3 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -173,7 +173,7 @@ func TestIpfsStressRead(t *testing.T) { t.Fatal(err) } - relpath := strings.Replace(item.String(), "/ipfs/", "/", 1) + relpath := strings.Replace(item.String(), item.Namespace(), "", 1) fname := path.Join(mnt.Dir, relpath) rbuf, err := ioutil.ReadFile(fname) From 71a177cf4bcffde193ac8136cee3fe3d6aeff0df Mon Sep 17 00:00:00 2001 From: Max Chechel Date: Mon, 14 Jan 2019 21:56:39 +0300 Subject: [PATCH 3/3] goroutines are back License: MIT Signed-off-by: Max Chechel --- fuse/readonly/ipfs_test.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index f7d5fb2e0b3..4b4ef0fc6c8 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -5,13 +5,14 @@ package readonly import ( "bytes" "context" + "errors" "fmt" "io/ioutil" "math/rand" "os" "path" - "strconv" "strings" + "sync" "testing" core "github.com/ipfs/go-ipfs/core" @@ -163,22 +164,24 @@ func TestIpfsStressRead(t *testing.T) { paths = append(paths, npaths...) } - t.Parallel() + // Now read a bunch, concurrently + wg := sync.WaitGroup{} + errs := make(chan error) for s := 0; s < 4; s++ { - t.Run(strconv.Itoa(s), func(t *testing.T) { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 2000; i++ { - item, err := iface.ParsePath(paths[rand.Intn(len(paths))]) - if err != nil { - t.Fatal(err) - } + item, _ := iface.ParsePath(paths[rand.Intn(len(paths))]) relpath := strings.Replace(item.String(), item.Namespace(), "", 1) fname := path.Join(mnt.Dir, relpath) rbuf, err := ioutil.ReadFile(fname) if err != nil { - t.Fatal(err) + errs <- err } //nd.Context() is never closed which leads to @@ -187,21 +190,32 @@ func TestIpfsStressRead(t *testing.T) { read, err := api.Unixfs().Get(ctx, item) if err != nil { - t.Fatal(err) + errs <- err } data, err := ioutil.ReadAll(read.(files.File)) if err != nil { - t.Fatal(err) + errs <- err } cancelFunc() if !bytes.Equal(rbuf, data) { - t.Fatal("incorrect read") + errs <- errors.New("incorrect read") } } - }) + }() + } + + go func() { + wg.Wait() + close(errs) + }() + + for err := range errs { + if err != nil { + t.Fatal(err) + } } }