From cd10ff2083fc44ac8c0cfc83105fb7492101f139 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 12 Apr 2019 17:28:06 +0900 Subject: [PATCH 01/18] swarm/chunk: add tags struct swarm/chunk: address (some) PR comments swarm/chunk: add seen field swarm/chunk: change eta status call swarm/chunk: address pr comments swarm/chunk: address pr comments swarm/chunk: address pr comments swarm/chunk: remove ambiguity of total swarm/api: wip add chunk totals swarm/api: tag test-bed swarm/api, swarm/chunk: wip rudimentary implementation for tag increments swarm/api, swarm/storage: initial TestApiPut passes swarm/api: remove logline swarm: address PR comments cmd, swarm: fix compilation errors swarm/chunk: adjust test swarm/chunk: adjust tests swarm/chunk: pointers galore swarm: fix tests and compilation swarm/storage: change pyramid chunker append signature swarm/api: make api tags private swarm/api: add test for larger content cmd, swarm: add test for http upload swarm: remove deprecated and unused code, add swarm hash to DoneSplit signature swarm/chunk: add marshalling for address field swarm/api: add handle get tag handler, add tags for post raw cmd/swarm: reinstate recursive flag swarm/api: cleanup api test swarm/api: cleanup api test swarm/api/client: add client tag check to multipart upload swarm/api/client: add more tag assertions to tests swarm/api/http: test for correct tag size estimate according to Content-Length header swarm/api/http: document test swarm/api/http: uncomment test swarm/storage: address pr comments swarm/api: remove silly comment --- cmd/swarm/explore.go | 3 +- cmd/swarm/hash.go | 3 +- cmd/swarm/swarm-smoke/upload_and_sync.go | 2 +- swarm/api/api.go | 37 +++---- swarm/api/api_test.go | 107 +++++++++++++++++++-- swarm/api/client/client.go | 31 ++++++ swarm/api/client/client_test.go | 56 ++++++++++- swarm/api/filesystem_test.go | 3 +- swarm/api/http/middleware.go | 42 ++++++++ swarm/api/http/server.go | 52 +++++++++- swarm/api/http/server_test.go | 55 +++++++++++ swarm/api/http/test_server.go | 9 +- swarm/api/manifest_test.go | 7 +- swarm/api/storage.go | 85 ---------------- swarm/api/storage_test.go | 56 ----------- swarm/chunk/tag.go | 3 + swarm/fuse/swarmfs_test.go | 5 +- swarm/network/stream/common_test.go | 2 +- swarm/network/stream/delivery_test.go | 2 +- swarm/network/stream/snapshot_sync_test.go | 2 +- swarm/network_test.go | 32 +++++- swarm/storage/chunker_test.go | 21 ++-- swarm/storage/filestore.go | 31 ++++-- swarm/storage/filestore_test.go | 7 +- swarm/storage/hasherstore.go | 12 ++- swarm/storage/hasherstore_test.go | 2 +- swarm/storage/pyramid.go | 13 ++- swarm/swarm.go | 5 +- swarm/testutil/tag.go | 50 ++++++++++ 29 files changed, 507 insertions(+), 228 deletions(-) delete mode 100644 swarm/api/storage.go delete mode 100644 swarm/api/storage_test.go create mode 100644 swarm/testutil/tag.go diff --git a/cmd/swarm/explore.go b/cmd/swarm/explore.go index 5b5b8bf41f..9566213e41 100644 --- a/cmd/swarm/explore.go +++ b/cmd/swarm/explore.go @@ -23,6 +23,7 @@ import ( "os" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" "gopkg.in/urfave/cli.v1" ) @@ -47,7 +48,7 @@ func hashes(ctx *cli.Context) { } defer f.Close() - fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams(), chunk.NewTags()) refs, err := fileStore.GetAllReferences(context.TODO(), f, false) if err != nil { utils.Fatalf("%v\n", err) diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index 2df02c0ed7..ff786fa105 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" "gopkg.in/urfave/cli.v1" ) @@ -77,7 +78,7 @@ func hash(ctx *cli.Context) { defer f.Close() stat, _ := f.Stat() - fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams(), chunk.NewTags()) addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false) if err != nil { utils.Fatalf("%v\n", err) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index bbcf66b26b..d6eb87ace4 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -255,7 +255,7 @@ func getAllRefs(testData []byte) (storage.AddressCollection, error) { return nil, fmt.Errorf("unable to create temp dir: %v", err) } defer os.RemoveAll(datadir) - fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), chunk.NewTags()) if err != nil { return nil, err } diff --git a/swarm/api/api.go b/swarm/api/api.go index 86c1119232..a5579a0832 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -41,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" @@ -188,15 +189,17 @@ type API struct { feed *feed.Handler fileStore *storage.FileStore dns Resolver + tags *chunk.Tags Decryptor func(context.Context, string) DecryptFunc } // NewAPI the api constructor initialises a new API instance. -func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handler, pk *ecdsa.PrivateKey) (self *API) { +func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handler, pk *ecdsa.PrivateKey, tags *chunk.Tags) (self *API) { self = &API{ fileStore: fileStore, dns: dns, feed: feedHandler, + tags: tags, Decryptor: func(ctx context.Context, credentials string) DecryptFunc { return self.doDecrypt(ctx, credentials, pk) }, @@ -297,31 +300,6 @@ func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (sto return addr, nil } -// Put provides singleton manifest creation on top of FileStore store -func (a *API) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { - apiPutCount.Inc(1) - r := strings.NewReader(content) - key, waitContent, err := a.fileStore.Store(ctx, r, int64(len(content)), toEncrypt) - if err != nil { - apiPutFail.Inc(1) - return nil, nil, err - } - manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) - r = strings.NewReader(manifest) - key, waitManifest, err := a.fileStore.Store(ctx, r, int64(len(manifest)), toEncrypt) - if err != nil { - apiPutFail.Inc(1) - return nil, nil, err - } - return key, func(ctx context.Context) error { - err := waitContent(ctx) - if err != nil { - return err - } - return waitManifest(ctx) - }, nil -} - // Get uses iterative manifest retrieval and prefix matching // to resolve basePath to content using FileStore retrieve // it returns a section reader, mimeType, status, the key of the actual content and an error @@ -987,6 +965,13 @@ func (a *API) ResolveFeed(ctx context.Context, uri *URI, values feed.Values) (*f return fd, nil } +// NewTag exposes chunk.Tags New method for incoming upload requests +func (a *API) NewTag(s string, total int) (*chunk.Tag, error) { + return a.tags.New(s, total) +} + +func (a *API) GetTag(uid uint32) (*chunk.Tag, error) { return a.tags.Get(uid) } + // MimeOctetStream default value of http Content-Type header const MimeOctetStream = "application/octet-stream" diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index eb896f32aa..91bdeee139 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -19,6 +19,7 @@ package api import ( "bytes" "context" + crand "crypto/rand" "errors" "flag" "fmt" @@ -26,13 +27,16 @@ import ( "io/ioutil" "math/big" "os" + "strings" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func init() { @@ -41,19 +45,20 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func testAPI(t *testing.T, f func(*API, bool)) { +func testAPI(t *testing.T, f func(*API, *chunk.Tags, bool)) { datadir, err := ioutil.TempDir("", "bzz-test") if err != nil { t.Fatalf("unable to create temp dir: %v", err) } defer os.RemoveAll(datadir) - fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) + tags := chunk.NewTags() + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), tags) if err != nil { return } - api := NewAPI(fileStore, nil, nil, nil) - f(api, false) - f(api, true) + api := NewAPI(fileStore, nil, nil, nil, tags) + f(api, tags, false) + f(api, tags, true) } type testResponse struct { @@ -61,6 +66,13 @@ type testResponse struct { *Response } +type Response struct { + MimeType string + Status int + Size int64 + Content string +} + func checkResponse(t *testing.T, resp *testResponse, exp *Response) { if resp.MimeType != exp.MimeType { @@ -115,11 +127,11 @@ func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse { } func TestApiPut(t *testing.T) { - testAPI(t, func(api *API, toEncrypt bool) { + testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { content := "hello" exp := expResponse(content, "text/plain", 0) ctx := context.TODO() - addr, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt) + addr, wait, err := putString(ctx, api, content, exp.MimeType, toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -129,7 +141,26 @@ func TestApiPut(t *testing.T) { } resp := testGet(t, api, addr.Hex(), "") checkResponse(t, resp, exp) + testutil.CheckTag(t, tags, chunk.SPLIT, 2, 2) //1 chunk data, 1 chunk manifest + }) +} + +// TestApiTagLarge tests that the the number of chunks counted is larger for a larger input +func TestApiTagLarge(t *testing.T) { + testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { + ctx := context.TODO() + _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", toEncrypt) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = wait(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + testutil.CheckTag(t, tags, chunk.SPLIT, 4129, 4129) //11 chunks random data, 1 chunk manifest + testutil.CheckTag(t, tags, chunk.SEEN, 0, 4129) //0 chunks seen, 12 total }) + } // testResolver implements the Resolver interface and either returns the given @@ -391,7 +422,7 @@ func TestDecryptOriginForbidden(t *testing.T) { Access: &AccessEntry{Type: AccessTypePass}, } - api := NewAPI(nil, nil, nil, nil) + api := NewAPI(nil, nil, nil, nil, chunk.NewTags()) f := api.Decryptor(ctx, "") err := f(me) @@ -425,7 +456,7 @@ func TestDecryptOrigin(t *testing.T) { Access: &AccessEntry{Type: AccessTypePass}, } - api := NewAPI(nil, nil, nil, nil) + api := NewAPI(nil, nil, nil, nil, chunk.NewTags()) f := api.Decryptor(ctx, "") err := f(me) @@ -500,3 +531,61 @@ func TestDetectContentType(t *testing.T) { }) } } + +// putRandomContent provides singleton manifest creation on top of API. it uploads an arbitrary byte stream +// of the desired contentLength and wraps it in a manifest +func putRandomContent(ctx context.Context, a *API, contentLength int, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { + randomContentReader := io.LimitReader(crand.Reader, int64(contentLength)) + + tag, err := a.NewTag("unnamed-tag", 0) + + log.Trace("created new tag", "uid", tag.Uid) + + cCtx := sctx.SetTag(ctx, tag.Uid) + key, waitContent, err := a.Store(cCtx, randomContentReader, int64(contentLength), toEncrypt) + if err != nil { + return nil, nil, err + } + manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) + r := strings.NewReader(manifest) + key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) + if err != nil { + return nil, nil, err + } + tag.DoneSplit(key) + return key, func(ctx context.Context) error { + err := waitContent(ctx) + if err != nil { + return err + } + return waitManifest(ctx) + }, nil +} + +// putString provides singleton manifest creation on top of api.API +func putString(ctx context.Context, a *API, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { + r := strings.NewReader(content) + tag, err := a.NewTag("unnamed-tag", 0) + + log.Trace("created new tag", "uid", tag.Uid) + + cCtx := sctx.SetTag(ctx, tag.Uid) + key, waitContent, err := a.Store(cCtx, r, int64(len(content)), toEncrypt) + if err != nil { + return nil, nil, err + } + manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) + r = strings.NewReader(manifest) + key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) + if err != nil { + return nil, nil, err + } + tag.DoneSplit(key) + return key, func(ctx context.Context) error { + err := waitContent(ctx) + if err != nil { + return err + } + return waitManifest(ctx) + }, nil +} diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 5e293cca72..a5a61c6851 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -75,6 +75,8 @@ func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, err return "", err } req.ContentLength = size + req.Header.Set("x-swarm-tag", fmt.Sprintf("raw_upload_%d", time.Now().Unix())) + res, err := http.DefaultClient.Do(req) if err != nil { return "", err @@ -111,6 +113,7 @@ func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { type File struct { io.ReadCloser api.ManifestEntry + Tag string } // Open opens a local file which can then be passed to client.Upload to upload @@ -139,6 +142,7 @@ func Open(path string) (*File, error) { Size: stat.Size(), ModTime: stat.ModTime(), }, + Tag: filepath.Base(path), }, nil } @@ -422,6 +426,7 @@ func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, erro // Uploader uploads files to swarm using a provided UploadFn type Uploader interface { Upload(UploadFn) error + Tag() string } type UploaderFunc func(UploadFn) error @@ -430,12 +435,23 @@ func (u UploaderFunc) Upload(upload UploadFn) error { return u(upload) } +func (u UploaderFunc) Tag() string { + return fmt.Sprintf("multipart_upload_%d", time.Now().Unix()) +} + +// DirectoryUploader implements Uploader +var _ Uploader = &DirectoryUploader{} + // DirectoryUploader uploads all files in a directory, optionally uploading // a file to the default path type DirectoryUploader struct { Dir string } +func (d *DirectoryUploader) Tag() string { + return filepath.Base(d.Dir) +} + // Upload performs the upload of the directory and default path func (d *DirectoryUploader) Upload(upload UploadFn) error { return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { @@ -458,11 +474,17 @@ func (d *DirectoryUploader) Upload(upload UploadFn) error { }) } +var _ Uploader = &FileUploader{} + // FileUploader uploads a single file type FileUploader struct { File *File } +func (f *FileUploader) Tag() string { + return f.File.Tag +} + // Upload performs the upload of the file func (f *FileUploader) Upload(upload UploadFn) error { return upload(f.File) @@ -509,6 +531,14 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t req.URL.RawQuery = q.Encode() } + tag := uploader.Tag() + if tag == "" { + tag = "unnamed_tag_" + fmt.Sprintf("%d", time.Now().Unix()) + } + log.Trace("setting upload tag", "tag", tag) + + req.Header.Set("x-swarm-tag", tag) + // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request req.Header.Set("Expect", "100-continue") @@ -574,6 +604,7 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) mw := multipart.NewWriter(reqW) req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) + req.Header.Set("x-swarm-tag", fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) // define an UploadFn which adds files to the multipart form uploadFn := func(file *File) error { diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 9c9bde5d67..78ff6a8ea8 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -25,16 +25,15 @@ import ( "sort" "testing" - "github.com/ethereum/go-ethereum/swarm/testutil" - - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func serverFunc(api *api.API) swarmhttp.TestServer { @@ -68,6 +67,9 @@ func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { t.Fatal(err) } + // check the tag was created successfully + checkTag(t, srv.Tags, 1, 1, 0, 1) + // check we can download the same data res, isEncrypted, err := client.DownloadRaw(hash) if err != nil { @@ -209,6 +211,9 @@ func TestClientUploadDownloadDirectory(t *testing.T) { t.Fatalf("error uploading directory: %s", err) } + // check the tag was created successfully + checkTag(t, srv.Tags, 9, 9, 0, 9) + // check we can download the individual files checkDownloadFile := func(path string, expected []byte) { file, err := client.Download(hash, path) @@ -323,6 +328,7 @@ func TestClientMultipartUpload(t *testing.T) { defer srv.Close() // define an uploader which uploads testDirFiles with some data + // note: this test should result in SEEN chunks. assert accordingly data := []byte("some-data") uploader := UploaderFunc(func(upload UploadFn) error { for _, name := range testDirFiles { @@ -348,6 +354,9 @@ func TestClientMultipartUpload(t *testing.T) { t.Fatal(err) } + // check the tag was created successfully + checkTag(t, srv.Tags, 9, 7, 7, 9) + // check we can download the individual files checkDownloadFile := func(path string) { file, err := client.Download(hash, path) @@ -595,3 +604,40 @@ func TestClientCreateUpdateFeed(t *testing.T) { t.Fatalf("Expected: %v, got %v", databytes, gotData) } } + +func checkTag(t *testing.T, tags *chunk.Tags, split, stored, seen, total int) { + i := 0 + // check that the tag was created and incremented accordingly + tags.Range(func(k, v interface{}) bool { + vv, ok := v.(*chunk.Tag) + if !ok { + t.Fatal("error unmarshalling tag pointer") + } + + tSplit := vv.Get(chunk.SPLIT) + if tSplit != split { + t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) + } + + tSeen := vv.Get(chunk.SEEN) + if tSeen != seen { + t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) + } + + tStored := vv.Get(chunk.STORED) + if tStored != stored { + t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) + } + + tTotal := vv.Total() + if tTotal != total { + t.Fatalf("mismatch total chunks, got %d want %d", tTotal, total) + } + i++ + + return false + }) + if i == 0 { + t.Fatal("no tags found") + } +} diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 02f5bff658..b8f37fdd57 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -25,13 +25,14 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" ) var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") func testFileSystem(t *testing.T, f func(*FileSystem, bool)) { - testAPI(t, func(api *API, toEncrypt bool) { + testAPI(t, func(api *API, _ *chunk.Tags, toEncrypt bool) { f(NewFileSystem(api), toEncrypt) }) } diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 320da30462..3f698e7e72 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "runtime/debug" + "strconv" "strings" "time" @@ -86,6 +87,47 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { }) } +func InitUploadTag(h http.Handler, a *api.API) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + tagName string + err error + estimatedTotal = 0 + contentType = r.Header.Get("Content-Type") + contentLength = r.Header.Get("Content-Length") + headerTag = r.Header.Get(SwarmTagHeaderName) + ) + if headerTag != "" { + tagName = headerTag + log.Trace("got tag name from http header", "tagName", tagName) + } else { + tagName = fmt.Sprintf("unnamed_tag_%d", time.Now().Unix()) + } + + log.Trace("trying to estimate tag size", "contentType", contentType, "contentLength", contentLength, "cl", r.ContentLength) + + if !strings.Contains(contentType, "multipart") && contentLength != "" { + estimatedTotal, err = strconv.Atoi(contentLength) + if err != nil { + log.Error("error parsing content-length string, falling back to 0", "contentLength", contentLength) + estimatedTotal = 0 + } else { + estimatedTotal = estimatedTotal / 4096 + } + + } + log.Trace("creating tag", "tagName", tagName, "estimatedTotal", estimatedTotal) + + t, err := a.NewTag(tagName, estimatedTotal) + if err != nil { + log.Error("error creating tag", "err", err, "tagName", tagName) + } + ctx := sctx.SetTag(r.Context(), t.Uid) + + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} + func InstrumentOpenTracing(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { uri := GetURI(r.Context()) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 3c6735a73e..90d5fd7c15 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -38,7 +38,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/rs/cors" @@ -60,6 +62,8 @@ var ( getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) ) +const SwarmTagHeaderName = "x-swarm-tag" + type methodHandler map[string]http.Handler func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -102,7 +106,15 @@ func NewServer(api *api.API, corsString string) *Server { ), "POST": Adapt( http.HandlerFunc(server.HandlePostFiles), - defaultMiddlewares..., + Adapter(func(h http.Handler) http.Handler { + return InitUploadTag(h, api) + }), + RecoverPanic, + SetRequestID, + SetRequestHost, + InitLoggingResponseWriter, + ParseURI, + InstrumentOpenTracing, ), "DELETE": Adapt( http.HandlerFunc(server.HandleDelete), @@ -116,7 +128,15 @@ func NewServer(api *api.API, corsString string) *Server { ), "POST": Adapt( http.HandlerFunc(server.HandlePostRaw), - defaultMiddlewares..., + Adapter(func(h http.Handler) http.Handler { + return InitUploadTag(h, api) + }), + RecoverPanic, + SetRequestID, + SetRequestHost, + InitLoggingResponseWriter, + ParseURI, + InstrumentOpenTracing, ), }) mux.Handle("/bzz-immutable:/", methodHandler{ @@ -147,6 +167,12 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares..., ), }) + mux.Handle("/bzz-tag:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGetTag), + defaultMiddlewares..., + ), + }) mux.Handle("/", methodHandler{ "GET": Adapt( @@ -175,6 +201,11 @@ type Server struct { listenAddr string } +// HandleGetTag handles the HTTP GET for a certain (or all) tags +func (s *Server) HandleGetTag(w http.ResponseWriter, r *http.Request) { + log.Debug("handleGetTag", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) +} + func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) if r.Header.Get("Accept") == "application/x-tar" { @@ -230,6 +261,12 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) log.Debug("handle.post.raw", "ruid", ruid) + tagUid := sctx.GetTag(r.Context()) + tag, err := s.api.GetTag(tagUid) + if err != nil { + log.Error("handle post raw got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) + } + postRawCount.Inc(1) toEncrypt := false @@ -263,6 +300,8 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { return } + tag.DoneSplit(addr) + log.Debug("stored content", "ruid", ruid, "key", addr) w.Header().Set("Content-Type", "text/plain") @@ -333,6 +372,15 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } + tagUid := sctx.GetTag(r.Context()) + tag, err := s.api.GetTag(tagUid) + + if err != nil { + log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) + } + + log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.SPLIT), "TOTAL", tag.Total()) + tag.DoneSplit(newAddr) log.Debug("stored content", "ruid", ruid, "key", newAddr) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e82762ce05..3ee853024f 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -37,6 +37,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" @@ -755,6 +756,7 @@ func testBzzTar(encrypted bool, t *testing.T) { t.Fatal(err) } req.Header.Add("Content-Type", "application/x-tar") + req.Header.Add(SwarmTagHeaderName, "test-upload") client := &http.Client{} resp2, err := client.Do(req) if err != nil { @@ -763,6 +765,10 @@ func testBzzTar(encrypted bool, t *testing.T) { if resp2.StatusCode != http.StatusOK { t.Fatalf("err %s", resp2.Status) } + + // check that the tag was written correctly + testutil.CheckTag(t, srv.Tags, chunk.SPLIT, 4, 4) + swarmHash, err := ioutil.ReadAll(resp2.Body) resp2.Body.Close() if err != nil { @@ -834,6 +840,55 @@ func testBzzTar(encrypted bool, t *testing.T) { t.Fatalf("file %s did not pass content assertion", hdr.Name) } } + + // now check the tags endpoint +} + +// TestBzzCorrectTagEstimate checks that the HTTP middleware sets the total number of chunks +// in the tag according to an estimate from the HTTP request Content-Length header divided +// by chunk size (4096). It is needed to be checked BEFORE chunking is done, therefore +// concurrency was introduced to slow down the HTTP request +func TestBzzCorrectTagEstimate(t *testing.T) { + srv := NewTestSwarmServer(t, serverFunc, nil) + defer srv.Close() + + pr, pw := io.Pipe() + c := make(chan struct{}) + + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequest("POST", srv.URL+"/bzz:/", pr) + if err != nil { + t.Fatal(err) + } + + req = req.WithContext(ctx) + req.ContentLength = 1000000 + req.Header.Add("x-swarm-tag", "1000000") + + go func() { + for { + select { + case <-c: + return + default: + _, err := pw.Write([]byte{0}) + if err != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + } + }() + + client := &http.Client{} + _, err = client.Do(req) + if err != nil { + t.Log(err) + } + time.Sleep(100 * time.Millisecond) + testutil.CheckTag(t, srv.Tags, chunk.SEEN, 0, 244) + close(c) + cancel() } // TestBzzRootRedirect tests that getting the root path of a manifest without diff --git a/swarm/api/http/test_server.go b/swarm/api/http/test_server.go index 9f057d796a..542191cc0a 100644 --- a/swarm/api/http/test_server.go +++ b/swarm/api/http/test_server.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/localstore" @@ -44,7 +45,9 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso t.Fatal(err) } - fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams()) + tags := chunk.NewTags() + fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams(), tags) + // Swarm feeds test setup feedsDir, err := ioutil.TempDir("", "swarm-feeds-test") if err != nil { @@ -56,7 +59,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso t.Fatal(err) } - swarmApi := api.NewAPI(fileStore, resolver, feeds.Handler, nil) + swarmApi := api.NewAPI(fileStore, resolver, feeds.Handler, nil, tags) apiServer := httptest.NewServer(serverFunc(swarmApi)) tss := &TestSwarmServer{ @@ -72,6 +75,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso os.RemoveAll(feedsDir) }, CurrentTime: 42, + Tags: tags, } feed.TimestampProvider = tss return tss @@ -81,6 +85,7 @@ type TestSwarmServer struct { *httptest.Server Hasher storage.SwarmHash FileStore *storage.FileStore + Tags *chunk.Tags dir string cleanup func() CurrentTime uint64 diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 1c8e53c433..c193ebcb4d 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -25,6 +25,7 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -42,7 +43,7 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) { func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie { quitC := make(chan bool) - fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams(), chunk.NewTags()) ref := make([]byte, fileStore.HashSize()) trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt) if err != nil { @@ -99,7 +100,7 @@ func TestGetEntry(t *testing.T) { func TestExactMatch(t *testing.T) { quitC := make(chan bool) mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") - fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams(), chunk.NewTags()) ref := make([]byte, fileStore.HashSize()) trie, err := readManifest(mf, ref, fileStore, false, quitC, nil) if err != nil { @@ -132,7 +133,7 @@ func TestAddFileWithManifestPath(t *testing.T) { reader := &storage.LazyTestSectionReader{ SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))), } - fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams(), chunk.NewTags()) ref := make([]byte, fileStore.HashSize()) trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt) if err != nil { diff --git a/swarm/api/storage.go b/swarm/api/storage.go deleted file mode 100644 index 254375b777..0000000000 --- a/swarm/api/storage.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package api - -import ( - "context" - "path" - - "github.com/ethereum/go-ethereum/swarm/storage" -) - -type Response struct { - MimeType string - Status int - Size int64 - // Content []byte - Content string -} - -// implements a service -// -// DEPRECATED: Use the HTTP API instead -type Storage struct { - api *API -} - -func NewStorage(api *API) *Storage { - return &Storage{api} -} - -// Put uploads the content to the swarm with a simple manifest speficying -// its content type -// -// DEPRECATED: Use the HTTP API instead -func (s *Storage) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (storage.Address, func(context.Context) error, error) { - return s.api.Put(ctx, content, contentType, toEncrypt) -} - -// Get retrieves the content from bzzpath and reads the response in full -// It returns the Response object, which serialises containing the -// response body as the value of the Content field -// NOTE: if error is non-nil, sResponse may still have partial content -// the actual size of which is given in len(resp.Content), while the expected -// size is resp.Size -// -// DEPRECATED: Use the HTTP API instead -func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) { - uri, err := Parse(path.Join("bzz:/", bzzpath)) - if err != nil { - return nil, err - } - addr, err := s.api.Resolve(ctx, uri.Addr) - if err != nil { - return nil, err - } - reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path) - if err != nil { - return nil, err - } - quitC := make(chan bool) - expsize, err := reader.Size(ctx, quitC) - if err != nil { - return nil, err - } - body := make([]byte, expsize) - size, err := reader.Read(body) - if int64(size) == expsize { - err = nil - } - return &Response{mimeType, status, expsize, string(body[:size])}, err -} diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go deleted file mode 100644 index ef96972b68..0000000000 --- a/swarm/api/storage_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package api - -import ( - "context" - "testing" -) - -func testStorage(t *testing.T, f func(*Storage, bool)) { - testAPI(t, func(api *API, toEncrypt bool) { - f(NewStorage(api), toEncrypt) - }) -} - -func TestStoragePutGet(t *testing.T) { - testStorage(t, func(api *Storage, toEncrypt bool) { - content := "hello" - exp := expResponse(content, "text/plain", 0) - // exp := expResponse([]byte(content), "text/plain", 0) - ctx := context.TODO() - bzzkey, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - err = wait(ctx) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - bzzhash := bzzkey.Hex() - // to check put against the API#Get - resp0 := testGet(t, api.api, bzzhash, "") - checkResponse(t, resp0, exp) - - // check storage#Get - resp, err := api.Get(context.TODO(), bzzhash) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - checkResponse(t, &testResponse{nil, resp}, exp) - }) -} diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 359ac11acb..5e17b5a24e 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -28,6 +28,9 @@ var ( errNA = errors.New("not available yet") errNoETA = errors.New("unable to calculate ETA") errTagNotFound = errors.New("tag not found") + errExists = errors.New("already exists") + errNA = errors.New("not available yet") + errNoETA = errors.New("unable to calculate ETA") ) // State is the enum type for chunk states diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 460e31c4e9..77573f0fc7 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" @@ -1614,11 +1615,11 @@ func TestFUSE(t *testing.T) { } defer os.RemoveAll(datadir) - fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), chunk.NewTags()) if err != nil { t.Fatal(err) } - ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil)} + ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil, chunk.NewTags())} //run a short suite of tests //approx time: 28s diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 8e6be72b64..615b3b68f9 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -127,7 +127,7 @@ func netStoreAndDeliveryWithAddr(ctx *adapters.ServiceContext, bucket *sync.Map, return nil, nil, nil, err } - fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams(), chunk.NewTags()) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) delivery := NewDelivery(kad, netStore) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 4037243c17..e7b11592f5 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -380,7 +380,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) i++ } //...which then gets passed to the round-robin file store - roundRobinFileStore := storage.NewFileStore(newRoundRobinStore(stores...), storage.NewFileStoreParams()) + roundRobinFileStore := storage.NewFileStore(newRoundRobinStore(stores...), storage.NewFileStoreParams(), chunk.NewTags()) //now we can actually upload a (random) file to the round-robin store size := chunkCount * chunkSize log.Debug("Storing data to file store") diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index fefdb7c9f8..da4ff673b1 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -298,7 +298,7 @@ func mapKeysToNodes(conf *synctestConfig) { //upload a file(chunks) to a single local node store func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, store chunk.Store) ([]storage.Address, error) { log.Debug(fmt.Sprintf("Uploading to node id: %s", id)) - fileStore := storage.NewFileStore(store, storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(store, storage.NewFileStoreParams(), chunk.NewTags()) size := chunkSize var rootAddrs []storage.Address for i := 0; i < chunkCount; i++ { diff --git a/swarm/network_test.go b/swarm/network_test.go index 97bdd07b14..0b31b4bdc6 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -23,11 +23,13 @@ import ( "io/ioutil" "math/rand" "os" + "strings" "sync" "sync/atomic" "testing" "time" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/ethereum/go-ethereum/crypto" @@ -416,7 +418,7 @@ func uploadFile(swarm *Swarm) (storage.Address, string, error) { // uniqueness is very certain. data := fmt.Sprintf("test content %s %x", time.Now().Round(0), b) ctx := context.TODO() - k, wait, err := swarm.api.Put(ctx, data, "text/plain", false) + k, wait, err := putString(ctx, swarm.api, data, "text/plain", false) if err != nil { return nil, "", err } @@ -530,3 +532,31 @@ func retrieve( return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount) } + +// putString provides singleton manifest creation on top of api.API +func putString(ctx context.Context, a *api.API, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { + r := strings.NewReader(content) + tag, err := a.NewTag("unnamed-tag", 0) + + log.Trace("created new tag", "uid", tag.Uid) + + cCtx := sctx.SetTag(ctx, tag.Uid) + key, waitContent, err := a.Store(cCtx, r, int64(len(content)), toEncrypt) + if err != nil { + return nil, nil, err + } + manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) + r = strings.NewReader(manifest) + key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) + if err != nil { + return nil, nil, err + } + tag.DoneSplit(key) + return key, func(ctx context.Context) error { + err := waitContent(ctx) + if err != nil { + return err + } + return waitManifest(ctx) + }, nil +} diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 9a12594443..a0fe2e7697 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -24,6 +24,7 @@ import ( "io" "testing" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/testutil" "golang.org/x/crypto/sha3" ) @@ -42,8 +43,10 @@ type chunkerTester struct { t test } +var mockTag = chunk.NewTag(0, "mock-tag", 0) + func newTestHasherStore(store ChunkStore, hash string) *hasherStore { - return NewHasherStore(store, MakeHashFunc(hash), false) + return NewHasherStore(store, MakeHashFunc(hash), false, chunk.NewTag(0, "test-tag", 0)) } func testRandomBrokenData(n int, tester *chunkerTester) { @@ -91,7 +94,7 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) var err error ctx := context.TODO() if usePyramid { - addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter) + addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter, mockTag) } else { addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter) } @@ -188,7 +191,7 @@ func TestDataAppend(t *testing.T) { putGetter := newTestHasherStore(store, SHA3Hash) ctx := context.TODO() - addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter, mockTag) if err != nil { tester.t.Fatalf(err.Error()) } @@ -208,7 +211,7 @@ func TestDataAppend(t *testing.T) { } putGetter = newTestHasherStore(store, SHA3Hash) - newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter) + newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter, mockTag) if err != nil { tester.t.Fatalf(err.Error()) } @@ -278,7 +281,7 @@ func benchmarkSplitJoin(n int, t *testing.B) { putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) ctx := context.TODO() - key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter, mockTag) if err != nil { t.Fatalf(err.Error()) } @@ -335,7 +338,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() - _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter, mockTag) if err != nil { t.Fatalf(err.Error()) } @@ -353,7 +356,7 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() - _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter, mockTag) if err != nil { t.Fatalf(err.Error()) } @@ -374,7 +377,7 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { putGetter := newTestHasherStore(store, SHA3Hash) ctx := context.Background() - key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter, mockTag) if err != nil { t.Fatalf(err.Error()) } @@ -384,7 +387,7 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { } putGetter = newTestHasherStore(store, SHA3Hash) - _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter) + _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter, mockTag) if err != nil { t.Fatalf(err.Error()) } diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 2b15f7da68..2754cb3642 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/localstore" ) @@ -47,6 +48,7 @@ const ( type FileStore struct { ChunkStore hashFunc SwarmHasher + tags *chunk.Tags } type FileStoreParams struct { @@ -60,19 +62,20 @@ func NewFileStoreParams() *FileStoreParams { } // for testing locally -func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) { +func NewLocalFileStore(datadir string, basekey []byte, tags *chunk.Tags) (*FileStore, error) { localStore, err := localstore.New(datadir, basekey, nil) if err != nil { return nil, err } - return NewFileStore(chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams()), nil + return NewFileStore(chunk.NewValidatorStore(localStore, NewContentAddressValidator(MakeHashFunc(DefaultHash))), NewFileStoreParams(), tags), nil } -func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore { +func NewFileStore(store ChunkStore, params *FileStoreParams, tags *chunk.Tags) *FileStore { hashFunc := MakeHashFunc(params.Hash) return &FileStore{ ChunkStore: store, hashFunc: hashFunc, + tags: tags, } } @@ -83,7 +86,12 @@ func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore { // It returns a reader with the chunk data and whether the content was encrypted func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) { isEncrypted = len(addr) > f.hashFunc().Size() - getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted) + tag, err := f.tags.GetContext(ctx) + if err != nil { + tag = chunk.NewTag(0, "ephemeral", 0) //note: how to handle this. this function does not return an error + log.Error("did not find a tag for retrieve, creating ephemeral") + } + getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted, tag) reader = TreeJoin(ctx, addr, getter, 0) return } @@ -91,8 +99,13 @@ func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChu // Store is a public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) { - putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt) - return PyramidSplit(ctx, data, putter, putter) + tag, err := f.tags.GetContext(ctx) + if err != nil { + //tag = chunk.NewTag(0, "", 0) //create an ephemeral tag + return nil, nil, err + } + putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt, tag) + return PyramidSplit(ctx, data, putter, putter, tag) } func (f *FileStore) HashSize() int { @@ -101,12 +114,14 @@ func (f *FileStore) HashSize() int { // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { + tag := chunk.NewTag(0, "ephemeral-tag", 0) //this tag is just a mock ephemeral tag since we don't want to save these results + // create a special kind of putter, which only will store the references putter := &hashExplorer{ - hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), + hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt, tag), } // do the actual splitting anyway, no way around it - _, wait, err := PyramidSplit(ctx, data, putter, putter) + _, wait, err := PyramidSplit(ctx, data, putter, putter, tag) if err != nil { return nil, err } diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index fe01eed9aa..d0a167a244 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -25,6 +25,7 @@ import ( "path/filepath" "testing" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/localstore" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -48,7 +49,7 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, NewFileStoreParams(), chunk.NewTags()) slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() @@ -113,7 +114,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, NewFileStoreParams(), chunk.NewTags()) slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) @@ -182,7 +183,7 @@ func TestGetAllReferences(t *testing.T) { } defer localStore.Close() - fileStore := NewFileStore(localStore, NewFileStoreParams()) + fileStore := NewFileStore(localStore, NewFileStoreParams(), chunk.NewTags()) // testRuns[i] and expectedLen[i] are dataSize and expected length respectively testRuns := []int{1024, 8192, 16000, 30000, 1000000} diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 2e4a1c11b0..1c9d3bb711 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -22,12 +22,14 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) type hasherStore struct { store ChunkStore + tag *chunk.Tag toEncrypt bool hashFunc SwarmHasher hashSize int // content hash size @@ -44,7 +46,7 @@ type hasherStore struct { // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary -func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { +func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool, tag *chunk.Tag) *hasherStore { hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { @@ -53,6 +55,7 @@ func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *has h := &hasherStore{ store: store, + tag: tag, toEncrypt: toEncrypt, hashFunc: hashFunc, hashSize: hashSize, @@ -242,7 +245,12 @@ func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryptio func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { atomic.AddUint64(&h.nrChunks, 1) go func() { - _, err := h.store.Put(ctx, chunk.ModePutUpload, ch) + seen, err := h.store.Put(ctx, chunk.ModePutUpload, ch) + uid := sctx.GetTag(ctx) + h.tag.Inc(chunk.STORED) + if seen { + h.tag.Inc(chunk.SEEN) + } select { case h.errC <- err: case <-h.quitC: diff --git a/swarm/storage/hasherstore_test.go b/swarm/storage/hasherstore_test.go index c95537db73..9dfd7ab1d9 100644 --- a/swarm/storage/hasherstore_test.go +++ b/swarm/storage/hasherstore_test.go @@ -43,7 +43,7 @@ func TestHasherStore(t *testing.T) { for _, tt := range tests { chunkStore := NewMapChunkStore() - hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt) + hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt, chunk.NewTag(0, "test-tag", 0)) // Put two random chunks into the hasherStore chunkData1 := GenerateRandomChunk(int64(tt.chunkLength)).Data() diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 281bbe9fe3..cb731cb918 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -96,12 +96,12 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Address), the root hash of the entire content will fill this once processing finishes. New chunks to store are store using the putter which the caller provides. */ -func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx) +func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter, tag *chunk.Tag) (Address, func(context.Context) error, error) { + return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize), tag).Split(ctx) } -func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx) +func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter, tag *chunk.Tag) (Address, func(context.Context) error, error) { + return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize), tag).Append(ctx) } // Entry to create a tree node @@ -142,6 +142,7 @@ type PyramidChunker struct { putter Putter getter Getter key Address + tag *chunk.Tag workerCount int64 workerLock sync.RWMutex jobC chan *chunkJob @@ -152,7 +153,7 @@ type PyramidChunker struct { chunkLevel [][]*TreeEntry } -func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker) { +func NewPyramidSplitter(params *PyramidSplitterParams, tag *chunk.Tag) (pc *PyramidChunker) { pc = &PyramidChunker{} pc.reader = params.reader pc.hashSize = params.hashSize @@ -161,6 +162,7 @@ func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker) { pc.putter = params.putter pc.getter = params.getter pc.key = params.addr + pc.tag = tag pc.workerCount = 0 pc.jobC = make(chan *chunkJob, 2*ChunkProcessors) pc.wg = &sync.WaitGroup{} @@ -273,6 +275,7 @@ func (pc *PyramidChunker) processor(ctx context.Context, id int64) { return } pc.processChunk(ctx, id, job) + pc.tag.Inc(chunk.SPLIT) case <-pc.quitC: return } diff --git a/swarm/swarm.go b/swarm/swarm.go index 2f025d9cc9..d004bcd2f4 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -211,9 +211,10 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e MaxPeerServers: config.MaxStreamPeerServers, } self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, self.stateStore, registryOptions, self.swap) + tags := chunk.NewTags() //todo load from state store // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage - self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) + self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams, tags) log.Debug("Setup local storage") @@ -228,7 +229,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } - self.api = api.NewAPI(self.fileStore, self.dns, feedsHandler, self.privateKey) + self.api = api.NewAPI(self.fileStore, self.dns, feedsHandler, self.privateKey, tags) self.sfs = fuse.NewSwarmFS(self.api) log.Debug("Initialized FUSE filesystem") diff --git a/swarm/testutil/tag.go b/swarm/testutil/tag.go new file mode 100644 index 0000000000..2b8ca25690 --- /dev/null +++ b/swarm/testutil/tag.go @@ -0,0 +1,50 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testutil + +import ( + "testing" + + "github.com/ethereum/go-ethereum/swarm/chunk" +) + +// CheckTag checks the first tag in the api struct to be in a certain state +func CheckTag(t *testing.T, tags *chunk.Tags, state chunk.State, exp, expTotal int) { + t.Helper() + i := 0 + tags.Range(func(k, v interface{}) bool { + i++ + tag := v.(*chunk.Tag) + count, total, err := tag.Status(state) + if err != nil { + t.Fatal(err) + } + + if count != exp { + t.Fatalf("expected count to be %d, got %d", exp, count) + } + + if total != expTotal { + t.Fatalf("expected total to be %d, got %d", expTotal, total) + } + return false + }) + + if i == 0 { + t.Fatal("did not find any tags") + } +} From b0a891dce06c43db2ab2a6fae2131dcc352be0d6 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 30 Apr 2019 16:27:25 +0900 Subject: [PATCH 02/18] swarm: fix compile errors --- swarm/chunk/tag.go | 3 --- swarm/storage/hasherstore.go | 2 -- 2 files changed, 5 deletions(-) diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 5e17b5a24e..359ac11acb 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -28,9 +28,6 @@ var ( errNA = errors.New("not available yet") errNoETA = errors.New("unable to calculate ETA") errTagNotFound = errors.New("tag not found") - errExists = errors.New("already exists") - errNA = errors.New("not available yet") - errNoETA = errors.New("unable to calculate ETA") ) // State is the enum type for chunk states diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 1c9d3bb711..f4a811da74 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -22,7 +22,6 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) @@ -246,7 +245,6 @@ func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { atomic.AddUint64(&h.nrChunks, 1) go func() { seen, err := h.store.Put(ctx, chunk.ModePutUpload, ch) - uid := sctx.GetTag(ctx) h.tag.Inc(chunk.STORED) if seen { h.tag.Inc(chunk.SEEN) From 466387ed10215c08ba0a6a29c859426f1b660524 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 1 May 2019 10:37:25 +0900 Subject: [PATCH 03/18] swarm: fix tests, address pr comments --- swarm/api/api.go | 11 ++------- swarm/api/api_test.go | 6 ++--- swarm/api/client/client_test.go | 3 ++- swarm/api/http/middleware.go | 7 ++++-- swarm/api/http/response.go | 2 +- swarm/api/http/server.go | 44 +++++++++------------------------ swarm/api/http/test_server.go | 2 +- swarm/chunk/tag.go | 1 - swarm/network_test.go | 2 +- swarm/storage/filestore.go | 12 +++++---- swarm/swarm_test.go | 8 ++++-- 11 files changed, 39 insertions(+), 59 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index a5579a0832..7c49c24527 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -189,7 +189,7 @@ type API struct { feed *feed.Handler fileStore *storage.FileStore dns Resolver - tags *chunk.Tags + Tags *chunk.Tags Decryptor func(context.Context, string) DecryptFunc } @@ -199,7 +199,7 @@ func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handle fileStore: fileStore, dns: dns, feed: feedHandler, - tags: tags, + Tags: tags, Decryptor: func(ctx context.Context, credentials string) DecryptFunc { return self.doDecrypt(ctx, credentials, pk) }, @@ -965,13 +965,6 @@ func (a *API) ResolveFeed(ctx context.Context, uri *URI, values feed.Values) (*f return fd, nil } -// NewTag exposes chunk.Tags New method for incoming upload requests -func (a *API) NewTag(s string, total int) (*chunk.Tag, error) { - return a.tags.New(s, total) -} - -func (a *API) GetTag(uid uint32) (*chunk.Tag, error) { return a.tags.Get(uid) } - // MimeOctetStream default value of http Content-Type header const MimeOctetStream = "application/octet-stream" diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 91bdeee139..423cc68601 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -146,7 +146,7 @@ func TestApiPut(t *testing.T) { } // TestApiTagLarge tests that the the number of chunks counted is larger for a larger input -func TestApiTagLarge(t *testing.T) { +func xTestApiTagLarge(t *testing.T) { testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { ctx := context.TODO() _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", toEncrypt) @@ -537,7 +537,7 @@ func TestDetectContentType(t *testing.T) { func putRandomContent(ctx context.Context, a *API, contentLength int, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { randomContentReader := io.LimitReader(crand.Reader, int64(contentLength)) - tag, err := a.NewTag("unnamed-tag", 0) + tag, err := a.Tags.New("unnamed-tag", 0) log.Trace("created new tag", "uid", tag.Uid) @@ -565,7 +565,7 @@ func putRandomContent(ctx context.Context, a *API, contentLength int, contentTyp // putString provides singleton manifest creation on top of api.API func putString(ctx context.Context, a *API, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { r := strings.NewReader(content) - tag, err := a.NewTag("unnamed-tag", 0) + tag, err := a.Tags.New("unnamed-tag", 0) log.Trace("created new tag", "uid", tag.Uid) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 78ff6a8ea8..5d056b00b4 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -355,7 +355,7 @@ func TestClientMultipartUpload(t *testing.T) { } // check the tag was created successfully - checkTag(t, srv.Tags, 9, 7, 7, 9) + checkTag(t, srv.Tags, 9, 9, 7, 9) // check we can download the individual files checkDownloadFile := func(path string) { @@ -606,6 +606,7 @@ func TestClientCreateUpdateFeed(t *testing.T) { } func checkTag(t *testing.T, tags *chunk.Tags, split, stored, seen, total int) { + t.Helper() i := 0 // check that the tag was created and incremented accordingly tags.Range(func(k, v interface{}) bool { diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 3f698e7e72..2b071455aa 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/spancontext" @@ -87,7 +88,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { }) } -func InitUploadTag(h http.Handler, a *api.API) http.Handler { +func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var ( tagName string @@ -118,10 +119,12 @@ func InitUploadTag(h http.Handler, a *api.API) http.Handler { } log.Trace("creating tag", "tagName", tagName, "estimatedTotal", estimatedTotal) - t, err := a.NewTag(tagName, estimatedTotal) + t, err := tags.New(tagName, estimatedTotal) if err != nil { log.Error("error creating tag", "err", err, "tagName", tagName) } + + log.Trace("setting tag id to context", "uid", t.Uid) ctx := sctx.SetTag(r.Context(), t.Uid) h.ServeHTTP(w, r.WithContext(ctx)) diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index d4e81d7f67..c851a3992e 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -79,7 +79,7 @@ func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s } func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) + log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code, "msg", msg) respondTemplate(w, r, "error", msg, code) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 90d5fd7c15..4ad402ef93 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -98,6 +98,12 @@ func NewServer(api *api.API, corsString string) *Server { InstrumentOpenTracing, } + tagAdapter := Adapter(func(h http.Handler) http.Handler { + return InitUploadTag(h, api.Tags) + }) + + defaultPostMiddlewares := append(defaultMiddlewares, tagAdapter) + mux := http.NewServeMux() mux.Handle("/bzz:/", methodHandler{ "GET": Adapt( @@ -106,15 +112,7 @@ func NewServer(api *api.API, corsString string) *Server { ), "POST": Adapt( http.HandlerFunc(server.HandlePostFiles), - Adapter(func(h http.Handler) http.Handler { - return InitUploadTag(h, api) - }), - RecoverPanic, - SetRequestID, - SetRequestHost, - InitLoggingResponseWriter, - ParseURI, - InstrumentOpenTracing, + defaultPostMiddlewares..., ), "DELETE": Adapt( http.HandlerFunc(server.HandleDelete), @@ -128,15 +126,7 @@ func NewServer(api *api.API, corsString string) *Server { ), "POST": Adapt( http.HandlerFunc(server.HandlePostRaw), - Adapter(func(h http.Handler) http.Handler { - return InitUploadTag(h, api) - }), - RecoverPanic, - SetRequestID, - SetRequestHost, - InitLoggingResponseWriter, - ParseURI, - InstrumentOpenTracing, + defaultPostMiddlewares..., ), }) mux.Handle("/bzz-immutable:/", methodHandler{ @@ -167,12 +157,6 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares..., ), }) - mux.Handle("/bzz-tag:/", methodHandler{ - "GET": Adapt( - http.HandlerFunc(server.HandleGetTag), - defaultMiddlewares..., - ), - }) mux.Handle("/", methodHandler{ "GET": Adapt( @@ -201,11 +185,6 @@ type Server struct { listenAddr string } -// HandleGetTag handles the HTTP GET for a certain (or all) tags -func (s *Server) HandleGetTag(w http.ResponseWriter, r *http.Request) { - log.Debug("handleGetTag", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) -} - func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) if r.Header.Get("Accept") == "application/x-tar" { @@ -262,7 +241,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { log.Debug("handle.post.raw", "ruid", ruid) tagUid := sctx.GetTag(r.Context()) - tag, err := s.api.GetTag(tagUid) + tag, err := s.api.Tags.Get(tagUid) if err != nil { log.Error("handle post raw got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) } @@ -350,7 +329,6 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { } log.Debug("new manifest", "ruid", ruid, "key", addr) } - newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error { switch contentType { case "application/x-tar": @@ -373,7 +351,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { return } tagUid := sctx.GetTag(r.Context()) - tag, err := s.api.GetTag(tagUid) + tag, err := s.api.Tags.Get(tagUid) if err != nil { log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) @@ -390,7 +368,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) { - log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context())) + log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()), "tag", sctx.GetTag(r.Context())) defaultPath := r.URL.Query().Get("defaultpath") diff --git a/swarm/api/http/test_server.go b/swarm/api/http/test_server.go index 542191cc0a..a3be01e992 100644 --- a/swarm/api/http/test_server.go +++ b/swarm/api/http/test_server.go @@ -65,6 +65,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso tss := &TestSwarmServer{ Server: apiServer, FileStore: fileStore, + Tags: tags, dir: swarmDir, Hasher: storage.MakeHashFunc(storage.DefaultHash)(), cleanup: func() { @@ -75,7 +76,6 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso os.RemoveAll(feedsDir) }, CurrentTime: 42, - Tags: tags, } feed.TimestampProvider = tss return tss diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 359ac11acb..71b4857c15 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -202,7 +202,6 @@ func (tag *Tag) UnmarshalBinary(buffer []byte) error { tag.Name = string(buffer[t:]) return nil - } func encodeUint32Append(buffer *[]byte, val uint32) { diff --git a/swarm/network_test.go b/swarm/network_test.go index 0b31b4bdc6..1a8c992a3a 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -536,7 +536,7 @@ func retrieve( // putString provides singleton manifest creation on top of api.API func putString(ctx context.Context, a *api.API, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { r := strings.NewReader(content) - tag, err := a.NewTag("unnamed-tag", 0) + tag, err := a.Tags.New("unnamed-tag", 0) log.Trace("created new tag", "uid", tag.Uid) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 2754cb3642..70f1de1bc3 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -23,7 +23,6 @@ import ( "sync" "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/localstore" ) @@ -88,8 +87,7 @@ func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChu isEncrypted = len(addr) > f.hashFunc().Size() tag, err := f.tags.GetContext(ctx) if err != nil { - tag = chunk.NewTag(0, "ephemeral", 0) //note: how to handle this. this function does not return an error - log.Error("did not find a tag for retrieve, creating ephemeral") + tag = chunk.NewTag(0, "ephemeral-retrieval-tag", 0) } getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted, tag) reader = TreeJoin(ctx, addr, getter, 0) @@ -101,8 +99,12 @@ func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChu func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) { tag, err := f.tags.GetContext(ctx) if err != nil { - //tag = chunk.NewTag(0, "", 0) //create an ephemeral tag - return nil, nil, err + // some of the parts of the codebase, namely the manifest trie, do not store the context + // of the original request nor the tag with the trie, recalculating the trie hence + // loses the tag uid. thus we create an ephemeral tag here for that purpose + + tag = chunk.NewTag(0, "", 0) + //return nil, nil, err } putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt, tag) return PyramidSplit(ctx, data, putter, putter, tag) diff --git a/swarm/swarm_test.go b/swarm/swarm_test.go index 2a5b28513a..cf2afaec9e 100644 --- a/swarm/swarm_test.go +++ b/swarm/swarm_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/sctx" ) // TestNewSwarm validates Swarm fields in repsect to the provided configuration. @@ -352,8 +353,11 @@ func testLocalStoreAndRetrieve(t *testing.T, swarm *Swarm, n int, randomData boo rand.Read(slice) } dataPut := string(slice) - - ctx := context.TODO() + tag, err := swarm.api.Tags.New("test-local-store-and-retrieve", 0) + if err != nil { + t.Fatal(err) + } + ctx := sctx.SetTag(context.Background(), tag.Uid) k, wait, err := swarm.api.Store(ctx, strings.NewReader(dataPut), int64(len(dataPut)), false) if err != nil { t.Fatal(err) From 4061571aac7cc9b5a3084950dbbc9274358f987e Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 1 May 2019 11:05:13 +0900 Subject: [PATCH 04/18] swarm: fix linter, move stored increment so that tests dont flake --- swarm/api/api.go | 2 -- swarm/storage/hasherstore.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7c49c24527..96fb86e1cc 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -54,8 +54,6 @@ import ( var ( apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) - apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) - apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) apiGetHTTP300 = metrics.NewRegisteredCounter("api.get.http.300", nil) diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index f4a811da74..4a4aa1d703 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -243,9 +243,9 @@ func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryptio func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { atomic.AddUint64(&h.nrChunks, 1) + h.tag.Inc(chunk.STORED) // this has to be here otherwise tests flake go func() { seen, err := h.store.Put(ctx, chunk.ModePutUpload, ch) - h.tag.Inc(chunk.STORED) if seen { h.tag.Inc(chunk.SEEN) } From 8734b3b29b9c8d6f0d66e1e34ab3d2c9261ffd9f Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 2 May 2019 00:00:23 +0900 Subject: [PATCH 05/18] swarm: WIP address PR comments --- swarm/api/api_test.go | 33 ++++++++++++++---------- swarm/api/client/client.go | 8 +++--- swarm/api/client/client_test.go | 45 +++------------------------------ swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 9 +++---- swarm/storage/hasherstore.go | 2 +- swarm/testutil/tag.go | 33 +++++++++++++++--------- 7 files changed, 54 insertions(+), 78 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 423cc68601..d542be0f7d 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -141,15 +141,17 @@ func TestApiPut(t *testing.T) { } resp := testGet(t, api, addr.Hex(), "") checkResponse(t, resp, exp) - testutil.CheckTag(t, tags, chunk.SPLIT, 2, 2) //1 chunk data, 1 chunk manifest + testutil.CheckTag(t, tags, 2, 2, 0, 2) //1 chunk data, 1 chunk manifest }) } // TestApiTagLarge tests that the the number of chunks counted is larger for a larger input -func xTestApiTagLarge(t *testing.T) { +func TestApiTagLarge(t *testing.T) { testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { ctx := context.TODO() - _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", toEncrypt) + //(data length / 4096) + 128 + // nr of data chunks divided by 128 for unencrypted, 64 for encrypted, till u get to 1, add one root chunk + _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", true) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -157,10 +159,12 @@ func xTestApiTagLarge(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - testutil.CheckTag(t, tags, chunk.SPLIT, 4129, 4129) //11 chunks random data, 1 chunk manifest - testutil.CheckTag(t, tags, chunk.SEEN, 0, 4129) //0 chunks seen, 12 total + if toEncrypt { + } else { + testutil.CheckTag(t, tags, 4129, 4129, 0, 4129) + //testutil.CheckTag() //whatever + } }) - } // testResolver implements the Resolver interface and either returns the given @@ -546,19 +550,20 @@ func putRandomContent(ctx context.Context, a *API, contentLength int, contentTyp if err != nil { return nil, nil, err } - manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) - r := strings.NewReader(manifest) - key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) + //manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) + //r := strings.NewReader(manifest) + //key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) if err != nil { return nil, nil, err } tag.DoneSplit(key) return key, func(ctx context.Context) error { - err := waitContent(ctx) - if err != nil { - return err - } - return waitManifest(ctx) + return waitContent(ctx) + /* err := waitContent(ctx) + if err != nil { + return err + } + return waitManifest(ctx)*/ }, nil } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index a5a61c6851..acc14bbb9a 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -49,6 +49,8 @@ var ( ErrUnauthorized = errors.New("unauthorized") ) +const SwarmTagHeaderName = "x-swarm-tag" + func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, @@ -75,7 +77,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, err return "", err } req.ContentLength = size - req.Header.Set("x-swarm-tag", fmt.Sprintf("raw_upload_%d", time.Now().Unix())) + req.Header.Set(SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix())) res, err := http.DefaultClient.Do(req) if err != nil { @@ -537,7 +539,7 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t } log.Trace("setting upload tag", "tag", tag) - req.Header.Set("x-swarm-tag", tag) + req.Header.Set(SwarmTagHeaderName, tag) // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request @@ -604,7 +606,7 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) mw := multipart.NewWriter(reqW) req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) - req.Header.Set("x-swarm-tag", fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) + req.Header.Set(SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) // define an UploadFn which adds files to the multipart form uploadFn := func(file *File) error { diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 5d056b00b4..793cbd1dd6 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" @@ -68,7 +67,7 @@ func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { } // check the tag was created successfully - checkTag(t, srv.Tags, 1, 1, 0, 1) + testutil.CheckTag(t, srv.Tags, 1, 1, 0, 1) // check we can download the same data res, isEncrypted, err := client.DownloadRaw(hash) @@ -212,7 +211,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { } // check the tag was created successfully - checkTag(t, srv.Tags, 9, 9, 0, 9) + testutil.CheckTag(t, srv.Tags, 9, 9, 0, 9) // check we can download the individual files checkDownloadFile := func(path string, expected []byte) { @@ -355,7 +354,7 @@ func TestClientMultipartUpload(t *testing.T) { } // check the tag was created successfully - checkTag(t, srv.Tags, 9, 9, 7, 9) + testutil.CheckTag(t, srv.Tags, 9, 9, 7, 9) // check we can download the individual files checkDownloadFile := func(path string) { @@ -604,41 +603,3 @@ func TestClientCreateUpdateFeed(t *testing.T) { t.Fatalf("Expected: %v, got %v", databytes, gotData) } } - -func checkTag(t *testing.T, tags *chunk.Tags, split, stored, seen, total int) { - t.Helper() - i := 0 - // check that the tag was created and incremented accordingly - tags.Range(func(k, v interface{}) bool { - vv, ok := v.(*chunk.Tag) - if !ok { - t.Fatal("error unmarshalling tag pointer") - } - - tSplit := vv.Get(chunk.SPLIT) - if tSplit != split { - t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) - } - - tSeen := vv.Get(chunk.SEEN) - if tSeen != seen { - t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) - } - - tStored := vv.Get(chunk.STORED) - if tStored != stored { - t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) - } - - tTotal := vv.Total() - if tTotal != total { - t.Fatalf("mismatch total chunks, got %d want %d", tTotal, total) - } - i++ - - return false - }) - if i == 0 { - t.Fatal("no tags found") - } -} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 4ad402ef93..62510ce4cb 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -350,9 +350,9 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } + tagUid := sctx.GetTag(r.Context()) tag, err := s.api.Tags.Get(tagUid) - if err != nil { log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 3ee853024f..1275f69b3d 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -37,7 +37,6 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" @@ -767,7 +766,7 @@ func testBzzTar(encrypted bool, t *testing.T) { } // check that the tag was written correctly - testutil.CheckTag(t, srv.Tags, chunk.SPLIT, 4, 4) + testutil.CheckTag(t, srv.Tags, 4, 4, 0, 4) swarmHash, err := ioutil.ReadAll(resp2.Body) resp2.Body.Close() @@ -856,6 +855,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) { c := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() req, err := http.NewRequest("POST", srv.URL+"/bzz:/", pr) if err != nil { t.Fatal(err) @@ -863,7 +863,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) { req = req.WithContext(ctx) req.ContentLength = 1000000 - req.Header.Add("x-swarm-tag", "1000000") + req.Header.Add(SwarmTagHeaderName, "1000000") go func() { for { @@ -886,9 +886,8 @@ func TestBzzCorrectTagEstimate(t *testing.T) { t.Log(err) } time.Sleep(100 * time.Millisecond) - testutil.CheckTag(t, srv.Tags, chunk.SEEN, 0, 244) + testutil.CheckTag(t, srv.Tags, 0, 0, 0, 244) close(c) - cancel() } // TestBzzRootRedirect tests that getting the root path of a manifest without diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 4a4aa1d703..f4a811da74 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -243,9 +243,9 @@ func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryptio func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { atomic.AddUint64(&h.nrChunks, 1) - h.tag.Inc(chunk.STORED) // this has to be here otherwise tests flake go func() { seen, err := h.store.Put(ctx, chunk.ModePutUpload, ch) + h.tag.Inc(chunk.STORED) if seen { h.tag.Inc(chunk.SEEN) } diff --git a/swarm/testutil/tag.go b/swarm/testutil/tag.go index 2b8ca25690..21d83820de 100644 --- a/swarm/testutil/tag.go +++ b/swarm/testutil/tag.go @@ -23,28 +23,37 @@ import ( ) // CheckTag checks the first tag in the api struct to be in a certain state -func CheckTag(t *testing.T, tags *chunk.Tags, state chunk.State, exp, expTotal int) { +func CheckTag(t *testing.T, tags *chunk.Tags, split, stored, seen, total int) { t.Helper() i := 0 + // check that the tag was created and incremented accordingly tags.Range(func(k, v interface{}) bool { - i++ - tag := v.(*chunk.Tag) - count, total, err := tag.Status(state) - if err != nil { - t.Fatal(err) + vv := v.(*chunk.Tag) + + tSplit := vv.Get(chunk.SPLIT) + if tSplit != split { + t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) + } + + tSeen := vv.Get(chunk.SEEN) + if tSeen != seen { + t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) } - if count != exp { - t.Fatalf("expected count to be %d, got %d", exp, count) + tStored := vv.Get(chunk.STORED) + if tStored != stored { + t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) } - if total != expTotal { - t.Fatalf("expected total to be %d, got %d", expTotal, total) + tTotal := vv.Total() + if tTotal != total { + t.Fatalf("mismatch total chunks, got %d want %d", tTotal, total) } + i++ + return false }) - if i == 0 { - t.Fatal("did not find any tags") + t.Fatal("no tags found") } } From e4b472f5aff90bdcf46eaeb9b6cd663bc7176d6c Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 2 May 2019 08:55:07 +0900 Subject: [PATCH 06/18] swarm/chunk: add Tags.All() method --- swarm/chunk/tags.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index 07f9b8cd77..e33e79c1ac 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -55,6 +55,19 @@ func (ts *Tags) New(s string, total int) (*Tag, error) { return t, nil } +func (ts *Tags) All() []*Tag { + t := make([]*Tag, 0) + + ts.tags.Range(func(k, v interface{}) bool { + tag := v.(*Tag) + t = append(t, tag) + + return true + }) + + return t +} + // Get returns the undelying tag for the uid or an error if not found func (ts *Tags) Get(uid uint32) (*Tag, error) { t, ok := ts.tags.Load(uid) From d01da184b139b00dce6be3bb61a9e533f3b92e06 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 2 May 2019 09:04:38 +0900 Subject: [PATCH 07/18] swarm/api, swarm/testutil: mend to use Tags.All() when CheckTag runs --- swarm/api/api_test.go | 6 ++-- swarm/api/client/client_test.go | 9 ++++-- swarm/api/http/server_test.go | 6 ++-- swarm/chunk/tags_test.go | 47 ++++++++++++++++++++++++++++ swarm/testutil/tag.go | 54 ++++++++++++++------------------- 5 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 swarm/chunk/tags_test.go diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index d542be0f7d..836c082b93 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -141,7 +141,8 @@ func TestApiPut(t *testing.T) { } resp := testGet(t, api, addr.Hex(), "") checkResponse(t, resp, exp) - testutil.CheckTag(t, tags, 2, 2, 0, 2) //1 chunk data, 1 chunk manifest + tag := tags.All()[0] + testutil.CheckTag(t, tag, 2, 2, 0, 2) //1 chunk data, 1 chunk manifest }) } @@ -161,7 +162,8 @@ func TestApiTagLarge(t *testing.T) { } if toEncrypt { } else { - testutil.CheckTag(t, tags, 4129, 4129, 0, 4129) + tag := tags.All()[0] + testutil.CheckTag(t, tag, 4129, 4129, 0, 4129) //testutil.CheckTag() //whatever } }) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 793cbd1dd6..92489849c4 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -67,7 +67,8 @@ func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { } // check the tag was created successfully - testutil.CheckTag(t, srv.Tags, 1, 1, 0, 1) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 1, 1, 0, 1) // check we can download the same data res, isEncrypted, err := client.DownloadRaw(hash) @@ -211,7 +212,8 @@ func TestClientUploadDownloadDirectory(t *testing.T) { } // check the tag was created successfully - testutil.CheckTag(t, srv.Tags, 9, 9, 0, 9) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 9, 9, 0, 9) // check we can download the individual files checkDownloadFile := func(path string, expected []byte) { @@ -354,7 +356,8 @@ func TestClientMultipartUpload(t *testing.T) { } // check the tag was created successfully - testutil.CheckTag(t, srv.Tags, 9, 9, 7, 9) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 9, 9, 7, 9) // check we can download the individual files checkDownloadFile := func(path string) { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1275f69b3d..8c6e2a8414 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -766,7 +766,8 @@ func testBzzTar(encrypted bool, t *testing.T) { } // check that the tag was written correctly - testutil.CheckTag(t, srv.Tags, 4, 4, 0, 4) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 4, 4, 0, 4) swarmHash, err := ioutil.ReadAll(resp2.Body) resp2.Body.Close() @@ -886,7 +887,8 @@ func TestBzzCorrectTagEstimate(t *testing.T) { t.Log(err) } time.Sleep(100 * time.Millisecond) - testutil.CheckTag(t, srv.Tags, 0, 0, 0, 244) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 0, 0, 0, 244) close(c) } diff --git a/swarm/chunk/tags_test.go b/swarm/chunk/tags_test.go new file mode 100644 index 0000000000..b041d95a97 --- /dev/null +++ b/swarm/chunk/tags_test.go @@ -0,0 +1,47 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package chunk + +import "testing" + +func TestAll(t *testing.T) { + ts := NewTags() + + ts.New("1", 1) + ts.New("2", 1) + + all := ts.All() + + if len(all) != 2 { + t.Fatalf("expected length to be 2 got %d", len(all)) + } + + if all[0].Total() != 1 { + t.Fatal("mismatch") + } + if all[1].Total() != 1 { + t.Fatal("mismatch") + } + + ts.New("3", 1) + all = ts.All() + + if len(all) != 3 { + t.Fatalf("expected length to be 2 got %d", len(all)) + } + +} diff --git a/swarm/testutil/tag.go b/swarm/testutil/tag.go index 21d83820de..a3edee0900 100644 --- a/swarm/testutil/tag.go +++ b/swarm/testutil/tag.go @@ -23,37 +23,29 @@ import ( ) // CheckTag checks the first tag in the api struct to be in a certain state -func CheckTag(t *testing.T, tags *chunk.Tags, split, stored, seen, total int) { +func CheckTag(t *testing.T, tag *chunk.Tag, split, stored, seen, total int) { t.Helper() - i := 0 - // check that the tag was created and incremented accordingly - tags.Range(func(k, v interface{}) bool { - vv := v.(*chunk.Tag) - - tSplit := vv.Get(chunk.SPLIT) - if tSplit != split { - t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) - } - - tSeen := vv.Get(chunk.SEEN) - if tSeen != seen { - t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) - } - - tStored := vv.Get(chunk.STORED) - if tStored != stored { - t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) - } - - tTotal := vv.Total() - if tTotal != total { - t.Fatalf("mismatch total chunks, got %d want %d", tTotal, total) - } - i++ - - return false - }) - if i == 0 { - t.Fatal("no tags found") + if tag == nil { + t.Fatal("no tag found") + } + + tSplit := tag.Get(chunk.SPLIT) + if tSplit != split { + t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) + } + + tSeen := tag.Get(chunk.SEEN) + if tSeen != seen { + t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) + } + + tStored := tag.Get(chunk.STORED) + if tStored != stored { + t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) + } + + tTotal := tag.Total() + if tTotal != total { + t.Fatalf("mismatch total chunks, got %d want %d", tTotal, total) } } From 44469a21e31b03061f4a09d021ed664baa884fbf Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 2 May 2019 09:11:43 +0900 Subject: [PATCH 08/18] swarm: rename chunk states as per janos recommendation --- swarm/api/http/server.go | 2 +- swarm/chunk/tag.go | 34 +++++++++++++++++----------------- swarm/chunk/tag_test.go | 36 ++++++++++++++++++------------------ swarm/storage/hasherstore.go | 4 ++-- swarm/storage/pyramid.go | 2 +- swarm/testutil/tag.go | 6 +++--- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 62510ce4cb..a1cfde813c 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -357,7 +357,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err) } - log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.SPLIT), "TOTAL", tag.Total()) + log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.StateSplit), "TOTAL", tag.Total()) tag.DoneSplit(newAddr) log.Debug("stored content", "ruid", ruid, "key", newAddr) diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 71b4857c15..5e337d86d5 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -34,11 +34,11 @@ var ( type State = uint32 const ( - SPLIT State = iota // chunk has been processed by filehasher/swarm safe call - STORED // chunk stored locally - SEEN // chunk previously seen - SENT // chunk sent to neighbourhood - SYNCED // proof is received; chunk removed from sync db; chunk is available everywhere + StateSplit State = iota // chunk has been processed by filehasher/swarm safe call + StateStored // chunk stored locally + StateSeen // chunk previously seen + StateSent // chunk sent to neighbourhood + StateSynced // proof is received; chunk removed from sync db; chunk is available everywhere ) // Tag represents info on the status of new chunks @@ -71,15 +71,15 @@ func NewTag(uid uint32, s string, total uint32) *Tag { func (t *Tag) Inc(state State) { var v *uint32 switch state { - case SPLIT: + case StateSplit: v = &t.split - case STORED: + case StateStored: v = &t.stored - case SEEN: + case StateSeen: v = &t.seen - case SENT: + case StateSent: v = &t.sent - case SYNCED: + case StateSynced: v = &t.synced } atomic.AddUint32(v, 1) @@ -89,15 +89,15 @@ func (t *Tag) Inc(state State) { func (t *Tag) Get(state State) int { var v *uint32 switch state { - case SPLIT: + case StateSplit: v = &t.split - case STORED: + case StateStored: v = &t.stored - case SEEN: + case StateSeen: v = &t.seen - case SENT: + case StateSent: v = &t.sent - case SYNCED: + case StateSynced: v = &t.synced } return int(atomic.LoadUint32(v)) @@ -124,9 +124,9 @@ func (t *Tag) Status(state State) (int, int, error) { return count, total, errNA } switch state { - case SPLIT, STORED, SEEN: + case StateSplit, StateStored, StateSeen: return count, total, nil - case SENT, SYNCED: + case StateSent, StateSynced: stored := int(atomic.LoadUint32(&t.stored)) if stored < total { return count, total - seen, errNA diff --git a/swarm/chunk/tag_test.go b/swarm/chunk/tag_test.go index b3f3be2cad..ac2dcacd7e 100644 --- a/swarm/chunk/tag_test.go +++ b/swarm/chunk/tag_test.go @@ -24,7 +24,7 @@ import ( ) var ( - allStates = []State{SPLIT, STORED, SEEN, SENT, SYNCED} + allStates = []State{StateSplit, StateStored, StateSeen, StateSent, StateSynced} ) // TestTagSingleIncrements tests if Inc increments the tag state value @@ -37,11 +37,11 @@ func TestTagSingleIncrements(t *testing.T) { expcount int exptotal int }{ - {state: SPLIT, inc: 10, expcount: 10, exptotal: 10}, - {state: STORED, inc: 9, expcount: 9, exptotal: 9}, - {state: SEEN, inc: 1, expcount: 1, exptotal: 10}, - {state: SENT, inc: 9, expcount: 9, exptotal: 9}, - {state: SYNCED, inc: 9, expcount: 9, exptotal: 9}, + {state: StateSplit, inc: 10, expcount: 10, exptotal: 10}, + {state: StateStored, inc: 9, expcount: 9, exptotal: 9}, + {state: StateSeen, inc: 1, expcount: 1, exptotal: 10}, + {state: StateSent, inc: 9, expcount: 9, exptotal: 9}, + {state: StateSynced, inc: 9, expcount: 9, exptotal: 9}, } for _, tc := range tc { @@ -60,24 +60,24 @@ func TestTagSingleIncrements(t *testing.T) { // TestTagStatus is a unit test to cover Tag.Status method functionality func TestTagStatus(t *testing.T) { tg := &Tag{total: 10} - tg.Inc(SEEN) - tg.Inc(SENT) - tg.Inc(SYNCED) + tg.Inc(StateSeen) + tg.Inc(StateSent) + tg.Inc(StateSynced) for i := 0; i < 10; i++ { - tg.Inc(SPLIT) - tg.Inc(STORED) + tg.Inc(StateSplit) + tg.Inc(StateStored) } for _, v := range []struct { state State expVal int expTotal int }{ - {state: STORED, expVal: 10, expTotal: 10}, - {state: SPLIT, expVal: 10, expTotal: 10}, - {state: SEEN, expVal: 1, expTotal: 10}, - {state: SENT, expVal: 1, expTotal: 9}, - {state: SYNCED, expVal: 1, expTotal: 9}, + {state: StateStored, expVal: 10, expTotal: 10}, + {state: StateSplit, expVal: 10, expTotal: 10}, + {state: StateSeen, expVal: 1, expTotal: 10}, + {state: StateSent, expVal: 1, expTotal: 9}, + {state: StateSynced, expVal: 1, expTotal: 9}, } { val, total, err := tg.Status(v.state) if err != nil { @@ -98,8 +98,8 @@ func TestTagETA(t *testing.T) { maxDiff := 100000 // 100 microsecond tg := &Tag{total: 10, startedAt: now} time.Sleep(100 * time.Millisecond) - tg.Inc(SPLIT) - eta, err := tg.ETA(SPLIT) + tg.Inc(StateSplit) + eta, err := tg.ETA(StateSplit) if err != nil { t.Fatal(err) } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index f4a811da74..1e702f11ae 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -245,9 +245,9 @@ func (h *hasherStore) storeChunk(ctx context.Context, ch Chunk) { atomic.AddUint64(&h.nrChunks, 1) go func() { seen, err := h.store.Put(ctx, chunk.ModePutUpload, ch) - h.tag.Inc(chunk.STORED) + h.tag.Inc(chunk.StateStored) if seen { - h.tag.Inc(chunk.SEEN) + h.tag.Inc(chunk.StateSeen) } select { case h.errC <- err: diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index cb731cb918..9b0d5397b8 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -275,7 +275,7 @@ func (pc *PyramidChunker) processor(ctx context.Context, id int64) { return } pc.processChunk(ctx, id, job) - pc.tag.Inc(chunk.SPLIT) + pc.tag.Inc(chunk.StateSplit) case <-pc.quitC: return } diff --git a/swarm/testutil/tag.go b/swarm/testutil/tag.go index a3edee0900..8801b7ab60 100644 --- a/swarm/testutil/tag.go +++ b/swarm/testutil/tag.go @@ -29,17 +29,17 @@ func CheckTag(t *testing.T, tag *chunk.Tag, split, stored, seen, total int) { t.Fatal("no tag found") } - tSplit := tag.Get(chunk.SPLIT) + tSplit := tag.Get(chunk.StateSplit) if tSplit != split { t.Fatalf("should have had split chunks, got %d want %d", tSplit, split) } - tSeen := tag.Get(chunk.SEEN) + tSeen := tag.Get(chunk.StateSeen) if tSeen != seen { t.Fatalf("should have had seen chunks, got %d want %d", tSeen, seen) } - tStored := tag.Get(chunk.STORED) + tStored := tag.Get(chunk.StateStored) if tStored != stored { t.Fatalf("mismatch stored chunks, got %d want %d", tStored, stored) } From a6bc1672c05edf870f0bb5411d11767af7f1c2d2 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 2 May 2019 13:57:57 +0900 Subject: [PATCH 09/18] swarm/chunk: pr comments --- swarm/chunk/tags.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index e33e79c1ac..d9de27bdf7 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -55,12 +55,10 @@ func (ts *Tags) New(s string, total int) (*Tag, error) { return t, nil } -func (ts *Tags) All() []*Tag { - t := make([]*Tag, 0) - +// All returns all existing tags in Tags' sync.Map +func (ts *Tags) All() (t []*Tag) { ts.tags.Range(func(k, v interface{}) bool { - tag := v.(*Tag) - t = append(t, tag) + t = append(t, v.(*Tag)) return true }) From 212ebf22b7c821b8cb3368a6b0d905db1df80809 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 13:25:01 +0900 Subject: [PATCH 10/18] swarm/api/http: add function to calculate number of chunks for a certain content-length --- swarm/api/http/server.go | 22 +++++++++++++++++++ swarm/api/http/server_test.go | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index a1cfde813c..e652bcda1c 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -26,6 +26,7 @@ import ( "fmt" "io" "io/ioutil" + "math" "mime" "mime/multipart" "net/http" @@ -863,6 +864,27 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) } +func CalculateNumberOfChunks(contentLength int, isEncrypted bool) int { + if contentLength < 4096 { + return 1 + } + branchingFactor := 128 + if isEncrypted { + branchingFactor = 64 + } + + dataChunks := math.Ceil(float64(contentLength) / float64(4096)) + totalChunks := dataChunks + intermediate := float64(dataChunks) / float64(branchingFactor) + + for intermediate > 1 { + totalChunks += math.Ceil(intermediate) + intermediate = intermediate / float64(branchingFactor) + } + + return int(totalChunks) + 1 +} + // The size of buffer used for bufio.Reader on LazyChunkReader passed to // http.ServeContent in HandleGetFile. // Warning: This value influences the number of chunk requests and chunker join goroutines diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 8c6e2a8414..93439861ee 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -1339,6 +1339,46 @@ func TestBzzGetFileWithResolver(t *testing.T) { } } +// TestCalculateNumberOfChunks is a unit test for the chunk-number-according-to-content-length +// calculation +func TestCalculateNumberOfChunks(t *testing.T) { + + //test cases: + for _, tc := range []struct{ len, chunks int }{ + {len: 1000, chunks: 1}, + {len: 5000, chunks: 3}, + {len: 10000, chunks: 4}, + {len: 100000, chunks: 26}, + {len: 1000000, chunks: 248}, + {len: 325839339210, chunks: 79550620 + 621490 + 4856 + 38 + 1}, + } { + res := CalculateNumberOfChunks(tc.len, false) + if res != tc.chunks { + t.Fatalf("expected result for %d bytes to be %d got %d", tc.len, tc.chunks, res) + } + } +} + +// TestCalculateNumberOfChunksEncrypted is a unit test for the chunk-number-according-to-content-length +// calculation with encryption (branching factor=64) +func TestCalculateNumberOfChunksEncrypted(t *testing.T) { + + //test cases: + for _, tc := range []struct{ len, chunks int }{ + {len: 1000, chunks: 1}, + {len: 5000, chunks: 3}, + {len: 10000, chunks: 4}, + {len: 100000, chunks: 26}, + {len: 1000000, chunks: 245 + 4 + 1}, + {len: 325839339210, chunks: 79550620 + 1242979 + 19422 + 304 + 5 + 1}, + } { + res := CalculateNumberOfChunks(tc.len, true) + if res != tc.chunks { + t.Fatalf("expected result for %d bytes to be %d got %d", tc.len, tc.chunks, res) + } + } +} + // testResolver implements the Resolver interface and either returns the given // hash if it is set, or returns a "name not found" error type testResolveValidator struct { @@ -1364,6 +1404,7 @@ func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) { func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) { return } + func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) { return } From d26205435c07ab911789be66cfd687f49e902063 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 15:52:36 +0900 Subject: [PATCH 11/18] swarm: fix api test, fix chunk estimate test, move to int64 due to content length and number of chunks in a trie --- swarm/api/api_test.go | 37 ++++++++------- swarm/api/http/middleware.go | 29 ++++++------ swarm/api/http/server.go | 4 +- swarm/api/http/server_test.go | 6 +-- swarm/chunk/tag.go | 89 ++++++++++++++++++----------------- swarm/chunk/tag_test.go | 14 +++--- swarm/chunk/tags.go | 4 +- swarm/testutil/tag.go | 2 +- 8 files changed, 95 insertions(+), 90 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 836c082b93..c167fe9ca3 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -46,19 +46,20 @@ func init() { } func testAPI(t *testing.T, f func(*API, *chunk.Tags, bool)) { - datadir, err := ioutil.TempDir("", "bzz-test") - if err != nil { - t.Fatalf("unable to create temp dir: %v", err) - } - defer os.RemoveAll(datadir) - tags := chunk.NewTags() - fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), tags) - if err != nil { - return + for _, v := range []bool{true, false} { + datadir, err := ioutil.TempDir("", "bzz-test") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + defer os.RemoveAll(datadir) + tags := chunk.NewTags() + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), tags) + if err != nil { + return + } + api := NewAPI(fileStore, nil, nil, nil, tags) + f(api, tags, v) } - api := NewAPI(fileStore, nil, nil, nil, tags) - f(api, tags, false) - f(api, tags, true) } type testResponse struct { @@ -150,9 +151,7 @@ func TestApiPut(t *testing.T) { func TestApiTagLarge(t *testing.T) { testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { ctx := context.TODO() - //(data length / 4096) + 128 - // nr of data chunks divided by 128 for unencrypted, 64 for encrypted, till u get to 1, add one root chunk - _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", true) + _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -160,11 +159,15 @@ func TestApiTagLarge(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if toEncrypt { + tag := tags.All()[0] + expect := int64(4095 + 64 + 1) + testutil.CheckTag(t, tag, expect, expect, 0, expect) } else { tag := tags.All()[0] - testutil.CheckTag(t, tag, 4129, 4129, 0, 4129) - //testutil.CheckTag() //whatever + expect := int64(4095 + 32 + 1) + testutil.CheckTag(t, tag, expect, expect, 0, expect) } }) } diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 2b071455aa..a60b57bfef 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "runtime/debug" - "strconv" "strings" "time" @@ -93,10 +92,9 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler { var ( tagName string err error - estimatedTotal = 0 - contentType = r.Header.Get("Content-Type") - contentLength = r.Header.Get("Content-Length") - headerTag = r.Header.Get(SwarmTagHeaderName) + estimatedTotal int64 = 0 + contentType = r.Header.Get("Content-Type") + headerTag = r.Header.Get(SwarmTagHeaderName) ) if headerTag != "" { tagName = headerTag @@ -105,18 +103,19 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler { tagName = fmt.Sprintf("unnamed_tag_%d", time.Now().Unix()) } - log.Trace("trying to estimate tag size", "contentType", contentType, "contentLength", contentLength, "cl", r.ContentLength) - - if !strings.Contains(contentType, "multipart") && contentLength != "" { - estimatedTotal, err = strconv.Atoi(contentLength) - if err != nil { - log.Error("error parsing content-length string, falling back to 0", "contentLength", contentLength) - estimatedTotal = 0 - } else { - estimatedTotal = estimatedTotal / 4096 + if !strings.Contains(contentType, "multipart") && r.ContentLength > 0 { + log.Trace("calculating tag size", "contentType", contentType, "contentLength", r.ContentLength) + uri := GetURI(r.Context()) + if uri != nil { + log.Debug("got uri from context") + if uri.Addr == "encrypt" { + estimatedTotal = CalculateNumberOfChunks(r.ContentLength, true) + } else { + estimatedTotal = CalculateNumberOfChunks(r.ContentLength, false) + } } - } + log.Trace("creating tag", "tagName", tagName, "estimatedTotal", estimatedTotal) t, err := tags.New(tagName, estimatedTotal) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index e652bcda1c..fdcaa1ecf6 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -864,7 +864,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) } -func CalculateNumberOfChunks(contentLength int, isEncrypted bool) int { +func CalculateNumberOfChunks(contentLength int64, isEncrypted bool) int64 { if contentLength < 4096 { return 1 } @@ -882,7 +882,7 @@ func CalculateNumberOfChunks(contentLength int, isEncrypted bool) int { intermediate = intermediate / float64(branchingFactor) } - return int(totalChunks) + 1 + return int64(totalChunks) + 1 } // The size of buffer used for bufio.Reader on LazyChunkReader passed to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 93439861ee..0ff235586e 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -888,7 +888,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) { } time.Sleep(100 * time.Millisecond) tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 0, 0, 0, 244) + testutil.CheckTag(t, tag, 0, 0, 0, 248) close(c) } @@ -1344,7 +1344,7 @@ func TestBzzGetFileWithResolver(t *testing.T) { func TestCalculateNumberOfChunks(t *testing.T) { //test cases: - for _, tc := range []struct{ len, chunks int }{ + for _, tc := range []struct{ len, chunks int64 }{ {len: 1000, chunks: 1}, {len: 5000, chunks: 3}, {len: 10000, chunks: 4}, @@ -1364,7 +1364,7 @@ func TestCalculateNumberOfChunks(t *testing.T) { func TestCalculateNumberOfChunksEncrypted(t *testing.T) { //test cases: - for _, tc := range []struct{ len, chunks int }{ + for _, tc := range []struct{ len, chunks int64 }{ {len: 1000, chunks: 1}, {len: 5000, chunks: 3}, {len: 10000, chunks: 4}, diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 5e337d86d5..755864b10e 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -46,18 +46,18 @@ type Tag struct { Uid uint32 // a unique identifier for this tag Name string // a name tag for this tag Address Address // the associated swarm hash for this tag - total uint32 // total chunks belonging to a tag - split uint32 // number of chunks already processed by splitter for hashing - seen uint32 // number of chunks already seen - stored uint32 // number of chunks already stored locally - sent uint32 // number of chunks sent for push syncing - synced uint32 // number of chunks synced with proof + total int64 // total chunks belonging to a tag + split int64 // number of chunks already processed by splitter for hashing + seen int64 // number of chunks already seen + stored int64 // number of chunks already stored locally + sent int64 // number of chunks sent for push syncing + synced int64 // number of chunks synced with proof startedAt time.Time // tag started to calculate ETA } // New creates a new tag, stores it by the name and returns it // it returns an error if the tag with this name already exists -func NewTag(uid uint32, s string, total uint32) *Tag { +func NewTag(uid uint32, s string, total int64) *Tag { t := &Tag{ Uid: uid, Name: s, @@ -69,7 +69,7 @@ func NewTag(uid uint32, s string, total uint32) *Tag { // Inc increments the count for a state func (t *Tag) Inc(state State) { - var v *uint32 + var v *int64 switch state { case StateSplit: v = &t.split @@ -82,12 +82,12 @@ func (t *Tag) Inc(state State) { case StateSynced: v = &t.synced } - atomic.AddUint32(v, 1) + atomic.AddInt64(v, 1) } // Get returns the count for a state on a tag -func (t *Tag) Get(state State) int { - var v *uint32 +func (t *Tag) Get(state State) int64 { + var v *int64 switch state { case StateSplit: v = &t.split @@ -100,26 +100,26 @@ func (t *Tag) Get(state State) int { case StateSynced: v = &t.synced } - return int(atomic.LoadUint32(v)) + return atomic.LoadInt64(v) } // GetTotal returns the total count -func (t *Tag) Total() int { - return int(atomic.LoadUint32(&t.total)) +func (t *Tag) Total() int64 { + return atomic.LoadInt64(&t.total) } // DoneSplit sets total count to SPLIT count and sets the associated swarm hash for this tag // is meant to be called when splitter finishes for input streams of unknown size -func (t *Tag) DoneSplit(address Address) int { - total := atomic.LoadUint32(&t.split) - atomic.StoreUint32(&t.total, total) +func (t *Tag) DoneSplit(address Address) int64 { + total := atomic.LoadInt64(&t.split) + atomic.StoreInt64(&t.total, total) t.Address = address - return int(total) + return total } // Status returns the value of state and the total count -func (t *Tag) Status(state State) (int, int, error) { - count, seen, total := t.Get(state), int(atomic.LoadUint32(&t.seen)), int(atomic.LoadUint32(&t.total)) +func (t *Tag) Status(state State) (int64, int64, error) { + count, seen, total := t.Get(state), atomic.LoadInt64(&t.seen), atomic.LoadInt64(&t.total) if total == 0 { return count, total, errNA } @@ -127,7 +127,7 @@ func (t *Tag) Status(state State) (int, int, error) { case StateSplit, StateStored, StateSeen: return count, total, nil case StateSent, StateSynced: - stored := int(atomic.LoadUint32(&t.stored)) + stored := atomic.LoadInt64(&t.stored) if stored < total { return count, total - seen, errNA } @@ -152,14 +152,15 @@ func (t *Tag) ETA(state State) (time.Time, error) { // MarshalBinary marshals the tag into a byte slice func (tag *Tag) MarshalBinary() (data []byte, err error) { - buffer := make([]byte, 0) - encodeUint32Append(&buffer, tag.Uid) - encodeUint32Append(&buffer, tag.total) - encodeUint32Append(&buffer, tag.split) - encodeUint32Append(&buffer, tag.seen) - encodeUint32Append(&buffer, tag.stored) - encodeUint32Append(&buffer, tag.sent) - encodeUint32Append(&buffer, tag.synced) + buffer := make([]byte, 4) + binary.BigEndian.PutUint32(buffer, tag.Uid) + //encodeUint64Append(&buffer, tag.Uid) + encodeUint64Append(&buffer, tag.total) + encodeUint64Append(&buffer, tag.split) + encodeUint64Append(&buffer, tag.seen) + encodeUint64Append(&buffer, tag.stored) + encodeUint64Append(&buffer, tag.sent) + encodeUint64Append(&buffer, tag.synced) intBuffer := make([]byte, 8) @@ -181,14 +182,16 @@ func (tag *Tag) UnmarshalBinary(buffer []byte) error { if len(buffer) < 13 { return errors.New("buffer too short") } + tag.Uid = binary.BigEndian.Uint32(buffer) + buffer = buffer[4:] - tag.Uid = decodeUint32Splice(&buffer) - tag.total = decodeUint32Splice(&buffer) - tag.split = decodeUint32Splice(&buffer) - tag.seen = decodeUint32Splice(&buffer) - tag.stored = decodeUint32Splice(&buffer) - tag.sent = decodeUint32Splice(&buffer) - tag.synced = decodeUint32Splice(&buffer) + //tag.Uid = decodeInt64Splice(&buffer) + tag.total = decodeInt64Splice(&buffer) + tag.split = decodeInt64Splice(&buffer) + tag.seen = decodeInt64Splice(&buffer) + tag.stored = decodeInt64Splice(&buffer) + tag.sent = decodeInt64Splice(&buffer) + tag.synced = decodeInt64Splice(&buffer) t, n := binary.Varint(buffer) tag.startedAt = time.Unix(t, 0) @@ -204,14 +207,14 @@ func (tag *Tag) UnmarshalBinary(buffer []byte) error { return nil } -func encodeUint32Append(buffer *[]byte, val uint32) { - intBuffer := make([]byte, 4) - binary.BigEndian.PutUint32(intBuffer, val) - *buffer = append(*buffer, intBuffer...) +func encodeUint64Append(buffer *[]byte, val int64) { + intBuffer := make([]byte, 8) + n := binary.PutVarint(intBuffer, val) + *buffer = append(*buffer, intBuffer[:n]...) } -func decodeUint32Splice(buffer *[]byte) uint32 { - val := binary.BigEndian.Uint32((*buffer)[:4]) - *buffer = (*buffer)[4:] +func decodeInt64Splice(buffer *[]byte) int64 { + val, n := binary.Varint((*buffer)) + *buffer = (*buffer)[n:] return val } diff --git a/swarm/chunk/tag_test.go b/swarm/chunk/tag_test.go index ac2dcacd7e..e6acfb185b 100644 --- a/swarm/chunk/tag_test.go +++ b/swarm/chunk/tag_test.go @@ -34,8 +34,8 @@ func TestTagSingleIncrements(t *testing.T) { tc := []struct { state uint32 inc int - expcount int - exptotal int + expcount int64 + exptotal int64 }{ {state: StateSplit, inc: 10, expcount: 10, exptotal: 10}, {state: StateStored, inc: 9, expcount: 9, exptotal: 9}, @@ -70,8 +70,8 @@ func TestTagStatus(t *testing.T) { } for _, v := range []struct { state State - expVal int - expTotal int + expVal int64 + expTotal int64 }{ {state: StateStored, expVal: 10, expTotal: 10}, {state: StateSplit, expVal: 10, expTotal: 10}, @@ -128,7 +128,7 @@ func TestTagConcurrentIncrements(t *testing.T) { wg.Wait() for _, f := range allStates { v := tg.Get(f) - if v != n { + if v != int64(n) { t.Fatalf("expected state %v to be %v, got %v", f, n, v) } } @@ -142,7 +142,7 @@ func TestTagsMultipleConcurrentIncrementsSyncMap(t *testing.T) { wg.Add(10 * 5 * n) for i := 0; i < 10; i++ { s := string([]byte{uint8(i)}) - tag, err := ts.New(s, n) + tag, err := ts.New(s, int64(n)) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestTagsMultipleConcurrentIncrementsSyncMap(t *testing.T) { t.Fatal(err) } stateVal := tag.Get(f) - if stateVal != n { + if stateVal != int64(n) { t.Fatalf("expected tag %v state %v to be %v, got %v", uid, f, n, v) } } diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index d9de27bdf7..d5e3e64d48 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -42,12 +42,12 @@ func NewTags() *Tags { // New creates a new tag, stores it by the name and returns it // it returns an error if the tag with this name already exists -func (ts *Tags) New(s string, total int) (*Tag, error) { +func (ts *Tags) New(s string, total int64) (*Tag, error) { t := &Tag{ Uid: ts.rng.Uint32(), Name: s, startedAt: time.Now(), - total: uint32(total), + total: total, } if _, loaded := ts.tags.LoadOrStore(t.Uid, t); loaded { return nil, errExists diff --git a/swarm/testutil/tag.go b/swarm/testutil/tag.go index 8801b7ab60..d9908f11bd 100644 --- a/swarm/testutil/tag.go +++ b/swarm/testutil/tag.go @@ -23,7 +23,7 @@ import ( ) // CheckTag checks the first tag in the api struct to be in a certain state -func CheckTag(t *testing.T, tag *chunk.Tag, split, stored, seen, total int) { +func CheckTag(t *testing.T, tag *chunk.Tag, split, stored, seen, total int64) { t.Helper() if tag == nil { t.Fatal("no tag found") From 000dd3c9d36f1c1600b68f145093407b1ebaf700 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 16:43:57 +0900 Subject: [PATCH 12/18] swarm/api: mend flaking test, create test for ecnrypted chunk number assertion --- swarm/api/http/server.go | 4 +- swarm/api/http/server_test.go | 76 ++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index fdcaa1ecf6..6bd4217b4a 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -273,13 +273,15 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { return } - addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) + addr, wait, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) respondError(w, r, err.Error(), http.StatusInternalServerError) return } + wait(r.Context()) + tag.DoneSplit(addr) log.Debug("stored content", "ruid", ruid, "key", addr) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 0ff235586e..4d6437574e 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -849,47 +849,59 @@ func testBzzTar(encrypted bool, t *testing.T) { // by chunk size (4096). It is needed to be checked BEFORE chunking is done, therefore // concurrency was introduced to slow down the HTTP request func TestBzzCorrectTagEstimate(t *testing.T) { - srv := NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() + for _, v := range []struct { + toEncrypt bool + expChunks int64 + }{ + {toEncrypt: false, expChunks: 248}, + {toEncrypt: true, expChunks: 250}, + } { + srv := NewTestSwarmServer(t, serverFunc, nil) + defer srv.Close() - pr, pw := io.Pipe() - c := make(chan struct{}) + pr, pw := io.Pipe() + c := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - req, err := http.NewRequest("POST", srv.URL+"/bzz:/", pr) - if err != nil { - t.Fatal(err) - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + addr := "" + if v.toEncrypt { + addr = "encrypt" + } + req, err := http.NewRequest("POST", srv.URL+"/bzz:/"+addr, pr) + if err != nil { + t.Fatal(err) + } - req = req.WithContext(ctx) - req.ContentLength = 1000000 - req.Header.Add(SwarmTagHeaderName, "1000000") + req = req.WithContext(ctx) + req.ContentLength = 1000000 + req.Header.Add(SwarmTagHeaderName, "1000000") - go func() { - for { - select { - case <-c: - return - default: - _, err := pw.Write([]byte{0}) - if err != nil { + go func() { + for { + select { + case <-c: return + default: + _, err := pw.Write([]byte{0}) + if err != nil { + return + } + time.Sleep(100 * time.Millisecond) } - time.Sleep(100 * time.Millisecond) } - } - }() + }() - client := &http.Client{} - _, err = client.Do(req) - if err != nil { - t.Log(err) + client := &http.Client{} + _, err = client.Do(req) + if err != nil { + t.Log(err) + } + time.Sleep(100 * time.Millisecond) + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 0, 0, 0, v.expChunks) + close(c) } - time.Sleep(100 * time.Millisecond) - tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 0, 0, 0, 248) - close(c) } // TestBzzRootRedirect tests that getting the root path of a manifest without From 6cdf4d83cc0c4c8640388809336a18ef91a99a3b Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 17:13:53 +0900 Subject: [PATCH 13/18] swarm/api/http: inline needed functionality --- swarm/api/api_test.go | 49 +++++++++++-------------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index c167fe9ca3..4a5f923626 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -124,7 +124,6 @@ func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse { } reader.Seek(0, 0) return &testResponse{reader, &Response{mimeType, status, size, string(s)}} - // return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}} } func TestApiPut(t *testing.T) { @@ -149,16 +148,23 @@ func TestApiPut(t *testing.T) { // TestApiTagLarge tests that the the number of chunks counted is larger for a larger input func TestApiTagLarge(t *testing.T) { + const contentLength = 4096 * 4095 testAPI(t, func(api *API, tags *chunk.Tags, toEncrypt bool) { - ctx := context.TODO() - _, wait, err := putRandomContent(ctx, api, 4096*4095, "text/plain", toEncrypt) + randomContentReader := io.LimitReader(crand.Reader, int64(contentLength)) + tag, err := api.Tags.New("unnamed-tag", 0) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Fatal(err) } - err = wait(ctx) + ctx := sctx.SetTag(context.Background(), tag.Uid) + key, waitContent, err := api.Store(ctx, randomContentReader, int64(contentLength), toEncrypt) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Fatal(err) + } + err = waitContent(ctx) + if err != nil { + t.Fatal(err) } + tag.DoneSplit(key) if toEncrypt { tag := tags.All()[0] @@ -541,37 +547,6 @@ func TestDetectContentType(t *testing.T) { } } -// putRandomContent provides singleton manifest creation on top of API. it uploads an arbitrary byte stream -// of the desired contentLength and wraps it in a manifest -func putRandomContent(ctx context.Context, a *API, contentLength int, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { - randomContentReader := io.LimitReader(crand.Reader, int64(contentLength)) - - tag, err := a.Tags.New("unnamed-tag", 0) - - log.Trace("created new tag", "uid", tag.Uid) - - cCtx := sctx.SetTag(ctx, tag.Uid) - key, waitContent, err := a.Store(cCtx, randomContentReader, int64(contentLength), toEncrypt) - if err != nil { - return nil, nil, err - } - //manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) - //r := strings.NewReader(manifest) - //key, waitManifest, err := a.Store(cCtx, r, int64(len(manifest)), toEncrypt) - if err != nil { - return nil, nil, err - } - tag.DoneSplit(key) - return key, func(ctx context.Context) error { - return waitContent(ctx) - /* err := waitContent(ctx) - if err != nil { - return err - } - return waitManifest(ctx)*/ - }, nil -} - // putString provides singleton manifest creation on top of api.API func putString(ctx context.Context, a *API, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { r := strings.NewReader(content) From 4d0be7c6abe36f56d78631ed365c0f176c841b84 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 17:27:59 +0900 Subject: [PATCH 14/18] swarm/api/http: remove api client dependency --- swarm/api/http/server_test.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 4d6437574e..f2252b61a9 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -1114,21 +1114,10 @@ func TestGet(t *testing.T) { func TestModify(t *testing.T) { srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() - - swarmClient := swarm.NewClient(srv.URL) - data := []byte("data") - file := &swarm.File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), - ManifestEntry: api.ManifestEntry{ - Path: "", - ContentType: "text/plain", - Size: int64(len(data)), - }, - } - - hash, err := swarmClient.Upload(file, "", false) - if err != nil { - t.Fatal(err) + headers := map[string]string{"Content-Type": "text/plain"} + res, hash := httpDo("POST", srv.URL+"/bzz:/", bytes.NewReader([]byte("data")), headers, false, t) + if res.StatusCode != http.StatusOK { + t.Fatalf("unexpected status code from server %d want %d", res.StatusCode, http.StatusOK) } for _, testCase := range []struct { From c68107fc38cf3cbb99381f780e42a79d6e7c4be4 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 17:35:07 +0900 Subject: [PATCH 15/18] swarm/api/http: remove client dependency --- swarm/api/http/server_test.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index f2252b61a9..5e420c3783 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" @@ -919,19 +918,11 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { defer srv.Close() // create a manifest with some data at the root path - client := swarm.NewClient(srv.URL) data := []byte("data") - file := &swarm.File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), - ManifestEntry: api.ManifestEntry{ - Path: "", - ContentType: "text/plain", - Size: int64(len(data)), - }, - } - hash, err := client.Upload(file, "", toEncrypt) - if err != nil { - t.Fatal(err) + headers := map[string]string{"Content-Type": "text/plain"} + res, hash := httpDo("POST", srv.URL+"/bzz:/", bytes.NewReader([]byte("data")), headers, false, t) + if res.StatusCode != http.StatusOK { + t.Fatalf("unexpected status code from server %d want %d", res.StatusCode, http.StatusOK) } // define a CheckRedirect hook which ensures there is only a single From 683d0c2140962eafc80037610f0f6544cdd1cc50 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 17:39:44 +0900 Subject: [PATCH 16/18] swarm/api/client: fix header name ref to http package after cyclic dependency solved --- swarm/api/client/client.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index acc14bbb9a..9ad0948f43 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/pborman/uuid" @@ -49,8 +50,6 @@ var ( ErrUnauthorized = errors.New("unauthorized") ) -const SwarmTagHeaderName = "x-swarm-tag" - func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, @@ -77,7 +76,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, err return "", err } req.ContentLength = size - req.Header.Set(SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix())) + req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix())) res, err := http.DefaultClient.Do(req) if err != nil { @@ -539,7 +538,7 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t } log.Trace("setting upload tag", "tag", tag) - req.Header.Set(SwarmTagHeaderName, tag) + req.Header.Set(swarmhttp.SwarmTagHeaderName, tag) // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request @@ -606,7 +605,7 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) mw := multipart.NewWriter(reqW) req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) - req.Header.Set(SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) + req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) // define an UploadFn which adds files to the multipart form uploadFn := func(file *File) error { From 8c71b9f9cb71330784be918775f21153e2060aaa Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 3 May 2019 17:55:52 +0900 Subject: [PATCH 17/18] swarm/api/http: minor cosmetics --- swarm/api/http/server.go | 1 - swarm/api/http/server_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6bd4217b4a..fc8b71b895 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -281,7 +281,6 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { } wait(r.Context()) - tag.DoneSplit(addr) log.Debug("stored content", "ruid", ruid, "key", addr) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 5e420c3783..c3f2e98a58 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -920,7 +920,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { // create a manifest with some data at the root path data := []byte("data") headers := map[string]string{"Content-Type": "text/plain"} - res, hash := httpDo("POST", srv.URL+"/bzz:/", bytes.NewReader([]byte("data")), headers, false, t) + res, hash := httpDo("POST", srv.URL+"/bzz:/", bytes.NewReader(data), headers, false, t) if res.StatusCode != http.StatusOK { t.Fatalf("unexpected status code from server %d want %d", res.StatusCode, http.StatusOK) } From f628f74ce7cfe1acbace8bda2736a1a0b3edc83d Mon Sep 17 00:00:00 2001 From: Elad Date: Sat, 4 May 2019 11:06:53 +0200 Subject: [PATCH 18/18] swarm: address pr comments --- swarm/api/http/middleware.go | 9 +++++-- swarm/api/http/server.go | 5 ++-- swarm/api/http/server_test.go | 45 ++++++++++++++++++++--------------- swarm/chunk/tag.go | 16 ++++++------- swarm/chunk/tags.go | 9 +++++-- swarm/chunk/tags_test.go | 11 +++++---- swarm/storage/filestore.go | 4 ++-- 7 files changed, 58 insertions(+), 41 deletions(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index a60b57bfef..e6e263f4c0 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -87,6 +87,11 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { }) } +// InitUploadTag creates a new tag for an upload to the local HTTP proxy +// if a tag is not named using the SwarmTagHeaderName, a fallback name will be used +// when the Content-Length header is set, an ETA on chunking will be available since the +// number of chunks to be split is known in advance (not including enclosing manifest chunks) +// the tag can later be accessed using the appropriate identifier in the request context func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var ( @@ -109,9 +114,9 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler { if uri != nil { log.Debug("got uri from context") if uri.Addr == "encrypt" { - estimatedTotal = CalculateNumberOfChunks(r.ContentLength, true) + estimatedTotal = calculateNumberOfChunks(r.ContentLength, true) } else { - estimatedTotal = CalculateNumberOfChunks(r.ContentLength, false) + estimatedTotal = calculateNumberOfChunks(r.ContentLength, false) } } } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index fc8b71b895..a336bd82ff 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -865,7 +865,8 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) } -func CalculateNumberOfChunks(contentLength int64, isEncrypted bool) int64 { +// calculateNumberOfChunks calculates the number of chunks in an arbitrary content length +func calculateNumberOfChunks(contentLength int64, isEncrypted bool) int64 { if contentLength < 4096 { return 1 } @@ -876,7 +877,7 @@ func CalculateNumberOfChunks(contentLength int64, isEncrypted bool) int64 { dataChunks := math.Ceil(float64(contentLength) / float64(4096)) totalChunks := dataChunks - intermediate := float64(dataChunks) / float64(branchingFactor) + intermediate := dataChunks / float64(branchingFactor) for intermediate > 1 { totalChunks += math.Ceil(intermediate) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index c3f2e98a58..1de41d18d3 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -848,6 +848,9 @@ func testBzzTar(encrypted bool, t *testing.T) { // by chunk size (4096). It is needed to be checked BEFORE chunking is done, therefore // concurrency was introduced to slow down the HTTP request func TestBzzCorrectTagEstimate(t *testing.T) { + srv := NewTestSwarmServer(t, serverFunc, nil) + defer srv.Close() + for _, v := range []struct { toEncrypt bool expChunks int64 @@ -855,11 +858,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) { {toEncrypt: false, expChunks: 248}, {toEncrypt: true, expChunks: 250}, } { - srv := NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - pr, pw := io.Pipe() - c := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -879,27 +878,35 @@ func TestBzzCorrectTagEstimate(t *testing.T) { go func() { for { select { - case <-c: + case <-ctx.Done(): return - default: + case <-time.After(1 * time.Millisecond): _, err := pw.Write([]byte{0}) if err != nil { - return + t.Error(err) } - time.Sleep(100 * time.Millisecond) } } }() - - client := &http.Client{} - _, err = client.Do(req) - if err != nil { - t.Log(err) + go func() { + transport := http.DefaultTransport + _, err := transport.RoundTrip(req) + if err != nil { + t.Error(err) + } + }() + done := false + for !done { + switch len(srv.Tags.All()) { + case 0: + <-time.After(10 * time.Millisecond) + case 1: + tag := srv.Tags.All()[0] + testutil.CheckTag(t, tag, 0, 0, 0, v.expChunks) + srv.Tags.Delete(tag.Uid) + done = true + } } - time.Sleep(100 * time.Millisecond) - tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 0, 0, 0, v.expChunks) - close(c) } } @@ -1344,7 +1351,7 @@ func TestCalculateNumberOfChunks(t *testing.T) { {len: 1000000, chunks: 248}, {len: 325839339210, chunks: 79550620 + 621490 + 4856 + 38 + 1}, } { - res := CalculateNumberOfChunks(tc.len, false) + res := calculateNumberOfChunks(tc.len, false) if res != tc.chunks { t.Fatalf("expected result for %d bytes to be %d got %d", tc.len, tc.chunks, res) } @@ -1364,7 +1371,7 @@ func TestCalculateNumberOfChunksEncrypted(t *testing.T) { {len: 1000000, chunks: 245 + 4 + 1}, {len: 325839339210, chunks: 79550620 + 1242979 + 19422 + 304 + 5 + 1}, } { - res := CalculateNumberOfChunks(tc.len, true) + res := calculateNumberOfChunks(tc.len, true) if res != tc.chunks { t.Fatalf("expected result for %d bytes to be %d got %d", tc.len, tc.chunks, res) } diff --git a/swarm/chunk/tag.go b/swarm/chunk/tag.go index 755864b10e..ee700d22bf 100644 --- a/swarm/chunk/tag.go +++ b/swarm/chunk/tag.go @@ -154,13 +154,12 @@ func (t *Tag) ETA(state State) (time.Time, error) { func (tag *Tag) MarshalBinary() (data []byte, err error) { buffer := make([]byte, 4) binary.BigEndian.PutUint32(buffer, tag.Uid) - //encodeUint64Append(&buffer, tag.Uid) - encodeUint64Append(&buffer, tag.total) - encodeUint64Append(&buffer, tag.split) - encodeUint64Append(&buffer, tag.seen) - encodeUint64Append(&buffer, tag.stored) - encodeUint64Append(&buffer, tag.sent) - encodeUint64Append(&buffer, tag.synced) + encodeInt64Append(&buffer, tag.total) + encodeInt64Append(&buffer, tag.split) + encodeInt64Append(&buffer, tag.seen) + encodeInt64Append(&buffer, tag.stored) + encodeInt64Append(&buffer, tag.sent) + encodeInt64Append(&buffer, tag.synced) intBuffer := make([]byte, 8) @@ -185,7 +184,6 @@ func (tag *Tag) UnmarshalBinary(buffer []byte) error { tag.Uid = binary.BigEndian.Uint32(buffer) buffer = buffer[4:] - //tag.Uid = decodeInt64Splice(&buffer) tag.total = decodeInt64Splice(&buffer) tag.split = decodeInt64Splice(&buffer) tag.seen = decodeInt64Splice(&buffer) @@ -207,7 +205,7 @@ func (tag *Tag) UnmarshalBinary(buffer []byte) error { return nil } -func encodeUint64Append(buffer *[]byte, val int64) { +func encodeInt64Append(buffer *[]byte, val int64) { intBuffer := make([]byte, 8) n := binary.PutVarint(intBuffer, val) *buffer = append(*buffer, intBuffer[:n]...) diff --git a/swarm/chunk/tags.go b/swarm/chunk/tags.go index d5e3e64d48..435f5d7068 100644 --- a/swarm/chunk/tags.go +++ b/swarm/chunk/tags.go @@ -56,6 +56,7 @@ func (ts *Tags) New(s string, total int64) (*Tag, error) { } // All returns all existing tags in Tags' sync.Map +// Note that tags are returned in no particular order func (ts *Tags) All() (t []*Tag) { ts.tags.Range(func(k, v interface{}) bool { t = append(t, v.(*Tag)) @@ -75,8 +76,8 @@ func (ts *Tags) Get(uid uint32) (*Tag, error) { return t.(*Tag), nil } -// GetContext gets a tag from the tag uid stored in the context -func (ts *Tags) GetContext(ctx context.Context) (*Tag, error) { +// GetFromContext gets a tag from the tag uid stored in the context +func (ts *Tags) GetFromContext(ctx context.Context) (*Tag, error) { uid := sctx.GetTag(ctx) t, ok := ts.tags.Load(uid) if !ok { @@ -89,3 +90,7 @@ func (ts *Tags) GetContext(ctx context.Context) (*Tag, error) { func (ts *Tags) Range(fn func(k, v interface{}) bool) { ts.tags.Range(fn) } + +func (ts *Tags) Delete(k interface{}) { + ts.tags.Delete(k) +} diff --git a/swarm/chunk/tags_test.go b/swarm/chunk/tags_test.go index b041d95a97..f818c4c5ca 100644 --- a/swarm/chunk/tags_test.go +++ b/swarm/chunk/tags_test.go @@ -30,18 +30,19 @@ func TestAll(t *testing.T) { t.Fatalf("expected length to be 2 got %d", len(all)) } - if all[0].Total() != 1 { - t.Fatal("mismatch") + if n := all[0].Total(); n != 1 { + t.Fatalf("expected tag 0 total to be 1 got %d", n) } - if all[1].Total() != 1 { - t.Fatal("mismatch") + + if n := all[1].Total(); n != 1 { + t.Fatalf("expected tag 1 total to be 1 got %d", n) } ts.New("3", 1) all = ts.All() if len(all) != 3 { - t.Fatalf("expected length to be 2 got %d", len(all)) + t.Fatalf("expected length to be 3 got %d", len(all)) } } diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 70f1de1bc3..dc096e56cb 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -85,7 +85,7 @@ func NewFileStore(store ChunkStore, params *FileStoreParams, tags *chunk.Tags) * // It returns a reader with the chunk data and whether the content was encrypted func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) { isEncrypted = len(addr) > f.hashFunc().Size() - tag, err := f.tags.GetContext(ctx) + tag, err := f.tags.GetFromContext(ctx) if err != nil { tag = chunk.NewTag(0, "ephemeral-retrieval-tag", 0) } @@ -97,7 +97,7 @@ func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChu // Store is a public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) { - tag, err := f.tags.GetContext(ctx) + tag, err := f.tags.GetFromContext(ctx) if err != nil { // some of the parts of the codebase, namely the manifest trie, do not store the context // of the original request nor the tag with the trie, recalculating the trie hence