swarm: Added cache-control headers for immutable content (ethereum#383)

swarm/api/api.go

@@ -23,7 +23,6 @@ import (
 	"math/big"
 	"net/http"
 	"path"
-	"regexp"
 	"strings"
 
 	"bytes"
@@ -40,9 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-// TODO: this is bad, it should not be hardcoded how long is a hash
-var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?$")
-
 type ErrResourceReturn struct {
 	key string
 }
@@ -256,33 +252,37 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
 	apiResolveCount.Inc(1)
 	log.Trace("resolving", "uri", uri.Addr)
 
-	// if the URI is immutable, check if the address is a hash
-	isHash := hashMatcher.MatchString(uri.Addr)
+	// if the URI is immutable, check if the address looks like a hash
 	if uri.Immutable() {
-		if !isHash {
+		key := uri.Key()
+		if key == nil {
 			return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
 		}
-		return common.Hex2Bytes(uri.Addr), nil
+		return key, nil
 	}
 
 	// if DNS is not configured, check if the address is a hash
 	if self.dns == nil {
-		if !isHash {
+		key := uri.Key()
+		if key == nil {
 			apiResolveFail.Inc(1)
 			return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
 		}
-		return common.Hex2Bytes(uri.Addr), nil
+		return key, nil
 	}
 
 	// try and resolve the address
 	resolved, err := self.dns.Resolve(uri.Addr)
 	if err == nil {
 		return resolved[:], nil
-	} else if !isHash {
+	}
+
+	key := uri.Key()
+	if key == nil {
 		apiResolveFail.Inc(1)
 		return nil, err
 	}
-	return common.Hex2Bytes(uri.Addr), nil
+	return key, nil
 }
 
 // Put provides singleton manifest creation on top of dpa store
@@ -309,23 +309,23 @@ func (self *Api) Put(content, contentType string, toEncrypt bool) (k storage.Key
 
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve basePath to content using dpa retrieve
-// it returns a section reader, mimeType, status and an error
-func (self *Api) Get(key storage.Key, path string) (reader *storage.LazyChunkReader, mimeType string, status int, err error) {
-	log.Debug("api.get", "key", key, "path", path)
+// it returns a section reader, mimeType, status, the key of the actual content and an error
+func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.LazyChunkReader, mimeType string, status int, contentKey storage.Key, err error) {
+	log.Debug("api.get", "key", manifestKey, "path", path)
 	apiGetCount.Inc(1)
-	trie, err := loadManifest(self.dpa, key, nil)
+	trie, err := loadManifest(self.dpa, manifestKey, nil)
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
 		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
 		return
 	}
 
-	log.Debug("trie getting entry", "key", key, "path", path)
+	log.Debug("trie getting entry", "key", manifestKey, "path", path)
 	entry, _ := trie.getEntry(path)
 
 	if entry != nil {
-		log.Debug("trie got entry", "key", key, "path", path, "entry.Hash", entry.Hash)
+		log.Debug("trie got entry", "key", manifestKey, "path", path, "entry.Hash", entry.Hash)
 		// we want to be able to serve Mutable Resource Updates transparently using the bzz:// scheme
 		//
 		// we use a special manifest hack for this purpose, which is pathless and where the resource root key
@@ -339,7 +339,7 @@ func (self *Api) Get(key storage.Key, path string) (reader *storage.LazyChunkRea
 		// if the resource update is of multihash type:
 		// we validate the multihash and retrieve the manifest behind it, and resume normal operations from there
 		if entry.ContentType == ResourceContentType {
-			log.Trace("resource type", "key", key, "hash", entry.Hash)
+			log.Trace("resource type", "key", manifestKey, "hash", entry.Hash)
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			rsrc, err := self.resource.LookupLatestByName(ctx, entry.Hash, true, &storage.ResourceLookupParams{})
@@ -348,7 +348,7 @@ func (self *Api) Get(key storage.Key, path string) (reader *storage.LazyChunkRea
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
 				log.Warn(fmt.Sprintf("get resource content error: %v", err))
-				return reader, mimeType, status, err
+				return reader, mimeType, status, nil, err
 			}
 			if rsrc.Multihash {
 
@@ -358,58 +358,58 @@ func (self *Api) Get(key storage.Key, path string) (reader *storage.LazyChunkRea
 					apiGetInvalid.Inc(1)
 					status = http.StatusInternalServerError
 					log.Warn(fmt.Sprintf("could not decode resource multihash: %v", err))
-					return reader, mimeType, status, err
+					return reader, mimeType, status, nil, err
 				} else if decodedMultihash.Code != multihash.KECCAK_256 {
 					apiGetInvalid.Inc(1)
 					status = http.StatusUnprocessableEntity
 					log.Warn(fmt.Sprintf("invalid resource multihash code: %x", decodedMultihash.Code))
-					return reader, mimeType, status, err
+					return reader, mimeType, status, nil, err
 				}
-				key = storage.Key(decodedMultihash.Digest)
-				log.Trace("resource is multihash", "key", key)
+				manifestKey = storage.Key(decodedMultihash.Digest)
+				log.Trace("resource is multihash", "key", manifestKey)
 
 				// get the manifest the multihash digest points to
-				trie, err := loadManifest(self.dpa, key, nil)
+				trie, err := loadManifest(self.dpa, manifestKey, nil)
 				if err != nil {
 					apiGetNotFound.Inc(1)
 					status = http.StatusNotFound
 					log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
-					return reader, mimeType, status, err
+					return reader, mimeType, status, nil, err
 				}
 
-				log.Trace("trie getting resource multihash entry", "key", key, "path", path)
+				log.Trace("trie getting resource multihash entry", "key", manifestKey, "path", path)
 				var fullpath string
 				entry, fullpath = trie.getEntry(path)
-				log.Trace("trie got resource multihash entry", "key", key, "path", path, "entry", entry, "fullpath", fullpath)
+				log.Trace("trie got resource multihash entry", "key", manifestKey, "path", path, "entry", entry, "fullpath", fullpath)
 
 				if entry == nil {
 					status = http.StatusNotFound
 					apiGetNotFound.Inc(1)
 					err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
-					log.Trace("manifest (resource multihash) entry not found", "key", key, "path", path)
-					return reader, mimeType, status, err
+					log.Trace("manifest (resource multihash) entry not found", "key", manifestKey, "path", path)
+					return reader, mimeType, status, nil, err
 				}
 
 			} else {
-				return nil, entry.ContentType, http.StatusOK, &ErrResourceReturn{entry.Hash}
+				return nil, entry.ContentType, http.StatusOK, nil, &ErrResourceReturn{entry.Hash}
 			}
 		}
 
-		key = common.Hex2Bytes(entry.Hash)
+		contentKey = common.Hex2Bytes(entry.Hash)
 		status = entry.Status
 		if status == http.StatusMultipleChoices {
 			apiGetHttp300.Inc(1)
-			return nil, entry.ContentType, status, err
+			return nil, entry.ContentType, status, contentKey, err
 		} else {
 			mimeType = entry.ContentType
-			log.Debug("content lookup key", "key", key, "mimetype", mimeType)
-			reader, _ = self.dpa.Retrieve(key)
+			log.Debug("content lookup key", "key", contentKey, "mimetype", mimeType)
+			reader, _ = self.dpa.Retrieve(contentKey)
 		}
 	} else {
 		status = http.StatusNotFound
 		apiGetNotFound.Inc(1)
 		err = fmt.Errorf("manifest entry for '%s' not found", path)
-		log.Trace("manifest entry not found", "key", key, "path", path)
+		log.Trace("manifest entry not found", "key", contentKey, "path", path)
 	}
 	return
 }

swarm/api/api_test.go

@@ -86,7 +86,7 @@ func expResponse(content string, mimeType string, status int) *Response {
 // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
 func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
 	key := storage.Key(common.Hex2Bytes(bzzhash))
-	reader, mimeType, status, err := api.Get(key, path)
+	reader, mimeType, status, _, err := api.Get(key, path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

swarm/api/filesystem_test.go

@@ -63,7 +63,7 @@ func TestApiDirUpload0(t *testing.T) {
 		checkResponse(t, resp, exp)
 
 		key := storage.Key(common.Hex2Bytes(bzzhash))
-		_, _, _, err = api.Get(key, "")
+		_, _, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -137,7 +137,7 @@ func TestApiDirUploadModify(t *testing.T) {
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, err = api.Get(key, "")
+		_, _, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}

swarm/api/http/server.go

@@ -543,12 +543,22 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
 func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri)
 	getCount.Inc(1)
-	key, err := s.api.Resolve(r.uri)
-	if err != nil {
-		getFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
-		return
+	var err error
+	var resolvedKey bool // indicates if the key was resolved by DNS
+	var etag string
+	key := r.uri.Key()
+	if key == nil {
+		key, err = s.api.Resolve(r.uri)
+		if err != nil {
+			getFail.Inc(1)
+			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			return
+		}
+		resolvedKey = true // url was of type bzz://name.eth/path , so we won't set cache headers
+	} else {
+		resolvedKey = false // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
+
 	log.Debug("handle.get: resolved", "ruid", r.ruid, "key", key)
 
 	// if path is set, interpret <key> as a manifest and return the
@@ -590,6 +600,9 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 			return
 		}
 		key = storage.Key(common.Hex2Bytes(entry.Hash))
+		etag = entry.Hash
+	} else {
+		etag = common.Bytes2Hex(key)
 	}
 
 	// check the root chunk exists by retrieving the file's size
@@ -602,6 +615,11 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 
 	w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
 
+	if !resolvedKey { // url was of type bzz-raw://<hex key>, so we are sure it is immutable.
+		w.Header().Set("Cache-Control", "max-age=2147483648, immutable")
+	}
+	w.Header().Set("ETag", etag) // set etag to hash of actually delivered content
+
 	switch {
 	case r.uri.Raw():
 		// allow the request to overwrite the content type using a query
@@ -812,16 +830,25 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
+	var err error
+	manifestKey := r.uri.Key()
+	var resolvedKey bool // indicates if the key was resolved by DNS
 
-	key, err := s.api.Resolve(r.uri)
-	if err != nil {
-		getFileFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
-		return
+	if manifestKey == nil {
+		manifestKey, err = s.api.Resolve(r.uri)
+		if err != nil {
+			getFileFail.Inc(1)
+			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			return
+		}
+		resolvedKey = true // url was of type bzz://name.eth/path , so we won't set cache headers
+	} else {
+		resolvedKey = false // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
-	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", key)
 
-	reader, contentType, status, err := s.api.Get(key, r.uri.Path)
+	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestKey)
+
+	reader, contentType, status, contentKey, err := s.api.Get(manifestKey, r.uri.Path)
 
 	if err != nil {
 		// cheeky, cheeky hack. See swarm/api/api.go:Api.Get() for an explanation
@@ -844,7 +871,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.getManifestList(key, r.uri.Path)
+		list, err := s.getManifestList(manifestKey, r.uri.Path)
 
 		if err != nil {
 			getFileFail.Inc(1)
@@ -866,6 +893,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	}
 
 	w.Header().Set("Content-Type", contentType)
+	if !resolvedKey { // url was of type bzz://<hex key>/path, so we are sure it is immutable.
+		w.Header().Set("Cache-Control", "max-age=2147483648, immutable")
+	}
+	w.Header().Set("ETag", common.Bytes2Hex(contentKey)) // set etag to hash of actually delivered content
 
 	http.ServeContent(w, &r.Request, "", time.Now(), reader)
 }

swarm/api/storage.go

@@ -66,7 +66,7 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 	if err != nil {
 		return nil, err
 	}
-	reader, mimeType, status, err := self.api.Get(key, uri.Path)
+	reader, mimeType, status, _, err := self.api.Get(key, uri.Path)
 	if err != nil {
 		return nil, err
 	}

swarm/api/uri.go

@@ -19,9 +19,17 @@ package api
 import (
 	"fmt"
 	"net/url"
+	"regexp"
 	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
+//matches hex swarm hashes
+// TODO: this is bad, it should not be hardcoded how long is a hash
+var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?$")
+
 // URI is a reference to content stored in swarm.
 type URI struct {
 	// Scheme has one of the following values:
@@ -38,6 +46,9 @@ type URI struct {
 	// resolves to a storage key
 	Addr string
 
+	// key stores the parsed storage key
+	key storage.Key
+
 	// Path is the path to the content within a swarm manifest
 	Path string
 }
@@ -84,7 +95,6 @@ func Parse(rawuri string) (*URI, error) {
 	}
 	return uri, nil
 }
-
 func (u *URI) Resource() bool {
 	return u.Scheme == "bzz-resource"
 }
@@ -108,3 +118,14 @@ func (u *URI) Hash() bool {
 func (u *URI) String() string {
 	return u.Scheme + ":/" + u.Addr + "/" + u.Path
 }
+
+func (u *URI) Key() storage.Key {
+	if u.key != nil {
+		return u.key
+	}
+	if hashMatcher.MatchString(u.Addr) {
+		u.key = common.Hex2Bytes(u.Addr)
+		return u.key
+	}
+	return nil
+}