From 5f4c62b8c64f63df740638536e1d47a6a0c506d5 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Tue, 19 Dec 2023 18:15:24 +0900 Subject: [PATCH 1/5] Support extended rules like proxy_cache_valid of NGINX --- rfc9111/shared.go | 62 +++++++++++++++++++++++++++++++- rfc9111/shared_test.go | 82 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/rfc9111/shared.go b/rfc9111/shared.go index 8be9dfe..7e6cc12 100644 --- a/rfc9111/shared.go +++ b/rfc9111/shared.go @@ -15,6 +15,15 @@ type Shared struct { understoodStatusCodes []int heuristicallyCacheableStatusCodes []int heuristicExpirationRatio float64 + extendedRules []ExtendedRule +} + +// ExtendedRule is an extended rule. +// Like proxy_cache_valid of NGINX. +// THIS IS NOT RFC 9111. +type ExtendedRule interface { + // Cacheable returns true and and the expiration time if the response is cacheable. + Cacheable(req *http.Request, res *http.Response) (ok bool, age time.Duration) } // SharedOption is an option for Shared. @@ -55,6 +64,14 @@ func HeuristicExpirationRatio(ratio float64) SharedOption { } } +// ExtendedRules sets the extended rules. +func ExtendedRules(rules []ExtendedRule) SharedOption { + return func(s *Shared) error { + s.extendedRules = rules + return nil + } +} + // NewShared returns a new Shared cache handler. func NewShared(opts ...SharedOption) (*Shared, error) { s := &Shared{ @@ -87,6 +104,10 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) // 3. Storing Responses in Caches (https://httpwg.org/specs/rfc9111.html#rfc.section.3) // - the request method is understood by the cache; if !contains(req.Method, s.understoodMethods) { + rescc := ParseResponseCacheControlHeader(res.Header.Values("Cache-Control")) + if !rescc.NoStore && !rescc.Private { + return s.storableWithExtendedRules(req, res, now) + } return false, time.Time{} } @@ -97,6 +118,10 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusProcessing, http.StatusEarlyHints, }) { + rescc := ParseResponseCacheControlHeader(res.Header.Values("Cache-Control")) + if !rescc.NoStore && !rescc.Private { + return s.storableWithExtendedRules(req, res, now) + } return false, time.Time{} } @@ -107,6 +132,9 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusPartialContent, http.StatusNotModified, }) || (rescc.MustUnderstand && !contains(res.StatusCode, s.understoodStatusCodes)) { + if !rescc.NoStore && !rescc.Private { + return s.storableWithExtendedRules(req, res, now) + } return false, time.Time{} } @@ -128,6 +156,9 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) expires := CalclateExpires(rescc, res.Header, s.heuristicExpirationRatio, now) if expires.Sub(now) <= 0 { + if expires.Sub(time.Time{}) == 0 { + return s.storableWithExtendedRules(req, res, now) + } return false, time.Time{} } @@ -160,6 +191,11 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) return true, expires } + ok, _ := s.storableWithExtendedRules(req, res, now) + if ok { + return true, expires + } + return false, time.Time{} } @@ -276,6 +312,29 @@ func (s *Shared) Handle(req *http.Request, cachedReq *http.Request, cachedRes *h return false, res, err } +func (s *Shared) storableWithExtendedRules(req *http.Request, res *http.Response, now time.Time) (bool, time.Time) { + for _, rule := range s.extendedRules { + ok, age := rule.Cacheable(req, res) + if ok { + // Add Expires header field + var expires time.Time + if res.Header.Get("Date") != "" { + date, err := http.ParseTime(res.Header.Get("Date")) + if err == nil { + expires = date.Add(age) + } else { + expires = now.Add(age) + } + } else { + expires = now.Add(age) + } + res.Header.Set("Expires", expires.UTC().Format(http.TimeFormat)) + return true, expires + } + } + return false, time.Time{} +} + func CalclateExpires(d *ResponseDirectives, header http.Header, heuristicExpirationRatio float64, now time.Time) time.Time { // 4.2.1. Calculating Freshness Lifetime // A cache can calculate the freshness lifetime (denoted as freshness_lifetime) of a response by evaluating the following rules and using the first match: @@ -319,7 +378,8 @@ func CalclateExpires(d *ResponseDirectives, header http.Header, heuristicExpirat } } - return now + // Can't calculate expires + return time.Time{} } func contains[T comparable](v T, vv []T) bool { diff --git a/rfc9111/shared_test.go b/rfc9111/shared_test.go index 4122890..792f886 100644 --- a/rfc9111/shared_test.go +++ b/rfc9111/shared_test.go @@ -9,6 +9,25 @@ import ( "github.com/google/go-cmp/cmp" ) +type testRule struct { + cacheableMethods []string + cacheableStatus []int + age time.Duration +} + +func (r *testRule) Cacheable(req *http.Request, res *http.Response) (bool, time.Duration) { + for _, m := range r.cacheableMethods { + if req.Method == m { + for _, s := range r.cacheableStatus { + if res.StatusCode == s { + return true, r.age + } + } + } + } + return false, 0 +} + func TestShared_Storable(t *testing.T) { now := time.Date(2024, 12, 13, 14, 15, 16, 00, time.UTC) @@ -16,6 +35,7 @@ func TestShared_Storable(t *testing.T) { name string req *http.Request res *http.Response + rules []ExtendedRule wantOK bool wantExpires time.Time }{ @@ -30,6 +50,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"s-maxage=10"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 26, 00, time.UTC), }, @@ -44,6 +65,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"max-age=15"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 31, 00, time.UTC), }, @@ -58,6 +80,7 @@ func TestShared_Storable(t *testing.T) { "Expires": []string{"Mon, 13 Dec 2024 14:15:20 GMT"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 20, 00, time.UTC), }, @@ -73,6 +96,7 @@ func TestShared_Storable(t *testing.T) { "Date": []string{"Mon, 13 Dec 2024 13:15:20 GMT"}, }, }, + nil, true, time.Date(2024, 12, 13, 15, 15, 16, 00, time.UTC), }, @@ -88,6 +112,7 @@ func TestShared_Storable(t *testing.T) { "Date": []string{"Mon, 13 Dec 2024 14:15:20 GMT"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 21, 00, time.UTC), }, @@ -102,6 +127,7 @@ func TestShared_Storable(t *testing.T) { "Last-Modified": []string{"Mon, 13 Dec 2024 14:15:06 GMT"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 17, 00, time.UTC), }, @@ -116,6 +142,7 @@ func TestShared_Storable(t *testing.T) { "Last-Modified": []string{"Mon, 13 Dec 2024 14:15:06 GMT"}, }, }, + nil, false, time.Time{}, }, @@ -130,6 +157,7 @@ func TestShared_Storable(t *testing.T) { "Date": []string{"Mon, 13 Dec 2024 14:15:10 GMT"}, }, }, + nil, false, time.Time{}, }, @@ -144,6 +172,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"max-age=15"}, }, }, + nil, false, time.Time{}, }, @@ -158,6 +187,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"max-age=15"}, }, }, + nil, false, time.Time{}, }, @@ -172,6 +202,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"max-age=15"}, }, }, + nil, false, time.Time{}, }, @@ -186,6 +217,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"no-store"}, }, }, + nil, false, time.Time{}, }, @@ -200,6 +232,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"private"}, }, }, + nil, false, time.Time{}, }, @@ -215,6 +248,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"public"}, }, }, + nil, true, time.Date(2024, 12, 13, 14, 15, 17, 00, time.UTC), }, @@ -230,6 +264,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"public"}, }, }, + nil, false, time.Time{}, }, @@ -244,6 +279,7 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"public"}, }, }, + nil, false, time.Time{}, }, @@ -261,15 +297,59 @@ func TestShared_Storable(t *testing.T) { "Cache-Control": []string{"max-age=15"}, }, }, + nil, false, time.Time{}, }, + { + "ExtendedRule(+15s) GET 200 -> +15s", + &http.Request{ + Method: http.MethodGet, + }, + &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{ + "Date": []string{"Mon, 13 Dec 2024 14:15:10 GMT"}, + }, + }, + []ExtendedRule{ + &testRule{ + cacheableMethods: []string{http.MethodGet}, + cacheableStatus: []int{http.StatusOK}, + age: 15 * time.Second, + }, + }, + true, + time.Date(2024, 12, 13, 14, 15, 25, 00, time.UTC), + }, + { + "ExtendedRule(+15s) POST 201 Cache-Control: public, Last-Modified 2024-12-13 14:15:06 -> +15s", + &http.Request{ + Method: http.MethodPost, + }, + &http.Response{ + StatusCode: http.StatusCreated, + Header: http.Header{ + "Last-Modified": []string{"Mon, 13 Dec 2024 14:15:06 GMT"}, + "Cache-Control": []string{"public"}, + }, + }, + []ExtendedRule{ + &testRule{ + cacheableMethods: []string{http.MethodPost}, + cacheableStatus: []int{http.StatusCreated}, + age: 15 * time.Second, + }, + }, + true, + time.Date(2024, 12, 13, 14, 15, 31, 00, time.UTC), + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - s, err := NewShared() + s, err := NewShared(ExtendedRules(tt.rules)) if err != nil { t.Errorf("Shared.Storable() error = %v", err) return From 542af8656bf34837c56b7351a8c56584bff19959 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Wed, 20 Dec 2023 12:05:14 +0900 Subject: [PATCH 2/5] Fix rule --- rfc9111/shared.go | 16 +++++----------- rfc9111/shared_test.go | 7 ++----- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/rfc9111/shared.go b/rfc9111/shared.go index 7e6cc12..1892a71 100644 --- a/rfc9111/shared.go +++ b/rfc9111/shared.go @@ -20,6 +20,7 @@ type Shared struct { // ExtendedRule is an extended rule. // Like proxy_cache_valid of NGINX. +// Additional rules are applied only when there is no Cache-Control header and the expiration time cannot be calculated. // THIS IS NOT RFC 9111. type ExtendedRule interface { // Cacheable returns true and and the expiration time if the response is cacheable. @@ -104,8 +105,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) // 3. Storing Responses in Caches (https://httpwg.org/specs/rfc9111.html#rfc.section.3) // - the request method is understood by the cache; if !contains(req.Method, s.understoodMethods) { - rescc := ParseResponseCacheControlHeader(res.Header.Values("Cache-Control")) - if !rescc.NoStore && !rescc.Private { + if res.Header.Get("Cache-Control") == "" { return s.storableWithExtendedRules(req, res, now) } return false, time.Time{} @@ -118,8 +118,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusProcessing, http.StatusEarlyHints, }) { - rescc := ParseResponseCacheControlHeader(res.Header.Values("Cache-Control")) - if !rescc.NoStore && !rescc.Private { + if res.Header.Get("Cache-Control") == "" { return s.storableWithExtendedRules(req, res, now) } return false, time.Time{} @@ -132,7 +131,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusPartialContent, http.StatusNotModified, }) || (rescc.MustUnderstand && !contains(res.StatusCode, s.understoodStatusCodes)) { - if !rescc.NoStore && !rescc.Private { + if res.Header.Get("Cache-Control") == "" { return s.storableWithExtendedRules(req, res, now) } return false, time.Time{} @@ -156,7 +155,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) expires := CalclateExpires(rescc, res.Header, s.heuristicExpirationRatio, now) if expires.Sub(now) <= 0 { - if expires.Sub(time.Time{}) == 0 { + if expires.Sub(time.Time{}) == 0 && res.Header.Get("Cache-Control") == "" { return s.storableWithExtendedRules(req, res, now) } return false, time.Time{} @@ -191,11 +190,6 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) return true, expires } - ok, _ := s.storableWithExtendedRules(req, res, now) - if ok { - return true, expires - } - return false, time.Time{} } diff --git a/rfc9111/shared_test.go b/rfc9111/shared_test.go index 792f886..0a17128 100644 --- a/rfc9111/shared_test.go +++ b/rfc9111/shared_test.go @@ -323,16 +323,13 @@ func TestShared_Storable(t *testing.T) { time.Date(2024, 12, 13, 14, 15, 25, 00, time.UTC), }, { - "ExtendedRule(+15s) POST 201 Cache-Control: public, Last-Modified 2024-12-13 14:15:06 -> +15s", + "ExtendedRule(+15s) POST 201 -> +15s", &http.Request{ Method: http.MethodPost, }, &http.Response{ StatusCode: http.StatusCreated, - Header: http.Header{ - "Last-Modified": []string{"Mon, 13 Dec 2024 14:15:06 GMT"}, - "Cache-Control": []string{"public"}, - }, + Header: http.Header{}, }, []ExtendedRule{ &testRule{ From 94cacc5593d746730672e56e84a09f90431a7092 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Wed, 20 Dec 2023 12:09:49 +0900 Subject: [PATCH 3/5] bonsai --- rfc9111/shared.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rfc9111/shared.go b/rfc9111/shared.go index 1892a71..1e0c6c2 100644 --- a/rfc9111/shared.go +++ b/rfc9111/shared.go @@ -20,7 +20,7 @@ type Shared struct { // ExtendedRule is an extended rule. // Like proxy_cache_valid of NGINX. -// Additional rules are applied only when there is no Cache-Control header and the expiration time cannot be calculated. +// Rules are applied only when there is no Cache-Control header and the expiration time cannot be calculated. // THIS IS NOT RFC 9111. type ExtendedRule interface { // Cacheable returns true and and the expiration time if the response is cacheable. @@ -306,6 +306,7 @@ func (s *Shared) Handle(req *http.Request, cachedReq *http.Request, cachedRes *h return false, res, err } +// storableWithExtendedRules returns true if the response is storable with extended rules. func (s *Shared) storableWithExtendedRules(req *http.Request, res *http.Response, now time.Time) (bool, time.Time) { for _, rule := range s.extendedRules { ok, age := rule.Cacheable(req, res) From 80c85aa869afd885a4914c4eccdded0f3fdd5365 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Wed, 20 Dec 2023 12:19:18 +0900 Subject: [PATCH 4/5] Fix lint warn --- rfc9111/shared.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc9111/shared.go b/rfc9111/shared.go index 1e0c6c2..7dd7d29 100644 --- a/rfc9111/shared.go +++ b/rfc9111/shared.go @@ -22,7 +22,7 @@ type Shared struct { // Like proxy_cache_valid of NGINX. // Rules are applied only when there is no Cache-Control header and the expiration time cannot be calculated. // THIS IS NOT RFC 9111. -type ExtendedRule interface { +type ExtendedRule interface { //nostyle:ifacenames // Cacheable returns true and and the expiration time if the response is cacheable. Cacheable(req *http.Request, res *http.Response) (ok bool, age time.Duration) } From 80b6ad4d3d377b64a14cfbcf2569c2ded371ebf0 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Wed, 20 Dec 2023 12:21:22 +0900 Subject: [PATCH 5/5] bonsai --- rfc9111/shared.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/rfc9111/shared.go b/rfc9111/shared.go index 7dd7d29..743ca76 100644 --- a/rfc9111/shared.go +++ b/rfc9111/shared.go @@ -105,10 +105,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) // 3. Storing Responses in Caches (https://httpwg.org/specs/rfc9111.html#rfc.section.3) // - the request method is understood by the cache; if !contains(req.Method, s.understoodMethods) { - if res.Header.Get("Cache-Control") == "" { - return s.storableWithExtendedRules(req, res, now) - } - return false, time.Time{} + return s.storableWithExtendedRules(req, res, now) } // - the response status code is final (see https://httpwg.org/specs/rfc9110.html#rfc.section.15); @@ -118,10 +115,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusProcessing, http.StatusEarlyHints, }) { - if res.Header.Get("Cache-Control") == "" { - return s.storableWithExtendedRules(req, res, now) - } - return false, time.Time{} + return s.storableWithExtendedRules(req, res, now) } rescc := ParseResponseCacheControlHeader(res.Header.Values("Cache-Control")) @@ -131,10 +125,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) http.StatusPartialContent, http.StatusNotModified, }) || (rescc.MustUnderstand && !contains(res.StatusCode, s.understoodStatusCodes)) { - if res.Header.Get("Cache-Control") == "" { - return s.storableWithExtendedRules(req, res, now) - } - return false, time.Time{} + return s.storableWithExtendedRules(req, res, now) } // - the no-store cache directive is not present in the response (see https://httpwg.org/specs/rfc9111.html#rfc.section.5.2.2.5); @@ -155,7 +146,7 @@ func (s *Shared) Storable(req *http.Request, res *http.Response, now time.Time) expires := CalclateExpires(rescc, res.Header, s.heuristicExpirationRatio, now) if expires.Sub(now) <= 0 { - if expires.Sub(time.Time{}) == 0 && res.Header.Get("Cache-Control") == "" { + if expires.Sub(time.Time{}) == 0 { return s.storableWithExtendedRules(req, res, now) } return false, time.Time{} @@ -308,6 +299,10 @@ func (s *Shared) Handle(req *http.Request, cachedReq *http.Request, cachedRes *h // storableWithExtendedRules returns true if the response is storable with extended rules. func (s *Shared) storableWithExtendedRules(req *http.Request, res *http.Response, now time.Time) (bool, time.Time) { + if res.Header.Get("Cache-Control") != "" { + return false, time.Time{} + } + for _, rule := range s.extendedRules { ok, age := rule.Cacheable(req, res) if ok {