Skip to content

Commit

Permalink
Add resources.GetRemote
Browse files Browse the repository at this point in the history 
In Hugo 0.89 we added remote support to `resources.Get`.

In hindsight that was not a great idea, as a poll from many Hugo users showed. See Issue #9285 for more details.

After this commit `resources.Get` only supports local resource lookups. If you want to support both, you need to use a construct similar to:

Also improve some option case handling.

```
{{ resource := "" }}
{{ if (urls.Parse $url).IsAbs }}
{{ $resource = resources.GetRemote $url }}
{{ else }}
{{ $resource = resources.Get $url }}
{{ end }}
```

Fixes #9285
Fixes #9296
  • Loading branch information
@bep
bep committed Dec 17, 2021
1 parent 5758c37 commit 22ef5da
Show file tree
Hide file tree
Showing 7 changed files with 375 additions and 253 deletions.
30 changes: 15 additions & 15 deletions hugolib/resource_chain_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,10 +408,10 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }}
CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }}
{{ $rimg := resources.Get "%[1]s/sunset.jpg" }}
{{ $remotenotfound := resources.Get "%[1]s/notfound.jpg" }}
{{ $rimg := resources.GetRemote "%[1]s/sunset.jpg" }}
{{ $remotenotfound := resources.GetRemote "%[1]s/notfound.jpg" }}
{{ $localnotfound := resources.Get "images/notfound.jpg" }}
{{ $gopherprotocol := resources.Get "gopher://example.org" }}
{{ $gopherprotocol := resources.GetRemote "gopher://example.org" }}
{{ $rfit := $rimg.Fit "200x200" }}
{{ $rfit2 := $rfit.Fit "100x200" }}
{{ $rimg = $rimg | fingerprint }}
Expand Down Expand Up @@ -453,8 +453,8 @@ SUNSET REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0
FIT REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_0_200x200_fit_q75_box.jpg|200
REMOTE NOT FOUND: OK
LOCAL NOT FOUND: OK
PRINT PROTOCOL ERROR1: error calling resources.Get: Get "gopher://example.org": unsupported protocol scheme "gopher"
PRINT PROTOCOL ERROR2: error calling resources.Get: Get "gopher://example.org": unsupported protocol scheme "gopher"
PRINT PROTOCOL ERROR1: error calling resources.GetRemote: Get "gopher://example.org": unsupported protocol scheme "gopher"
PRINT PROTOCOL ERROR2: error calling resources.GetRemote: Get "gopher://example.org": unsupported protocol scheme "gopher"
`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]interface{}{})))
Expand Down Expand Up @@ -691,18 +691,18 @@ T6: {{ $bundle1.Permalink }}
`)
b.WithTemplates("home.html", fmt.Sprintf(`
Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }}
Min CSS Remote: {{ ( resources.Get "%[1]s/css/styles1.css" | minify ).Content }}
Min CSS Remote: {{ ( resources.GetRemote "%[1]s/css/styles1.css" | minify ).Content }}
Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }}
Min JS Remote: {{ ( resources.Get "%[1]s/js/script1.js" | minify ).Content }}
Min JS Remote: {{ ( resources.GetRemote "%[1]s/js/script1.js" | minify ).Content }}
Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min JSON Remote: {{ ( resources.Get "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min JSON Remote: {{ ( resources.GetRemote "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min XML Remote: {{ ( resources.Get "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min XML Remote: {{ ( resources.GetRemote "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG Remote: {{ ( resources.Get "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG Remote: {{ ( resources.GetRemote "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }}
Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
Min HTML Remote: {{ ( resources.GetRemote "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
`, ts.URL))
}, func(b *sitesBuilder) {
b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`)
Expand All @@ -722,13 +722,13 @@ Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify

{"remote", func() bool { return true }, func(b *sitesBuilder) {
b.WithTemplates("home.html", fmt.Sprintf(`
{{$js := resources.Get "%[1]s/js/script1.js" }}
{{$js := resources.GetRemote "%[1]s/js/script1.js" }}
Remote Filename: {{ $js.RelPermalink }}
{{$svg := resources.Get "%[1]s/mydata/svg1.svg" }}
{{$svg := resources.GetRemote "%[1]s/mydata/svg1.svg" }}
Remote Content-Disposition: {{ $svg.RelPermalink }}
{{$auth := resources.Get "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
{{$auth := resources.GetRemote "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
Remote Authorization: {{ $auth.Content }}
{{$post := resources.Get "%[1]s/post" (dict "method" "post" "body" "Request body") }}
{{$post := resources.GetRemote "%[1]s/post" (dict "method" "post" "body" "Request body") }}
Remote POST: {{ $post.Content }}
`, ts.URL))
}, func(b *sitesBuilder) {
Expand Down
12 changes: 6 additions & 6 deletions hugolib/securitypolicies_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,19 +148,19 @@ func TestSecurityPolicies(t *testing.T) {
testVariant(c, cb, `(?s).*"dart-sass-embedded" is not whitelisted in policy "security\.exec\.allow".*`)
})

c.Run("resources.Get, OK", func(c *qt.C) {
c.Run("resources.GetRemote, OK", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
})

c.Run("resources.Get, denied method", func(c *qt.C) {
c.Run("resources.GetRemote, denied method", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil)
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil)
})

c.Run("resources.Get, denied URL", func(c *qt.C) {
c.Run("resources.GetRemote, denied URL", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
func(b *sitesBuilder) {
b.WithConfigFile("toml", `
[security]
Expand Down
213 changes: 0 additions & 213 deletions resources/resource_factories/create/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,7 @@
package create

import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"net/http/httputil"
"net/url"
"path"
"path/filepath"
"strings"
Expand All @@ -36,13 +28,8 @@ import (

"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"

"github.com/pkg/errors"
)

// Client contains methods to create Resource objects.
Expand Down Expand Up @@ -150,203 +137,3 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
})
})
}

// FromRemote expects one or n-parts of a URL to a resource
// If you provide multiple parts they will be joined together to the final URL.
func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
if err := c.validateFromRemoteArgs(uri, options); err != nil {
return nil, err
}
rURL, err := url.Parse(uri)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
}

resourceID := helpers.HashString(uri, options)

_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
method, reqBody, err := getMethodAndBody(options)
if err != nil {
return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
}

req, err := http.NewRequest(method, uri, reqBody)
if err != nil {
return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
}
addDefaultHeaders(req)

if _, ok := options["headers"]; ok {
headers, err := maps.ToStringMapE(options["headers"])
if err != nil {
return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri)
}
addUserProvidedHeaders(headers, req)
}

res, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}

if res.StatusCode != http.StatusNotFound {
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
}
}

httpResponse, err := httputil.DumpResponse(res, true)
if err != nil {
return nil, err
}

return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
})
if err != nil {
return nil, err
}
defer httpResponse.Close()

res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
if err != nil {
return nil, err
}

if res.StatusCode == http.StatusNotFound {
// Not found. This matches how looksup for local resources work.
return nil, nil
}

body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
}

filename := path.Base(rURL.Path)
if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
if _, ok := params["filename"]; ok {
filename = params["filename"]
}
}

var extension string
if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
extension = arr[0]
}

// If extension was not determined by header, look for a file extention
if extension == "" {
if ext := path.Ext(filename); ext != "" {
extension = ext
}
}

// If extension was not determined by header or file extention, try using content itself
if extension == "" {
if ct := http.DetectContentType(body); ct != "application/octet-stream" {
if ct == "image/jpeg" {
extension = ".jpg"
} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
extension = arr[0]
}
}
}

resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension

return c.rs.New(
resources.ResourceSourceDescriptor{
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
},
RelTargetFilename: filepath.Clean(resourceID),
})

}

func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error {
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
return err
}

if method, ok := options["method"].(string); ok {
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil {
return err
}
}
return nil
}

func addDefaultHeaders(req *http.Request, accepts ...string) {
for _, accept := range accepts {
if !hasHeaderValue(req.Header, "Accept", accept) {
req.Header.Add("Accept", accept)
}
}
if !hasHeaderKey(req.Header, "User-Agent") {
req.Header.Add("User-Agent", "Hugo Static Site Generator")
}
}

func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
if headers == nil {
return
}
for key, val := range headers {
vals := types.ToStringSlicePreserveString(val)
for _, s := range vals {
req.Header.Add(key, s)
}
}
}

func hasHeaderValue(m http.Header, key, value string) bool {
var s []string
var ok bool

if s, ok = m[key]; !ok {
return false
}

for _, v := range s {
if v == value {
return true
}
}
return false
}

func hasHeaderKey(m http.Header, key string) bool {
_, ok := m[key]
return ok
}

func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) {
if options == nil {
return "GET", nil, nil
}

if method, ok := options["method"].(string); ok {
method = strings.ToUpper(method)
switch method {
case "GET", "DELETE", "HEAD", "OPTIONS":
return method, nil, nil
case "POST", "PUT", "PATCH":
var body []byte
if _, ok := options["body"]; ok {
switch b := options["body"].(type) {
case string:
body = []byte(b)
case []byte:
body = b
}
}
return method, bytes.NewBuffer(body), nil
}

return "", nil, fmt.Errorf("invalid HTTP method %q", method)
}

return "GET", nil, nil
}
Loading

0 comments on commit 22ef5da

Please sign in to comment.