commit 8aa7257f652ebea70dfbb819c301800f5c25b567
parent 75a823a36a75781c0c5d89fe9f328e3b9322d95f
Author: Paul van Brouwershaven <vanbroup@users.noreply.github.com>
Date: Tue, 30 Nov 2021 11:49:51 +0100
Add remote support to resources.Get
Closes #5255
Supports #9044
Diffstat:
4 files changed, 370 insertions(+), 16 deletions(-)
diff --git a/docs/content/en/hugo-pipes/introduction.md b/docs/content/en/hugo-pipes/introduction.md
@@ -20,12 +20,39 @@ aliases: [/assets/]
Asset files must be stored in the asset directory. This is `/assets` by default, but can be configured via the configuration file's `assetDir` key.
-### From file to resource
+### From file or URL to resource
-In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`, which takes one argument: the filepath of the file relative to the asset directory.
+In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`. The first argument can be the filepath of the file relative to the asset directory or the URL of the file.
```go-html-template
{{ $style := resources.Get "sass/main.scss" }}
+{{ $remoteStyle := resources.Get "https://www.example.com/styles.scss" }}
+```
+
+#### Request options
+
+When using an URL, the `resources.Get` function takes an optional options map as the last argument, e.g.:
+
+```
+{{ $resource := resources.Get "https://example.org/api" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
+```
+
+If you need multiple values for the same header key, use a slice:
+
+```
+{{ $resource := resources.Get "https://example.org/api" (dict "headers" (dict "X-List" (slice "a" "b" "c"))) }}
+```
+
+You can also change the request method and set the request body:
+
+```
+{{ $postResponse := resources.Get "https://example.org/api" (dict
+ "method" "post"
+ "body" `{"complete": true}`
+ "headers" (dict
+ "Content-Type" "application/json"
+ )
+)}}
```
### Asset publishing
diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go
@@ -17,7 +17,10 @@ import (
"bytes"
"fmt"
"io"
+ "io/ioutil"
"math/rand"
+ "net/http"
+ "net/http/httptest"
"os"
"github.com/gohugoio/hugo/config"
@@ -35,6 +38,7 @@ import (
"github.com/gohugoio/hugo/common/herrors"
+ "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/htesting"
qt "github.com/frankban/quicktest"
@@ -385,8 +389,13 @@ T1: {{ $r.Content }}
func TestResourceChainBasic(t *testing.T) {
t.Parallel()
+ ts := httptest.NewServer(http.FileServer(http.Dir("testdata/")))
+ t.Cleanup(func() {
+ ts.Close()
+ })
+
b := newTestSitesBuilder(t)
- b.WithTemplatesAdded("index.html", `
+ b.WithTemplatesAdded("index.html", fmt.Sprintf(`
{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }}
{{ $cssFingerprinted1 := "body { background-color: lightblue; }" | resources.FromString "styles.css" | minify | fingerprint }}
{{ $cssFingerprinted2 := "body { background-color: orange; }" | resources.FromString "styles2.css" | minify | fingerprint }}
@@ -403,7 +412,14 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }}
CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }}
-`)
+{{ $rimg := resources.Get "%[1]s/sunset.jpg" }}
+{{ $rfit := $rimg.Fit "200x200" }}
+{{ $rfit2 := $rfit.Fit "100x200" }}
+{{ $rimg = $rimg | fingerprint }}
+SUNSET REMOTE: {{ $rimg.Name }}|{{ $rimg.RelPermalink }}|{{ $rimg.Width }}|{{ len $rimg.Content }}
+FIT REMOTE: {{ $rfit.Name }}|{{ $rfit.RelPermalink }}|{{ $rfit.Width }}
+
+`, ts.URL))
fs := b.Fs.Source
@@ -424,12 +440,16 @@ CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerpri
b.Build(BuildCfg{})
b.AssertFileContent("public/index.html",
- `
+ fmt.Sprintf(`
SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200
CSS integrity Data first: sha256-od9YaHw8nMOL8mUy97Sy8sKwMV3N4hI3aVmZXATxH+8= /styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css
CSS integrity Data last: /styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css sha256-HPxSmGg2QF03+ZmKY/1t2GCOjEEOXj2x2qow94vCc7o=
-`)
+
+SUNSET REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
+FIT REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_0_200x200_fit_q75_box.jpg|200
+
+`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]interface{}{})))
b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}")
b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}")
@@ -537,6 +557,89 @@ func TestResourceChains(t *testing.T) {
c := qt.New(t)
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/css/styles1.css":
+ w.Header().Set("Content-Type", "text/css")
+ w.Write([]byte(`h1 {
+ font-style: bold;
+ }`))
+ return
+
+ case "/js/script1.js":
+ w.Write([]byte(`var x; x = 5, document.getElementById("demo").innerHTML = x * 10`))
+ return
+
+ case "/mydata/json1.json":
+ w.Write([]byte(`{
+ "employees": [
+ {
+ "firstName": "John",
+ "lastName": "Doe"
+ },
+ {
+ "firstName": "Anna",
+ "lastName": "Smith"
+ },
+ {
+ "firstName": "Peter",
+ "lastName": "Jones"
+ }
+ ]
+ }`))
+ return
+
+ case "/mydata/xml1.xml":
+ w.Write([]byte(`
+ <hello>
+ <world>Hugo Rocks!</<world>
+ </hello>`))
+ return
+
+ case "/mydata/svg1.svg":
+ w.Header().Set("Content-Disposition", `attachment; filename="image.svg"`)
+ w.Write([]byte(`
+ <svg height="100" width="100">
+ <path d="M1e2 1e2H3e2 2e2z"/>
+ </svg>`))
+ return
+
+ case "/mydata/html1.html":
+ w.Write([]byte(`
+ <html>
+ <a href=#>Cool</a>
+ </html>`))
+ return
+
+ case "/authenticated/":
+ if r.Header.Get("Authorization") == "Bearer abcd" {
+ w.Write([]byte(`Welcome`))
+ return
+ }
+ http.Error(w, "Forbidden", http.StatusForbidden)
+ return
+
+ case "/post":
+ if r.Method == http.MethodPost {
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
+ w.Write(body)
+ return
+ }
+ http.Error(w, "Bad request", http.StatusBadRequest)
+ return
+ }
+
+ http.Error(w, "Not found", http.StatusNotFound)
+ return
+ }))
+ t.Cleanup(func() {
+ ts.Close()
+ })
+
tests := []struct {
name string
shouldRun func() bool
@@ -578,25 +681,53 @@ T6: {{ $bundle1.Permalink }}
[minify.tdewolff.html]
keepWhitespace = false
`)
- b.WithTemplates("home.html", `
+ b.WithTemplates("home.html", fmt.Sprintf(`
Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }}
+Min CSS Remote: {{ ( resources.Get "%[1]s/css/styles1.css" | minify ).Content }}
Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }}
+Min JS Remote: {{ ( resources.Get "%[1]s/js/script1.js" | minify ).Content }}
Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }}
+Min JSON Remote: {{ ( resources.Get "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
+Min XML Remote: {{ ( resources.Get "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
+Min SVG Remote: {{ ( resources.Get "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }}
-
-
-`)
+Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
+`, ts.URL))
}, func(b *sitesBuilder) {
b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`)
+ b.AssertFileContent("public/index.html", `Min CSS Remote: h1{font-style:bold}`)
b.AssertFileContent("public/index.html", `Min JS: var x;x=5,document.getElementById("demo").innerHTML=x*10`)
+ b.AssertFileContent("public/index.html", `Min JS Remote: var x;x=5,document.getElementById("demo").innerHTML=x*10`)
b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
+ b.AssertFileContent("public/index.html", `Min JSON Remote: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`)
+ b.AssertFileContent("public/index.html", `Min XML Remote: <hello><world>Hugo Rocks!</<world></hello>`)
b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
+ b.AssertFileContent("public/index.html", `Min SVG Remote: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
b.AssertFileContent("public/index.html", `Min HTML: <html><a href=#>Cool</a></html>`)
+ b.AssertFileContent("public/index.html", `Min HTML Remote: <html><a href=#>Cool</a></html>`)
+ }},
+
+ {"remote", func() bool { return true }, func(b *sitesBuilder) {
+ b.WithTemplates("home.html", fmt.Sprintf(`
+{{$js := resources.Get "%[1]s/js/script1.js" }}
+Remote Filename: {{ $js.RelPermalink }}
+{{$svg := resources.Get "%[1]s/mydata/svg1.svg" }}
+Remote Content-Disposition: {{ $svg.RelPermalink }}
+{{$auth := resources.Get "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
+Remote Authorization: {{ $auth.Content }}
+{{$post := resources.Get "%[1]s/post" (dict "method" "post" "body" "Request body") }}
+Remote POST: {{ $post.Content }}
+`, ts.URL))
+ }, func(b *sitesBuilder) {
+ b.AssertFileContent("public/index.html", `Remote Filename: /script1_`)
+ b.AssertFileContent("public/index.html", `Remote Content-Disposition: /image_`)
+ b.AssertFileContent("public/index.html", `Remote Authorization: Welcome`)
+ b.AssertFileContent("public/index.html", `Remote POST: Request body`)
}},
{"concat", func() bool { return true }, func(b *sitesBuilder) {
diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
@@ -16,28 +16,47 @@
package create
import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "mime"
+ "net/http"
+ "net/url"
"path"
"path/filepath"
"strings"
+ "time"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/common/types"
+ "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
+
+ "github.com/pkg/errors"
)
// Client contains methods to create Resource objects.
// tasks to Resource objects.
type Client struct {
- rs *resources.Spec
+ rs *resources.Spec
+ httpClient *http.Client
}
// New creates a new Client with the given specification.
func New(rs *resources.Spec) *Client {
- return &Client{rs: rs}
+ return &Client{
+ rs: rs,
+ httpClient: &http.Client{
+ Timeout: 10 * time.Second,
+ },
+ }
}
// Get creates a new Resource by opening the given filename in the assets filesystem.
@@ -126,3 +145,161 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
})
})
}
+
+// FromRemote expects one or n-parts of a URL to a resource
+// If you provide multiple parts they will be joined together to the final URL.
+func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
+ rURL, err := url.Parse(uri)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
+ }
+
+ resourceID := helpers.HashString(uri, options)
+
+ return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(resourceID), func() (resource.Resource, error) {
+ method, reqBody, err := getMethodAndBody(options)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
+ }
+
+ req, err := http.NewRequest(method, uri, reqBody)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
+ }
+ addDefaultHeaders(req)
+
+ if _, ok := options["headers"]; ok {
+ headers, err := maps.ToStringMapE(options["headers"])
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri)
+ }
+ addUserProvidedHeaders(headers, req)
+ }
+ res, err := c.httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode < 200 || res.StatusCode > 299 {
+ return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
+ }
+
+ body, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+ }
+
+ filename := path.Base(rURL.Path)
+ if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+ if _, ok := params["filename"]; ok {
+ filename = params["filename"]
+ }
+ }
+
+ var contentType string
+ if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
+ contentType = arr[0]
+ }
+
+ // If content type was not determined by header, look for a file extention
+ if contentType == "" {
+ if ext := path.Ext(filename); ext != "" {
+ contentType = ext
+ }
+ }
+
+ // If content type was not determined by header or file extention, try using content itself
+ if contentType == "" {
+ if ct := http.DetectContentType(body); ct != "application/octet-stream" {
+ if arr, _ := mime.ExtensionsByType(ct); arr != nil {
+ contentType = arr[0]
+ }
+ }
+ }
+
+ resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType
+
+ return c.rs.New(
+ resources.ResourceSourceDescriptor{
+ Fs: c.rs.FileCaches.AssetsCache().Fs,
+ LazyPublish: true,
+ OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+ return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+ },
+ RelTargetFilename: filepath.Clean(resourceID),
+ })
+ })
+}
+
+func addDefaultHeaders(req *http.Request, accepts ...string) {
+ for _, accept := range accepts {
+ if !hasHeaderValue(req.Header, "Accept", accept) {
+ req.Header.Add("Accept", accept)
+ }
+ }
+ if !hasHeaderKey(req.Header, "User-Agent") {
+ req.Header.Add("User-Agent", "Hugo Static Site Generator")
+ }
+}
+
+func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
+ if headers == nil {
+ return
+ }
+ for key, val := range headers {
+ vals := types.ToStringSlicePreserveString(val)
+ for _, s := range vals {
+ req.Header.Add(key, s)
+ }
+ }
+}
+
+func hasHeaderValue(m http.Header, key, value string) bool {
+ var s []string
+ var ok bool
+
+ if s, ok = m[key]; !ok {
+ return false
+ }
+
+ for _, v := range s {
+ if v == value {
+ return true
+ }
+ }
+ return false
+}
+
+func hasHeaderKey(m http.Header, key string) bool {
+ _, ok := m[key]
+ return ok
+}
+
+func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) {
+ if options == nil {
+ return "GET", nil, nil
+ }
+
+ if method, ok := options["method"].(string); ok {
+ method = strings.ToUpper(method)
+ switch method {
+ case "GET", "DELETE", "HEAD", "OPTIONS":
+ return method, nil, nil
+ case "POST", "PUT", "PATCH":
+ var body []byte
+ if _, ok := options["body"]; ok {
+ switch b := options["body"].(type) {
+ case string:
+ body = []byte(b)
+ case []byte:
+ body = b
+ }
+ }
+ return method, bytes.NewBuffer(body), nil
+ }
+
+ return "", nil, fmt.Errorf("invalid HTTP method %q", method)
+ }
+
+ return "GET", nil, nil
+}
diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go
@@ -16,6 +16,7 @@ package resources
import (
"fmt"
+ "net/url"
"path/filepath"
"sync"
@@ -107,14 +108,32 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) {
return ns.scssClientDartSass, err
}
-// Get locates the filename given in Hugo's assets filesystem
-// and creates a Resource object that can be used for further transformations.
-func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
- filenamestr, err := cast.ToStringE(filename)
+// Get locates the filename given in Hugo's assets filesystem or downloads
+// a file from an URL and creates a Resource object that can be used for
+// further transformations.
+//
+// For URLs an additional argument with options can be provided.
+func (ns *Namespace) Get(args ...interface{}) (resource.Resource, error) {
+ if len(args) != 1 && len(args) != 2 {
+ return nil, errors.New("must provide a filename or URL")
+ }
+
+ filenamestr, err := cast.ToStringE(args[0])
if err != nil {
return nil, err
}
+ if u, err := url.Parse(filenamestr); err == nil && u.Scheme != "" {
+ if len(args) == 2 {
+ options, err := maps.ToStringMapE(args[1])
+ if err != nil {
+ return nil, err
+ }
+ return ns.createClient.FromRemote(filenamestr, options)
+ }
+ return ns.createClient.FromRemote(filenamestr, nil)
+ }
+
filenamestr = filepath.Clean(filenamestr)
return ns.createClient.Get(filenamestr)