commit fcd63de3a54fadcd30972654d8eb86dc4d889784
parent 150d75738b54acddc485d363436757189144da6a
Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Sat, 5 Jun 2021 12:44:45 +0200
tpl/data: Misc header improvements, tests, allow multiple headers of same key
Closes #5617
Diffstat:
6 files changed, 291 insertions(+), 175 deletions(-)
diff --git a/common/types/convert.go b/common/types/convert.go
@@ -15,21 +15,52 @@ package types
import (
"encoding/json"
+ "fmt"
"html/template"
+ "reflect"
"github.com/spf13/cast"
)
-// ToStringSlicePreserveString converts v to a string slice.
-// If v is a string, it will be wrapped in a string slice.
+// ToStringSlicePreserveString is the same as ToStringSlicePreserveStringE,
+// but it never fails.
func ToStringSlicePreserveString(v interface{}) []string {
+ vv, _ := ToStringSlicePreserveStringE(v)
+ return vv
+}
+
+// ToStringSlicePreserveStringE converts v to a string slice.
+// If v is a string, it will be wrapped in a string slice.
+func ToStringSlicePreserveStringE(v interface{}) ([]string, error) {
if v == nil {
- return nil
+ return nil, nil
}
if sds, ok := v.(string); ok {
- return []string{sds}
+ return []string{sds}, nil
+ }
+ result, err := cast.ToStringSliceE(v)
+ if err == nil {
+ return result, nil
}
- return cast.ToStringSlice(v)
+
+ // Probably []int or similar. Fall back to reflect.
+ vv := reflect.ValueOf(v)
+
+ switch vv.Kind() {
+ case reflect.Slice, reflect.Array:
+ result = make([]string, vv.Len())
+ for i := 0; i < vv.Len(); i++ {
+ s, err := cast.ToStringE(vv.Index(i).Interface())
+ if err != nil {
+ return nil, err
+ }
+ result[i] = s
+ }
+ return result, nil
+ default:
+ return nil, fmt.Errorf("failed to convert %T to a string slice", v)
+ }
+
}
// TypeToString converts v to a string if it's a valid string type.
diff --git a/common/types/convert_test.go b/common/types/convert_test.go
@@ -24,7 +24,9 @@ func TestToStringSlicePreserveString(t *testing.T) {
c := qt.New(t)
c.Assert(ToStringSlicePreserveString("Hugo"), qt.DeepEquals, []string{"Hugo"})
+ c.Assert(ToStringSlicePreserveString(qt.Commentf("Hugo")), qt.DeepEquals, []string{"Hugo"})
c.Assert(ToStringSlicePreserveString([]interface{}{"A", "B"}), qt.DeepEquals, []string{"A", "B"})
+ c.Assert(ToStringSlicePreserveString([]int{1, 3}), qt.DeepEquals, []string{"1", "3"})
c.Assert(ToStringSlicePreserveString(nil), qt.IsNil)
}
diff --git a/docs/content/en/templates/data-templates.md b/docs/content/en/templates/data-templates.md
@@ -114,19 +114,10 @@ You can use the following code to render the `Short Description` in your layout:
Note the use of the [`markdownify` template function][markdownify]. This will send the description through the Blackfriday Markdown rendering engine.
-<!-- begin "Data-drive Content" page -->
-## Data-Driven Content
+## Get Remote Data
-In addition to the [data files](/extras/datafiles/) feature, Hugo also has a "data-driven content" feature, which lets you load any [JSON](https://www.json.org/) or [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) file from nearly any resource.
-
-Data-driven content currently consists of two functions, `getJSON` and `getCSV`, which are available in all template files.
-
-## Implementation details
-
-### Call the Functions with a URL
-
-In your template, call the functions like this:
+Use `getJSON` or `getCSV` to get remote data:
```
{{ $dataJ := getJSON "url" }}
@@ -155,19 +146,18 @@ This will resolve internally to the following:
{{ $gistJ := getJSON "https://api.github.com/users/GITHUB_USERNAME/gists" }}
```
-Finally, you can range over an array. This example will output the
-first 5 gists for a GitHub user:
+### Add HTTP headers
+
+{{< new-in "0.84.0" >}} Both `getJSON` and `getCSV` takes an optional map as the last argument, e.g.:
```
-<ul>
- {{ $urlPre := "https://api.github.com" }}
- {{ $gistJ := getJSON $urlPre "/users/GITHUB_USERNAME/gists" }}
- {{ range first 5 $gistJ }}
- {{ if .public }}
- <li><a href="{{ .html_url }}" target="_blank">{{ .description }}</a></li>
- {{ end }}
- {{ end }}
-</ul>
+{{ $data := getJSON "https://example.org/api" (dict "Authorization" "Bearer abcd") }}
+```
+
+If you need multiple values for the same header key, use a slice:
+
+```
+{{ $data := getJSON "https://example.org/api" (dict "X-List" (slice "a" "b" "c")) }}
```
### Example for CSV files
diff --git a/tpl/data/data.go b/tpl/data/data.go
@@ -23,6 +23,10 @@ import (
"net/http"
"strings"
+ "github.com/gohugoio/hugo/common/maps"
+
+ "github.com/gohugoio/hugo/common/types"
+
"github.com/gohugoio/hugo/common/constants"
"github.com/gohugoio/hugo/common/loggers"
@@ -59,14 +63,10 @@ type Namespace struct {
// If you provide multiple parts for the URL they will be joined together to the final URL.
// GetCSV returns nil or a slice slice to use in a short code.
func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err error) {
- url := joinURL(args)
+ url, headers := toURLAndHeaders(args)
cache := ns.cacheGetCSV
unmarshal := func(b []byte) (bool, error) {
- if !bytes.Contains(b, []byte(sep)) {
- return false, _errors.Errorf("cannot find separator %s in CSV for %s", sep, url)
- }
-
if d, err = parseCSV(b, sep); err != nil {
err = _errors.Wrapf(err, "failed to parse CSV file %s", url)
@@ -82,17 +82,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url)
}
- req.Header.Add("Accept", "text/csv")
- req.Header.Add("Accept", "text/plain")
-
- // Add custom user headers to the get request
- finalArg := args[len(args)-1]
-
- if userHeaders, ok := finalArg.(map[string]interface{}); ok {
- for key, val := range userHeaders {
- req.Header.Add(key, val.(string))
- }
- }
+ // Add custom user headers.
+ addUserProvidedHeaders(headers, req)
+ addDefaultHeaders(req, "text/csv", "text/plain")
err = ns.getResource(cache, unmarshal, req)
if err != nil {
@@ -108,7 +100,7 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
// GetJSON returns nil or parsed JSON to use in a short code.
func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
var v interface{}
- url := joinURL(args)
+ url, headers := toURLAndHeaders(args)
cache := ns.cacheGetJSON
req, err := http.NewRequest("GET", url, nil)
@@ -124,17 +116,8 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
return false, nil
}
- req.Header.Add("Accept", "application/json")
- req.Header.Add("User-Agent", "Hugo Static Site Generator")
-
- // Add custom user headers to the get request
- finalArg := args[len(args)-1]
-
- if userHeaders, ok := finalArg.(map[string]interface{}); ok {
- for key, val := range userHeaders {
- req.Header.Add(key, val.(string))
- }
- }
+ addUserProvidedHeaders(headers, req)
+ addDefaultHeaders(req, "application/json")
err = ns.getResource(cache, unmarshal, req)
if err != nil {
@@ -145,8 +128,64 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
return v, nil
}
-func joinURL(urlParts []interface{}) string {
- return strings.Join(cast.ToStringSlice(urlParts), "")
+func addDefaultHeaders(req *http.Request, accepts ...string) {
+ for _, accept := range accepts {
+ if !hasHeaderValue(req.Header, "Accept", accept) {
+ req.Header.Add("Accept", accept)
+ }
+ }
+ if !hasHeaderKey(req.Header, "User-Agent") {
+ req.Header.Add("User-Agent", "Hugo Static Site Generator")
+ }
+}
+
+func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
+ if headers == nil {
+ return
+ }
+ for key, val := range headers {
+ vals := types.ToStringSlicePreserveString(val)
+ for _, s := range vals {
+ req.Header.Add(key, s)
+ }
+ }
+}
+
+func hasHeaderValue(m http.Header, key, value string) bool {
+ var s []string
+ var ok bool
+
+ if s, ok = m[key]; !ok {
+ return false
+ }
+
+ for _, v := range s {
+ if v == value {
+ return true
+ }
+ }
+ return false
+}
+
+func hasHeaderKey(m http.Header, key string) bool {
+ _, ok := m[key]
+ return ok
+}
+
+func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) {
+ if len(urlParts) == 0 {
+ return "", nil
+ }
+
+ // The last argument may be a map.
+ headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
+ if err == nil {
+ urlParts = urlParts[:len(urlParts)-1]
+ } else {
+ headers = nil
+ }
+
+ return strings.Join(cast.ToStringSlice(urlParts), ""), headers
}
// parseCSV parses bytes of CSV data into a slice slice string or an error
diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go
@@ -14,12 +14,16 @@
package data
import (
+ "bytes"
+ "html/template"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
+ "github.com/gohugoio/hugo/common/maps"
+
qt "github.com/frankban/quicktest"
)
@@ -48,12 +52,6 @@ func TestGetCSV(t *testing.T) {
},
{
",",
- `http://error.no.sep/`,
- "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
- false,
- },
- {
- ",",
`http://nofound/404`,
``,
false,
@@ -73,66 +71,54 @@ func TestGetCSV(t *testing.T) {
false,
},
} {
- msg := qt.Commentf("Test %d", i)
- ns := newTestNs()
-
- // Setup HTTP test server
- var srv *httptest.Server
- srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
- if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") {
- http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
- return
- }
+ c.Run(test.url, func(c *qt.C) {
+ msg := qt.Commentf("Test %d", i)
- if r.URL.Path == "/404" {
- http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
- return
- }
+ ns := newTestNs()
- w.Header().Add("Content-type", "text/csv")
+ // Setup HTTP test server
+ var srv *httptest.Server
+ srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ if !hasHeaderValue(r.Header, "Accept", "text/csv") && !hasHeaderValue(r.Header, "Accept", "text/plain") {
+ http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+ return
+ }
- w.Write([]byte(test.content))
- })
- defer func() { srv.Close() }()
+ if r.URL.Path == "/404" {
+ http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+ return
+ }
- // Setup local test file for schema-less URLs
- if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
- f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
- c.Assert(err, qt.IsNil, msg)
- f.WriteString(test.content)
- f.Close()
- }
+ w.Header().Add("Content-type", "text/csv")
- // Get on with it
- got, err := ns.GetCSV(test.sep, test.url)
+ w.Write([]byte(test.content))
+ })
+ defer func() { srv.Close() }()
- if _, ok := test.expect.(bool); ok {
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
- // c.Assert(err, msg, qt.Not(qt.IsNil))
- c.Assert(got, qt.IsNil)
- continue
- }
+ // Setup local test file for schema-less URLs
+ if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+ f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+ c.Assert(err, qt.IsNil, msg)
+ f.WriteString(test.content)
+ f.Close()
+ }
- c.Assert(err, qt.IsNil, msg)
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
- c.Assert(got, qt.Not(qt.IsNil), msg)
- c.Assert(got, qt.DeepEquals, test.expect, msg)
+ // Get on with it
+ got, err := ns.GetCSV(test.sep, test.url)
- // Test user-defined headers as well
- gotHeader, _ := ns.GetCSV(test.sep, test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"})
+ if _, ok := test.expect.(bool); ok {
+ c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
+ c.Assert(got, qt.IsNil)
+ return
+ }
- if _, ok := test.expect.(bool); ok {
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
- // c.Assert(err, msg, qt.Not(qt.IsNil))
- c.Assert(got, qt.IsNil)
- continue
- }
+ c.Assert(err, qt.IsNil, msg)
+ c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
+ c.Assert(got, qt.Not(qt.IsNil), msg)
+ c.Assert(got, qt.DeepEquals, test.expect, msg)
+ })
- c.Assert(err, qt.IsNil, msg)
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
- c.Assert(gotHeader, qt.Not(qt.IsNil), msg)
- c.Assert(gotHeader, qt.DeepEquals, test.expect, msg)
}
}
@@ -178,68 +164,153 @@ func TestGetJSON(t *testing.T) {
},
} {
- msg := qt.Commentf("Test %d", i)
- ns := newTestNs()
+ c.Run(test.url, func(c *qt.C) {
- // Setup HTTP test server
- var srv *httptest.Server
- srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
- if !haveHeader(r.Header, "Accept", "application/json") {
- http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
- return
+ msg := qt.Commentf("Test %d", i)
+ ns := newTestNs()
+
+ // Setup HTTP test server
+ var srv *httptest.Server
+ srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ if !hasHeaderValue(r.Header, "Accept", "application/json") {
+ http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+ return
+ }
+
+ if r.URL.Path == "/404" {
+ http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+ return
+ }
+
+ w.Header().Add("Content-type", "application/json")
+
+ w.Write([]byte(test.content))
+ })
+ defer func() { srv.Close() }()
+
+ // Setup local test file for schema-less URLs
+ if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+ f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+ c.Assert(err, qt.IsNil, msg)
+ f.WriteString(test.content)
+ f.Close()
}
- if r.URL.Path == "/404" {
- http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+ // Get on with it
+ got, _ := ns.GetJSON(test.url)
+
+ if _, ok := test.expect.(bool); ok {
+ c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
return
}
- w.Header().Add("Content-type", "application/json")
+ c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
+ c.Assert(got, qt.Not(qt.IsNil), msg)
+ c.Assert(got, qt.DeepEquals, test.expect)
- w.Write([]byte(test.content))
})
- defer func() { srv.Close() }()
+ }
+}
- // Setup local test file for schema-less URLs
- if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
- f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
- c.Assert(err, qt.IsNil, msg)
- f.WriteString(test.content)
- f.Close()
- }
+func TestHeaders(t *testing.T) {
+ t.Parallel()
+ c := qt.New(t)
- // Get on with it
- got, _ := ns.GetJSON(test.url)
+ for _, test := range []struct {
+ name string
+ headers interface{}
+ assert func(c *qt.C, headers string)
+ }{
+ {
+ `Misc header variants`,
+ map[string]interface{}{
+ "Accept-Charset": "utf-8",
+ "Max-forwards": "10",
+ "X-Int": 32,
+ "X-Templ": template.HTML("a"),
+ "X-Multiple": []string{"a", "b"},
+ "X-MultipleInt": []int{3, 4},
+ },
+ func(c *qt.C, headers string) {
+ c.Assert(headers, qt.Contains, "Accept-Charset: utf-8")
+ c.Assert(headers, qt.Contains, "Max-Forwards: 10")
+ c.Assert(headers, qt.Contains, "X-Int: 32")
+ c.Assert(headers, qt.Contains, "X-Templ: a")
+ c.Assert(headers, qt.Contains, "X-Multiple: a")
+ c.Assert(headers, qt.Contains, "X-Multiple: b")
+ c.Assert(headers, qt.Contains, "X-Multipleint: 3")
+ c.Assert(headers, qt.Contains, "X-Multipleint: 4")
+ c.Assert(headers, qt.Contains, "User-Agent: Hugo Static Site Generator")
+ },
+ },
+ {
+ `Params`,
+ maps.Params{
+ "Accept-Charset": "utf-8",
+ },
+ func(c *qt.C, headers string) {
+ c.Assert(headers, qt.Contains, "Accept-Charset: utf-8")
+ },
+ },
+ {
+ `Override User-Agent`,
+ map[string]interface{}{
+ "User-Agent": "007",
+ },
+ func(c *qt.C, headers string) {
+ c.Assert(headers, qt.Contains, "User-Agent: 007")
+ },
+ },
+ } {
- if _, ok := test.expect.(bool); ok {
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
- // c.Assert(err, msg, qt.Not(qt.IsNil))
- continue
- }
+ c.Run(test.name, func(c *qt.C) {
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
- c.Assert(got, qt.Not(qt.IsNil), msg)
- c.Assert(got, qt.DeepEquals, test.expect)
+ ns := newTestNs()
- // Test user-defined headers as well
- gotHeader, _ := ns.GetJSON(test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"})
+ // Setup HTTP test server
+ var srv *httptest.Server
+ var headers bytes.Buffer
+ srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ c.Assert(r.URL.String(), qt.Equals, "http://gohugo.io/api?foo")
+ w.Write([]byte("{}"))
+ r.Header.Write(&headers)
- if _, ok := test.expect.(bool); ok {
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
- // c.Assert(err, msg, qt.Not(qt.IsNil))
- continue
- }
+ })
+ defer func() { srv.Close() }()
+
+ testFunc := func(fn func(args ...interface{}) error) {
+ defer headers.Reset()
+ err := fn("http://example.org/api", "?foo", test.headers)
+
+ c.Assert(err, qt.IsNil)
+ c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
+ test.assert(c, headers.String())
+ }
+
+ testFunc(func(args ...interface{}) error {
+ _, err := ns.GetJSON(args...)
+ return err
+ })
+ testFunc(func(args ...interface{}) error {
+ _, err := ns.GetCSV(",", args...)
+ return err
+ })
+
+ })
- c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
- c.Assert(gotHeader, qt.Not(qt.IsNil), msg)
- c.Assert(gotHeader, qt.DeepEquals, test.expect)
}
}
-func TestJoinURL(t *testing.T) {
+func TestToURLAndHeaders(t *testing.T) {
t.Parallel()
c := qt.New(t)
- c.Assert(joinURL([]interface{}{"https://foo?id=", 32}), qt.Equals, "https://foo?id=32")
+ url, headers := toURLAndHeaders([]interface{}{"https://foo?id=", 32})
+ c.Assert(url, qt.Equals, "https://foo?id=32")
+ c.Assert(headers, qt.IsNil)
+
+ url, headers = toURLAndHeaders([]interface{}{"https://foo?id=", 32, map[string]interface{}{"a": "b"}})
+ c.Assert(url, qt.Equals, "https://foo?id=32")
+ c.Assert(headers, qt.DeepEquals, map[string]interface{}{"a": "b"})
}
func TestParseCSV(t *testing.T) {
@@ -276,19 +347,3 @@ func TestParseCSV(t *testing.T) {
c.Assert(act, qt.Equals, test.exp, msg)
}
}
-
-func haveHeader(m http.Header, key, needle string) bool {
- var s []string
- var ok bool
-
- if s, ok = m[key]; !ok {
- return false
- }
-
- for _, v := range s {
- if v == needle {
- return true
- }
- }
- return false
-}
diff --git a/tpl/data/resources.go b/tpl/data/resources.go
@@ -14,6 +14,7 @@
package data
import (
+ "bytes"
"io/ioutil"
"net/http"
"net/url"
@@ -37,7 +38,9 @@ var (
// getRemote loads the content of a remote file. This method is thread safe.
func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
url := req.URL.String()
- id := helpers.MD5String(url)
+ var headers bytes.Buffer
+ req.Header.Write(&headers)
+ id := helpers.MD5String(url + headers.String())
var handled bool
var retry bool
@@ -94,10 +97,6 @@ func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (b
// getLocal loads the content of a local file
func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
filename := filepath.Join(cfg.GetString("workingDir"), url)
- if e, err := helpers.Exists(filename, fs); !e {
- return nil, err
- }
-
return afero.ReadFile(fs, filename)
}