data.go (5527B)
1 // Copyright 2017 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 // Package data provides template functions for working with external data
15 // sources.
16 package data
17
18 import (
19 "bytes"
20 "encoding/csv"
21 "encoding/json"
22 "errors"
23 "fmt"
24 "net/http"
25 "strings"
26
27 "github.com/gohugoio/hugo/common/maps"
28 "github.com/gohugoio/hugo/config/security"
29
30 "github.com/gohugoio/hugo/common/types"
31
32 "github.com/gohugoio/hugo/common/constants"
33 "github.com/gohugoio/hugo/common/loggers"
34
35 "github.com/spf13/cast"
36
37 "github.com/gohugoio/hugo/cache/filecache"
38 "github.com/gohugoio/hugo/deps"
39 )
40
41 // New returns a new instance of the data-namespaced template functions.
42 func New(deps *deps.Deps) *Namespace {
43 return &Namespace{
44 deps: deps,
45 cacheGetCSV: deps.FileCaches.GetCSVCache(),
46 cacheGetJSON: deps.FileCaches.GetJSONCache(),
47 client: http.DefaultClient,
48 }
49 }
50
51 // Namespace provides template functions for the "data" namespace.
52 type Namespace struct {
53 deps *deps.Deps
54
55 cacheGetJSON *filecache.Cache
56 cacheGetCSV *filecache.Cache
57
58 client *http.Client
59 }
60
61 // GetCSV expects a data separator and one or n-parts of a URL to a resource which
62 // can either be a local or a remote one.
63 // The data separator can be a comma, semi-colon, pipe, etc, but only one character.
64 // If you provide multiple parts for the URL they will be joined together to the final URL.
65 // GetCSV returns nil or a slice slice to use in a short code.
66 func (ns *Namespace) GetCSV(sep string, args ...any) (d [][]string, err error) {
67 url, headers := toURLAndHeaders(args)
68 cache := ns.cacheGetCSV
69
70 unmarshal := func(b []byte) (bool, error) {
71 if d, err = parseCSV(b, sep); err != nil {
72 err = fmt.Errorf("failed to parse CSV file %s: %w", url, err)
73
74 return true, err
75 }
76
77 return false, nil
78 }
79
80 var req *http.Request
81 req, err = http.NewRequest("GET", url, nil)
82 if err != nil {
83 return nil, fmt.Errorf("failed to create request for getCSV for resource %s: %w", url, err)
84 }
85
86 // Add custom user headers.
87 addUserProvidedHeaders(headers, req)
88 addDefaultHeaders(req, "text/csv", "text/plain")
89
90 err = ns.getResource(cache, unmarshal, req)
91 if err != nil {
92 if security.IsAccessDenied(err) {
93 return nil, err
94 }
95 ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err)
96 return nil, nil
97 }
98
99 return
100 }
101
102 // GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
103 // If you provide multiple parts they will be joined together to the final URL.
104 // GetJSON returns nil or parsed JSON to use in a short code.
105 func (ns *Namespace) GetJSON(args ...any) (any, error) {
106 var v any
107 url, headers := toURLAndHeaders(args)
108 cache := ns.cacheGetJSON
109
110 req, err := http.NewRequest("GET", url, nil)
111 if err != nil {
112 return nil, fmt.Errorf("Failed to create request for getJSON resource %s: %w", url, err)
113 }
114
115 unmarshal := func(b []byte) (bool, error) {
116 err := json.Unmarshal(b, &v)
117 if err != nil {
118 return true, err
119 }
120 return false, nil
121 }
122
123 addUserProvidedHeaders(headers, req)
124 addDefaultHeaders(req, "application/json")
125
126 err = ns.getResource(cache, unmarshal, req)
127 if err != nil {
128 if security.IsAccessDenied(err) {
129 return nil, err
130 }
131 ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err)
132 return nil, nil
133 }
134
135 return v, nil
136 }
137
138 func addDefaultHeaders(req *http.Request, accepts ...string) {
139 for _, accept := range accepts {
140 if !hasHeaderValue(req.Header, "Accept", accept) {
141 req.Header.Add("Accept", accept)
142 }
143 }
144 if !hasHeaderKey(req.Header, "User-Agent") {
145 req.Header.Add("User-Agent", "Hugo Static Site Generator")
146 }
147 }
148
149 func addUserProvidedHeaders(headers map[string]any, req *http.Request) {
150 if headers == nil {
151 return
152 }
153 for key, val := range headers {
154 vals := types.ToStringSlicePreserveString(val)
155 for _, s := range vals {
156 req.Header.Add(key, s)
157 }
158 }
159 }
160
161 func hasHeaderValue(m http.Header, key, value string) bool {
162 var s []string
163 var ok bool
164
165 if s, ok = m[key]; !ok {
166 return false
167 }
168
169 for _, v := range s {
170 if v == value {
171 return true
172 }
173 }
174 return false
175 }
176
177 func hasHeaderKey(m http.Header, key string) bool {
178 _, ok := m[key]
179 return ok
180 }
181
182 func toURLAndHeaders(urlParts []any) (string, map[string]any) {
183 if len(urlParts) == 0 {
184 return "", nil
185 }
186
187 // The last argument may be a map.
188 headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
189 if err == nil {
190 urlParts = urlParts[:len(urlParts)-1]
191 } else {
192 headers = nil
193 }
194
195 return strings.Join(cast.ToStringSlice(urlParts), ""), headers
196 }
197
198 // parseCSV parses bytes of CSV data into a slice slice string or an error
199 func parseCSV(c []byte, sep string) ([][]string, error) {
200 if len(sep) != 1 {
201 return nil, errors.New("Incorrect length of CSV separator: " + sep)
202 }
203 b := bytes.NewReader(c)
204 r := csv.NewReader(b)
205 rSep := []rune(sep)
206 r.Comma = rSep[0]
207 r.FieldsPerRecord = 0
208 return r.ReadAll()
209 }