remote.go (7055B)
1 // Copyright 2021 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package create
15
16 import (
17 "bufio"
18 "bytes"
19 "fmt"
20 "io"
21 "io/ioutil"
22 "mime"
23 "net/http"
24 "net/http/httputil"
25 "net/url"
26 "path"
27 "path/filepath"
28 "strings"
29
30 "github.com/gohugoio/hugo/common/hugio"
31 "github.com/gohugoio/hugo/common/maps"
32 "github.com/gohugoio/hugo/common/types"
33 "github.com/gohugoio/hugo/helpers"
34 "github.com/gohugoio/hugo/media"
35 "github.com/gohugoio/hugo/resources"
36 "github.com/gohugoio/hugo/resources/resource"
37 "github.com/mitchellh/mapstructure"
38 )
39
40 type HTTPError struct {
41 error
42 Data map[string]any
43
44 StatusCode int
45 Body string
46 }
47
48 func toHTTPError(err error, res *http.Response) *HTTPError {
49 if err == nil {
50 panic("err is nil")
51 }
52 if res == nil {
53 return &HTTPError{
54 error: err,
55 Data: map[string]any{},
56 }
57 }
58
59 var body []byte
60 body, _ = ioutil.ReadAll(res.Body)
61
62 return &HTTPError{
63 error: err,
64 Data: map[string]any{
65 "StatusCode": res.StatusCode,
66 "Status": res.Status,
67 "Body": string(body),
68 "TransferEncoding": res.TransferEncoding,
69 "ContentLength": res.ContentLength,
70 "ContentType": res.Header.Get("Content-Type"),
71 },
72 }
73 }
74
75 // FromRemote expects one or n-parts of a URL to a resource
76 // If you provide multiple parts they will be joined together to the final URL.
77 func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) {
78 rURL, err := url.Parse(uri)
79 if err != nil {
80 return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err)
81 }
82
83 resourceID := calculateResourceID(uri, optionsm)
84
85 _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
86 options, err := decodeRemoteOptions(optionsm)
87 if err != nil {
88 return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err)
89 }
90 if err := c.validateFromRemoteArgs(uri, options); err != nil {
91 return nil, err
92 }
93
94 req, err := http.NewRequest(options.Method, uri, options.BodyReader())
95 if err != nil {
96 return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
97 }
98 addDefaultHeaders(req)
99
100 if options.Headers != nil {
101 addUserProvidedHeaders(options.Headers, req)
102 }
103
104 res, err := c.httpClient.Do(req)
105 if err != nil {
106 return nil, err
107 }
108
109 httpResponse, err := httputil.DumpResponse(res, true)
110 if err != nil {
111 return nil, toHTTPError(err, res)
112 }
113
114 if res.StatusCode != http.StatusNotFound {
115 if res.StatusCode < 200 || res.StatusCode > 299 {
116 return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res)
117
118 }
119 }
120
121 return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
122 })
123 if err != nil {
124 return nil, err
125 }
126 defer httpResponse.Close()
127
128 res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
129 if err != nil {
130 return nil, err
131 }
132
133 if res.StatusCode == http.StatusNotFound {
134 // Not found. This matches how looksup for local resources work.
135 return nil, nil
136 }
137
138 body, err := ioutil.ReadAll(res.Body)
139 if err != nil {
140 return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
141 }
142
143 filename := path.Base(rURL.Path)
144 if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
145 if _, ok := params["filename"]; ok {
146 filename = params["filename"]
147 }
148 }
149
150 var extensionHints []string
151
152 contentType := res.Header.Get("Content-Type")
153
154 // mime.ExtensionsByType gives a long list of extensions for text/plain,
155 // just use ".txt".
156 if strings.HasPrefix(contentType, "text/plain") {
157 extensionHints = []string{".txt"}
158 } else {
159 exts, _ := mime.ExtensionsByType(contentType)
160 if exts != nil {
161 extensionHints = exts
162 }
163 }
164
165 // Look for a file extension. If it's .txt, look for a more specific.
166 if extensionHints == nil || extensionHints[0] == ".txt" {
167 if ext := path.Ext(filename); ext != "" {
168 extensionHints = []string{ext}
169 }
170 }
171
172 // Now resolve the media type primarily using the content.
173 mediaType := media.FromContent(c.rs.MediaTypes, extensionHints, body)
174 if mediaType.IsZero() {
175 return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
176 }
177
178 resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
179
180 return c.rs.New(
181 resources.ResourceSourceDescriptor{
182 MediaType: mediaType,
183 LazyPublish: true,
184 OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
185 return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
186 },
187 RelTargetFilename: filepath.Clean(resourceID),
188 })
189 }
190
191 func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
192 if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
193 return err
194 }
195
196 if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(options.Method); err != nil {
197 return err
198 }
199
200 return nil
201 }
202
203 func calculateResourceID(uri string, optionsm map[string]any) string {
204 if key, found := maps.LookupEqualFold(optionsm, "key"); found {
205 return helpers.HashString(key)
206 }
207 return helpers.HashString(uri, optionsm)
208 }
209
210 func addDefaultHeaders(req *http.Request, accepts ...string) {
211 for _, accept := range accepts {
212 if !hasHeaderValue(req.Header, "Accept", accept) {
213 req.Header.Add("Accept", accept)
214 }
215 }
216 if !hasHeaderKey(req.Header, "User-Agent") {
217 req.Header.Add("User-Agent", "Hugo Static Site Generator")
218 }
219 }
220
221 func addUserProvidedHeaders(headers map[string]any, req *http.Request) {
222 if headers == nil {
223 return
224 }
225 for key, val := range headers {
226 vals := types.ToStringSlicePreserveString(val)
227 for _, s := range vals {
228 req.Header.Add(key, s)
229 }
230 }
231 }
232
233 func hasHeaderValue(m http.Header, key, value string) bool {
234 var s []string
235 var ok bool
236
237 if s, ok = m[key]; !ok {
238 return false
239 }
240
241 for _, v := range s {
242 if v == value {
243 return true
244 }
245 }
246 return false
247 }
248
249 func hasHeaderKey(m http.Header, key string) bool {
250 _, ok := m[key]
251 return ok
252 }
253
254 type fromRemoteOptions struct {
255 Method string
256 Headers map[string]any
257 Body []byte
258 }
259
260 func (o fromRemoteOptions) BodyReader() io.Reader {
261 if o.Body == nil {
262 return nil
263 }
264 return bytes.NewBuffer(o.Body)
265 }
266
267 func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) {
268 options := fromRemoteOptions{
269 Method: "GET",
270 }
271
272 err := mapstructure.WeakDecode(optionsm, &options)
273 if err != nil {
274 return options, err
275 }
276 options.Method = strings.ToUpper(options.Method)
277
278 return options, nil
279 }