resources.go (3476B)
1 // Copyright 2016 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package data
15
16 import (
17 "bytes"
18 "fmt"
19 "io/ioutil"
20 "net/http"
21 "net/url"
22 "path/filepath"
23 "time"
24
25 "github.com/gohugoio/hugo/cache/filecache"
26
27 "github.com/gohugoio/hugo/config"
28 "github.com/gohugoio/hugo/helpers"
29 "github.com/spf13/afero"
30 )
31
32 var (
33 resSleep = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
34 resRetries = 1 // number of retries to load the JSON from URL
35 )
36
37 // getRemote loads the content of a remote file. This method is thread safe.
38 func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
39 url := req.URL.String()
40 if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil {
41 return err
42 }
43 if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil {
44 return err
45 }
46
47 var headers bytes.Buffer
48 req.Header.Write(&headers)
49 id := helpers.MD5String(url + headers.String())
50 var handled bool
51 var retry bool
52
53 _, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) {
54 var err error
55 handled = true
56 for i := 0; i <= resRetries; i++ {
57 ns.deps.Log.Infof("Downloading: %s ...", url)
58 var res *http.Response
59 res, err = ns.client.Do(req)
60 if err != nil {
61 return nil, err
62 }
63
64 var b []byte
65 b, err = ioutil.ReadAll(res.Body)
66 if err != nil {
67 return nil, err
68 }
69 res.Body.Close()
70
71 if isHTTPError(res) {
72 return nil, fmt.Errorf("Failed to retrieve remote file: %s, body: %q", http.StatusText(res.StatusCode), b)
73 }
74
75 retry, err = unmarshal(b)
76
77 if err == nil {
78 // Return it so it can be cached.
79 return b, nil
80 }
81
82 if !retry {
83 return nil, err
84 }
85
86 ns.deps.Log.Infof("Cannot read remote resource %s: %s", url, err)
87 ns.deps.Log.Infof("Retry #%d for %s and sleeping for %s", i+1, url, resSleep)
88 time.Sleep(resSleep)
89 }
90
91 return nil, err
92 })
93
94 if !handled {
95 // This is cached content and should be correct.
96 _, err = unmarshal(b)
97 }
98
99 return err
100 }
101
102 // getLocal loads the content of a local file
103 func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
104 filename := filepath.Join(cfg.GetString("workingDir"), url)
105 return afero.ReadFile(fs, filename)
106 }
107
108 // getResource loads the content of a local or remote file and returns its content and the
109 // cache ID used, if relevant.
110 func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (bool, error), req *http.Request) error {
111 switch req.URL.Scheme {
112 case "":
113 url, err := url.QueryUnescape(req.URL.String())
114 if err != nil {
115 return err
116 }
117 b, err := getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
118 if err != nil {
119 return err
120 }
121 _, err = unmarshal(b)
122 return err
123 default:
124 return ns.getRemote(cache, unmarshal, req)
125 }
126 }
127
128 func isHTTPError(res *http.Response) bool {
129 return res.StatusCode < 200 || res.StatusCode > 299
130 }