hugo

Fork of github.com/gohugoio/hugo with reverse pagination support

git clone git://git.shimmy1996.com/hugo.git

remote.go (7055B)

    1 // Copyright 2021 The Hugo Authors. All rights reserved.
    2 //
    3 // Licensed under the Apache License, Version 2.0 (the "License");
    4 // you may not use this file except in compliance with the License.
    5 // You may obtain a copy of the License at
    6 // http://www.apache.org/licenses/LICENSE-2.0
    7 //
    8 // Unless required by applicable law or agreed to in writing, software
    9 // distributed under the License is distributed on an "AS IS" BASIS,
   10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   11 // See the License for the specific language governing permissions and
   12 // limitations under the License.
   13 
   14 package create
   15 
   16 import (
   17 	"bufio"
   18 	"bytes"
   19 	"fmt"
   20 	"io"
   21 	"io/ioutil"
   22 	"mime"
   23 	"net/http"
   24 	"net/http/httputil"
   25 	"net/url"
   26 	"path"
   27 	"path/filepath"
   28 	"strings"
   29 
   30 	"github.com/gohugoio/hugo/common/hugio"
   31 	"github.com/gohugoio/hugo/common/maps"
   32 	"github.com/gohugoio/hugo/common/types"
   33 	"github.com/gohugoio/hugo/helpers"
   34 	"github.com/gohugoio/hugo/media"
   35 	"github.com/gohugoio/hugo/resources"
   36 	"github.com/gohugoio/hugo/resources/resource"
   37 	"github.com/mitchellh/mapstructure"
   38 )
   39 
   40 type HTTPError struct {
   41 	error
   42 	Data map[string]any
   43 
   44 	StatusCode int
   45 	Body       string
   46 }
   47 
   48 func toHTTPError(err error, res *http.Response) *HTTPError {
   49 	if err == nil {
   50 		panic("err is nil")
   51 	}
   52 	if res == nil {
   53 		return &HTTPError{
   54 			error: err,
   55 			Data:  map[string]any{},
   56 		}
   57 	}
   58 
   59 	var body []byte
   60 	body, _ = ioutil.ReadAll(res.Body)
   61 
   62 	return &HTTPError{
   63 		error: err,
   64 		Data: map[string]any{
   65 			"StatusCode":       res.StatusCode,
   66 			"Status":           res.Status,
   67 			"Body":             string(body),
   68 			"TransferEncoding": res.TransferEncoding,
   69 			"ContentLength":    res.ContentLength,
   70 			"ContentType":      res.Header.Get("Content-Type"),
   71 		},
   72 	}
   73 }
   74 
   75 // FromRemote expects one or n-parts of a URL to a resource
   76 // If you provide multiple parts they will be joined together to the final URL.
   77 func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) {
   78 	rURL, err := url.Parse(uri)
   79 	if err != nil {
   80 		return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err)
   81 	}
   82 
   83 	resourceID := calculateResourceID(uri, optionsm)
   84 
   85 	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
   86 		options, err := decodeRemoteOptions(optionsm)
   87 		if err != nil {
   88 			return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err)
   89 		}
   90 		if err := c.validateFromRemoteArgs(uri, options); err != nil {
   91 			return nil, err
   92 		}
   93 
   94 		req, err := http.NewRequest(options.Method, uri, options.BodyReader())
   95 		if err != nil {
   96 			return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
   97 		}
   98 		addDefaultHeaders(req)
   99 
  100 		if options.Headers != nil {
  101 			addUserProvidedHeaders(options.Headers, req)
  102 		}
  103 
  104 		res, err := c.httpClient.Do(req)
  105 		if err != nil {
  106 			return nil, err
  107 		}
  108 
  109 		httpResponse, err := httputil.DumpResponse(res, true)
  110 		if err != nil {
  111 			return nil, toHTTPError(err, res)
  112 		}
  113 
  114 		if res.StatusCode != http.StatusNotFound {
  115 			if res.StatusCode < 200 || res.StatusCode > 299 {
  116 				return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res)
  117 
  118 			}
  119 		}
  120 
  121 		return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
  122 	})
  123 	if err != nil {
  124 		return nil, err
  125 	}
  126 	defer httpResponse.Close()
  127 
  128 	res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
  129 	if err != nil {
  130 		return nil, err
  131 	}
  132 
  133 	if res.StatusCode == http.StatusNotFound {
  134 		// Not found. This matches how looksup for local resources work.
  135 		return nil, nil
  136 	}
  137 
  138 	body, err := ioutil.ReadAll(res.Body)
  139 	if err != nil {
  140 		return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
  141 	}
  142 
  143 	filename := path.Base(rURL.Path)
  144 	if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
  145 		if _, ok := params["filename"]; ok {
  146 			filename = params["filename"]
  147 		}
  148 	}
  149 
  150 	var extensionHints []string
  151 
  152 	contentType := res.Header.Get("Content-Type")
  153 
  154 	// mime.ExtensionsByType gives a long list of extensions for text/plain,
  155 	// just use ".txt".
  156 	if strings.HasPrefix(contentType, "text/plain") {
  157 		extensionHints = []string{".txt"}
  158 	} else {
  159 		exts, _ := mime.ExtensionsByType(contentType)
  160 		if exts != nil {
  161 			extensionHints = exts
  162 		}
  163 	}
  164 
  165 	// Look for a file extension. If it's .txt, look for a more specific.
  166 	if extensionHints == nil || extensionHints[0] == ".txt" {
  167 		if ext := path.Ext(filename); ext != "" {
  168 			extensionHints = []string{ext}
  169 		}
  170 	}
  171 
  172 	// Now resolve the media type primarily using the content.
  173 	mediaType := media.FromContent(c.rs.MediaTypes, extensionHints, body)
  174 	if mediaType.IsZero() {
  175 		return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
  176 	}
  177 
  178 	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
  179 
  180 	return c.rs.New(
  181 		resources.ResourceSourceDescriptor{
  182 			MediaType:   mediaType,
  183 			LazyPublish: true,
  184 			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
  185 				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
  186 			},
  187 			RelTargetFilename: filepath.Clean(resourceID),
  188 		})
  189 }
  190 
  191 func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
  192 	if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
  193 		return err
  194 	}
  195 
  196 	if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(options.Method); err != nil {
  197 		return err
  198 	}
  199 
  200 	return nil
  201 }
  202 
  203 func calculateResourceID(uri string, optionsm map[string]any) string {
  204 	if key, found := maps.LookupEqualFold(optionsm, "key"); found {
  205 		return helpers.HashString(key)
  206 	}
  207 	return helpers.HashString(uri, optionsm)
  208 }
  209 
  210 func addDefaultHeaders(req *http.Request, accepts ...string) {
  211 	for _, accept := range accepts {
  212 		if !hasHeaderValue(req.Header, "Accept", accept) {
  213 			req.Header.Add("Accept", accept)
  214 		}
  215 	}
  216 	if !hasHeaderKey(req.Header, "User-Agent") {
  217 		req.Header.Add("User-Agent", "Hugo Static Site Generator")
  218 	}
  219 }
  220 
  221 func addUserProvidedHeaders(headers map[string]any, req *http.Request) {
  222 	if headers == nil {
  223 		return
  224 	}
  225 	for key, val := range headers {
  226 		vals := types.ToStringSlicePreserveString(val)
  227 		for _, s := range vals {
  228 			req.Header.Add(key, s)
  229 		}
  230 	}
  231 }
  232 
  233 func hasHeaderValue(m http.Header, key, value string) bool {
  234 	var s []string
  235 	var ok bool
  236 
  237 	if s, ok = m[key]; !ok {
  238 		return false
  239 	}
  240 
  241 	for _, v := range s {
  242 		if v == value {
  243 			return true
  244 		}
  245 	}
  246 	return false
  247 }
  248 
  249 func hasHeaderKey(m http.Header, key string) bool {
  250 	_, ok := m[key]
  251 	return ok
  252 }
  253 
  254 type fromRemoteOptions struct {
  255 	Method  string
  256 	Headers map[string]any
  257 	Body    []byte
  258 }
  259 
  260 func (o fromRemoteOptions) BodyReader() io.Reader {
  261 	if o.Body == nil {
  262 		return nil
  263 	}
  264 	return bytes.NewBuffer(o.Body)
  265 }
  266 
  267 func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) {
  268 	options := fromRemoteOptions{
  269 		Method: "GET",
  270 	}
  271 
  272 	err := mapstructure.WeakDecode(optionsm, &options)
  273 	if err != nil {
  274 		return options, err
  275 	}
  276 	options.Method = strings.ToUpper(options.Method)
  277 
  278 	return options, nil
  279 }