transform_test.go (14646B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package resources
15
16 import (
17 "encoding/base64"
18 "fmt"
19 "io"
20 "path/filepath"
21 "strconv"
22 "strings"
23 "sync"
24 "testing"
25
26 "github.com/gohugoio/hugo/htesting"
27
28 "github.com/gohugoio/hugo/common/herrors"
29 "github.com/gohugoio/hugo/hugofs"
30
31 "github.com/gohugoio/hugo/media"
32 "github.com/gohugoio/hugo/resources/images"
33 "github.com/gohugoio/hugo/resources/internal"
34
35 "github.com/gohugoio/hugo/helpers"
36
37 "github.com/gohugoio/hugo/resources/resource"
38 "github.com/spf13/afero"
39
40 qt "github.com/frankban/quicktest"
41 )
42
43 const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
44
45 func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
46
47 func TestTransform(t *testing.T) {
48 c := qt.New(t)
49
50 createTransformer := func(spec *Spec, filename, content string) Transformer {
51 filename = filepath.FromSlash(filename)
52 fs := spec.Fs.Source
53 afero.WriteFile(fs, filename, []byte(content), 0777)
54 r, _ := spec.New(ResourceSourceDescriptor{Fs: fs, SourceFilename: filename})
55 return r.(Transformer)
56 }
57
58 createContentReplacer := func(name, old, new string) ResourceTransformation {
59 return &testTransformation{
60 name: name,
61 transform: func(ctx *ResourceTransformationCtx) error {
62 in := helpers.ReaderToString(ctx.From)
63 in = strings.Replace(in, old, new, 1)
64 ctx.AddOutPathIdentifier("." + name)
65 fmt.Fprint(ctx.To, in)
66 return nil
67 },
68 }
69 }
70
71 // Verify that we publish the same file once only.
72 assertNoDuplicateWrites := func(c *qt.C, spec *Spec) {
73 c.Helper()
74 d := spec.Fs.PublishDir.(hugofs.DuplicatesReporter)
75 c.Assert(d.ReportDuplicates(), qt.Equals, "")
76 }
77
78 assertShouldExist := func(c *qt.C, spec *Spec, filename string, should bool) {
79 c.Helper()
80 exists, _ := helpers.Exists(filepath.FromSlash(filename), spec.Fs.WorkingDirReadOnly)
81 c.Assert(exists, qt.Equals, should)
82 }
83
84 c.Run("All values", func(c *qt.C) {
85 c.Parallel()
86
87 spec := newTestResourceSpec(specDescriptor{c: c})
88
89 transformation := &testTransformation{
90 name: "test",
91 transform: func(ctx *ResourceTransformationCtx) error {
92 // Content
93 in := helpers.ReaderToString(ctx.From)
94 in = strings.Replace(in, "blue", "green", 1)
95 fmt.Fprint(ctx.To, in)
96
97 // Media type
98 ctx.OutMediaType = media.CSVType
99
100 // Change target
101 ctx.ReplaceOutPathExtension(".csv")
102
103 // Add some data to context
104 ctx.Data["mydata"] = "Hugo Rocks!"
105
106 return nil
107 },
108 }
109
110 r := createTransformer(spec, "f1.txt", "color is blue")
111
112 tr, err := r.Transform(transformation)
113 c.Assert(err, qt.IsNil)
114 content, err := tr.(resource.ContentProvider).Content()
115 c.Assert(err, qt.IsNil)
116
117 c.Assert(content, qt.Equals, "color is green")
118 c.Assert(tr.MediaType(), eq, media.CSVType)
119 c.Assert(tr.RelPermalink(), qt.Equals, "/f1.csv")
120 assertShouldExist(c, spec, "public/f1.csv", true)
121
122 data := tr.Data().(map[string]any)
123 c.Assert(data["mydata"], qt.Equals, "Hugo Rocks!")
124
125 assertNoDuplicateWrites(c, spec)
126 })
127
128 c.Run("Meta only", func(c *qt.C) {
129 c.Parallel()
130
131 spec := newTestResourceSpec(specDescriptor{c: c})
132
133 transformation := &testTransformation{
134 name: "test",
135 transform: func(ctx *ResourceTransformationCtx) error {
136 // Change media type only
137 ctx.OutMediaType = media.CSVType
138 ctx.ReplaceOutPathExtension(".csv")
139
140 return nil
141 },
142 }
143
144 r := createTransformer(spec, "f1.txt", "color is blue")
145
146 tr, err := r.Transform(transformation)
147 c.Assert(err, qt.IsNil)
148 content, err := tr.(resource.ContentProvider).Content()
149 c.Assert(err, qt.IsNil)
150
151 c.Assert(content, qt.Equals, "color is blue")
152 c.Assert(tr.MediaType(), eq, media.CSVType)
153
154 // The transformed file should only be published if RelPermalink
155 // or Permalink is called.
156 n := htesting.Rnd.Intn(3)
157 shouldExist := true
158 switch n {
159 case 0:
160 tr.RelPermalink()
161 case 1:
162 tr.Permalink()
163 default:
164 shouldExist = false
165 }
166
167 assertShouldExist(c, spec, "public/f1.csv", shouldExist)
168 assertNoDuplicateWrites(c, spec)
169 })
170
171 c.Run("Memory-cached transformation", func(c *qt.C) {
172 c.Parallel()
173
174 spec := newTestResourceSpec(specDescriptor{c: c})
175
176 // Two transformations with same id, different behaviour.
177 t1 := createContentReplacer("t1", "blue", "green")
178 t2 := createContentReplacer("t1", "color", "car")
179
180 for i, transformation := range []ResourceTransformation{t1, t2} {
181 r := createTransformer(spec, "f1.txt", "color is blue")
182 tr, _ := r.Transform(transformation)
183 content, err := tr.(resource.ContentProvider).Content()
184 c.Assert(err, qt.IsNil)
185 c.Assert(content, qt.Equals, "color is green", qt.Commentf("i=%d", i))
186
187 assertShouldExist(c, spec, "public/f1.t1.txt", false)
188 }
189
190 assertNoDuplicateWrites(c, spec)
191 })
192
193 c.Run("File-cached transformation", func(c *qt.C) {
194 c.Parallel()
195
196 fs := afero.NewMemMapFs()
197
198 for i := 0; i < 2; i++ {
199 spec := newTestResourceSpec(specDescriptor{c: c, fs: fs})
200
201 r := createTransformer(spec, "f1.txt", "color is blue")
202
203 var transformation ResourceTransformation
204
205 if i == 0 {
206 // There is currently a hardcoded list of transformations that we
207 // persist to disk (tocss, postcss).
208 transformation = &testTransformation{
209 name: "tocss",
210 transform: func(ctx *ResourceTransformationCtx) error {
211 in := helpers.ReaderToString(ctx.From)
212 in = strings.Replace(in, "blue", "green", 1)
213 ctx.AddOutPathIdentifier("." + "cached")
214 ctx.OutMediaType = media.CSVType
215 ctx.Data = map[string]any{
216 "Hugo": "Rocks!",
217 }
218 fmt.Fprint(ctx.To, in)
219 return nil
220 },
221 }
222 } else {
223 // Force read from file cache.
224 transformation = &testTransformation{
225 name: "tocss",
226 transform: func(ctx *ResourceTransformationCtx) error {
227 return herrors.ErrFeatureNotAvailable
228 },
229 }
230 }
231
232 msg := qt.Commentf("i=%d", i)
233
234 tr, _ := r.Transform(transformation)
235 c.Assert(tr.RelPermalink(), qt.Equals, "/f1.cached.txt", msg)
236 content, err := tr.(resource.ContentProvider).Content()
237 c.Assert(err, qt.IsNil)
238 c.Assert(content, qt.Equals, "color is green", msg)
239 c.Assert(tr.MediaType(), eq, media.CSVType)
240 c.Assert(tr.Data(), qt.DeepEquals, map[string]any{
241 "Hugo": "Rocks!",
242 })
243
244 assertNoDuplicateWrites(c, spec)
245 assertShouldExist(c, spec, "public/f1.cached.txt", true)
246
247 }
248 })
249
250 c.Run("Access RelPermalink first", func(c *qt.C) {
251 c.Parallel()
252
253 spec := newTestResourceSpec(specDescriptor{c: c})
254
255 t1 := createContentReplacer("t1", "blue", "green")
256
257 r := createTransformer(spec, "f1.txt", "color is blue")
258
259 tr, _ := r.Transform(t1)
260
261 relPermalink := tr.RelPermalink()
262
263 content, err := tr.(resource.ContentProvider).Content()
264 c.Assert(err, qt.IsNil)
265
266 c.Assert(relPermalink, qt.Equals, "/f1.t1.txt")
267 c.Assert(content, qt.Equals, "color is green")
268 c.Assert(tr.MediaType(), eq, media.TextType)
269
270 assertNoDuplicateWrites(c, spec)
271 assertShouldExist(c, spec, "public/f1.t1.txt", true)
272 })
273
274 c.Run("Content two", func(c *qt.C) {
275 c.Parallel()
276
277 spec := newTestResourceSpec(specDescriptor{c: c})
278
279 t1 := createContentReplacer("t1", "blue", "green")
280 t2 := createContentReplacer("t1", "color", "car")
281
282 r := createTransformer(spec, "f1.txt", "color is blue")
283
284 tr, _ := r.Transform(t1, t2)
285 content, err := tr.(resource.ContentProvider).Content()
286 c.Assert(err, qt.IsNil)
287
288 c.Assert(content, qt.Equals, "car is green")
289 c.Assert(tr.MediaType(), eq, media.TextType)
290
291 assertNoDuplicateWrites(c, spec)
292 })
293
294 c.Run("Content two chained", func(c *qt.C) {
295 c.Parallel()
296
297 spec := newTestResourceSpec(specDescriptor{c: c})
298
299 t1 := createContentReplacer("t1", "blue", "green")
300 t2 := createContentReplacer("t2", "color", "car")
301
302 r := createTransformer(spec, "f1.txt", "color is blue")
303
304 tr1, _ := r.Transform(t1)
305 tr2, _ := tr1.Transform(t2)
306
307 content1, err := tr1.(resource.ContentProvider).Content()
308 c.Assert(err, qt.IsNil)
309 content2, err := tr2.(resource.ContentProvider).Content()
310 c.Assert(err, qt.IsNil)
311
312 c.Assert(content1, qt.Equals, "color is green")
313 c.Assert(content2, qt.Equals, "car is green")
314
315 assertNoDuplicateWrites(c, spec)
316 })
317
318 c.Run("Content many", func(c *qt.C) {
319 c.Parallel()
320
321 spec := newTestResourceSpec(specDescriptor{c: c})
322
323 const count = 26 // A-Z
324
325 transformations := make([]ResourceTransformation, count)
326 for i := 0; i < count; i++ {
327 transformations[i] = createContentReplacer(fmt.Sprintf("t%d", i), fmt.Sprint(i), string(rune(i+65)))
328 }
329
330 var countstr strings.Builder
331 for i := 0; i < count; i++ {
332 countstr.WriteString(fmt.Sprint(i))
333 }
334
335 r := createTransformer(spec, "f1.txt", countstr.String())
336
337 tr, _ := r.Transform(transformations...)
338 content, err := tr.(resource.ContentProvider).Content()
339 c.Assert(err, qt.IsNil)
340
341 c.Assert(content, qt.Equals, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
342
343 assertNoDuplicateWrites(c, spec)
344 })
345
346 c.Run("Image", func(c *qt.C) {
347 c.Parallel()
348
349 spec := newTestResourceSpec(specDescriptor{c: c})
350
351 transformation := &testTransformation{
352 name: "test",
353 transform: func(ctx *ResourceTransformationCtx) error {
354 ctx.AddOutPathIdentifier(".changed")
355 return nil
356 },
357 }
358
359 r := createTransformer(spec, "gopher.png", helpers.ReaderToString(gopherPNG()))
360
361 tr, err := r.Transform(transformation)
362 c.Assert(err, qt.IsNil)
363 c.Assert(tr.MediaType(), eq, media.PNGType)
364
365 img, ok := tr.(images.ImageResource)
366 c.Assert(ok, qt.Equals, true)
367
368 c.Assert(img.Width(), qt.Equals, 75)
369 c.Assert(img.Height(), qt.Equals, 60)
370
371 // RelPermalink called.
372 resizedPublished1, err := img.Resize("40x40")
373 c.Assert(err, qt.IsNil)
374 c.Assert(resizedPublished1.Height(), qt.Equals, 40)
375 c.Assert(resizedPublished1.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_3.png")
376 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_3.png", true)
377
378 // Permalink called.
379 resizedPublished2, err := img.Resize("30x30")
380 c.Assert(err, qt.IsNil)
381 c.Assert(resizedPublished2.Height(), qt.Equals, 30)
382 c.Assert(resizedPublished2.Permalink(), qt.Equals, "https://example.com/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_3.png")
383 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_3.png", true)
384
385 // Not published because none of RelPermalink or Permalink was called.
386 resizedNotPublished, err := img.Resize("50x50")
387 c.Assert(err, qt.IsNil)
388 c.Assert(resizedNotPublished.Height(), qt.Equals, 50)
389 // c.Assert(resized.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_2.png")
390 assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_3.png", false)
391
392 assertNoDuplicateWrites(c, spec)
393 })
394
395 c.Run("Concurrent", func(c *qt.C) {
396 spec := newTestResourceSpec(specDescriptor{c: c})
397
398 transformers := make([]Transformer, 10)
399 transformations := make([]ResourceTransformation, 10)
400
401 for i := 0; i < 10; i++ {
402 transformers[i] = createTransformer(spec, fmt.Sprintf("f%d.txt", i), fmt.Sprintf("color is %d", i))
403 transformations[i] = createContentReplacer("test", strconv.Itoa(i), "blue")
404 }
405
406 var wg sync.WaitGroup
407
408 for i := 0; i < 13; i++ {
409 wg.Add(1)
410 go func(i int) {
411 defer wg.Done()
412 for j := 0; j < 23; j++ {
413 id := (i + j) % 10
414 tr, err := transformers[id].Transform(transformations[id])
415 c.Assert(err, qt.IsNil)
416 content, err := tr.(resource.ContentProvider).Content()
417 c.Assert(err, qt.IsNil)
418 c.Assert(content, qt.Equals, "color is blue")
419 c.Assert(tr.RelPermalink(), qt.Equals, fmt.Sprintf("/f%d.test.txt", id))
420 }
421 }(i)
422 }
423 wg.Wait()
424
425 assertNoDuplicateWrites(c, spec)
426 })
427 }
428
429 type testTransformation struct {
430 name string
431 transform func(ctx *ResourceTransformationCtx) error
432 }
433
434 func (t *testTransformation) Key() internal.ResourceTransformationKey {
435 return internal.NewResourceTransformationKey(t.name)
436 }
437
438 func (t *testTransformation) Transform(ctx *ResourceTransformationCtx) error {
439 return t.transform(ctx)
440 }