datafiles_test.go (12703B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "fmt"
18 "path/filepath"
19 "reflect"
20 "runtime"
21 "testing"
22
23 "github.com/gohugoio/hugo/common/loggers"
24
25 "github.com/gohugoio/hugo/deps"
26
27 qt "github.com/frankban/quicktest"
28 )
29
30 func TestDataFromTheme(t *testing.T) {
31 t.Parallel()
32
33 files := `
34 -- config.toml --
35 [module]
36 [[module.imports]]
37 path = "mytheme"
38 -- data/a.toml --
39 d1 = "d1main"
40 d2 = "d2main"
41 -- themes/mytheme/data/a.toml --
42 d1 = "d1theme"
43 d2 = "d2theme"
44 d3 = "d3theme"
45 -- layouts/index.html --
46 d1: {{ site.Data.a.d1 }}|d2: {{ site.Data.a.d2 }}|d3: {{ site.Data.a.d3 }}
47
48 `
49
50 b := NewIntegrationTestBuilder(
51 IntegrationTestConfig{
52 T: t,
53 TxtarString: files,
54 },
55 ).Build()
56
57 b.AssertFileContent("public/index.html", `
58 d1: d1main|d2: d2main|d3: d3theme
59 `)
60 }
61
62 func TestDataDir(t *testing.T) {
63 t.Parallel()
64 equivDataDirs := make([]dataDir, 3)
65 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": "red" , "c2": "blue" } }`)
66 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: red\n c2: blue")
67 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = \"red\"\nc2 = \"blue\"\n")
68 expected := map[string]any{
69 "test": map[string]any{
70 "a": map[string]any{
71 "b": map[string]any{
72 "c1": "red",
73 "c2": "blue",
74 },
75 },
76 },
77 }
78 doTestEquivalentDataDirs(t, equivDataDirs, expected)
79 }
80
81 // Unable to enforce equivalency for int values as
82 // the JSON, YAML and TOML parsers return
83 // float64, int, int64 respectively. They all return
84 // float64 for float values though:
85 func TestDataDirNumeric(t *testing.T) {
86 t.Parallel()
87 equivDataDirs := make([]dataDir, 3)
88 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": 1.7 , "c2": 2.9 } }`)
89 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: 1.7\n c2: 2.9")
90 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = 1.7\nc2 = 2.9\n")
91 expected := map[string]any{
92 "test": map[string]any{
93 "a": map[string]any{
94 "b": map[string]any{
95 "c1": 1.7,
96 "c2": 2.9,
97 },
98 },
99 },
100 }
101 doTestEquivalentDataDirs(t, equivDataDirs, expected)
102 }
103
104 func TestDataDirBoolean(t *testing.T) {
105 t.Parallel()
106 equivDataDirs := make([]dataDir, 3)
107 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": true , "c2": false } }`)
108 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: true\n c2: false")
109 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = true\nc2 = false\n")
110 expected := map[string]any{
111 "test": map[string]any{
112 "a": map[string]any{
113 "b": map[string]any{
114 "c1": true,
115 "c2": false,
116 },
117 },
118 },
119 }
120 doTestEquivalentDataDirs(t, equivDataDirs, expected)
121 }
122
123 func TestDataDirTwoFiles(t *testing.T) {
124 t.Parallel()
125 equivDataDirs := make([]dataDir, 3)
126
127 equivDataDirs[0].addSource("data/test/foo.json", `{ "bar": "foofoo" }`)
128 equivDataDirs[0].addSource("data/test.json", `{ "hello": [ "world", "foo" ] }`)
129
130 equivDataDirs[1].addSource("data/test/foo.yaml", "bar: foofoo")
131 equivDataDirs[1].addSource("data/test.yaml", "hello:\n- world\n- foo")
132
133 equivDataDirs[2].addSource("data/test/foo.toml", "bar = \"foofoo\"")
134 equivDataDirs[2].addSource("data/test.toml", "hello = [\"world\", \"foo\"]")
135
136 expected :=
137 map[string]any{
138 "test": map[string]any{
139 "hello": []any{
140 "world",
141 "foo",
142 },
143 "foo": map[string]any{
144 "bar": "foofoo",
145 },
146 },
147 }
148
149 doTestEquivalentDataDirs(t, equivDataDirs, expected)
150 }
151
152 func TestDataDirOverriddenValue(t *testing.T) {
153 t.Parallel()
154 equivDataDirs := make([]dataDir, 3)
155
156 // filepath.Walk walks the files in lexical order, '/' comes before '.'. Simulate this:
157 equivDataDirs[0].addSource("data/a.json", `{"a": "1"}`)
158 equivDataDirs[0].addSource("data/test/v1.json", `{"v1-2": "2"}`)
159 equivDataDirs[0].addSource("data/test/v2.json", `{"v2": ["2", "3"]}`)
160 equivDataDirs[0].addSource("data/test.json", `{"v1": "1"}`)
161
162 equivDataDirs[1].addSource("data/a.yaml", "a: \"1\"")
163 equivDataDirs[1].addSource("data/test/v1.yaml", "v1-2: \"2\"")
164 equivDataDirs[1].addSource("data/test/v2.yaml", "v2:\n- \"2\"\n- \"3\"")
165 equivDataDirs[1].addSource("data/test.yaml", "v1: \"1\"")
166
167 equivDataDirs[2].addSource("data/a.toml", "a = \"1\"")
168 equivDataDirs[2].addSource("data/test/v1.toml", "v1-2 = \"2\"")
169 equivDataDirs[2].addSource("data/test/v2.toml", "v2 = [\"2\", \"3\"]")
170 equivDataDirs[2].addSource("data/test.toml", "v1 = \"1\"")
171
172 expected :=
173 map[string]any{
174 "a": map[string]any{"a": "1"},
175 "test": map[string]any{
176 "v1": map[string]any{"v1-2": "2"},
177 "v2": map[string]any{"v2": []any{"2", "3"}},
178 },
179 }
180
181 doTestEquivalentDataDirs(t, equivDataDirs, expected)
182 }
183
184 // Issue #4361, #3890
185 func TestDataDirArrayAtTopLevelOfFile(t *testing.T) {
186 t.Parallel()
187 equivDataDirs := make([]dataDir, 2)
188
189 equivDataDirs[0].addSource("data/test.json", `[ { "hello": "world" }, { "what": "time" }, { "is": "lunch?" } ]`)
190 equivDataDirs[1].addSource("data/test.yaml", `
191 - hello: world
192 - what: time
193 - is: lunch?
194 `)
195
196 expected :=
197 map[string]any{
198 "test": []any{
199 map[string]any{"hello": "world"},
200 map[string]any{"what": "time"},
201 map[string]any{"is": "lunch?"},
202 },
203 }
204
205 doTestEquivalentDataDirs(t, equivDataDirs, expected)
206 }
207
208 // Issue #892
209 func TestDataDirMultipleSources(t *testing.T) {
210 t.Parallel()
211
212 var dd dataDir
213 dd.addSource("data/test/first.yaml", "bar: 1")
214 dd.addSource("themes/mytheme/data/test/first.yaml", "bar: 2")
215 dd.addSource("data/test/second.yaml", "tender: 2")
216
217 expected :=
218 map[string]any{
219 "test": map[string]any{
220 "first": map[string]any{
221 "bar": 1,
222 },
223 "second": map[string]any{
224 "tender": 2,
225 },
226 },
227 }
228
229 doTestDataDir(t, dd, expected,
230 "theme", "mytheme")
231 }
232
233 // test (and show) the way values from four different sources,
234 // including theme data, commingle and override
235 func TestDataDirMultipleSourcesCommingled(t *testing.T) {
236 t.Parallel()
237
238 var dd dataDir
239 dd.addSource("data/a.json", `{ "b1" : { "c1": "data/a" }, "b2": "data/a", "b3": ["x", "y", "z"] }`)
240 dd.addSource("themes/mytheme/data/a.json", `{ "b1": "mytheme/data/a", "b2": "mytheme/data/a", "b3": "mytheme/data/a" }`)
241 dd.addSource("themes/mytheme/data/a/b1.json", `{ "c1": "mytheme/data/a/b1", "c2": "mytheme/data/a/b1" }`)
242 dd.addSource("data/a/b1.json", `{ "c1": "data/a/b1" }`)
243
244 // Per handleDataFile() comment:
245 // 1. A theme uses the same key; the main data folder wins
246 // 2. A sub folder uses the same key: the sub folder wins
247 expected :=
248 map[string]any{
249 "a": map[string]any{
250 "b1": map[string]any{
251 "c1": "data/a/b1",
252 "c2": "mytheme/data/a/b1",
253 },
254 "b2": "data/a",
255 "b3": []any{"x", "y", "z"},
256 },
257 }
258
259 doTestDataDir(t, dd, expected, "theme", "mytheme")
260 }
261
262 func TestDataDirCollidingChildArrays(t *testing.T) {
263 t.Parallel()
264
265 var dd dataDir
266 dd.addSource("themes/mytheme/data/a/b2.json", `["Q", "R", "S"]`)
267 dd.addSource("data/a.json", `{ "b1" : "data/a", "b2" : ["x", "y", "z"] }`)
268 dd.addSource("data/a/b2.json", `["1", "2", "3"]`)
269
270 // Per handleDataFile() comment:
271 // 1. A theme uses the same key; the main data folder wins
272 // 2. A sub folder uses the same key: the sub folder wins
273 expected :=
274 map[string]any{
275 "a": map[string]any{
276 "b1": "data/a",
277 "b2": []any{"1", "2", "3"},
278 },
279 }
280
281 doTestDataDir(t, dd, expected, "theme", "mytheme")
282 }
283
284 func TestDataDirCollidingTopLevelArrays(t *testing.T) {
285 t.Parallel()
286
287 var dd dataDir
288 dd.addSource("themes/mytheme/data/a/b1.json", `["x", "y", "z"]`)
289 dd.addSource("data/a/b1.json", `["1", "2", "3"]`)
290
291 expected :=
292 map[string]any{
293 "a": map[string]any{
294 "b1": []any{"1", "2", "3"},
295 },
296 }
297
298 doTestDataDir(t, dd, expected, "theme", "mytheme")
299 }
300
301 func TestDataDirCollidingMapsAndArrays(t *testing.T) {
302 t.Parallel()
303
304 var dd dataDir
305 // on
306 dd.addSource("themes/mytheme/data/a.json", `["1", "2", "3"]`)
307 dd.addSource("themes/mytheme/data/b.json", `{ "film" : "Logan Lucky" }`)
308 dd.addSource("data/a.json", `{ "music" : "Queen's Rebuke" }`)
309 dd.addSource("data/b.json", `["x", "y", "z"]`)
310
311 expected :=
312 map[string]any{
313 "a": map[string]any{
314 "music": "Queen's Rebuke",
315 },
316 "b": []any{"x", "y", "z"},
317 }
318
319 doTestDataDir(t, dd, expected, "theme", "mytheme")
320 }
321
322 // https://discourse.gohugo.io/t/recursive-data-file-parsing/26192
323 func TestDataDirNestedDirectories(t *testing.T) {
324 t.Parallel()
325
326 var dd dataDir
327 dd.addSource("themes/mytheme/data/a.json", `["1", "2", "3"]`)
328 dd.addSource("data/test1/20/06/a.json", `{ "artist" : "Michael Brecker" }`)
329 dd.addSource("data/test1/20/05/b.json", `{ "artist" : "Charlie Parker" }`)
330
331 expected :=
332 map[string]any{
333 "a": []any{"1", "2", "3"},
334 "test1": map[string]any{"20": map[string]any{"05": map[string]any{"b": map[string]any{"artist": "Charlie Parker"}}, "06": map[string]any{"a": map[string]any{"artist": "Michael Brecker"}}}},
335 }
336
337 doTestDataDir(t, dd, expected, "theme", "mytheme")
338 }
339
340 type dataDir struct {
341 sources [][2]string
342 }
343
344 func (d *dataDir) addSource(path, content string) {
345 d.sources = append(d.sources, [2]string{path, content})
346 }
347
348 func doTestEquivalentDataDirs(t *testing.T, equivDataDirs []dataDir, expected any, configKeyValues ...any) {
349 for i, dd := range equivDataDirs {
350 err := doTestDataDirImpl(t, dd, expected, configKeyValues...)
351 if err != "" {
352 t.Errorf("equivDataDirs[%d]: %s", i, err)
353 }
354 }
355 }
356
357 func doTestDataDir(t *testing.T, dd dataDir, expected any, configKeyValues ...any) {
358 err := doTestDataDirImpl(t, dd, expected, configKeyValues...)
359 if err != "" {
360 t.Error(err)
361 }
362 }
363
364 func doTestDataDirImpl(t *testing.T, dd dataDir, expected any, configKeyValues ...any) (err string) {
365 cfg, fs := newTestCfg()
366
367 for i := 0; i < len(configKeyValues); i += 2 {
368 cfg.Set(configKeyValues[i].(string), configKeyValues[i+1])
369 }
370
371 var (
372 logger = loggers.NewErrorLogger()
373 depsCfg = deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: logger}
374 )
375
376 writeSource(t, fs, filepath.Join("content", "dummy.md"), "content")
377 writeSourcesToSource(t, "", fs, dd.sources...)
378
379 expectBuildError := false
380
381 if ok, shouldFail := expected.(bool); ok && shouldFail {
382 expectBuildError = true
383 }
384
385 // trap and report panics as unmarshaling errors so that test suit can complete
386 defer func() {
387 if r := recover(); r != nil {
388 // Capture the stack trace
389 buf := make([]byte, 10000)
390 runtime.Stack(buf, false)
391 t.Errorf("PANIC: %s\n\nStack Trace : %s", r, string(buf))
392 }
393 }()
394
395 s := buildSingleSiteExpected(t, false, expectBuildError, depsCfg, BuildCfg{SkipRender: true})
396
397 if !expectBuildError && !reflect.DeepEqual(expected, s.h.Data()) {
398 // This disabled code detects the situation described in the WARNING message below.
399 // The situation seems to only occur for TOML data with integer values.
400 // Perhaps the TOML parser returns ints in another type.
401 // Re-enable temporarily to debug fails that should be passing.
402 // Re-enable permanently if reflect.DeepEqual is simply too strict.
403 /*
404 exp := fmt.Sprintf("%#v", expected)
405 got := fmt.Sprintf("%#v", s.Data)
406 if exp == got {
407 t.Logf("WARNING: reflect.DeepEqual returned FALSE for values that appear equal.\n"+
408 "Treating as equal for the purpose of the test, but this maybe should be investigated.\n"+
409 "Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.Data)
410 return
411 }
412 */
413
414 return fmt.Sprintf("Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.h.Data())
415 }
416
417 return
418 }
419
420 func TestDataFromShortcode(t *testing.T) {
421 t.Parallel()
422
423 var (
424 cfg, fs = newTestCfg()
425 c = qt.New(t)
426 )
427
428 writeSource(t, fs, "data/hugo.toml", "slogan = \"Hugo Rocks!\"")
429 writeSource(t, fs, "layouts/_default/single.html", `
430 * Slogan from template: {{ .Site.Data.hugo.slogan }}
431 * {{ .Content }}`)
432 writeSource(t, fs, "layouts/shortcodes/d.html", `{{ .Page.Site.Data.hugo.slogan }}`)
433 writeSource(t, fs, "content/c.md", `---
434 ---
435 Slogan from shortcode: {{< d >}}
436 `)
437
438 buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
439
440 content := readSource(t, fs, "public/c/index.html")
441
442 c.Assert(content, qt.Contains, "Slogan from template: Hugo Rocks!")
443 c.Assert(content, qt.Contains, "Slogan from shortcode: Hugo Rocks!")
444 }