datafiles_test.go (12703B)
1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "fmt" 18 "path/filepath" 19 "reflect" 20 "runtime" 21 "testing" 22 23 "github.com/gohugoio/hugo/common/loggers" 24 25 "github.com/gohugoio/hugo/deps" 26 27 qt "github.com/frankban/quicktest" 28 ) 29 30 func TestDataFromTheme(t *testing.T) { 31 t.Parallel() 32 33 files := ` 34 -- config.toml -- 35 [module] 36 [[module.imports]] 37 path = "mytheme" 38 -- data/a.toml -- 39 d1 = "d1main" 40 d2 = "d2main" 41 -- themes/mytheme/data/a.toml -- 42 d1 = "d1theme" 43 d2 = "d2theme" 44 d3 = "d3theme" 45 -- layouts/index.html -- 46 d1: {{ site.Data.a.d1 }}|d2: {{ site.Data.a.d2 }}|d3: {{ site.Data.a.d3 }} 47 48 ` 49 50 b := NewIntegrationTestBuilder( 51 IntegrationTestConfig{ 52 T: t, 53 TxtarString: files, 54 }, 55 ).Build() 56 57 b.AssertFileContent("public/index.html", ` 58 d1: d1main|d2: d2main|d3: d3theme 59 `) 60 } 61 62 func TestDataDir(t *testing.T) { 63 t.Parallel() 64 equivDataDirs := make([]dataDir, 3) 65 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": "red" , "c2": "blue" } }`) 66 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: red\n c2: blue") 67 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = \"red\"\nc2 = \"blue\"\n") 68 expected := map[string]any{ 69 "test": map[string]any{ 70 "a": map[string]any{ 71 "b": map[string]any{ 72 "c1": "red", 73 "c2": "blue", 74 }, 75 }, 76 }, 77 } 78 doTestEquivalentDataDirs(t, equivDataDirs, expected) 79 } 80 81 // Unable to enforce equivalency for int values as 82 // the JSON, YAML and TOML parsers return 83 // float64, int, int64 respectively. They all return 84 // float64 for float values though: 85 func TestDataDirNumeric(t *testing.T) { 86 t.Parallel() 87 equivDataDirs := make([]dataDir, 3) 88 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": 1.7 , "c2": 2.9 } }`) 89 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: 1.7\n c2: 2.9") 90 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = 1.7\nc2 = 2.9\n") 91 expected := map[string]any{ 92 "test": map[string]any{ 93 "a": map[string]any{ 94 "b": map[string]any{ 95 "c1": 1.7, 96 "c2": 2.9, 97 }, 98 }, 99 }, 100 } 101 doTestEquivalentDataDirs(t, equivDataDirs, expected) 102 } 103 104 func TestDataDirBoolean(t *testing.T) { 105 t.Parallel() 106 equivDataDirs := make([]dataDir, 3) 107 equivDataDirs[0].addSource("data/test/a.json", `{ "b" : { "c1": true , "c2": false } }`) 108 equivDataDirs[1].addSource("data/test/a.yaml", "b:\n c1: true\n c2: false") 109 equivDataDirs[2].addSource("data/test/a.toml", "[b]\nc1 = true\nc2 = false\n") 110 expected := map[string]any{ 111 "test": map[string]any{ 112 "a": map[string]any{ 113 "b": map[string]any{ 114 "c1": true, 115 "c2": false, 116 }, 117 }, 118 }, 119 } 120 doTestEquivalentDataDirs(t, equivDataDirs, expected) 121 } 122 123 func TestDataDirTwoFiles(t *testing.T) { 124 t.Parallel() 125 equivDataDirs := make([]dataDir, 3) 126 127 equivDataDirs[0].addSource("data/test/foo.json", `{ "bar": "foofoo" }`) 128 equivDataDirs[0].addSource("data/test.json", `{ "hello": [ "world", "foo" ] }`) 129 130 equivDataDirs[1].addSource("data/test/foo.yaml", "bar: foofoo") 131 equivDataDirs[1].addSource("data/test.yaml", "hello:\n- world\n- foo") 132 133 equivDataDirs[2].addSource("data/test/foo.toml", "bar = \"foofoo\"") 134 equivDataDirs[2].addSource("data/test.toml", "hello = [\"world\", \"foo\"]") 135 136 expected := 137 map[string]any{ 138 "test": map[string]any{ 139 "hello": []any{ 140 "world", 141 "foo", 142 }, 143 "foo": map[string]any{ 144 "bar": "foofoo", 145 }, 146 }, 147 } 148 149 doTestEquivalentDataDirs(t, equivDataDirs, expected) 150 } 151 152 func TestDataDirOverriddenValue(t *testing.T) { 153 t.Parallel() 154 equivDataDirs := make([]dataDir, 3) 155 156 // filepath.Walk walks the files in lexical order, '/' comes before '.'. Simulate this: 157 equivDataDirs[0].addSource("data/a.json", `{"a": "1"}`) 158 equivDataDirs[0].addSource("data/test/v1.json", `{"v1-2": "2"}`) 159 equivDataDirs[0].addSource("data/test/v2.json", `{"v2": ["2", "3"]}`) 160 equivDataDirs[0].addSource("data/test.json", `{"v1": "1"}`) 161 162 equivDataDirs[1].addSource("data/a.yaml", "a: \"1\"") 163 equivDataDirs[1].addSource("data/test/v1.yaml", "v1-2: \"2\"") 164 equivDataDirs[1].addSource("data/test/v2.yaml", "v2:\n- \"2\"\n- \"3\"") 165 equivDataDirs[1].addSource("data/test.yaml", "v1: \"1\"") 166 167 equivDataDirs[2].addSource("data/a.toml", "a = \"1\"") 168 equivDataDirs[2].addSource("data/test/v1.toml", "v1-2 = \"2\"") 169 equivDataDirs[2].addSource("data/test/v2.toml", "v2 = [\"2\", \"3\"]") 170 equivDataDirs[2].addSource("data/test.toml", "v1 = \"1\"") 171 172 expected := 173 map[string]any{ 174 "a": map[string]any{"a": "1"}, 175 "test": map[string]any{ 176 "v1": map[string]any{"v1-2": "2"}, 177 "v2": map[string]any{"v2": []any{"2", "3"}}, 178 }, 179 } 180 181 doTestEquivalentDataDirs(t, equivDataDirs, expected) 182 } 183 184 // Issue #4361, #3890 185 func TestDataDirArrayAtTopLevelOfFile(t *testing.T) { 186 t.Parallel() 187 equivDataDirs := make([]dataDir, 2) 188 189 equivDataDirs[0].addSource("data/test.json", `[ { "hello": "world" }, { "what": "time" }, { "is": "lunch?" } ]`) 190 equivDataDirs[1].addSource("data/test.yaml", ` 191 - hello: world 192 - what: time 193 - is: lunch? 194 `) 195 196 expected := 197 map[string]any{ 198 "test": []any{ 199 map[string]any{"hello": "world"}, 200 map[string]any{"what": "time"}, 201 map[string]any{"is": "lunch?"}, 202 }, 203 } 204 205 doTestEquivalentDataDirs(t, equivDataDirs, expected) 206 } 207 208 // Issue #892 209 func TestDataDirMultipleSources(t *testing.T) { 210 t.Parallel() 211 212 var dd dataDir 213 dd.addSource("data/test/first.yaml", "bar: 1") 214 dd.addSource("themes/mytheme/data/test/first.yaml", "bar: 2") 215 dd.addSource("data/test/second.yaml", "tender: 2") 216 217 expected := 218 map[string]any{ 219 "test": map[string]any{ 220 "first": map[string]any{ 221 "bar": 1, 222 }, 223 "second": map[string]any{ 224 "tender": 2, 225 }, 226 }, 227 } 228 229 doTestDataDir(t, dd, expected, 230 "theme", "mytheme") 231 } 232 233 // test (and show) the way values from four different sources, 234 // including theme data, commingle and override 235 func TestDataDirMultipleSourcesCommingled(t *testing.T) { 236 t.Parallel() 237 238 var dd dataDir 239 dd.addSource("data/a.json", `{ "b1" : { "c1": "data/a" }, "b2": "data/a", "b3": ["x", "y", "z"] }`) 240 dd.addSource("themes/mytheme/data/a.json", `{ "b1": "mytheme/data/a", "b2": "mytheme/data/a", "b3": "mytheme/data/a" }`) 241 dd.addSource("themes/mytheme/data/a/b1.json", `{ "c1": "mytheme/data/a/b1", "c2": "mytheme/data/a/b1" }`) 242 dd.addSource("data/a/b1.json", `{ "c1": "data/a/b1" }`) 243 244 // Per handleDataFile() comment: 245 // 1. A theme uses the same key; the main data folder wins 246 // 2. A sub folder uses the same key: the sub folder wins 247 expected := 248 map[string]any{ 249 "a": map[string]any{ 250 "b1": map[string]any{ 251 "c1": "data/a/b1", 252 "c2": "mytheme/data/a/b1", 253 }, 254 "b2": "data/a", 255 "b3": []any{"x", "y", "z"}, 256 }, 257 } 258 259 doTestDataDir(t, dd, expected, "theme", "mytheme") 260 } 261 262 func TestDataDirCollidingChildArrays(t *testing.T) { 263 t.Parallel() 264 265 var dd dataDir 266 dd.addSource("themes/mytheme/data/a/b2.json", `["Q", "R", "S"]`) 267 dd.addSource("data/a.json", `{ "b1" : "data/a", "b2" : ["x", "y", "z"] }`) 268 dd.addSource("data/a/b2.json", `["1", "2", "3"]`) 269 270 // Per handleDataFile() comment: 271 // 1. A theme uses the same key; the main data folder wins 272 // 2. A sub folder uses the same key: the sub folder wins 273 expected := 274 map[string]any{ 275 "a": map[string]any{ 276 "b1": "data/a", 277 "b2": []any{"1", "2", "3"}, 278 }, 279 } 280 281 doTestDataDir(t, dd, expected, "theme", "mytheme") 282 } 283 284 func TestDataDirCollidingTopLevelArrays(t *testing.T) { 285 t.Parallel() 286 287 var dd dataDir 288 dd.addSource("themes/mytheme/data/a/b1.json", `["x", "y", "z"]`) 289 dd.addSource("data/a/b1.json", `["1", "2", "3"]`) 290 291 expected := 292 map[string]any{ 293 "a": map[string]any{ 294 "b1": []any{"1", "2", "3"}, 295 }, 296 } 297 298 doTestDataDir(t, dd, expected, "theme", "mytheme") 299 } 300 301 func TestDataDirCollidingMapsAndArrays(t *testing.T) { 302 t.Parallel() 303 304 var dd dataDir 305 // on 306 dd.addSource("themes/mytheme/data/a.json", `["1", "2", "3"]`) 307 dd.addSource("themes/mytheme/data/b.json", `{ "film" : "Logan Lucky" }`) 308 dd.addSource("data/a.json", `{ "music" : "Queen's Rebuke" }`) 309 dd.addSource("data/b.json", `["x", "y", "z"]`) 310 311 expected := 312 map[string]any{ 313 "a": map[string]any{ 314 "music": "Queen's Rebuke", 315 }, 316 "b": []any{"x", "y", "z"}, 317 } 318 319 doTestDataDir(t, dd, expected, "theme", "mytheme") 320 } 321 322 // https://discourse.gohugo.io/t/recursive-data-file-parsing/26192 323 func TestDataDirNestedDirectories(t *testing.T) { 324 t.Parallel() 325 326 var dd dataDir 327 dd.addSource("themes/mytheme/data/a.json", `["1", "2", "3"]`) 328 dd.addSource("data/test1/20/06/a.json", `{ "artist" : "Michael Brecker" }`) 329 dd.addSource("data/test1/20/05/b.json", `{ "artist" : "Charlie Parker" }`) 330 331 expected := 332 map[string]any{ 333 "a": []any{"1", "2", "3"}, 334 "test1": map[string]any{"20": map[string]any{"05": map[string]any{"b": map[string]any{"artist": "Charlie Parker"}}, "06": map[string]any{"a": map[string]any{"artist": "Michael Brecker"}}}}, 335 } 336 337 doTestDataDir(t, dd, expected, "theme", "mytheme") 338 } 339 340 type dataDir struct { 341 sources [][2]string 342 } 343 344 func (d *dataDir) addSource(path, content string) { 345 d.sources = append(d.sources, [2]string{path, content}) 346 } 347 348 func doTestEquivalentDataDirs(t *testing.T, equivDataDirs []dataDir, expected any, configKeyValues ...any) { 349 for i, dd := range equivDataDirs { 350 err := doTestDataDirImpl(t, dd, expected, configKeyValues...) 351 if err != "" { 352 t.Errorf("equivDataDirs[%d]: %s", i, err) 353 } 354 } 355 } 356 357 func doTestDataDir(t *testing.T, dd dataDir, expected any, configKeyValues ...any) { 358 err := doTestDataDirImpl(t, dd, expected, configKeyValues...) 359 if err != "" { 360 t.Error(err) 361 } 362 } 363 364 func doTestDataDirImpl(t *testing.T, dd dataDir, expected any, configKeyValues ...any) (err string) { 365 cfg, fs := newTestCfg() 366 367 for i := 0; i < len(configKeyValues); i += 2 { 368 cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) 369 } 370 371 var ( 372 logger = loggers.NewErrorLogger() 373 depsCfg = deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: logger} 374 ) 375 376 writeSource(t, fs, filepath.Join("content", "dummy.md"), "content") 377 writeSourcesToSource(t, "", fs, dd.sources...) 378 379 expectBuildError := false 380 381 if ok, shouldFail := expected.(bool); ok && shouldFail { 382 expectBuildError = true 383 } 384 385 // trap and report panics as unmarshaling errors so that test suit can complete 386 defer func() { 387 if r := recover(); r != nil { 388 // Capture the stack trace 389 buf := make([]byte, 10000) 390 runtime.Stack(buf, false) 391 t.Errorf("PANIC: %s\n\nStack Trace : %s", r, string(buf)) 392 } 393 }() 394 395 s := buildSingleSiteExpected(t, false, expectBuildError, depsCfg, BuildCfg{SkipRender: true}) 396 397 if !expectBuildError && !reflect.DeepEqual(expected, s.h.Data()) { 398 // This disabled code detects the situation described in the WARNING message below. 399 // The situation seems to only occur for TOML data with integer values. 400 // Perhaps the TOML parser returns ints in another type. 401 // Re-enable temporarily to debug fails that should be passing. 402 // Re-enable permanently if reflect.DeepEqual is simply too strict. 403 /* 404 exp := fmt.Sprintf("%#v", expected) 405 got := fmt.Sprintf("%#v", s.Data) 406 if exp == got { 407 t.Logf("WARNING: reflect.DeepEqual returned FALSE for values that appear equal.\n"+ 408 "Treating as equal for the purpose of the test, but this maybe should be investigated.\n"+ 409 "Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.Data) 410 return 411 } 412 */ 413 414 return fmt.Sprintf("Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.h.Data()) 415 } 416 417 return 418 } 419 420 func TestDataFromShortcode(t *testing.T) { 421 t.Parallel() 422 423 var ( 424 cfg, fs = newTestCfg() 425 c = qt.New(t) 426 ) 427 428 writeSource(t, fs, "data/hugo.toml", "slogan = \"Hugo Rocks!\"") 429 writeSource(t, fs, "layouts/_default/single.html", ` 430 * Slogan from template: {{ .Site.Data.hugo.slogan }} 431 * {{ .Content }}`) 432 writeSource(t, fs, "layouts/shortcodes/d.html", `{{ .Page.Site.Data.hugo.slogan }}`) 433 writeSource(t, fs, "content/c.md", `--- 434 --- 435 Slogan from shortcode: {{< d >}} 436 `) 437 438 buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) 439 440 content := readSource(t, fs, "public/c/index.html") 441 442 c.Assert(content, qt.Contains, "Slogan from template: Hugo Rocks!") 443 c.Assert(content, qt.Contains, "Slogan from shortcode: Hugo Rocks!") 444 }