hugo_sites.go (26263B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "context"
18 "fmt"
19 "io"
20 "path/filepath"
21 "sort"
22 "strings"
23 "sync"
24 "sync/atomic"
25
26 "github.com/gohugoio/hugo/hugofs/glob"
27
28 "github.com/fsnotify/fsnotify"
29
30 "github.com/gohugoio/hugo/identity"
31
32 radix "github.com/armon/go-radix"
33
34 "github.com/gohugoio/hugo/output"
35 "github.com/gohugoio/hugo/parser/metadecoders"
36
37 "errors"
38
39 "github.com/gohugoio/hugo/common/para"
40 "github.com/gohugoio/hugo/hugofs"
41
42 "github.com/gohugoio/hugo/source"
43
44 "github.com/bep/gitmap"
45 "github.com/gohugoio/hugo/config"
46
47 "github.com/gohugoio/hugo/publisher"
48
49 "github.com/gohugoio/hugo/common/herrors"
50 "github.com/gohugoio/hugo/common/loggers"
51 "github.com/gohugoio/hugo/deps"
52 "github.com/gohugoio/hugo/helpers"
53 "github.com/gohugoio/hugo/langs"
54 "github.com/gohugoio/hugo/lazy"
55
56 "github.com/gohugoio/hugo/langs/i18n"
57 "github.com/gohugoio/hugo/resources/page"
58 "github.com/gohugoio/hugo/resources/page/pagemeta"
59 "github.com/gohugoio/hugo/tpl"
60 "github.com/gohugoio/hugo/tpl/tplimpl"
61 )
62
63 // HugoSites represents the sites to build. Each site represents a language.
64 type HugoSites struct {
65 Sites []*Site
66
67 multilingual *Multilingual
68
69 // Multihost is set if multilingual and baseURL set on the language level.
70 multihost bool
71
72 // If this is running in the dev server.
73 running bool
74
75 // Render output formats for all sites.
76 renderFormats output.Formats
77
78 // The currently rendered Site.
79 currentSite *Site
80
81 *deps.Deps
82
83 gitInfo *gitInfo
84 codeownerInfo *codeownerInfo
85
86 // As loaded from the /data dirs
87 data map[string]any
88
89 contentInit sync.Once
90 content *pageMaps
91
92 // Keeps track of bundle directories and symlinks to enable partial rebuilding.
93 ContentChanges *contentChangeMap
94
95 // File change events with filename stored in this map will be skipped.
96 skipRebuildForFilenamesMu sync.Mutex
97 skipRebuildForFilenames map[string]bool
98
99 init *hugoSitesInit
100
101 workers *para.Workers
102 numWorkers int
103
104 *fatalErrorHandler
105 *testCounters
106 }
107
108 // ShouldSkipFileChangeEvent allows skipping filesystem event early before
109 // the build is started.
110 func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
111 h.skipRebuildForFilenamesMu.Lock()
112 defer h.skipRebuildForFilenamesMu.Unlock()
113 return h.skipRebuildForFilenames[ev.Name]
114 }
115
116 func (h *HugoSites) getContentMaps() *pageMaps {
117 h.contentInit.Do(func() {
118 h.content = newPageMaps(h)
119 })
120 return h.content
121 }
122
123 // Only used in tests.
124 type testCounters struct {
125 contentRenderCounter uint64
126 pageRenderCounter uint64
127 }
128
129 func (h *testCounters) IncrContentRender() {
130 if h == nil {
131 return
132 }
133 atomic.AddUint64(&h.contentRenderCounter, 1)
134 }
135
136 func (h *testCounters) IncrPageRender() {
137 if h == nil {
138 return
139 }
140 atomic.AddUint64(&h.pageRenderCounter, 1)
141 }
142
143 type fatalErrorHandler struct {
144 mu sync.Mutex
145
146 h *HugoSites
147
148 err error
149
150 done bool
151 donec chan bool // will be closed when done
152 }
153
154 // FatalError error is used in some rare situations where it does not make sense to
155 // continue processing, to abort as soon as possible and log the error.
156 func (f *fatalErrorHandler) FatalError(err error) {
157 f.mu.Lock()
158 defer f.mu.Unlock()
159 if !f.done {
160 f.done = true
161 close(f.donec)
162 }
163 f.err = err
164 }
165
166 func (f *fatalErrorHandler) getErr() error {
167 f.mu.Lock()
168 defer f.mu.Unlock()
169 return f.err
170 }
171
172 func (f *fatalErrorHandler) Done() <-chan bool {
173 return f.donec
174 }
175
176 type hugoSitesInit struct {
177 // Loads the data from all of the /data folders.
178 data *lazy.Init
179
180 // Performs late initialization (before render) of the templates.
181 layouts *lazy.Init
182
183 // Loads the Git info and CODEOWNERS for all the pages if enabled.
184 gitInfo *lazy.Init
185
186 // Maps page translations.
187 translations *lazy.Init
188 }
189
190 func (h *hugoSitesInit) Reset() {
191 h.data.Reset()
192 h.layouts.Reset()
193 h.gitInfo.Reset()
194 h.translations.Reset()
195 }
196
197 func (h *HugoSites) Data() map[string]any {
198 if _, err := h.init.data.Do(); err != nil {
199 h.SendError(fmt.Errorf("failed to load data: %w", err))
200 return nil
201 }
202 return h.data
203 }
204
205 func (h *HugoSites) gitInfoForPage(p page.Page) (*gitmap.GitInfo, error) {
206 if _, err := h.init.gitInfo.Do(); err != nil {
207 return nil, err
208 }
209
210 if h.gitInfo == nil {
211 return nil, nil
212 }
213
214 return h.gitInfo.forPage(p), nil
215 }
216
217 func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) {
218 if _, err := h.init.gitInfo.Do(); err != nil {
219 return nil, err
220 }
221
222 if h.codeownerInfo == nil {
223 return nil, nil
224 }
225
226 return h.codeownerInfo.forPage(p), nil
227 }
228
229 func (h *HugoSites) siteInfos() page.Sites {
230 infos := make(page.Sites, len(h.Sites))
231 for i, site := range h.Sites {
232 infos[i] = site.Info
233 }
234 return infos
235 }
236
237 func (h *HugoSites) pickOneAndLogTheRest(errors []error) error {
238 if len(errors) == 0 {
239 return nil
240 }
241
242 var i int
243
244 for j, err := range errors {
245 // If this is in server mode, we want to return an error to the client
246 // with a file context, if possible.
247 if herrors.UnwrapFileError(err) != nil {
248 i = j
249 break
250 }
251 }
252
253 // Log the rest, but add a threshold to avoid flooding the log.
254 const errLogThreshold = 5
255
256 for j, err := range errors {
257 if j == i || err == nil {
258 continue
259 }
260
261 if j >= errLogThreshold {
262 break
263 }
264
265 h.Log.Errorln(err)
266 }
267
268 return errors[i]
269 }
270
271 func (h *HugoSites) IsMultihost() bool {
272 return h != nil && h.multihost
273 }
274
275 // TODO(bep) consolidate
276 func (h *HugoSites) LanguageSet() map[string]int {
277 set := make(map[string]int)
278 for i, s := range h.Sites {
279 set[s.language.Lang] = i
280 }
281 return set
282 }
283
284 func (h *HugoSites) NumLogErrors() int {
285 if h == nil {
286 return 0
287 }
288 return int(h.Log.LogCounters().ErrorCounter.Count())
289 }
290
291 func (h *HugoSites) PrintProcessingStats(w io.Writer) {
292 stats := make([]*helpers.ProcessingStats, len(h.Sites))
293 for i := 0; i < len(h.Sites); i++ {
294 stats[i] = h.Sites[i].PathSpec.ProcessingStats
295 }
296 helpers.ProcessingStatsTable(w, stats...)
297 }
298
299 // GetContentPage finds a Page with content given the absolute filename.
300 // Returns nil if none found.
301 func (h *HugoSites) GetContentPage(filename string) page.Page {
302 var p page.Page
303
304 h.getContentMaps().walkBundles(func(b *contentNode) bool {
305 if b.p == nil || b.fi == nil {
306 return false
307 }
308
309 if b.fi.Meta().Filename == filename {
310 p = b.p
311 return true
312 }
313
314 return false
315 })
316
317 return p
318 }
319
320 // NewHugoSites creates a new collection of sites given the input sites, building
321 // a language configuration based on those.
322 func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
323 if cfg.Language != nil {
324 return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
325 }
326
327 // Return error at the end. Make the caller decide if it's fatal or not.
328 var initErr error
329
330 langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)
331 if err != nil {
332 return nil, fmt.Errorf("failed to create language config: %w", err)
333 }
334
335 var contentChangeTracker *contentChangeMap
336
337 numWorkers := config.GetNumWorkerMultiplier()
338 if numWorkers > len(sites) {
339 numWorkers = len(sites)
340 }
341 var workers *para.Workers
342 if numWorkers > 1 {
343 workers = para.New(numWorkers)
344 }
345
346 h := &HugoSites{
347 running: cfg.Running,
348 multilingual: langConfig,
349 multihost: cfg.Cfg.GetBool("multihost"),
350 Sites: sites,
351 workers: workers,
352 numWorkers: numWorkers,
353 skipRebuildForFilenames: make(map[string]bool),
354 init: &hugoSitesInit{
355 data: lazy.New(),
356 layouts: lazy.New(),
357 gitInfo: lazy.New(),
358 translations: lazy.New(),
359 },
360 }
361
362 h.fatalErrorHandler = &fatalErrorHandler{
363 h: h,
364 donec: make(chan bool),
365 }
366
367 h.init.data.Add(func() (any, error) {
368 err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)
369 if err != nil {
370 return nil, fmt.Errorf("failed to load data: %w", err)
371 }
372 return nil, nil
373 })
374
375 h.init.layouts.Add(func() (any, error) {
376 for _, s := range h.Sites {
377 if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil {
378 return nil, err
379 }
380 }
381 return nil, nil
382 })
383
384 h.init.translations.Add(func() (any, error) {
385 if len(h.Sites) > 1 {
386 allTranslations := pagesToTranslationsMap(h.Sites)
387 assignTranslationsToPages(allTranslations, h.Sites)
388 }
389
390 return nil, nil
391 })
392
393 h.init.gitInfo.Add(func() (any, error) {
394 err := h.loadGitInfo()
395 if err != nil {
396 return nil, fmt.Errorf("failed to load Git info: %w", err)
397 }
398 return nil, nil
399 })
400
401 for _, s := range sites {
402 s.h = h
403 }
404
405 var l configLoader
406 if err := l.applyDeps(cfg, sites...); err != nil {
407 initErr = fmt.Errorf("add site dependencies: %w", err)
408 }
409
410 h.Deps = sites[0].Deps
411 if h.Deps == nil {
412 return nil, initErr
413 }
414
415 // Only needed in server mode.
416 // TODO(bep) clean up the running vs watching terms
417 if cfg.Running {
418 contentChangeTracker = &contentChangeMap{
419 pathSpec: h.PathSpec,
420 symContent: make(map[string]map[string]bool),
421 leafBundles: radix.New(),
422 branchBundles: make(map[string]bool),
423 }
424 h.ContentChanges = contentChangeTracker
425 }
426
427 return h, initErr
428 }
429
430 func (h *HugoSites) loadGitInfo() error {
431 if h.Cfg.GetBool("enableGitInfo") {
432 gi, err := newGitInfo(h.Cfg)
433 if err != nil {
434 h.Log.Errorln("Failed to read Git log:", err)
435 } else {
436 h.gitInfo = gi
437 }
438
439 co, err := newCodeOwners(h.Cfg)
440 if err != nil {
441 h.Log.Errorln("Failed to read CODEOWNERS:", err)
442 } else {
443 h.codeownerInfo = co
444 }
445 }
446 return nil
447 }
448
449 func (l configLoader) applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
450 if cfg.TemplateProvider == nil {
451 cfg.TemplateProvider = tplimpl.DefaultTemplateProvider
452 }
453
454 if cfg.TranslationProvider == nil {
455 cfg.TranslationProvider = i18n.NewTranslationProvider()
456 }
457
458 var (
459 d *deps.Deps
460 err error
461 )
462
463 for _, s := range sites {
464 if s.Deps != nil {
465 continue
466 }
467
468 onCreated := func(d *deps.Deps) error {
469 s.Deps = d
470
471 // Set up the main publishing chain.
472 pub, err := publisher.NewDestinationPublisher(
473 d.ResourceSpec,
474 s.outputFormatsConfig,
475 s.mediaTypesConfig,
476 )
477 if err != nil {
478 return err
479 }
480 s.publisher = pub
481
482 if err := s.initializeSiteInfo(); err != nil {
483 return err
484 }
485
486 d.Site = s.Info
487
488 siteConfig, err := l.loadSiteConfig(s.language)
489 if err != nil {
490 return fmt.Errorf("load site config: %w", err)
491 }
492 s.siteConfigConfig = siteConfig
493
494 pm := &pageMap{
495 contentMap: newContentMap(contentMapConfig{
496 lang: s.Lang(),
497 taxonomyConfig: s.siteCfg.taxonomiesConfig.Values(),
498 taxonomyDisabled: !s.isEnabled(page.KindTerm),
499 taxonomyTermDisabled: !s.isEnabled(page.KindTaxonomy),
500 pageDisabled: !s.isEnabled(page.KindPage),
501 }),
502 s: s,
503 }
504
505 s.PageCollections = newPageCollections(pm)
506
507 s.siteRefLinker, err = newSiteRefLinker(s.language, s)
508 return err
509 }
510
511 cfg.Language = s.language
512 cfg.MediaTypes = s.mediaTypesConfig
513 cfg.OutputFormats = s.outputFormatsConfig
514
515 if d == nil {
516 cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)
517
518 var err error
519 d, err = deps.New(cfg)
520 if err != nil {
521 return fmt.Errorf("create deps: %w", err)
522 }
523
524 d.OutputFormatsConfig = s.outputFormatsConfig
525
526 if err := onCreated(d); err != nil {
527 return fmt.Errorf("on created: %w", err)
528 }
529
530 if err = d.LoadResources(); err != nil {
531 return fmt.Errorf("load resources: %w", err)
532 }
533
534 } else {
535 d, err = d.ForLanguage(cfg, onCreated)
536 if err != nil {
537 return err
538 }
539 d.OutputFormatsConfig = s.outputFormatsConfig
540 }
541 }
542
543 return nil
544 }
545
546 // NewHugoSites creates HugoSites from the given config.
547 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
548 if cfg.Logger == nil {
549 cfg.Logger = loggers.NewErrorLogger()
550 }
551 sites, err := createSitesFromConfig(cfg)
552 if err != nil {
553 return nil, fmt.Errorf("from config: %w", err)
554 }
555 return newHugoSites(cfg, sites...)
556 }
557
558 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error {
559 return func(templ tpl.TemplateManager) error {
560 for _, wt := range withTemplates {
561 if wt == nil {
562 continue
563 }
564 if err := wt(templ); err != nil {
565 return err
566 }
567 }
568
569 return nil
570 }
571 }
572
573 func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
574 var sites []*Site
575
576 languages := getLanguages(cfg.Cfg)
577
578 for _, lang := range languages {
579 if lang.Disabled {
580 continue
581 }
582 var s *Site
583 var err error
584 cfg.Language = lang
585 s, err = newSite(cfg)
586
587 if err != nil {
588 return nil, err
589 }
590
591 sites = append(sites, s)
592 }
593
594 return sites, nil
595 }
596
597 // Reset resets the sites and template caches etc., making it ready for a full rebuild.
598 func (h *HugoSites) reset(config *BuildCfg) {
599 if config.ResetState {
600 for i, s := range h.Sites {
601 h.Sites[i] = s.reset()
602 if r, ok := s.Fs.PublishDir.(hugofs.Reseter); ok {
603 r.Reset()
604 }
605 }
606 }
607
608 h.fatalErrorHandler = &fatalErrorHandler{
609 h: h,
610 donec: make(chan bool),
611 }
612
613 h.init.Reset()
614 }
615
616 // resetLogs resets the log counters etc. Used to do a new build on the same sites.
617 func (h *HugoSites) resetLogs() {
618 h.Log.Reset()
619 loggers.GlobalErrorCounter.Reset()
620 for _, s := range h.Sites {
621 s.Deps.Log.Reset()
622 s.Deps.LogDistinct.Reset()
623 }
624 }
625
626 func (h *HugoSites) withSite(fn func(s *Site) error) error {
627 if h.workers == nil {
628 for _, s := range h.Sites {
629 if err := fn(s); err != nil {
630 return err
631 }
632 }
633 return nil
634 }
635
636 g, _ := h.workers.Start(context.Background())
637 for _, s := range h.Sites {
638 s := s
639 g.Run(func() error {
640 return fn(s)
641 })
642 }
643 return g.Wait()
644 }
645
646 func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error {
647 oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages)
648
649 l := configLoader{cfg: h.Cfg}
650 if err := l.loadLanguageSettings(oldLangs); err != nil {
651 return err
652 }
653
654 depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: l.cfg}
655
656 sites, err := createSitesFromConfig(depsCfg)
657 if err != nil {
658 return err
659 }
660
661 langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...)
662 if err != nil {
663 return err
664 }
665
666 h.Sites = sites
667
668 for _, s := range sites {
669 s.h = h
670 }
671
672 var cl configLoader
673 if err := cl.applyDeps(depsCfg, sites...); err != nil {
674 return err
675 }
676
677 h.Deps = sites[0].Deps
678
679 h.multilingual = langConfig
680 h.multihost = h.Deps.Cfg.GetBool("multihost")
681
682 return nil
683 }
684
685 func (h *HugoSites) toSiteInfos() []*SiteInfo {
686 infos := make([]*SiteInfo, len(h.Sites))
687 for i, s := range h.Sites {
688 infos[i] = s.Info
689 }
690 return infos
691 }
692
693 // BuildCfg holds build options used to, as an example, skip the render step.
694 type BuildCfg struct {
695 // Reset site state before build. Use to force full rebuilds.
696 ResetState bool
697 // If set, we re-create the sites from the given configuration before a build.
698 // This is needed if new languages are added.
699 NewConfig config.Provider
700 // Skip rendering. Useful for testing.
701 SkipRender bool
702 // Use this to indicate what changed (for rebuilds).
703 whatChanged *whatChanged
704
705 // This is a partial re-render of some selected pages. This means
706 // we should skip most of the processing.
707 PartialReRender bool
708
709 // Set in server mode when the last build failed for some reason.
710 ErrRecovery bool
711
712 // Recently visited URLs. This is used for partial re-rendering.
713 RecentlyVisited map[string]bool
714
715 // Can be set to build only with a sub set of the content source.
716 ContentInclusionFilter *glob.FilenameFilter
717
718 // Set when the buildlock is already acquired (e.g. the archetype content builder).
719 NoBuildLock bool
720
721 testCounters *testCounters
722 }
723
724 // shouldRender is used in the Fast Render Mode to determine if we need to re-render
725 // a Page: If it is recently visited (the home pages will always be in this set) or changed.
726 // Note that a page does not have to have a content page / file.
727 // For regular builds, this will allways return true.
728 // TODO(bep) rename/work this.
729 func (cfg *BuildCfg) shouldRender(p *pageState) bool {
730 if p == nil {
731 return false
732 }
733
734 if p.forceRender {
735 return true
736 }
737
738 if len(cfg.RecentlyVisited) == 0 {
739 return true
740 }
741
742 if cfg.RecentlyVisited[p.RelPermalink()] {
743 return true
744 }
745
746 if cfg.whatChanged != nil && !p.File().IsZero() {
747 return cfg.whatChanged.files[p.File().Filename()]
748 }
749
750 return false
751 }
752
753 func (h *HugoSites) renderCrossSitesSitemap() error {
754 if !h.multilingual.enabled() || h.IsMultihost() {
755 return nil
756 }
757
758 sitemapEnabled := false
759 for _, s := range h.Sites {
760 if s.isEnabled(kindSitemap) {
761 sitemapEnabled = true
762 break
763 }
764 }
765
766 if !sitemapEnabled {
767 return nil
768 }
769
770 s := h.Sites[0]
771
772 templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml")
773
774 return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex",
775 s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ)
776 }
777
778 func (h *HugoSites) renderCrossSitesRobotsTXT() error {
779 if h.multihost {
780 return nil
781 }
782 if !h.Cfg.GetBool("enableRobotsTXT") {
783 return nil
784 }
785
786 s := h.Sites[0]
787
788 p, err := newPageStandalone(&pageMeta{
789 s: s,
790 kind: kindRobotsTXT,
791 urlPaths: pagemeta.URLPath{
792 URL: "robots.txt",
793 },
794 },
795 output.RobotsTxtFormat)
796 if err != nil {
797 return err
798 }
799
800 if !p.render {
801 return nil
802 }
803
804 templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt")
805
806 return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ)
807 }
808
809 func (h *HugoSites) removePageByFilename(filename string) {
810 h.getContentMaps().withMaps(func(m *pageMap) error {
811 m.deleteBundleMatching(func(b *contentNode) bool {
812 if b.p == nil {
813 return false
814 }
815
816 if b.fi == nil {
817 return false
818 }
819
820 return b.fi.Meta().Filename == filename
821 })
822 return nil
823 })
824 }
825
826 func (h *HugoSites) createPageCollections() error {
827 allPages := newLazyPagesFactory(func() page.Pages {
828 var pages page.Pages
829 for _, s := range h.Sites {
830 pages = append(pages, s.Pages()...)
831 }
832
833 page.SortByDefault(pages)
834
835 return pages
836 })
837
838 allRegularPages := newLazyPagesFactory(func() page.Pages {
839 return h.findPagesByKindIn(page.KindPage, allPages.get())
840 })
841
842 for _, s := range h.Sites {
843 s.PageCollections.allPages = allPages
844 s.PageCollections.allRegularPages = allRegularPages
845 }
846
847 return nil
848 }
849
850 func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error {
851 var err error
852 s.pageMap.withEveryBundlePage(func(p *pageState) bool {
853 if err = p.initOutputFormat(isRenderingSite, idx); err != nil {
854 return true
855 }
856 return false
857 })
858 return nil
859 }
860
861 // Pages returns all pages for all sites.
862 func (h *HugoSites) Pages() page.Pages {
863 return h.Sites[0].AllPages()
864 }
865
866 func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) {
867 spec := source.NewSourceSpec(h.PathSpec, nil, nil)
868
869 h.data = make(map[string]any)
870 for _, fi := range fis {
871 fileSystem := spec.NewFilesystemFromFileMetaInfo(fi)
872 files, err := fileSystem.Files()
873 if err != nil {
874 return err
875 }
876 for _, r := range files {
877 if err := h.handleDataFile(r); err != nil {
878 return err
879 }
880 }
881 }
882
883 return
884 }
885
886 func (h *HugoSites) handleDataFile(r source.File) error {
887 var current map[string]any
888
889 f, err := r.FileInfo().Meta().Open()
890 if err != nil {
891 return fmt.Errorf("data: failed to open %q: %w", r.LogicalName(), err)
892 }
893 defer f.Close()
894
895 // Crawl in data tree to insert data
896 current = h.data
897 keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator)
898
899 for _, key := range keyParts {
900 if key != "" {
901 if _, ok := current[key]; !ok {
902 current[key] = make(map[string]any)
903 }
904 current = current[key].(map[string]any)
905 }
906 }
907
908 data, err := h.readData(r)
909 if err != nil {
910 return h.errWithFileContext(err, r)
911 }
912
913 if data == nil {
914 return nil
915 }
916
917 // filepath.Walk walks the files in lexical order, '/' comes before '.'
918 higherPrecedentData := current[r.BaseFileName()]
919
920 switch data.(type) {
921 case nil:
922 case map[string]any:
923
924 switch higherPrecedentData.(type) {
925 case nil:
926 current[r.BaseFileName()] = data
927 case map[string]any:
928 // merge maps: insert entries from data for keys that
929 // don't already exist in higherPrecedentData
930 higherPrecedentMap := higherPrecedentData.(map[string]any)
931 for key, value := range data.(map[string]any) {
932 if _, exists := higherPrecedentMap[key]; exists {
933 // this warning could happen if
934 // 1. A theme uses the same key; the main data folder wins
935 // 2. A sub folder uses the same key: the sub folder wins
936 // TODO(bep) figure out a way to detect 2) above and make that a WARN
937 h.Log.Infof("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())
938 } else {
939 higherPrecedentMap[key] = value
940 }
941 }
942 default:
943 // can't merge: higherPrecedentData is not a map
944 h.Log.Warnf("The %T data from '%s' overridden by "+
945 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
946 }
947
948 case []any:
949 if higherPrecedentData == nil {
950 current[r.BaseFileName()] = data
951 } else {
952 // we don't merge array data
953 h.Log.Warnf("The %T data from '%s' overridden by "+
954 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
955 }
956
957 default:
958 h.Log.Errorf("unexpected data type %T in file %s", data, r.LogicalName())
959 }
960
961 return nil
962 }
963
964 func (h *HugoSites) errWithFileContext(err error, f source.File) error {
965 fim, ok := f.FileInfo().(hugofs.FileMetaInfo)
966 if !ok {
967 return err
968 }
969 realFilename := fim.Meta().Filename
970
971 return herrors.NewFileErrorFromFile(err, realFilename, h.SourceSpec.Fs.Source, nil)
972
973 }
974
975 func (h *HugoSites) readData(f source.File) (any, error) {
976 file, err := f.FileInfo().Meta().Open()
977 if err != nil {
978 return nil, fmt.Errorf("readData: failed to open data file: %w", err)
979 }
980 defer file.Close()
981 content := helpers.ReaderToBytes(file)
982
983 format := metadecoders.FormatFromString(f.Ext())
984 return metadecoders.Default.Unmarshal(content, format)
985 }
986
987 func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Pages {
988 return h.Sites[0].findPagesByKindIn(kind, inPages)
989 }
990
991 func (h *HugoSites) resetPageState() {
992 h.getContentMaps().walkBundles(func(n *contentNode) bool {
993 if n.p == nil {
994 return false
995 }
996 p := n.p
997 for _, po := range p.pageOutputs {
998 if po.cp == nil {
999 continue
1000 }
1001 po.cp.Reset()
1002 }
1003
1004 return false
1005 })
1006 }
1007
1008 func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) {
1009 h.getContentMaps().walkBundles(func(n *contentNode) bool {
1010 if n.p == nil {
1011 return false
1012 }
1013 p := n.p
1014 OUTPUTS:
1015 for _, po := range p.pageOutputs {
1016 if po.cp == nil {
1017 continue
1018 }
1019 for id := range idset {
1020 if po.cp.dependencyTracker.Search(id) != nil {
1021 po.cp.Reset()
1022 continue OUTPUTS
1023 }
1024 }
1025 }
1026
1027 if p.shortcodeState == nil {
1028 return false
1029 }
1030
1031 for _, s := range p.shortcodeState.shortcodes {
1032 for _, templ := range s.templs {
1033 sid := templ.(identity.Manager)
1034 for id := range idset {
1035 if sid.Search(id) != nil {
1036 for _, po := range p.pageOutputs {
1037 if po.cp != nil {
1038 po.cp.Reset()
1039 }
1040 }
1041 return false
1042 }
1043 }
1044 }
1045 }
1046 return false
1047 })
1048 }
1049
1050 // Used in partial reloading to determine if the change is in a bundle.
1051 type contentChangeMap struct {
1052 mu sync.RWMutex
1053
1054 // Holds directories with leaf bundles.
1055 leafBundles *radix.Tree
1056
1057 // Holds directories with branch bundles.
1058 branchBundles map[string]bool
1059
1060 pathSpec *helpers.PathSpec
1061
1062 // Hugo supports symlinked content (both directories and files). This
1063 // can lead to situations where the same file can be referenced from several
1064 // locations in /content -- which is really cool, but also means we have to
1065 // go an extra mile to handle changes.
1066 // This map is only used in watch mode.
1067 // It maps either file to files or the real dir to a set of content directories
1068 // where it is in use.
1069 symContentMu sync.Mutex
1070 symContent map[string]map[string]bool
1071 }
1072
1073 func (m *contentChangeMap) add(dirname string, tp bundleDirType) {
1074 m.mu.Lock()
1075 if !strings.HasSuffix(dirname, helpers.FilePathSeparator) {
1076 dirname += helpers.FilePathSeparator
1077 }
1078 switch tp {
1079 case bundleBranch:
1080 m.branchBundles[dirname] = true
1081 case bundleLeaf:
1082 m.leafBundles.Insert(dirname, true)
1083 default:
1084 m.mu.Unlock()
1085 panic("invalid bundle type")
1086 }
1087 m.mu.Unlock()
1088 }
1089
1090 func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) {
1091 m.mu.RLock()
1092 defer m.mu.RUnlock()
1093
1094 // Bundles share resources, so we need to start from the virtual root.
1095 relFilename := m.pathSpec.RelContentDir(filename)
1096 dir, name := filepath.Split(relFilename)
1097 if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
1098 dir += helpers.FilePathSeparator
1099 }
1100
1101 if _, found := m.branchBundles[dir]; found {
1102 delete(m.branchBundles, dir)
1103 return dir, bundleBranch
1104 }
1105
1106 if key, _, found := m.leafBundles.LongestPrefix(dir); found {
1107 m.leafBundles.Delete(key)
1108 dir = string(key)
1109 return dir, bundleLeaf
1110 }
1111
1112 fileTp, isContent := classifyBundledFile(name)
1113 if isContent && fileTp != bundleNot {
1114 // A new bundle.
1115 return dir, fileTp
1116 }
1117
1118 return dir, bundleNot
1119 }
1120
1121 func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) {
1122 meta := fim.Meta()
1123 if !meta.IsSymlink {
1124 return
1125 }
1126 m.symContentMu.Lock()
1127
1128 from, to := meta.Filename, meta.OriginalFilename
1129 if fim.IsDir() {
1130 if !strings.HasSuffix(from, helpers.FilePathSeparator) {
1131 from += helpers.FilePathSeparator
1132 }
1133 }
1134
1135 mm, found := m.symContent[from]
1136
1137 if !found {
1138 mm = make(map[string]bool)
1139 m.symContent[from] = mm
1140 }
1141 mm[to] = true
1142 m.symContentMu.Unlock()
1143 }
1144
1145 func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string {
1146 mm, found := m.symContent[dir]
1147 if !found {
1148 return nil
1149 }
1150 dirs := make([]string, len(mm))
1151 i := 0
1152 for dir := range mm {
1153 dirs[i] = dir
1154 i++
1155 }
1156
1157 sort.Strings(dirs)
1158
1159 return dirs
1160 }