hugo

pages_capture.go (13084B)

 // Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 // http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
 package hugolib
 
 import (
 	"context"
 	"fmt"
 	"os"
 	pth "path"
 	"path/filepath"
 	"reflect"
 
 	"github.com/gohugoio/hugo/common/maps"
 
 	"github.com/gohugoio/hugo/parser/pageparser"
 
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/gohugoio/hugo/source"
 
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/afero"
 )
 
 const (
 	walkIsRootFileMetaKey = "walkIsRootFileMetaKey"
 )
 
 func newPagesCollector(
 	sp *source.SourceSpec,
 	contentMap *pageMaps,
 	logger loggers.Logger,
 	contentTracker *contentChangeMap,
 	proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector {
 	return &pagesCollector{
 		fs:         sp.SourceFs,
 		contentMap: contentMap,
 		proc:       proc,
 		sp:         sp,
 		logger:     logger,
 		filenames:  filenames,
 		tracker:    contentTracker,
 	}
 }
 
 type contentDirKey struct {
 	dirname  string
 	filename string
 	tp       bundleDirType
 }
 
 type fileinfoBundle struct {
 	header    hugofs.FileMetaInfo
 	resources []hugofs.FileMetaInfo
 }
 
 func (b *fileinfoBundle) containsResource(name string) bool {
 	for _, r := range b.resources {
 		if r.Name() == name {
 			return true
 		}
 	}
 
 	return false
 }
 
 type pageBundles map[string]*fileinfoBundle
 
 type pagesCollector struct {
 	sp     *source.SourceSpec
 	fs     afero.Fs
 	logger loggers.Logger
 
 	contentMap *pageMaps
 
 	// Ordered list (bundle headers first) used in partial builds.
 	filenames []string
 
 	// Content files tracker used in partial builds.
 	tracker *contentChangeMap
 
 	proc pagesCollectorProcessorProvider
 }
 
 // isCascadingEdit returns whether the dir represents a cascading edit.
 // That is, if a front matter cascade section is removed, added or edited.
 // If this is the case we must re-evaluate its descendants.
 func (c *pagesCollector) isCascadingEdit(dir contentDirKey) (bool, string) {
 	// This is either a section or a taxonomy node. Find it.
 	prefix := cleanTreeKey(dir.dirname)
 
 	section := "/"
 	var isCascade bool
 
 	c.contentMap.walkBranchesPrefix(prefix, func(s string, n *contentNode) bool {
 		if n.fi == nil || dir.filename != n.fi.Meta().Filename {
 			return false
 		}
 
 		f, err := n.fi.Meta().Open()
 		if err != nil {
 			// File may have been removed, assume a cascading edit.
 			// Some false positives is not too bad.
 			isCascade = true
 			return true
 		}
 
 		pf, err := pageparser.ParseFrontMatterAndContent(f)
 		f.Close()
 		if err != nil {
 			isCascade = true
 			return true
 		}
 
 		if n.p == nil || n.p.bucket == nil {
 			return true
 		}
 
 		section = s
 
 		maps.PrepareParams(pf.FrontMatter)
 		cascade1, ok := pf.FrontMatter["cascade"]
 		hasCascade := n.p.bucket.cascade != nil && len(n.p.bucket.cascade) > 0
 		if !ok {
 			isCascade = hasCascade
 
 			return true
 		}
 
 		if !hasCascade {
 			isCascade = true
 			return true
 		}
 
 		for _, v := range n.p.bucket.cascade {
 			isCascade = !reflect.DeepEqual(cascade1, v)
 			if isCascade {
 				break
 			}
 		}
 
 		return true
 	})
 
 	return isCascade, section
 }
 
 // Collect.
 func (c *pagesCollector) Collect() (collectErr error) {
 	c.proc.Start(context.Background())
 	defer func() {
 		err := c.proc.Wait()
 		if collectErr == nil {
 			collectErr = err
 		}
 	}()
 
 	if len(c.filenames) == 0 {
 		// Collect everything.
 		collectErr = c.collectDir("", false, nil)
 	} else {
 		for _, pm := range c.contentMap.pmaps {
 			pm.cfg.isRebuild = true
 		}
 		dirs := make(map[contentDirKey]bool)
 		for _, filename := range c.filenames {
 			dir, btype := c.tracker.resolveAndRemove(filename)
 			dirs[contentDirKey{dir, filename, btype}] = true
 		}
 
 		for dir := range dirs {
 			for _, pm := range c.contentMap.pmaps {
 				pm.s.ResourceSpec.DeleteBySubstring(dir.dirname)
 			}
 
 			switch dir.tp {
 			case bundleLeaf:
 				collectErr = c.collectDir(dir.dirname, true, nil)
 			case bundleBranch:
 				isCascading, section := c.isCascadingEdit(dir)
 
 				if isCascading {
 					c.contentMap.deleteSection(section)
 				}
 				collectErr = c.collectDir(dir.dirname, !isCascading, nil)
 			default:
 				// We always start from a directory.
 				collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool {
 					return dir.filename == fim.Meta().Filename
 				})
 			}
 
 			if collectErr != nil {
 				break
 			}
 		}
 
 	}
 
 	return
 }
 
 func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool {
 	class := fi.Meta().Classifier
 	return class == files.ContentClassLeaf || class == files.ContentClassBranch
 }
 
 func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string {
 	lang := fi.Meta().Lang
 	if lang != "" {
 		return lang
 	}
 
 	return c.sp.DefaultContentLanguage
 }
 
 func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error {
 	getBundle := func(lang string) *fileinfoBundle {
 		return bundles[lang]
 	}
 
 	cloneBundle := func(lang string) *fileinfoBundle {
 		// Every bundled content file needs a content file header.
 		// Use the default content language if found, else just
 		// pick one.
 		var (
 			source *fileinfoBundle
 			found  bool
 		)
 
 		source, found = bundles[c.sp.DefaultContentLanguage]
 		if !found {
 			for _, b := range bundles {
 				source = b
 				break
 			}
 		}
 
 		if source == nil {
 			panic(fmt.Sprintf("no source found, %d", len(bundles)))
 		}
 
 		clone := c.cloneFileInfo(source.header)
 		clone.Meta().Lang = lang
 
 		return &fileinfoBundle{
 			header: clone,
 		}
 	}
 
 	lang := c.getLang(info)
 	bundle := getBundle(lang)
 	isBundleHeader := c.isBundleHeader(info)
 	if bundle != nil && isBundleHeader {
 		// index.md file inside a bundle, see issue 6208.
 		info.Meta().Classifier = files.ContentClassContent
 		isBundleHeader = false
 	}
 	classifier := info.Meta().Classifier
 	isContent := classifier == files.ContentClassContent
 	if bundle == nil {
 		if isBundleHeader {
 			bundle = &fileinfoBundle{header: info}
 			bundles[lang] = bundle
 		} else {
 			if btyp == bundleBranch {
 				// No special logic for branch bundles.
 				// Every language needs its own _index.md file.
 				// Also, we only clone bundle headers for lonesome, bundled,
 				// content files.
 				return c.handleFiles(info)
 			}
 
 			if isContent {
 				bundle = cloneBundle(lang)
 				bundles[lang] = bundle
 			}
 		}
 	}
 
 	if !isBundleHeader && bundle != nil {
 		bundle.resources = append(bundle.resources, info)
 	}
 
 	if classifier == files.ContentClassFile {
 		translations := info.Meta().Translations
 
 		for lang, b := range bundles {
 			if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) {
 
 				// Clone and add it to the bundle.
 				clone := c.cloneFileInfo(info)
 				clone.Meta().Lang = lang
 				b.resources = append(b.resources, clone)
 			}
 		}
 	}
 
 	return nil
 }
 
 func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo {
 	return hugofs.NewFileMetaInfo(fi, hugofs.NewFileMeta())
 }
 
 func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error {
 	fi, err := c.fs.Stat(dirname)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// May have been deleted.
 			return nil
 		}
 		return err
 	}
 
 	handleDir := func(
 		btype bundleDirType,
 		dir hugofs.FileMetaInfo,
 		path string,
 		readdir []hugofs.FileMetaInfo) error {
 		if btype > bundleNot && c.tracker != nil {
 			c.tracker.add(path, btype)
 		}
 
 		if btype == bundleBranch {
 			if err := c.handleBundleBranch(readdir); err != nil {
 				return err
 			}
 			// A branch bundle is only this directory level, so keep walking.
 			return nil
 		} else if btype == bundleLeaf {
 			if err := c.handleBundleLeaf(dir, path, readdir); err != nil {
 				return err
 			}
 
 			return nil
 		}
 
 		if err := c.handleFiles(readdir...); err != nil {
 			return err
 		}
 
 		return nil
 	}
 
 	filter := func(fim hugofs.FileMetaInfo) bool {
 		if fim.Meta().SkipDir {
 			return false
 		}
 
 		if c.sp.IgnoreFile(fim.Meta().Filename) {
 			return false
 		}
 
 		if inFilter != nil {
 			return inFilter(fim)
 		}
 		return true
 	}
 
 	preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {
 		var btype bundleDirType
 
 		filtered := readdir[:0]
 		for _, fi := range readdir {
 			if filter(fi) {
 				filtered = append(filtered, fi)
 
 				if c.tracker != nil {
 					// Track symlinks.
 					c.tracker.addSymbolicLinkMapping(fi)
 				}
 			}
 		}
 		walkRoot := dir.Meta().IsRootFile
 		readdir = filtered
 
 		// We merge language directories, so there can be duplicates, but they
 		// will be ordered, most important first.
 		var duplicates []int
 		seen := make(map[string]bool)
 
 		for i, fi := range readdir {
 
 			if fi.IsDir() {
 				continue
 			}
 
 			meta := fi.Meta()
 			meta.IsRootFile = walkRoot
 			class := meta.Classifier
 			translationBase := meta.TranslationBaseNameWithExt
 			key := pth.Join(meta.Lang, translationBase)
 
 			if seen[key] {
 				duplicates = append(duplicates, i)
 				continue
 			}
 			seen[key] = true
 
 			var thisBtype bundleDirType
 
 			switch class {
 			case files.ContentClassLeaf:
 				thisBtype = bundleLeaf
 			case files.ContentClassBranch:
 				thisBtype = bundleBranch
 			}
 
 			// Folders with both index.md and _index.md type of files have
 			// undefined behaviour and can never work.
 			// The branch variant will win because of sort order, but log
 			// a warning about it.
 			if thisBtype > bundleNot && btype > bundleNot && thisBtype != btype {
 				c.logger.Warnf("Content directory %q have both index.* and _index.* files, pick one.", dir.Meta().Filename)
 				// Reclassify it so it will be handled as a content file inside the
 				// section, which is in line with the <= 0.55 behaviour.
 				meta.Classifier = files.ContentClassContent
 			} else if thisBtype > bundleNot {
 				btype = thisBtype
 			}
 
 		}
 
 		if len(duplicates) > 0 {
 			for i := len(duplicates) - 1; i >= 0; i-- {
 				idx := duplicates[i]
 				readdir = append(readdir[:idx], readdir[idx+1:]...)
 			}
 		}
 
 		err := handleDir(btype, dir, path, readdir)
 		if err != nil {
 			return nil, err
 		}
 
 		if btype == bundleLeaf || partial {
 			return nil, filepath.SkipDir
 		}
 
 		// Keep walking.
 		return readdir, nil
 	}
 
 	var postHook hugofs.WalkHook
 	if c.tracker != nil {
 		postHook = func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {
 			if c.tracker == nil {
 				// Nothing to do.
 				return readdir, nil
 			}
 
 			return readdir, nil
 		}
 	}
 
 	wfn := func(path string, info hugofs.FileMetaInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
 		return nil
 	}
 
 	fim := fi.(hugofs.FileMetaInfo)
 	// Make sure the pages in this directory gets re-rendered,
 	// even in fast render mode.
 	fim.Meta().IsRootFile = true
 
 	w := hugofs.NewWalkway(hugofs.WalkwayConfig{
 		Fs:       c.fs,
 		Logger:   c.logger,
 		Root:     dirname,
 		Info:     fim,
 		HookPre:  preHook,
 		HookPost: postHook,
 		WalkFn:   wfn,
 	})
 
 	return w.Walk()
 }
 
 func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error {
 	// Maps bundles to its language.
 	bundles := pageBundles{}
 
 	var contentFiles []hugofs.FileMetaInfo
 
 	for _, fim := range readdir {
 
 		if fim.IsDir() {
 			continue
 		}
 
 		meta := fim.Meta()
 
 		switch meta.Classifier {
 		case files.ContentClassContent:
 			contentFiles = append(contentFiles, fim)
 		default:
 			if err := c.addToBundle(fim, bundleBranch, bundles); err != nil {
 				return err
 			}
 		}
 
 	}
 
 	// Make sure the section is created before its pages.
 	if err := c.proc.Process(bundles); err != nil {
 		return err
 	}
 
 	return c.handleFiles(contentFiles...)
 }
 
 func (c *pagesCollector) handleBundleLeaf(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) error {
 	// Maps bundles to its language.
 	bundles := pageBundles{}
 
 	walk := func(path string, info hugofs.FileMetaInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if info.IsDir() {
 			return nil
 		}
 
 		return c.addToBundle(info, bundleLeaf, bundles)
 	}
 
 	// Start a new walker from the given path.
 	w := hugofs.NewWalkway(hugofs.WalkwayConfig{
 		Root:       path,
 		Fs:         c.fs,
 		Logger:     c.logger,
 		Info:       dir,
 		DirEntries: readdir,
 		WalkFn:     walk,
 	})
 
 	if err := w.Walk(); err != nil {
 		return err
 	}
 
 	return c.proc.Process(bundles)
 }
 
 func (c *pagesCollector) handleFiles(fis ...hugofs.FileMetaInfo) error {
 	for _, fi := range fis {
 		if fi.IsDir() {
 			continue
 		}
 
 		if err := c.proc.Process(fi); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
 func stringSliceContains(k string, values ...string) bool {
 	for _, v := range values {
 		if k == v {
 			return true
 		}
 	}
 	return false
 }