hugo

Fork of github.com/gohugoio/hugo with reverse pagination support

git clone git://git.shimmy1996.com/hugo.git

basefs.go (25054B)

    1 // Copyright 2018 The Hugo Authors. All rights reserved.
    2 //
    3 // Licensed under the Apache License, Version 2.0 (the "License");
    4 // you may not use this file except in compliance with the License.
    5 // You may obtain a copy of the License at
    6 // http://www.apache.org/licenses/LICENSE-2.0
    7 //
    8 // Unless required by applicable law or agreed to in writing, software
    9 // distributed under the License is distributed on an "AS IS" BASIS,
   10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   11 // See the License for the specific language governing permissions and
   12 // limitations under the License.
   13 
   14 // Package filesystems provides the fine grained file systems used by Hugo. These
   15 // are typically virtual filesystems that are composites of project and theme content.
   16 package filesystems
   17 
   18 import (
   19 	"fmt"
   20 	"io"
   21 	"os"
   22 	"path"
   23 	"path/filepath"
   24 	"strings"
   25 	"sync"
   26 
   27 	"github.com/bep/overlayfs"
   28 	"github.com/gohugoio/hugo/htesting"
   29 	"github.com/gohugoio/hugo/hugofs/glob"
   30 
   31 	"github.com/gohugoio/hugo/common/types"
   32 
   33 	"github.com/gohugoio/hugo/common/loggers"
   34 	"github.com/rogpeppe/go-internal/lockedfile"
   35 
   36 	"github.com/gohugoio/hugo/hugofs/files"
   37 
   38 	"github.com/gohugoio/hugo/modules"
   39 
   40 	hpaths "github.com/gohugoio/hugo/common/paths"
   41 	"github.com/gohugoio/hugo/hugofs"
   42 	"github.com/gohugoio/hugo/hugolib/paths"
   43 	"github.com/spf13/afero"
   44 )
   45 
   46 const (
   47 	// Used to control concurrency between multiple Hugo instances, e.g.
   48 	// a running server and building new content with 'hugo new'.
   49 	// It's placed in the project root.
   50 	lockFileBuild = ".hugo_build.lock"
   51 )
   52 
   53 var filePathSeparator = string(filepath.Separator)
   54 
   55 // BaseFs contains the core base filesystems used by Hugo. The name "base" is used
   56 // to underline that even if they can be composites, they all have a base path set to a specific
   57 // resource folder, e.g "/my-project/content". So, no absolute filenames needed.
   58 type BaseFs struct {
   59 
   60 	// SourceFilesystems contains the different source file systems.
   61 	*SourceFilesystems
   62 
   63 	// The project source.
   64 	SourceFs afero.Fs
   65 
   66 	// The filesystem used to publish the rendered site.
   67 	// This usually maps to /my-project/public.
   68 	PublishFs afero.Fs
   69 
   70 	// The filesystem used for renderStaticToDisk.
   71 	PublishFsStatic afero.Fs
   72 
   73 	// A read-only filesystem starting from the project workDir.
   74 	WorkDir afero.Fs
   75 
   76 	theBigFs *filesystemsCollector
   77 
   78 	// Locks.
   79 	buildMu Lockable // <project>/.hugo_build.lock
   80 }
   81 
   82 type Lockable interface {
   83 	Lock() (unlock func(), err error)
   84 }
   85 
   86 type fakeLockfileMutex struct {
   87 	mu sync.Mutex
   88 }
   89 
   90 func (f *fakeLockfileMutex) Lock() (func(), error) {
   91 	f.mu.Lock()
   92 	return func() { f.mu.Unlock() }, nil
   93 }
   94 
   95 // Tries to acquire a build lock.
   96 func (fs *BaseFs) LockBuild() (unlock func(), err error) {
   97 	return fs.buildMu.Lock()
   98 }
   99 
  100 // TODO(bep) we can get regular files in here and that is fine, but
  101 // we need to clean up the naming.
  102 func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo {
  103 	var dirs []hugofs.FileMetaInfo
  104 	for _, dir := range fs.AllDirs() {
  105 		if dir.Meta().Watch {
  106 			dirs = append(dirs, dir)
  107 		}
  108 	}
  109 	return dirs
  110 }
  111 
  112 func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo {
  113 	var dirs []hugofs.FileMetaInfo
  114 	for _, dirSet := range [][]hugofs.FileMetaInfo{
  115 		fs.Archetypes.Dirs,
  116 		fs.I18n.Dirs,
  117 		fs.Data.Dirs,
  118 		fs.Content.Dirs,
  119 		fs.Assets.Dirs,
  120 		fs.Layouts.Dirs,
  121 		// fs.Resources.Dirs,
  122 		fs.StaticDirs,
  123 	} {
  124 		dirs = append(dirs, dirSet...)
  125 	}
  126 
  127 	return dirs
  128 }
  129 
  130 // RelContentDir tries to create a path relative to the content root from
  131 // the given filename. The return value is the path and language code.
  132 func (b *BaseFs) RelContentDir(filename string) string {
  133 	for _, dir := range b.SourceFilesystems.Content.Dirs {
  134 		dirname := dir.Meta().Filename
  135 		if strings.HasPrefix(filename, dirname) {
  136 			rel := path.Join(dir.Meta().Path, strings.TrimPrefix(filename, dirname))
  137 			return strings.TrimPrefix(rel, filePathSeparator)
  138 		}
  139 	}
  140 	// Either not a content dir or already relative.
  141 	return filename
  142 }
  143 
  144 // AbsProjectContentDir tries to construct a filename below the most
  145 // relevant content directory.
  146 func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
  147 	isAbs := filepath.IsAbs(filename)
  148 	for _, dir := range b.SourceFilesystems.Content.Dirs {
  149 		meta := dir.Meta()
  150 		if !meta.IsProject {
  151 			continue
  152 		}
  153 
  154 		if isAbs {
  155 			if strings.HasPrefix(filename, meta.Filename) {
  156 				return strings.TrimPrefix(filename, meta.Filename), filename, nil
  157 			}
  158 		} else {
  159 			contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + filePathSeparator
  160 
  161 			if strings.HasPrefix(filename, contentDir) {
  162 				relFilename := strings.TrimPrefix(filename, contentDir)
  163 				absFilename := filepath.Join(meta.Filename, relFilename)
  164 				return relFilename, absFilename, nil
  165 			}
  166 		}
  167 
  168 	}
  169 
  170 	if !isAbs {
  171 		// A filename on the form "posts/mypage.md", put it inside
  172 		// the first content folder, usually <workDir>/content.
  173 		// Pick the first project dir (which is probably the most important one).
  174 		for _, dir := range b.SourceFilesystems.Content.Dirs {
  175 			meta := dir.Meta()
  176 			if meta.IsProject {
  177 				return filename, filepath.Join(meta.Filename, filename), nil
  178 			}
  179 		}
  180 
  181 	}
  182 
  183 	return "", "", fmt.Errorf("could not determine content directory for %q", filename)
  184 }
  185 
  186 // ResolveJSConfigFile resolves the JS-related config file to a absolute
  187 // filename. One example of such would be postcss.config.js.
  188 func (fs *BaseFs) ResolveJSConfigFile(name string) string {
  189 	// First look in assets/_jsconfig
  190 	fi, err := fs.Assets.Fs.Stat(filepath.Join(files.FolderJSConfig, name))
  191 	if err == nil {
  192 		return fi.(hugofs.FileMetaInfo).Meta().Filename
  193 	}
  194 	// Fall back to the work dir.
  195 	fi, err = fs.Work.Stat(name)
  196 	if err == nil {
  197 		return fi.(hugofs.FileMetaInfo).Meta().Filename
  198 	}
  199 
  200 	return ""
  201 }
  202 
  203 // SourceFilesystems contains the different source file systems. These can be
  204 // composite file systems (theme and project etc.), and they have all root
  205 // set to the source type the provides: data, i18n, static, layouts.
  206 type SourceFilesystems struct {
  207 	Content    *SourceFilesystem
  208 	Data       *SourceFilesystem
  209 	I18n       *SourceFilesystem
  210 	Layouts    *SourceFilesystem
  211 	Archetypes *SourceFilesystem
  212 	Assets     *SourceFilesystem
  213 
  214 	// Writable filesystem on top the project's resources directory,
  215 	// with any sub module's resource fs layered below.
  216 	ResourcesCache afero.Fs
  217 
  218 	// The work folder (may be a composite of project and theme components).
  219 	Work afero.Fs
  220 
  221 	// When in multihost we have one static filesystem per language. The sync
  222 	// static files is currently done outside of the Hugo build (where there is
  223 	// a concept of a site per language).
  224 	// When in non-multihost mode there will be one entry in this map with a blank key.
  225 	Static map[string]*SourceFilesystem
  226 
  227 	// All the /static dirs (including themes/modules).
  228 	StaticDirs []hugofs.FileMetaInfo
  229 }
  230 
  231 // FileSystems returns the FileSystems relevant for the change detection
  232 // in server mode.
  233 // Note: This does currently not return any static fs.
  234 func (s *SourceFilesystems) FileSystems() []*SourceFilesystem {
  235 	return []*SourceFilesystem{
  236 		s.Content,
  237 		s.Data,
  238 		s.I18n,
  239 		s.Layouts,
  240 		s.Archetypes,
  241 		// TODO(bep) static
  242 	}
  243 }
  244 
  245 // A SourceFilesystem holds the filesystem for a given source type in Hugo (data,
  246 // i18n, layouts, static) and additional metadata to be able to use that filesystem
  247 // in server mode.
  248 type SourceFilesystem struct {
  249 	// Name matches one in files.ComponentFolders
  250 	Name string
  251 
  252 	// This is a virtual composite filesystem. It expects path relative to a context.
  253 	Fs afero.Fs
  254 
  255 	// This filesystem as separate root directories, starting from project and down
  256 	// to the themes/modules.
  257 	Dirs []hugofs.FileMetaInfo
  258 
  259 	// When syncing a source folder to the target (e.g. /public), this may
  260 	// be set to publish into a subfolder. This is used for static syncing
  261 	// in multihost mode.
  262 	PublishFolder string
  263 }
  264 
  265 // ContentStaticAssetFs will create a new composite filesystem from the content,
  266 // static, and asset filesystems. The site language is needed to pick the correct static filesystem.
  267 // The order is content, static and then assets.
  268 // TODO(bep) check usage
  269 func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs {
  270 	return overlayfs.New(
  271 		overlayfs.Options{
  272 			Fss: []afero.Fs{
  273 				s.Content.Fs,
  274 				s.StaticFs(lang),
  275 				s.Assets.Fs,
  276 			},
  277 		},
  278 	)
  279 
  280 }
  281 
  282 // StaticFs returns the static filesystem for the given language.
  283 // This can be a composite filesystem.
  284 func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
  285 	var staticFs afero.Fs = hugofs.NoOpFs
  286 
  287 	if fs, ok := s.Static[lang]; ok {
  288 		staticFs = fs.Fs
  289 	} else if fs, ok := s.Static[""]; ok {
  290 		staticFs = fs.Fs
  291 	}
  292 
  293 	return staticFs
  294 }
  295 
  296 // StatResource looks for a resource in these filesystems in order: static, assets and finally content.
  297 // If found in any of them, it returns FileInfo and the relevant filesystem.
  298 // Any non os.IsNotExist error will be returned.
  299 // An os.IsNotExist error wil be returned only if all filesystems return such an error.
  300 // Note that if we only wanted to find the file, we could create a composite Afero fs,
  301 // but we also need to know which filesystem root it lives in.
  302 func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) {
  303 	for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} {
  304 		fs = fsToCheck
  305 		fi, err = fs.Stat(filename)
  306 		if err == nil || !os.IsNotExist(err) {
  307 			return
  308 		}
  309 	}
  310 	// Not found.
  311 	return
  312 }
  313 
  314 // IsStatic returns true if the given filename is a member of one of the static
  315 // filesystems.
  316 func (s SourceFilesystems) IsStatic(filename string) bool {
  317 	for _, staticFs := range s.Static {
  318 		if staticFs.Contains(filename) {
  319 			return true
  320 		}
  321 	}
  322 	return false
  323 }
  324 
  325 // IsContent returns true if the given filename is a member of the content filesystem.
  326 func (s SourceFilesystems) IsContent(filename string) bool {
  327 	return s.Content.Contains(filename)
  328 }
  329 
  330 // IsLayout returns true if the given filename is a member of the layouts filesystem.
  331 func (s SourceFilesystems) IsLayout(filename string) bool {
  332 	return s.Layouts.Contains(filename)
  333 }
  334 
  335 // IsData returns true if the given filename is a member of the data filesystem.
  336 func (s SourceFilesystems) IsData(filename string) bool {
  337 	return s.Data.Contains(filename)
  338 }
  339 
  340 // IsAsset returns true if the given filename is a member of the asset filesystem.
  341 func (s SourceFilesystems) IsAsset(filename string) bool {
  342 	return s.Assets.Contains(filename)
  343 }
  344 
  345 // IsI18n returns true if the given filename is a member of the i18n filesystem.
  346 func (s SourceFilesystems) IsI18n(filename string) bool {
  347 	return s.I18n.Contains(filename)
  348 }
  349 
  350 // MakeStaticPathRelative makes an absolute static filename into a relative one.
  351 // It will return an empty string if the filename is not a member of a static filesystem.
  352 func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
  353 	for _, staticFs := range s.Static {
  354 		rel, _ := staticFs.MakePathRelative(filename)
  355 		if rel != "" {
  356 			return rel
  357 		}
  358 	}
  359 	return ""
  360 }
  361 
  362 // MakePathRelative creates a relative path from the given filename.
  363 func (d *SourceFilesystem) MakePathRelative(filename string) (string, bool) {
  364 	for _, dir := range d.Dirs {
  365 		meta := dir.(hugofs.FileMetaInfo).Meta()
  366 		currentPath := meta.Filename
  367 
  368 		if strings.HasPrefix(filename, currentPath) {
  369 			rel := strings.TrimPrefix(filename, currentPath)
  370 			if mp := meta.Path; mp != "" {
  371 				rel = filepath.Join(mp, rel)
  372 			}
  373 			return strings.TrimPrefix(rel, filePathSeparator), true
  374 		}
  375 	}
  376 	return "", false
  377 }
  378 
  379 func (d *SourceFilesystem) RealFilename(rel string) string {
  380 	fi, err := d.Fs.Stat(rel)
  381 	if err != nil {
  382 		return rel
  383 	}
  384 	if realfi, ok := fi.(hugofs.FileMetaInfo); ok {
  385 		return realfi.Meta().Filename
  386 	}
  387 
  388 	return rel
  389 }
  390 
  391 // Contains returns whether the given filename is a member of the current filesystem.
  392 func (d *SourceFilesystem) Contains(filename string) bool {
  393 	for _, dir := range d.Dirs {
  394 		if strings.HasPrefix(filename, dir.Meta().Filename) {
  395 			return true
  396 		}
  397 	}
  398 	return false
  399 }
  400 
  401 // Path returns the mount relative path to the given filename if it is a member of
  402 // of the current filesystem, an empty string if not.
  403 func (d *SourceFilesystem) Path(filename string) string {
  404 	for _, dir := range d.Dirs {
  405 		meta := dir.Meta()
  406 		if strings.HasPrefix(filename, meta.Filename) {
  407 			p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename), filePathSeparator)
  408 			if mountRoot := meta.MountRoot; mountRoot != "" {
  409 				return filepath.Join(mountRoot, p)
  410 			}
  411 			return p
  412 		}
  413 	}
  414 	return ""
  415 }
  416 
  417 // RealDirs gets a list of absolute paths to directories starting from the given
  418 // path.
  419 func (d *SourceFilesystem) RealDirs(from string) []string {
  420 	var dirnames []string
  421 	for _, dir := range d.Dirs {
  422 		meta := dir.Meta()
  423 		dirname := filepath.Join(meta.Filename, from)
  424 		_, err := meta.Fs.Stat(from)
  425 
  426 		if err == nil {
  427 			dirnames = append(dirnames, dirname)
  428 		}
  429 	}
  430 	return dirnames
  431 }
  432 
  433 // WithBaseFs allows reuse of some potentially expensive to create parts that remain
  434 // the same across sites/languages.
  435 func WithBaseFs(b *BaseFs) func(*BaseFs) error {
  436 	return func(bb *BaseFs) error {
  437 		bb.theBigFs = b.theBigFs
  438 		bb.SourceFilesystems = b.SourceFilesystems
  439 		return nil
  440 	}
  441 }
  442 
  443 // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
  444 func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) {
  445 	fs := p.Fs
  446 	if logger == nil {
  447 		logger = loggers.NewWarningLogger()
  448 	}
  449 
  450 	publishFs := hugofs.NewBaseFileDecorator(fs.PublishDir)
  451 	sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.WorkingDir))
  452 	publishFsStatic := fs.PublishDirStatic
  453 
  454 	var buildMu Lockable
  455 	if p.Cfg.GetBool("noBuildLock") || htesting.IsTest {
  456 		buildMu = &fakeLockfileMutex{}
  457 	} else {
  458 		buildMu = lockedfile.MutexAt(filepath.Join(p.WorkingDir, lockFileBuild))
  459 	}
  460 
  461 	b := &BaseFs{
  462 		SourceFs:        sourceFs,
  463 		WorkDir:         fs.WorkingDirReadOnly,
  464 		PublishFs:       publishFs,
  465 		PublishFsStatic: publishFsStatic,
  466 		buildMu:         buildMu,
  467 	}
  468 
  469 	for _, opt := range options {
  470 		if err := opt(b); err != nil {
  471 			return nil, err
  472 		}
  473 	}
  474 
  475 	if b.theBigFs != nil && b.SourceFilesystems != nil {
  476 		return b, nil
  477 	}
  478 
  479 	builder := newSourceFilesystemsBuilder(p, logger, b)
  480 	sourceFilesystems, err := builder.Build()
  481 	if err != nil {
  482 		return nil, fmt.Errorf("build filesystems: %w", err)
  483 	}
  484 
  485 	b.SourceFilesystems = sourceFilesystems
  486 	b.theBigFs = builder.theBigFs
  487 
  488 	return b, nil
  489 }
  490 
  491 type sourceFilesystemsBuilder struct {
  492 	logger   loggers.Logger
  493 	p        *paths.Paths
  494 	sourceFs afero.Fs
  495 	result   *SourceFilesystems
  496 	theBigFs *filesystemsCollector
  497 }
  498 
  499 func newSourceFilesystemsBuilder(p *paths.Paths, logger loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder {
  500 	sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source)
  501 	return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}}
  502 }
  503 
  504 func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem {
  505 	return &SourceFilesystem{
  506 		Name: name,
  507 		Fs:   fs,
  508 		Dirs: dirs,
  509 	}
  510 }
  511 
  512 func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
  513 	if b.theBigFs == nil {
  514 		theBigFs, err := b.createMainOverlayFs(b.p)
  515 		if err != nil {
  516 			return nil, fmt.Errorf("create main fs: %w", err)
  517 		}
  518 
  519 		b.theBigFs = theBigFs
  520 	}
  521 
  522 	createView := func(componentID string) *SourceFilesystem {
  523 		if b.theBigFs == nil || b.theBigFs.overlayMounts == nil {
  524 			return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil)
  525 		}
  526 
  527 		dirs := b.theBigFs.overlayDirs[componentID]
  528 
  529 		return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs)
  530 	}
  531 
  532 	b.result.Archetypes = createView(files.ComponentFolderArchetypes)
  533 	b.result.Layouts = createView(files.ComponentFolderLayouts)
  534 	b.result.Assets = createView(files.ComponentFolderAssets)
  535 	b.result.ResourcesCache = b.theBigFs.overlayResources
  536 
  537 	// Data, i18n and content cannot use the overlay fs
  538 	dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData]
  539 	dataFs, err := hugofs.NewSliceFs(dataDirs...)
  540 	if err != nil {
  541 		return nil, err
  542 	}
  543 
  544 	b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs)
  545 
  546 	i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n]
  547 	i18nFs, err := hugofs.NewSliceFs(i18nDirs...)
  548 	if err != nil {
  549 		return nil, err
  550 	}
  551 	b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs)
  552 
  553 	contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent]
  554 	contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent)
  555 
  556 	contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs)
  557 	if err != nil {
  558 		return nil, fmt.Errorf("create content filesystem: %w", err)
  559 	}
  560 
  561 	b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs)
  562 
  563 	b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull)
  564 
  565 	// Create static filesystem(s)
  566 	ms := make(map[string]*SourceFilesystem)
  567 	b.result.Static = ms
  568 	b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic]
  569 
  570 	if b.theBigFs.staticPerLanguage != nil {
  571 		// Multihost mode
  572 		for k, v := range b.theBigFs.staticPerLanguage {
  573 			sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs)
  574 			sfs.PublishFolder = k
  575 			ms[k] = sfs
  576 		}
  577 	} else {
  578 		bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic)
  579 		ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs)
  580 	}
  581 
  582 	return b.result, nil
  583 }
  584 
  585 func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) {
  586 	var staticFsMap map[string]*overlayfs.OverlayFs
  587 	if b.p.Cfg.GetBool("multihost") {
  588 		staticFsMap = make(map[string]*overlayfs.OverlayFs)
  589 		for _, l := range b.p.Languages {
  590 			staticFsMap[l.Lang] = overlayfs.New(overlayfs.Options{})
  591 		}
  592 	}
  593 
  594 	collector := &filesystemsCollector{
  595 		sourceProject:     b.sourceFs,
  596 		sourceModules:     hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false),
  597 		overlayDirs:       make(map[string][]hugofs.FileMetaInfo),
  598 		staticPerLanguage: staticFsMap,
  599 
  600 		overlayMounts:        overlayfs.New(overlayfs.Options{}),
  601 		overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
  602 		overlayMountsStatic:  overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
  603 		overlayFull:          overlayfs.New(overlayfs.Options{}),
  604 		overlayResources:     overlayfs.New(overlayfs.Options{FirstWritable: true}),
  605 	}
  606 
  607 	mods := p.AllModules
  608 
  609 	mounts := make([]mountsDescriptor, len(mods))
  610 
  611 	for i := 0; i < len(mods); i++ {
  612 		mod := mods[i]
  613 		dir := mod.Dir()
  614 
  615 		isMainProject := mod.Owner() == nil
  616 		mounts[i] = mountsDescriptor{
  617 			Module:        mod,
  618 			dir:           dir,
  619 			isMainProject: isMainProject,
  620 			ordinal:       i,
  621 		}
  622 
  623 	}
  624 
  625 	err := b.createOverlayFs(collector, mounts)
  626 
  627 	return collector, err
  628 }
  629 
  630 func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool {
  631 	return strings.HasPrefix(mnt.Target, files.ComponentFolderContent)
  632 }
  633 
  634 func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool {
  635 	return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic)
  636 }
  637 
  638 func (b *sourceFilesystemsBuilder) createOverlayFs(
  639 	collector *filesystemsCollector,
  640 	mounts []mountsDescriptor) error {
  641 
  642 	if len(mounts) == 0 {
  643 		appendNopIfEmpty := func(ofs *overlayfs.OverlayFs) *overlayfs.OverlayFs {
  644 			if ofs.NumFilesystems() > 0 {
  645 				return ofs
  646 			}
  647 			return ofs.Append(hugofs.NoOpFs)
  648 		}
  649 		collector.overlayMounts = appendNopIfEmpty(collector.overlayMounts)
  650 		collector.overlayMountsContent = appendNopIfEmpty(collector.overlayMountsContent)
  651 		collector.overlayMountsStatic = appendNopIfEmpty(collector.overlayMountsStatic)
  652 		collector.overlayFull = appendNopIfEmpty(collector.overlayFull)
  653 		collector.overlayResources = appendNopIfEmpty(collector.overlayResources)
  654 
  655 		return nil
  656 	}
  657 
  658 	for _, md := range mounts {
  659 		var (
  660 			fromTo        []hugofs.RootMapping
  661 			fromToContent []hugofs.RootMapping
  662 			fromToStatic  []hugofs.RootMapping
  663 		)
  664 
  665 		absPathify := func(path string) (string, string) {
  666 			if filepath.IsAbs(path) {
  667 				return "", path
  668 			}
  669 			return md.dir, hpaths.AbsPathify(md.dir, path)
  670 		}
  671 
  672 		for i, mount := range md.Mounts() {
  673 
  674 			// Add more weight to early mounts.
  675 			// When two mounts contain the same filename,
  676 			// the first entry wins.
  677 			mountWeight := (10 + md.ordinal) * (len(md.Mounts()) - i)
  678 
  679 			inclusionFilter, err := glob.NewFilenameFilter(
  680 				types.ToStringSlicePreserveString(mount.IncludeFiles),
  681 				types.ToStringSlicePreserveString(mount.ExcludeFiles),
  682 			)
  683 			if err != nil {
  684 				return err
  685 			}
  686 
  687 			base, filename := absPathify(mount.Source)
  688 
  689 			rm := hugofs.RootMapping{
  690 				From:      mount.Target,
  691 				To:        filename,
  692 				ToBasedir: base,
  693 				Module:    md.Module.Path(),
  694 				IsProject: md.isMainProject,
  695 				Meta: &hugofs.FileMeta{
  696 					Watch:           md.Watch(),
  697 					Weight:          mountWeight,
  698 					Classifier:      files.ContentClassContent,
  699 					InclusionFilter: inclusionFilter,
  700 				},
  701 			}
  702 
  703 			isContentMount := b.isContentMount(mount)
  704 
  705 			lang := mount.Lang
  706 			if lang == "" && isContentMount {
  707 				lang = b.p.DefaultContentLanguage
  708 			}
  709 
  710 			rm.Meta.Lang = lang
  711 
  712 			if isContentMount {
  713 				fromToContent = append(fromToContent, rm)
  714 			} else if b.isStaticMount(mount) {
  715 				fromToStatic = append(fromToStatic, rm)
  716 			} else {
  717 				fromTo = append(fromTo, rm)
  718 			}
  719 		}
  720 
  721 		modBase := collector.sourceProject
  722 		if !md.isMainProject {
  723 			modBase = collector.sourceModules
  724 		}
  725 		sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true)
  726 
  727 		rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
  728 		if err != nil {
  729 			return err
  730 		}
  731 		rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
  732 		if err != nil {
  733 			return err
  734 		}
  735 		rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)
  736 		if err != nil {
  737 			return err
  738 		}
  739 
  740 		// We need to keep the ordered list of directories for watching and
  741 		// some special merge operations (data, i18n).
  742 		collector.addDirs(rmfs)
  743 		collector.addDirs(rmfsContent)
  744 		collector.addDirs(rmfsStatic)
  745 
  746 		if collector.staticPerLanguage != nil {
  747 			for _, l := range b.p.Languages {
  748 				lang := l.Lang
  749 
  750 				lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool {
  751 					rlang := rm.Meta.Lang
  752 					return rlang == "" || rlang == lang
  753 				})
  754 
  755 				bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic)
  756 				collector.staticPerLanguage[lang] = collector.staticPerLanguage[lang].Append(bfs)
  757 
  758 			}
  759 		}
  760 
  761 		getResourcesDir := func() string {
  762 			if md.isMainProject {
  763 				return b.p.AbsResourcesDir
  764 			}
  765 			_, filename := absPathify(files.FolderResources)
  766 			return filename
  767 		}
  768 
  769 		collector.overlayMounts = collector.overlayMounts.Append(rmfs)
  770 		collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
  771 		collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
  772 		collector.overlayFull = collector.overlayFull.Append(afero.NewBasePathFs(modBase, md.dir))
  773 		collector.overlayResources = collector.overlayResources.Append(afero.NewBasePathFs(modBase, getResourcesDir()))
  774 
  775 	}
  776 
  777 	return nil
  778 }
  779 
  780 func printFs(fs afero.Fs, path string, w io.Writer) {
  781 	if fs == nil {
  782 		return
  783 	}
  784 	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
  785 		if err != nil {
  786 			return err
  787 		}
  788 		if info.IsDir() {
  789 			return nil
  790 		}
  791 		var filename string
  792 		if fim, ok := info.(hugofs.FileMetaInfo); ok {
  793 			filename = fim.Meta().Filename
  794 		}
  795 		fmt.Fprintf(w, "    %q %q\n", path, filename)
  796 		return nil
  797 	})
  798 }
  799 
  800 type filesystemsCollector struct {
  801 	sourceProject afero.Fs // Source for project folders
  802 	sourceModules afero.Fs // Source for modules/themes
  803 
  804 	overlayMounts        *overlayfs.OverlayFs
  805 	overlayMountsContent *overlayfs.OverlayFs
  806 	overlayMountsStatic  *overlayfs.OverlayFs
  807 	overlayFull          *overlayfs.OverlayFs
  808 	overlayResources     *overlayfs.OverlayFs
  809 
  810 	// Maps component type (layouts, static, content etc.) an ordered list of
  811 	// directories representing the overlay filesystems above.
  812 	overlayDirs map[string][]hugofs.FileMetaInfo
  813 
  814 	// Set if in multihost mode
  815 	staticPerLanguage map[string]*overlayfs.OverlayFs
  816 
  817 	finalizerInit sync.Once
  818 }
  819 
  820 func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) {
  821 	for _, componentFolder := range files.ComponentFolders {
  822 		c.addDir(rfs, componentFolder)
  823 	}
  824 }
  825 
  826 func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder string) {
  827 	dirs, err := rfs.Dirs(componentFolder)
  828 
  829 	if err == nil {
  830 		c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...)
  831 	}
  832 }
  833 
  834 func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) {
  835 	for i := len(fis)/2 - 1; i >= 0; i-- {
  836 		opp := len(fis) - 1 - i
  837 		fis[i], fis[opp] = fis[opp], fis[i]
  838 	}
  839 }
  840 
  841 type mountsDescriptor struct {
  842 	modules.Module
  843 	dir           string
  844 	isMainProject bool
  845 	ordinal       int
  846 }