hugo

Fork of github.com/gohugoio/hugo with reverse pagination support

git clone git://git.shimmy1996.com/hugo.git

hugo.go (30957B)

    1 // Copyright 2019 The Hugo Authors. All rights reserved.
    2 //
    3 // Licensed under the Apache License, Version 2.0 (the "License");
    4 // you may not use this file except in compliance with the License.
    5 // You may obtain a copy of the License at
    6 // http://www.apache.org/licenses/LICENSE-2.0
    7 //
    8 // Unless required by applicable law or agreed to in writing, software
    9 // distributed under the License is distributed on an "AS IS" BASIS,
   10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   11 // See the License for the specific language governing permissions and
   12 // limitations under the License.
   13 
   14 // Package commands defines and implements command-line commands and flags
   15 // used by Hugo. Commands and flags are implemented using Cobra.
   16 package commands
   17 
   18 import (
   19 	"context"
   20 	"fmt"
   21 	"io/ioutil"
   22 	"os"
   23 	"os/signal"
   24 	"path/filepath"
   25 	"runtime"
   26 	"runtime/pprof"
   27 	"runtime/trace"
   28 	"strings"
   29 	"sync/atomic"
   30 	"syscall"
   31 	"time"
   32 
   33 	"github.com/gohugoio/hugo/hugofs/files"
   34 	"github.com/gohugoio/hugo/tpl"
   35 
   36 	"github.com/gohugoio/hugo/common/herrors"
   37 	"github.com/gohugoio/hugo/common/htime"
   38 	"github.com/gohugoio/hugo/common/types"
   39 
   40 	"github.com/gohugoio/hugo/hugofs"
   41 
   42 	"github.com/gohugoio/hugo/resources/page"
   43 
   44 	"github.com/gohugoio/hugo/common/hugo"
   45 	"github.com/gohugoio/hugo/common/loggers"
   46 	"github.com/gohugoio/hugo/common/terminal"
   47 
   48 	"github.com/gohugoio/hugo/hugolib/filesystems"
   49 
   50 	"golang.org/x/sync/errgroup"
   51 
   52 	"github.com/gohugoio/hugo/config"
   53 
   54 	flag "github.com/spf13/pflag"
   55 
   56 	"github.com/fsnotify/fsnotify"
   57 	"github.com/gohugoio/hugo/helpers"
   58 	"github.com/gohugoio/hugo/hugolib"
   59 	"github.com/gohugoio/hugo/livereload"
   60 	"github.com/gohugoio/hugo/watcher"
   61 	"github.com/spf13/afero"
   62 	"github.com/spf13/cobra"
   63 	"github.com/spf13/fsync"
   64 	jww "github.com/spf13/jwalterweatherman"
   65 )
   66 
   67 // The Response value from Execute.
   68 type Response struct {
   69 	// The build Result will only be set in the hugo build command.
   70 	Result *hugolib.HugoSites
   71 
   72 	// Err is set when the command failed to execute.
   73 	Err error
   74 
   75 	// The command that was executed.
   76 	Cmd *cobra.Command
   77 }
   78 
   79 // IsUserError returns true is the Response error is a user error rather than a
   80 // system error.
   81 func (r Response) IsUserError() bool {
   82 	return r.Err != nil && isUserError(r.Err)
   83 }
   84 
   85 // Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
   86 // The args are usually filled with os.Args[1:].
   87 func Execute(args []string) Response {
   88 	hugoCmd := newCommandsBuilder().addAll().build()
   89 	cmd := hugoCmd.getCommand()
   90 	cmd.SetArgs(args)
   91 
   92 	c, err := cmd.ExecuteC()
   93 
   94 	var resp Response
   95 
   96 	if c == cmd && hugoCmd.c != nil {
   97 		// Root command executed
   98 		resp.Result = hugoCmd.c.hugo()
   99 	}
  100 
  101 	if err == nil {
  102 		errCount := int(loggers.GlobalErrorCounter.Count())
  103 		if errCount > 0 {
  104 			err = fmt.Errorf("logged %d errors", errCount)
  105 		} else if resp.Result != nil {
  106 			errCount = resp.Result.NumLogErrors()
  107 			if errCount > 0 {
  108 				err = fmt.Errorf("logged %d errors", errCount)
  109 			}
  110 		}
  111 
  112 	}
  113 
  114 	resp.Err = err
  115 	resp.Cmd = c
  116 
  117 	return resp
  118 }
  119 
  120 // InitializeConfig initializes a config file with sensible default configuration flags.
  121 func initializeConfig(mustHaveConfigFile, failOnInitErr, running bool,
  122 	h *hugoBuilderCommon,
  123 	f flagsToConfigHandler,
  124 	cfgInit func(c *commandeer) error) (*commandeer, error) {
  125 	c, err := newCommandeer(mustHaveConfigFile, failOnInitErr, running, h, f, cfgInit)
  126 	if err != nil {
  127 		return nil, err
  128 	}
  129 
  130 	if h := c.hugoTry(); h != nil {
  131 		for _, s := range h.Sites {
  132 			s.RegisterMediaTypes()
  133 		}
  134 	}
  135 
  136 	return c, nil
  137 }
  138 
  139 func (c *commandeer) createLogger(cfg config.Provider) (loggers.Logger, error) {
  140 	var (
  141 		logHandle       = ioutil.Discard
  142 		logThreshold    = jww.LevelWarn
  143 		logFile         = cfg.GetString("logFile")
  144 		outHandle       = ioutil.Discard
  145 		stdoutThreshold = jww.LevelWarn
  146 	)
  147 
  148 	if !c.h.quiet {
  149 		outHandle = os.Stdout
  150 	}
  151 
  152 	if c.h.verboseLog || c.h.logging || (c.h.logFile != "") {
  153 		var err error
  154 		if logFile != "" {
  155 			logHandle, err = os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
  156 			if err != nil {
  157 				return nil, newSystemError("Failed to open log file:", logFile, err)
  158 			}
  159 		} else {
  160 			logHandle, err = ioutil.TempFile("", "hugo")
  161 			if err != nil {
  162 				return nil, newSystemError(err)
  163 			}
  164 		}
  165 	} else if !c.h.quiet && cfg.GetBool("verbose") {
  166 		stdoutThreshold = jww.LevelInfo
  167 	}
  168 
  169 	if cfg.GetBool("debug") {
  170 		stdoutThreshold = jww.LevelDebug
  171 	}
  172 
  173 	if c.h.verboseLog {
  174 		logThreshold = jww.LevelInfo
  175 		if cfg.GetBool("debug") {
  176 			logThreshold = jww.LevelDebug
  177 		}
  178 	}
  179 
  180 	loggers.InitGlobalLogger(stdoutThreshold, logThreshold, outHandle, logHandle)
  181 	helpers.InitLoggers()
  182 
  183 	return loggers.NewLogger(stdoutThreshold, logThreshold, outHandle, logHandle, c.running), nil
  184 }
  185 
  186 func initializeFlags(cmd *cobra.Command, cfg config.Provider) {
  187 	persFlagKeys := []string{
  188 		"debug",
  189 		"verbose",
  190 		"logFile",
  191 		// Moved from vars
  192 	}
  193 	flagKeys := []string{
  194 		"cleanDestinationDir",
  195 		"buildDrafts",
  196 		"buildFuture",
  197 		"buildExpired",
  198 		"clock",
  199 		"uglyURLs",
  200 		"canonifyURLs",
  201 		"enableRobotsTXT",
  202 		"enableGitInfo",
  203 		"pluralizeListTitles",
  204 		"preserveTaxonomyNames",
  205 		"ignoreCache",
  206 		"forceSyncStatic",
  207 		"noTimes",
  208 		"noChmod",
  209 		"noBuildLock",
  210 		"ignoreVendorPaths",
  211 		"templateMetrics",
  212 		"templateMetricsHints",
  213 
  214 		// Moved from vars.
  215 		"baseURL",
  216 		"buildWatch",
  217 		"cacheDir",
  218 		"cfgFile",
  219 		"confirm",
  220 		"contentDir",
  221 		"debug",
  222 		"destination",
  223 		"disableKinds",
  224 		"dryRun",
  225 		"force",
  226 		"gc",
  227 		"printI18nWarnings",
  228 		"printUnusedTemplates",
  229 		"invalidateCDN",
  230 		"layoutDir",
  231 		"logFile",
  232 		"maxDeletes",
  233 		"quiet",
  234 		"renderToMemory",
  235 		"source",
  236 		"target",
  237 		"theme",
  238 		"themesDir",
  239 		"verbose",
  240 		"verboseLog",
  241 		"duplicateTargetPaths",
  242 	}
  243 
  244 	for _, key := range persFlagKeys {
  245 		setValueFromFlag(cmd.PersistentFlags(), key, cfg, "", false)
  246 	}
  247 	for _, key := range flagKeys {
  248 		setValueFromFlag(cmd.Flags(), key, cfg, "", false)
  249 	}
  250 
  251 	setValueFromFlag(cmd.Flags(), "minify", cfg, "minifyOutput", true)
  252 
  253 	// Set some "config aliases"
  254 	setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir", false)
  255 	setValueFromFlag(cmd.Flags(), "printI18nWarnings", cfg, "logI18nWarnings", false)
  256 	setValueFromFlag(cmd.Flags(), "printPathWarnings", cfg, "logPathWarnings", false)
  257 }
  258 
  259 func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string, force bool) {
  260 	key = strings.TrimSpace(key)
  261 	if (force && flags.Lookup(key) != nil) || flags.Changed(key) {
  262 		f := flags.Lookup(key)
  263 		configKey := key
  264 		if targetKey != "" {
  265 			configKey = targetKey
  266 		}
  267 		// Gotta love this API.
  268 		switch f.Value.Type() {
  269 		case "bool":
  270 			bv, _ := flags.GetBool(key)
  271 			cfg.Set(configKey, bv)
  272 		case "string":
  273 			cfg.Set(configKey, f.Value.String())
  274 		case "stringSlice":
  275 			bv, _ := flags.GetStringSlice(key)
  276 			cfg.Set(configKey, bv)
  277 		case "int":
  278 			iv, _ := flags.GetInt(key)
  279 			cfg.Set(configKey, iv)
  280 		default:
  281 			panic(fmt.Sprintf("update switch with %s", f.Value.Type()))
  282 		}
  283 
  284 	}
  285 }
  286 
  287 func (c *commandeer) fullBuild(noBuildLock bool) error {
  288 	var (
  289 		g         errgroup.Group
  290 		langCount map[string]uint64
  291 	)
  292 
  293 	if !c.h.quiet {
  294 		fmt.Println("Start building sites … ")
  295 		fmt.Println(hugo.BuildVersionString())
  296 		if terminal.IsTerminal(os.Stdout) {
  297 			defer func() {
  298 				fmt.Print(showCursor + clearLine)
  299 			}()
  300 		}
  301 	}
  302 
  303 	copyStaticFunc := func() error {
  304 		cnt, err := c.copyStatic()
  305 		if err != nil {
  306 			return fmt.Errorf("Error copying static files: %w", err)
  307 		}
  308 		langCount = cnt
  309 		return nil
  310 	}
  311 	buildSitesFunc := func() error {
  312 		if err := c.buildSites(noBuildLock); err != nil {
  313 			return fmt.Errorf("Error building site: %w", err)
  314 		}
  315 		return nil
  316 	}
  317 	// Do not copy static files and build sites in parallel if cleanDestinationDir is enabled.
  318 	// This flag deletes all static resources in /public folder that are missing in /static,
  319 	// and it does so at the end of copyStatic() call.
  320 	if c.Cfg.GetBool("cleanDestinationDir") {
  321 		if err := copyStaticFunc(); err != nil {
  322 			return err
  323 		}
  324 		if err := buildSitesFunc(); err != nil {
  325 			return err
  326 		}
  327 	} else {
  328 		g.Go(copyStaticFunc)
  329 		g.Go(buildSitesFunc)
  330 		if err := g.Wait(); err != nil {
  331 			return err
  332 		}
  333 	}
  334 
  335 	for _, s := range c.hugo().Sites {
  336 		s.ProcessingStats.Static = langCount[s.Language().Lang]
  337 	}
  338 
  339 	if c.h.gc {
  340 		count, err := c.hugo().GC()
  341 		if err != nil {
  342 			return err
  343 		}
  344 		for _, s := range c.hugo().Sites {
  345 			// We have no way of knowing what site the garbage belonged to.
  346 			s.ProcessingStats.Cleaned = uint64(count)
  347 		}
  348 	}
  349 
  350 	return nil
  351 }
  352 
  353 func (c *commandeer) initCPUProfile() (func(), error) {
  354 	if c.h.cpuprofile == "" {
  355 		return nil, nil
  356 	}
  357 
  358 	f, err := os.Create(c.h.cpuprofile)
  359 	if err != nil {
  360 		return nil, fmt.Errorf("failed to create CPU profile: %w", err)
  361 	}
  362 	if err := pprof.StartCPUProfile(f); err != nil {
  363 		return nil, fmt.Errorf("failed to start CPU profile: %w", err)
  364 	}
  365 	return func() {
  366 		pprof.StopCPUProfile()
  367 		f.Close()
  368 	}, nil
  369 }
  370 
  371 func (c *commandeer) initMemProfile() {
  372 	if c.h.memprofile == "" {
  373 		return
  374 	}
  375 
  376 	f, err := os.Create(c.h.memprofile)
  377 	if err != nil {
  378 		c.logger.Errorf("could not create memory profile: ", err)
  379 	}
  380 	defer f.Close()
  381 	runtime.GC() // get up-to-date statistics
  382 	if err := pprof.WriteHeapProfile(f); err != nil {
  383 		c.logger.Errorf("could not write memory profile: ", err)
  384 	}
  385 }
  386 
  387 func (c *commandeer) initTraceProfile() (func(), error) {
  388 	if c.h.traceprofile == "" {
  389 		return nil, nil
  390 	}
  391 
  392 	f, err := os.Create(c.h.traceprofile)
  393 	if err != nil {
  394 		return nil, fmt.Errorf("failed to create trace file: %w", err)
  395 	}
  396 
  397 	if err := trace.Start(f); err != nil {
  398 		return nil, fmt.Errorf("failed to start trace: %w", err)
  399 	}
  400 
  401 	return func() {
  402 		trace.Stop()
  403 		f.Close()
  404 	}, nil
  405 }
  406 
  407 func (c *commandeer) initMutexProfile() (func(), error) {
  408 	if c.h.mutexprofile == "" {
  409 		return nil, nil
  410 	}
  411 
  412 	f, err := os.Create(c.h.mutexprofile)
  413 	if err != nil {
  414 		return nil, err
  415 	}
  416 
  417 	runtime.SetMutexProfileFraction(1)
  418 
  419 	return func() {
  420 		pprof.Lookup("mutex").WriteTo(f, 0)
  421 		f.Close()
  422 	}, nil
  423 }
  424 
  425 func (c *commandeer) initMemTicker() func() {
  426 	memticker := time.NewTicker(5 * time.Second)
  427 	quit := make(chan struct{})
  428 	printMem := func() {
  429 		var m runtime.MemStats
  430 		runtime.ReadMemStats(&m)
  431 		fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC)
  432 	}
  433 
  434 	go func() {
  435 		for {
  436 			select {
  437 			case <-memticker.C:
  438 				printMem()
  439 			case <-quit:
  440 				memticker.Stop()
  441 				printMem()
  442 				return
  443 			}
  444 		}
  445 	}()
  446 
  447 	return func() {
  448 		close(quit)
  449 	}
  450 }
  451 
  452 func (c *commandeer) initProfiling() (func(), error) {
  453 	stopCPUProf, err := c.initCPUProfile()
  454 	if err != nil {
  455 		return nil, err
  456 	}
  457 
  458 	stopMutexProf, err := c.initMutexProfile()
  459 	if err != nil {
  460 		return nil, err
  461 	}
  462 
  463 	stopTraceProf, err := c.initTraceProfile()
  464 	if err != nil {
  465 		return nil, err
  466 	}
  467 
  468 	var stopMemTicker func()
  469 	if c.h.printm {
  470 		stopMemTicker = c.initMemTicker()
  471 	}
  472 
  473 	return func() {
  474 		c.initMemProfile()
  475 
  476 		if stopCPUProf != nil {
  477 			stopCPUProf()
  478 		}
  479 		if stopMutexProf != nil {
  480 			stopMutexProf()
  481 		}
  482 
  483 		if stopTraceProf != nil {
  484 			stopTraceProf()
  485 		}
  486 
  487 		if stopMemTicker != nil {
  488 			stopMemTicker()
  489 		}
  490 	}, nil
  491 }
  492 
  493 func (c *commandeer) build() error {
  494 	stopProfiling, err := c.initProfiling()
  495 	if err != nil {
  496 		return err
  497 	}
  498 
  499 	defer func() {
  500 		if stopProfiling != nil {
  501 			stopProfiling()
  502 		}
  503 	}()
  504 
  505 	if err := c.fullBuild(false); err != nil {
  506 		return err
  507 	}
  508 
  509 	if !c.h.quiet {
  510 		fmt.Println()
  511 		c.hugo().PrintProcessingStats(os.Stdout)
  512 		fmt.Println()
  513 
  514 		if createCounter, ok := c.publishDirFs.(hugofs.DuplicatesReporter); ok {
  515 			dupes := createCounter.ReportDuplicates()
  516 			if dupes != "" {
  517 				c.logger.Warnln("Duplicate target paths:", dupes)
  518 			}
  519 		}
  520 
  521 		unusedTemplates := c.hugo().Tmpl().(tpl.UnusedTemplatesProvider).UnusedTemplates()
  522 		for _, unusedTemplate := range unusedTemplates {
  523 			c.logger.Warnf("Template %s is unused, source file %s", unusedTemplate.Name(), unusedTemplate.Filename())
  524 		}
  525 	}
  526 
  527 	if c.h.buildWatch {
  528 		watchDirs, err := c.getDirList()
  529 		if err != nil {
  530 			return err
  531 		}
  532 
  533 		baseWatchDir := c.Cfg.GetString("workingDir")
  534 		rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs)
  535 
  536 		c.logger.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
  537 		c.logger.Println("Press Ctrl+C to stop")
  538 		watcher, err := c.newWatcher(c.h.poll, watchDirs...)
  539 		checkErr(c.Logger, err)
  540 		defer watcher.Close()
  541 
  542 		sigs := make(chan os.Signal, 1)
  543 		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
  544 
  545 		<-sigs
  546 	}
  547 
  548 	return nil
  549 }
  550 
  551 func (c *commandeer) serverBuild() error {
  552 	stopProfiling, err := c.initProfiling()
  553 	if err != nil {
  554 		return err
  555 	}
  556 
  557 	defer func() {
  558 		if stopProfiling != nil {
  559 			stopProfiling()
  560 		}
  561 	}()
  562 
  563 	if err := c.fullBuild(false); err != nil {
  564 		return err
  565 	}
  566 
  567 	// TODO(bep) Feedback?
  568 	if !c.h.quiet {
  569 		fmt.Println()
  570 		c.hugo().PrintProcessingStats(os.Stdout)
  571 		fmt.Println()
  572 	}
  573 
  574 	return nil
  575 }
  576 
  577 func (c *commandeer) copyStatic() (map[string]uint64, error) {
  578 	m, err := c.doWithPublishDirs(c.copyStaticTo)
  579 	if err == nil || os.IsNotExist(err) {
  580 		return m, nil
  581 	}
  582 	return m, err
  583 }
  584 
  585 func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) {
  586 	langCount := make(map[string]uint64)
  587 
  588 	staticFilesystems := c.hugo().BaseFs.SourceFilesystems.Static
  589 
  590 	if len(staticFilesystems) == 0 {
  591 		c.logger.Infoln("No static directories found to sync")
  592 		return langCount, nil
  593 	}
  594 
  595 	for lang, fs := range staticFilesystems {
  596 		cnt, err := f(fs)
  597 		if err != nil {
  598 			return langCount, err
  599 		}
  600 
  601 		if lang == "" {
  602 			// Not multihost
  603 			for _, l := range c.languages {
  604 				langCount[l.Lang] = cnt
  605 			}
  606 		} else {
  607 			langCount[lang] = cnt
  608 		}
  609 	}
  610 
  611 	return langCount, nil
  612 }
  613 
  614 type countingStatFs struct {
  615 	afero.Fs
  616 	statCounter uint64
  617 }
  618 
  619 func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) {
  620 	f, err := fs.Fs.Stat(name)
  621 	if err == nil {
  622 		if !f.IsDir() {
  623 			atomic.AddUint64(&fs.statCounter, 1)
  624 		}
  625 	}
  626 	return f, err
  627 }
  628 
  629 func chmodFilter(dst, src os.FileInfo) bool {
  630 	// Hugo publishes data from multiple sources, potentially
  631 	// with overlapping directory structures. We cannot sync permissions
  632 	// for directories as that would mean that we might end up with write-protected
  633 	// directories inside /public.
  634 	// One example of this would be syncing from the Go Module cache,
  635 	// which have 0555 directories.
  636 	return src.IsDir()
  637 }
  638 
  639 func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
  640 	publishDir := helpers.FilePathSeparator
  641 
  642 	if sourceFs.PublishFolder != "" {
  643 		publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
  644 	}
  645 
  646 	fs := &countingStatFs{Fs: sourceFs.Fs}
  647 
  648 	syncer := fsync.NewSyncer()
  649 	syncer.NoTimes = c.Cfg.GetBool("noTimes")
  650 	syncer.NoChmod = c.Cfg.GetBool("noChmod")
  651 	syncer.ChmodFilter = chmodFilter
  652 	syncer.SrcFs = fs
  653 	syncer.DestFs = c.Fs.PublishDir
  654 	if c.renderStaticToDisk {
  655 		syncer.DestFs = c.Fs.PublishDirStatic
  656 	}
  657 	// Now that we are using a unionFs for the static directories
  658 	// We can effectively clean the publishDir on initial sync
  659 	syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")
  660 
  661 	if syncer.Delete {
  662 		c.logger.Infoln("removing all files from destination that don't exist in static dirs")
  663 
  664 		syncer.DeleteFilter = func(f os.FileInfo) bool {
  665 			return f.IsDir() && strings.HasPrefix(f.Name(), ".")
  666 		}
  667 	}
  668 	c.logger.Infoln("syncing static files to", publishDir)
  669 
  670 	// because we are using a baseFs (to get the union right).
  671 	// set sync src to root
  672 	err := syncer.Sync(publishDir, helpers.FilePathSeparator)
  673 	if err != nil {
  674 		return 0, err
  675 	}
  676 
  677 	// Sync runs Stat 3 times for every source file (which sounds much)
  678 	numFiles := fs.statCounter / 3
  679 
  680 	return numFiles, err
  681 }
  682 
  683 func (c *commandeer) firstPathSpec() *helpers.PathSpec {
  684 	return c.hugo().Sites[0].PathSpec
  685 }
  686 
  687 func (c *commandeer) timeTrack(start time.Time, name string) {
  688 	// Note the use of time.Since here and time.Now in the callers.
  689 	// We have a htime.Sinnce, but that may be adjusted to the future,
  690 	// and that does not make sense here, esp. when used before the
  691 	// global Clock is initialized.
  692 	elapsed := time.Since(start)
  693 	c.logger.Printf("%s in %v ms", name, int(1000*elapsed.Seconds()))
  694 }
  695 
  696 // getDirList provides NewWatcher() with a list of directories to watch for changes.
  697 func (c *commandeer) getDirList() ([]string, error) {
  698 	var filenames []string
  699 
  700 	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
  701 		if err != nil {
  702 			c.logger.Errorln("walker: ", err)
  703 			return nil
  704 		}
  705 
  706 		if fi.IsDir() {
  707 			if fi.Name() == ".git" ||
  708 				fi.Name() == "node_modules" || fi.Name() == "bower_components" {
  709 				return filepath.SkipDir
  710 			}
  711 
  712 			filenames = append(filenames, fi.Meta().Filename)
  713 		}
  714 
  715 		return nil
  716 	}
  717 
  718 	watchFiles := c.hugo().PathSpec.BaseFs.WatchDirs()
  719 	for _, fi := range watchFiles {
  720 		if !fi.IsDir() {
  721 			filenames = append(filenames, fi.Meta().Filename)
  722 			continue
  723 		}
  724 
  725 		w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: fi, WalkFn: walkFn})
  726 		if err := w.Walk(); err != nil {
  727 			c.logger.Errorln("walker: ", err)
  728 		}
  729 	}
  730 
  731 	filenames = helpers.UniqueStringsSorted(filenames)
  732 
  733 	return filenames, nil
  734 }
  735 
  736 func (c *commandeer) buildSites(noBuildLock bool) (err error) {
  737 	return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: noBuildLock})
  738 }
  739 
  740 func (c *commandeer) handleBuildErr(err error, msg string) {
  741 	c.buildErr = err
  742 	c.logger.Errorln(msg + ": " + cleanErrorLog(err.Error()))
  743 }
  744 
  745 func (c *commandeer) rebuildSites(events []fsnotify.Event) error {
  746 	if c.buildErr != nil {
  747 		ferrs := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr)
  748 		for _, err := range ferrs {
  749 			events = append(events, fsnotify.Event{Name: err.Position().Filename, Op: fsnotify.Write})
  750 		}
  751 	}
  752 	c.buildErr = nil
  753 	visited := c.visitedURLs.PeekAllSet()
  754 	if c.fastRenderMode {
  755 		// Make sure we always render the home pages
  756 		for _, l := range c.languages {
  757 			langPath := c.hugo().PathSpec.GetLangSubDir(l.Lang)
  758 			if langPath != "" {
  759 				langPath = langPath + "/"
  760 			}
  761 			home := c.hugo().PathSpec.PrependBasePath("/"+langPath, false)
  762 			visited[home] = true
  763 		}
  764 	}
  765 	return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: visited, ErrRecovery: c.wasError}, events...)
  766 }
  767 
  768 func (c *commandeer) partialReRender(urls ...string) error {
  769 	defer func() {
  770 		c.wasError = false
  771 	}()
  772 	c.buildErr = nil
  773 	visited := make(map[string]bool)
  774 	for _, url := range urls {
  775 		visited[url] = true
  776 	}
  777 
  778 	// Note: We do not set NoBuildLock as the file lock is not acquired at this stage.
  779 	return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyVisited: visited, PartialReRender: true, ErrRecovery: c.wasError})
  780 }
  781 
  782 func (c *commandeer) fullRebuild(changeType string) {
  783 	if changeType == configChangeGoMod {
  784 		// go.mod may be changed during the build itself, and
  785 		// we really want to prevent superfluous builds.
  786 		if !c.fullRebuildSem.TryAcquire(1) {
  787 			return
  788 		}
  789 		c.fullRebuildSem.Release(1)
  790 	}
  791 
  792 	c.fullRebuildSem.Acquire(context.Background(), 1)
  793 
  794 	go func() {
  795 		defer c.fullRebuildSem.Release(1)
  796 
  797 		c.printChangeDetected(changeType)
  798 
  799 		defer func() {
  800 			// Allow any file system events to arrive back.
  801 			// This will block any rebuild on config changes for the
  802 			// duration of the sleep.
  803 			time.Sleep(2 * time.Second)
  804 		}()
  805 
  806 		defer c.timeTrack(time.Now(), "Rebuilt")
  807 
  808 		c.commandeerHugoState = newCommandeerHugoState()
  809 		err := c.loadConfig()
  810 		if err != nil {
  811 			// Set the processing on pause until the state is recovered.
  812 			c.paused = true
  813 			c.handleBuildErr(err, "Failed to reload config")
  814 
  815 		} else {
  816 			c.paused = false
  817 		}
  818 
  819 		if !c.paused {
  820 			_, err := c.copyStatic()
  821 			if err != nil {
  822 				c.logger.Errorln(err)
  823 				return
  824 			}
  825 
  826 			err = c.buildSites(true)
  827 			if err != nil {
  828 				c.logger.Errorln(err)
  829 			} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
  830 				livereload.ForceRefresh()
  831 			}
  832 		}
  833 	}()
  834 }
  835 
  836 // newWatcher creates a new watcher to watch filesystem events.
  837 func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*watcher.Batcher, error) {
  838 	if runtime.GOOS == "darwin" {
  839 		tweakLimit()
  840 	}
  841 
  842 	staticSyncer, err := newStaticSyncer(c)
  843 	if err != nil {
  844 		return nil, err
  845 	}
  846 
  847 	var pollInterval time.Duration
  848 	poll := pollIntervalStr != ""
  849 	if poll {
  850 		pollInterval, err = types.ToDurationE(pollIntervalStr)
  851 		if err != nil {
  852 			return nil, fmt.Errorf("invalid value for flag poll: %s", err)
  853 		}
  854 		c.logger.Printf("Use watcher with poll interval %v", pollInterval)
  855 	}
  856 
  857 	if pollInterval == 0 {
  858 		pollInterval = 500 * time.Millisecond
  859 	}
  860 
  861 	watcher, err := watcher.New(500*time.Millisecond, pollInterval, poll)
  862 	if err != nil {
  863 		return nil, err
  864 	}
  865 
  866 	spec := c.hugo().Deps.SourceSpec
  867 
  868 	for _, d := range dirList {
  869 		if d != "" {
  870 			if spec.IgnoreFile(d) {
  871 				continue
  872 			}
  873 			_ = watcher.Add(d)
  874 		}
  875 	}
  876 
  877 	// Identifies changes to config (config.toml) files.
  878 	configSet := make(map[string]bool)
  879 
  880 	c.logger.Println("Watching for config changes in", strings.Join(c.configFiles, ", "))
  881 	for _, configFile := range c.configFiles {
  882 		watcher.Add(configFile)
  883 		configSet[configFile] = true
  884 	}
  885 
  886 	go func() {
  887 		for {
  888 			select {
  889 			case evs := <-watcher.Events:
  890 				unlock, err := c.buildLock()
  891 				if err != nil {
  892 					c.logger.Errorln("Failed to acquire a build lock: %s", err)
  893 					return
  894 				}
  895 				c.handleEvents(watcher, staticSyncer, evs, configSet)
  896 				if c.showErrorInBrowser && c.errCount() > 0 {
  897 					// Need to reload browser to show the error
  898 					livereload.ForceRefresh()
  899 				}
  900 				unlock()
  901 			case err := <-watcher.Errors():
  902 				if err != nil && !os.IsNotExist(err) {
  903 					c.logger.Errorln("Error while watching:", err)
  904 				}
  905 			}
  906 		}
  907 	}()
  908 
  909 	return watcher, nil
  910 }
  911 
  912 func (c *commandeer) printChangeDetected(typ string) {
  913 	msg := "\nChange"
  914 	if typ != "" {
  915 		msg += " of " + typ
  916 	}
  917 	msg += " detected, rebuilding site."
  918 
  919 	c.logger.Println(msg)
  920 	const layout = "2006-01-02 15:04:05.000 -0700"
  921 	c.logger.Println(htime.Now().Format(layout))
  922 }
  923 
  924 const (
  925 	configChangeConfig = "config file"
  926 	configChangeGoMod  = "go.mod file"
  927 )
  928 
  929 func (c *commandeer) handleEvents(watcher *watcher.Batcher,
  930 	staticSyncer *staticSyncer,
  931 	evs []fsnotify.Event,
  932 	configSet map[string]bool) {
  933 	defer func() {
  934 		c.wasError = false
  935 	}()
  936 
  937 	var isHandled bool
  938 
  939 	for _, ev := range evs {
  940 		isConfig := configSet[ev.Name]
  941 		configChangeType := configChangeConfig
  942 		if isConfig {
  943 			if strings.Contains(ev.Name, "go.mod") {
  944 				configChangeType = configChangeGoMod
  945 			}
  946 		}
  947 		if !isConfig {
  948 			// It may be one of the /config folders
  949 			dirname := filepath.Dir(ev.Name)
  950 			if dirname != "." && configSet[dirname] {
  951 				isConfig = true
  952 			}
  953 		}
  954 
  955 		if isConfig {
  956 			isHandled = true
  957 
  958 			if ev.Op&fsnotify.Chmod == fsnotify.Chmod {
  959 				continue
  960 			}
  961 
  962 			if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename {
  963 				for _, configFile := range c.configFiles {
  964 					counter := 0
  965 					for watcher.Add(configFile) != nil {
  966 						counter++
  967 						if counter >= 100 {
  968 							break
  969 						}
  970 						time.Sleep(100 * time.Millisecond)
  971 					}
  972 				}
  973 			}
  974 
  975 			// Config file(s) changed. Need full rebuild.
  976 			c.fullRebuild(configChangeType)
  977 
  978 			return
  979 		}
  980 	}
  981 
  982 	if isHandled {
  983 		return
  984 	}
  985 
  986 	if c.paused {
  987 		// Wait for the server to get into a consistent state before
  988 		// we continue with processing.
  989 		return
  990 	}
  991 
  992 	if len(evs) > 50 {
  993 		// This is probably a mass edit of the content dir.
  994 		// Schedule a full rebuild for when it slows down.
  995 		c.debounce(func() {
  996 			c.fullRebuild("")
  997 		})
  998 		return
  999 	}
 1000 
 1001 	c.logger.Infoln("Received System Events:", evs)
 1002 
 1003 	staticEvents := []fsnotify.Event{}
 1004 	dynamicEvents := []fsnotify.Event{}
 1005 
 1006 	filtered := []fsnotify.Event{}
 1007 	for _, ev := range evs {
 1008 		if c.hugo().ShouldSkipFileChangeEvent(ev) {
 1009 			continue
 1010 		}
 1011 		// Check the most specific first, i.e. files.
 1012 		contentMapped := c.hugo().ContentChanges.GetSymbolicLinkMappings(ev.Name)
 1013 		if len(contentMapped) > 0 {
 1014 			for _, mapped := range contentMapped {
 1015 				filtered = append(filtered, fsnotify.Event{Name: mapped, Op: ev.Op})
 1016 			}
 1017 			continue
 1018 		}
 1019 
 1020 		// Check for any symbolic directory mapping.
 1021 
 1022 		dir, name := filepath.Split(ev.Name)
 1023 
 1024 		contentMapped = c.hugo().ContentChanges.GetSymbolicLinkMappings(dir)
 1025 
 1026 		if len(contentMapped) == 0 {
 1027 			filtered = append(filtered, ev)
 1028 			continue
 1029 		}
 1030 
 1031 		for _, mapped := range contentMapped {
 1032 			mappedFilename := filepath.Join(mapped, name)
 1033 			filtered = append(filtered, fsnotify.Event{Name: mappedFilename, Op: ev.Op})
 1034 		}
 1035 	}
 1036 
 1037 	evs = filtered
 1038 
 1039 	for _, ev := range evs {
 1040 		ext := filepath.Ext(ev.Name)
 1041 		baseName := filepath.Base(ev.Name)
 1042 		istemp := strings.HasSuffix(ext, "~") ||
 1043 			(ext == ".swp") || // vim
 1044 			(ext == ".swx") || // vim
 1045 			(ext == ".tmp") || // generic temp file
 1046 			(ext == ".DS_Store") || // OSX Thumbnail
 1047 			baseName == "4913" || // vim
 1048 			strings.HasPrefix(ext, ".goutputstream") || // gnome
 1049 			strings.HasSuffix(ext, "jb_old___") || // intelliJ
 1050 			strings.HasSuffix(ext, "jb_tmp___") || // intelliJ
 1051 			strings.HasSuffix(ext, "jb_bak___") || // intelliJ
 1052 			strings.HasPrefix(ext, ".sb-") || // byword
 1053 			strings.HasPrefix(baseName, ".#") || // emacs
 1054 			strings.HasPrefix(baseName, "#") // emacs
 1055 		if istemp {
 1056 			continue
 1057 		}
 1058 		if c.hugo().Deps.SourceSpec.IgnoreFile(ev.Name) {
 1059 			continue
 1060 		}
 1061 		// Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these
 1062 		if ev.Name == "" {
 1063 			continue
 1064 		}
 1065 
 1066 		// Write and rename operations are often followed by CHMOD.
 1067 		// There may be valid use cases for rebuilding the site on CHMOD,
 1068 		// but that will require more complex logic than this simple conditional.
 1069 		// On OS X this seems to be related to Spotlight, see:
 1070 		// https://github.com/go-fsnotify/fsnotify/issues/15
 1071 		// A workaround is to put your site(s) on the Spotlight exception list,
 1072 		// but that may be a little mysterious for most end users.
 1073 		// So, for now, we skip reload on CHMOD.
 1074 		// We do have to check for WRITE though. On slower laptops a Chmod
 1075 		// could be aggregated with other important events, and we still want
 1076 		// to rebuild on those
 1077 		if ev.Op&(fsnotify.Chmod|fsnotify.Write|fsnotify.Create) == fsnotify.Chmod {
 1078 			continue
 1079 		}
 1080 
 1081 		walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error {
 1082 			if f.IsDir() {
 1083 				c.logger.Println("adding created directory to watchlist", path)
 1084 				if err := watcher.Add(path); err != nil {
 1085 					return err
 1086 				}
 1087 			} else if !staticSyncer.isStatic(path) {
 1088 				// Hugo's rebuilding logic is entirely file based. When you drop a new folder into
 1089 				// /content on OSX, the above logic will handle future watching of those files,
 1090 				// but the initial CREATE is lost.
 1091 				dynamicEvents = append(dynamicEvents, fsnotify.Event{Name: path, Op: fsnotify.Create})
 1092 			}
 1093 			return nil
 1094 		}
 1095 
 1096 		// recursively add new directories to watch list
 1097 		// When mkdir -p is used, only the top directory triggers an event (at least on OSX)
 1098 		if ev.Op&fsnotify.Create == fsnotify.Create {
 1099 			if s, err := c.Fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() {
 1100 				_ = helpers.SymbolicWalk(c.Fs.Source, ev.Name, walkAdder)
 1101 			}
 1102 		}
 1103 
 1104 		if staticSyncer.isStatic(ev.Name) {
 1105 			staticEvents = append(staticEvents, ev)
 1106 		} else {
 1107 			dynamicEvents = append(dynamicEvents, ev)
 1108 		}
 1109 	}
 1110 
 1111 	if len(staticEvents) > 0 {
 1112 		c.printChangeDetected("Static files")
 1113 
 1114 		if c.Cfg.GetBool("forceSyncStatic") {
 1115 			c.logger.Printf("Syncing all static files\n")
 1116 			_, err := c.copyStatic()
 1117 			if err != nil {
 1118 				c.logger.Errorln("Error copying static files to publish dir:", err)
 1119 				return
 1120 			}
 1121 		} else {
 1122 			if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil {
 1123 				c.logger.Errorln("Error syncing static files to publish dir:", err)
 1124 				return
 1125 			}
 1126 		}
 1127 
 1128 		if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
 1129 			// Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized
 1130 
 1131 			// force refresh when more than one file
 1132 			if !c.wasError && len(staticEvents) == 1 {
 1133 				ev := staticEvents[0]
 1134 				path := c.hugo().BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name)
 1135 				path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false)
 1136 
 1137 				livereload.RefreshPath(path)
 1138 			} else {
 1139 				livereload.ForceRefresh()
 1140 			}
 1141 		}
 1142 	}
 1143 
 1144 	if len(dynamicEvents) > 0 {
 1145 		partitionedEvents := partitionDynamicEvents(
 1146 			c.firstPathSpec().BaseFs.SourceFilesystems,
 1147 			dynamicEvents)
 1148 
 1149 		doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
 1150 		onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)
 1151 
 1152 		c.printChangeDetected("")
 1153 		c.changeDetector.PrepareNew()
 1154 
 1155 		func() {
 1156 			defer c.timeTrack(time.Now(), "Total")
 1157 			if err := c.rebuildSites(dynamicEvents); err != nil {
 1158 				c.handleBuildErr(err, "Rebuild failed")
 1159 			}
 1160 		}()
 1161 
 1162 		if doLiveReload {
 1163 			if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 {
 1164 				if c.wasError {
 1165 					livereload.ForceRefresh()
 1166 					return
 1167 				}
 1168 				changed := c.changeDetector.changed()
 1169 				if c.changeDetector != nil && len(changed) == 0 {
 1170 					// Nothing has changed.
 1171 					return
 1172 				} else if len(changed) == 1 {
 1173 					pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false)
 1174 					livereload.RefreshPath(pathToRefresh)
 1175 				} else {
 1176 					livereload.ForceRefresh()
 1177 				}
 1178 			}
 1179 
 1180 			if len(partitionedEvents.ContentEvents) > 0 {
 1181 
 1182 				navigate := c.Cfg.GetBool("navigateToChanged")
 1183 				// We have fetched the same page above, but it may have
 1184 				// changed.
 1185 				var p page.Page
 1186 
 1187 				if navigate {
 1188 					if onePageName != "" {
 1189 						p = c.hugo().GetContentPage(onePageName)
 1190 					}
 1191 				}
 1192 
 1193 				if p != nil {
 1194 					livereload.NavigateToPathForPort(p.RelPermalink(), p.Site().ServerPort())
 1195 				} else {
 1196 					livereload.ForceRefresh()
 1197 				}
 1198 			}
 1199 		}
 1200 	}
 1201 }
 1202 
 1203 // dynamicEvents contains events that is considered dynamic, as in "not static".
 1204 // Both of these categories will trigger a new build, but the asset events
 1205 // does not fit into the "navigate to changed" logic.
 1206 type dynamicEvents struct {
 1207 	ContentEvents []fsnotify.Event
 1208 	AssetEvents   []fsnotify.Event
 1209 }
 1210 
 1211 func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) {
 1212 	for _, e := range events {
 1213 		if sourceFs.IsAsset(e.Name) {
 1214 			de.AssetEvents = append(de.AssetEvents, e)
 1215 		} else {
 1216 			de.ContentEvents = append(de.ContentEvents, e)
 1217 		}
 1218 	}
 1219 	return
 1220 }
 1221 
 1222 func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
 1223 	name := ""
 1224 
 1225 	for _, ev := range events {
 1226 		if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create {
 1227 			if files.IsIndexContentFile(ev.Name) {
 1228 				return ev.Name
 1229 			}
 1230 
 1231 			if files.IsContentFile(ev.Name) {
 1232 				name = ev.Name
 1233 			}
 1234 
 1235 		}
 1236 	}
 1237 
 1238 	return name
 1239 }
 1240 
 1241 func formatByteCount(b uint64) string {
 1242 	const unit = 1000
 1243 	if b < unit {
 1244 		return fmt.Sprintf("%d B", b)
 1245 	}
 1246 	div, exp := int64(unit), 0
 1247 	for n := b / unit; n >= unit; n /= unit {
 1248 		div *= unit
 1249 		exp++
 1250 	}
 1251 	return fmt.Sprintf("%.1f %cB",
 1252 		float64(b)/float64(div), "kMGTPE"[exp])
 1253 }