hugo

Fork of github.com/gohugoio/hugo with reverse pagination support

git clone git://git.shimmy1996.com/hugo.git

integrationtest_builder.go (12227B)

    1 package hugolib
    2 
    3 import (
    4 	"bytes"
    5 	"encoding/base64"
    6 	"fmt"
    7 	"io"
    8 	"os"
    9 	"path/filepath"
   10 	"regexp"
   11 	"strings"
   12 	"sync"
   13 	"testing"
   14 
   15 	jww "github.com/spf13/jwalterweatherman"
   16 
   17 	qt "github.com/frankban/quicktest"
   18 	"github.com/fsnotify/fsnotify"
   19 	"github.com/gohugoio/hugo/common/herrors"
   20 	"github.com/gohugoio/hugo/common/hexec"
   21 	"github.com/gohugoio/hugo/common/loggers"
   22 	"github.com/gohugoio/hugo/config"
   23 	"github.com/gohugoio/hugo/config/security"
   24 	"github.com/gohugoio/hugo/deps"
   25 	"github.com/gohugoio/hugo/helpers"
   26 	"github.com/gohugoio/hugo/htesting"
   27 	"github.com/gohugoio/hugo/hugofs"
   28 	"github.com/spf13/afero"
   29 	"golang.org/x/tools/txtar"
   30 )
   31 
   32 func NewIntegrationTestBuilder(conf IntegrationTestConfig) *IntegrationTestBuilder {
   33 	// Code fences.
   34 	conf.TxtarString = strings.ReplaceAll(conf.TxtarString, "§§§", "```")
   35 
   36 	data := txtar.Parse([]byte(conf.TxtarString))
   37 
   38 	c, ok := conf.T.(*qt.C)
   39 	if !ok {
   40 		c = qt.New(conf.T)
   41 	}
   42 
   43 	if conf.NeedsOsFS {
   44 		if !filepath.IsAbs(conf.WorkingDir) {
   45 			tempDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-integration-test")
   46 			c.Assert(err, qt.IsNil)
   47 			conf.WorkingDir = filepath.Join(tempDir, conf.WorkingDir)
   48 			if !conf.PrintAndKeepTempDir {
   49 				c.Cleanup(clean)
   50 			} else {
   51 				fmt.Println("\nUsing WorkingDir dir:", conf.WorkingDir)
   52 			}
   53 		}
   54 	} else if conf.WorkingDir == "" {
   55 		conf.WorkingDir = helpers.FilePathSeparator
   56 	}
   57 
   58 	return &IntegrationTestBuilder{
   59 		Cfg:  conf,
   60 		C:    c,
   61 		data: data,
   62 	}
   63 }
   64 
   65 // IntegrationTestBuilder is a (partial) rewrite of sitesBuilder.
   66 // The main problem with the "old" one was that it was that the test data was often a little hidden,
   67 // so it became hard to look at a test and determine what it should do, especially coming back to the
   68 // test after a year or so.
   69 type IntegrationTestBuilder struct {
   70 	*qt.C
   71 
   72 	data *txtar.Archive
   73 
   74 	fs *hugofs.Fs
   75 	H  *HugoSites
   76 
   77 	Cfg IntegrationTestConfig
   78 
   79 	changedFiles []string
   80 	createdFiles []string
   81 	removedFiles []string
   82 	renamedFiles []string
   83 
   84 	buildCount int
   85 	counters   *testCounters
   86 	logBuff    lockingBuffer
   87 
   88 	builderInit sync.Once
   89 }
   90 
   91 type lockingBuffer struct {
   92 	sync.Mutex
   93 	bytes.Buffer
   94 }
   95 
   96 func (b *lockingBuffer) Write(p []byte) (n int, err error) {
   97 	b.Lock()
   98 	n, err = b.Buffer.Write(p)
   99 	b.Unlock()
  100 	return
  101 }
  102 
  103 func (s *IntegrationTestBuilder) AssertLogContains(text string) {
  104 	s.Helper()
  105 	s.Assert(s.logBuff.String(), qt.Contains, text)
  106 }
  107 
  108 func (s *IntegrationTestBuilder) AssertLogMatches(expression string) {
  109 	s.Helper()
  110 	re := regexp.MustCompile(expression)
  111 	s.Assert(re.MatchString(s.logBuff.String()), qt.IsTrue, qt.Commentf(s.logBuff.String()))
  112 }
  113 
  114 func (s *IntegrationTestBuilder) AssertBuildCountData(count int) {
  115 	s.Helper()
  116 	s.Assert(s.H.init.data.InitCount(), qt.Equals, count)
  117 }
  118 
  119 func (s *IntegrationTestBuilder) AssertBuildCountGitInfo(count int) {
  120 	s.Helper()
  121 	s.Assert(s.H.init.gitInfo.InitCount(), qt.Equals, count)
  122 }
  123 
  124 func (s *IntegrationTestBuilder) AssertBuildCountLayouts(count int) {
  125 	s.Helper()
  126 	s.Assert(s.H.init.layouts.InitCount(), qt.Equals, count)
  127 }
  128 
  129 func (s *IntegrationTestBuilder) AssertBuildCountTranslations(count int) {
  130 	s.Helper()
  131 	s.Assert(s.H.init.translations.InitCount(), qt.Equals, count)
  132 }
  133 
  134 func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) {
  135 	s.Helper()
  136 	content := strings.TrimSpace(s.FileContent(filename))
  137 	for _, m := range matches {
  138 		lines := strings.Split(m, "\n")
  139 		for _, match := range lines {
  140 			match = strings.TrimSpace(match)
  141 			if match == "" || strings.HasPrefix(match, "#") {
  142 				continue
  143 			}
  144 			s.Assert(content, qt.Contains, match, qt.Commentf(m))
  145 		}
  146 	}
  147 }
  148 
  149 func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) {
  150 	s.Helper()
  151 	content := s.FileContent(filename)
  152 	for _, m := range matches {
  153 		s.Assert(content, qt.Contains, m, qt.Commentf(m))
  154 	}
  155 }
  156 
  157 func (s *IntegrationTestBuilder) AssertDestinationExists(filename string, b bool) {
  158 	checker := qt.IsTrue
  159 	if !b {
  160 		checker = qt.IsFalse
  161 	}
  162 	s.Assert(s.destinationExists(filepath.Clean(filename)), checker)
  163 }
  164 
  165 func (s *IntegrationTestBuilder) destinationExists(filename string) bool {
  166 	b, err := helpers.Exists(filename, s.fs.PublishDir)
  167 	if err != nil {
  168 		panic(err)
  169 	}
  170 	return b
  171 }
  172 
  173 func (s *IntegrationTestBuilder) AssertIsFileError(err error) herrors.FileError {
  174 	s.Assert(err, qt.ErrorAs, new(herrors.FileError))
  175 	return herrors.UnwrapFileError(err)
  176 }
  177 
  178 func (s *IntegrationTestBuilder) AssertRenderCountContent(count int) {
  179 	s.Helper()
  180 	s.Assert(s.counters.contentRenderCounter, qt.Equals, uint64(count))
  181 }
  182 
  183 func (s *IntegrationTestBuilder) AssertRenderCountPage(count int) {
  184 	s.Helper()
  185 	s.Assert(s.counters.pageRenderCounter, qt.Equals, uint64(count))
  186 }
  187 
  188 func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder {
  189 	s.Helper()
  190 	_, err := s.BuildE()
  191 	if s.Cfg.Verbose || err != nil {
  192 		fmt.Println(s.logBuff.String())
  193 	}
  194 	s.Assert(err, qt.IsNil)
  195 	return s
  196 }
  197 
  198 func (s *IntegrationTestBuilder) BuildE() (*IntegrationTestBuilder, error) {
  199 	s.Helper()
  200 	if err := s.initBuilder(); err != nil {
  201 		return s, err
  202 	}
  203 
  204 	err := s.build(BuildCfg{})
  205 	return s, err
  206 }
  207 
  208 type IntegrationTestDebugConfig struct {
  209 	Out io.Writer
  210 
  211 	PrintDestinationFs bool
  212 	PrintPagemap       bool
  213 
  214 	PrefixDestinationFs string
  215 	PrefixPagemap       string
  216 }
  217 
  218 func (s *IntegrationTestBuilder) EditFileReplace(filename string, replacementFunc func(s string) string) *IntegrationTestBuilder {
  219 	absFilename := s.absFilename(filename)
  220 	b, err := afero.ReadFile(s.fs.Source, absFilename)
  221 	s.Assert(err, qt.IsNil)
  222 	s.changedFiles = append(s.changedFiles, absFilename)
  223 	oldContent := string(b)
  224 	s.writeSource(absFilename, replacementFunc(oldContent))
  225 	return s
  226 }
  227 
  228 func (s *IntegrationTestBuilder) EditFiles(filenameContent ...string) *IntegrationTestBuilder {
  229 	for i := 0; i < len(filenameContent); i += 2 {
  230 		filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
  231 		absFilename := s.absFilename(filename)
  232 		s.changedFiles = append(s.changedFiles, absFilename)
  233 		s.writeSource(absFilename, content)
  234 	}
  235 	return s
  236 }
  237 
  238 func (s *IntegrationTestBuilder) AddFiles(filenameContent ...string) *IntegrationTestBuilder {
  239 	for i := 0; i < len(filenameContent); i += 2 {
  240 		filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
  241 		absFilename := s.absFilename(filename)
  242 		s.createdFiles = append(s.createdFiles, absFilename)
  243 		s.writeSource(absFilename, content)
  244 	}
  245 	return s
  246 }
  247 
  248 func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTestBuilder {
  249 	for _, filename := range filenames {
  250 		absFilename := s.absFilename(filename)
  251 		s.removedFiles = append(s.removedFiles, absFilename)
  252 		s.Assert(s.fs.Source.Remove(absFilename), qt.IsNil)
  253 
  254 	}
  255 
  256 	return s
  257 }
  258 
  259 func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBuilder {
  260 	absOldFilename := s.absFilename(old)
  261 	absNewFilename := s.absFilename(new)
  262 	s.renamedFiles = append(s.renamedFiles, absOldFilename)
  263 	s.createdFiles = append(s.createdFiles, absNewFilename)
  264 	s.Assert(s.fs.Source.Rename(absOldFilename, absNewFilename), qt.IsNil)
  265 	return s
  266 }
  267 
  268 func (s *IntegrationTestBuilder) FileContent(filename string) string {
  269 	s.Helper()
  270 	return s.readWorkingDir(s, s.fs, filepath.FromSlash(filename))
  271 }
  272 
  273 func (s *IntegrationTestBuilder) initBuilder() error {
  274 	var initErr error
  275 	s.builderInit.Do(func() {
  276 		var afs afero.Fs
  277 		if s.Cfg.NeedsOsFS {
  278 			afs = afero.NewOsFs()
  279 		} else {
  280 			afs = afero.NewMemMapFs()
  281 		}
  282 
  283 		if s.Cfg.LogLevel == 0 {
  284 			s.Cfg.LogLevel = jww.LevelWarn
  285 		}
  286 
  287 		logger := loggers.NewBasicLoggerForWriter(s.Cfg.LogLevel, &s.logBuff)
  288 
  289 		isBinaryRe := regexp.MustCompile(`^(.*)(\.png|\.jpg)$`)
  290 
  291 		for _, f := range s.data.Files {
  292 			filename := filepath.Join(s.Cfg.WorkingDir, f.Name)
  293 			data := bytes.TrimSuffix(f.Data, []byte("\n"))
  294 			if isBinaryRe.MatchString(filename) {
  295 				var err error
  296 				data, err = base64.StdEncoding.DecodeString(string(data))
  297 				s.Assert(err, qt.IsNil)
  298 
  299 			}
  300 			s.Assert(afs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil)
  301 			s.Assert(afero.WriteFile(afs, filename, data, 0666), qt.IsNil)
  302 		}
  303 
  304 		cfg, _, err := LoadConfig(
  305 			ConfigSourceDescriptor{
  306 				WorkingDir: s.Cfg.WorkingDir,
  307 				Fs:         afs,
  308 				Logger:     logger,
  309 				Environ:    []string{},
  310 				Filename:   "config.toml",
  311 			},
  312 			func(cfg config.Provider) error {
  313 				return nil
  314 			},
  315 		)
  316 
  317 		s.Assert(err, qt.IsNil)
  318 
  319 		cfg.Set("workingDir", s.Cfg.WorkingDir)
  320 
  321 		fs := hugofs.NewFrom(afs, cfg)
  322 
  323 		s.Assert(err, qt.IsNil)
  324 
  325 		depsCfg := deps.DepsCfg{Cfg: cfg, Fs: fs, Running: s.Cfg.Running, Logger: logger}
  326 		sites, err := NewHugoSites(depsCfg)
  327 		if err != nil {
  328 			initErr = err
  329 			return
  330 		}
  331 
  332 		s.H = sites
  333 		s.fs = fs
  334 
  335 		if s.Cfg.NeedsNpmInstall {
  336 			wd, _ := os.Getwd()
  337 			s.Assert(os.Chdir(s.Cfg.WorkingDir), qt.IsNil)
  338 			s.C.Cleanup(func() { os.Chdir(wd) })
  339 			sc := security.DefaultConfig
  340 			sc.Exec.Allow = security.NewWhitelist("npm")
  341 			ex := hexec.New(sc)
  342 			command, err := ex.New("npm", "install")
  343 			s.Assert(err, qt.IsNil)
  344 			s.Assert(command.Run(), qt.IsNil)
  345 
  346 		}
  347 	})
  348 
  349 	return initErr
  350 }
  351 
  352 func (s *IntegrationTestBuilder) absFilename(filename string) string {
  353 	filename = filepath.FromSlash(filename)
  354 	if filepath.IsAbs(filename) {
  355 		return filename
  356 	}
  357 	if s.Cfg.WorkingDir != "" && !strings.HasPrefix(filename, s.Cfg.WorkingDir) {
  358 		filename = filepath.Join(s.Cfg.WorkingDir, filename)
  359 	}
  360 	return filename
  361 }
  362 
  363 func (s *IntegrationTestBuilder) build(cfg BuildCfg) error {
  364 	s.Helper()
  365 	defer func() {
  366 		s.changedFiles = nil
  367 		s.createdFiles = nil
  368 		s.removedFiles = nil
  369 		s.renamedFiles = nil
  370 	}()
  371 
  372 	changeEvents := s.changeEvents()
  373 	s.logBuff.Reset()
  374 	s.counters = &testCounters{}
  375 	cfg.testCounters = s.counters
  376 
  377 	if s.buildCount > 0 && (len(changeEvents) == 0) {
  378 		return nil
  379 	}
  380 
  381 	s.buildCount++
  382 
  383 	err := s.H.Build(cfg, changeEvents...)
  384 	if err != nil {
  385 		return err
  386 	}
  387 	logErrorCount := s.H.NumLogErrors()
  388 	if logErrorCount > 0 {
  389 		return fmt.Errorf("logged %d error(s): %s", logErrorCount, s.logBuff.String())
  390 	}
  391 
  392 	return nil
  393 }
  394 
  395 func (s *IntegrationTestBuilder) changeEvents() []fsnotify.Event {
  396 	var events []fsnotify.Event
  397 	for _, v := range s.removedFiles {
  398 		events = append(events, fsnotify.Event{
  399 			Name: v,
  400 			Op:   fsnotify.Remove,
  401 		})
  402 	}
  403 	for _, v := range s.renamedFiles {
  404 		events = append(events, fsnotify.Event{
  405 			Name: v,
  406 			Op:   fsnotify.Rename,
  407 		})
  408 	}
  409 	for _, v := range s.changedFiles {
  410 		events = append(events, fsnotify.Event{
  411 			Name: v,
  412 			Op:   fsnotify.Write,
  413 		})
  414 	}
  415 	for _, v := range s.createdFiles {
  416 		events = append(events, fsnotify.Event{
  417 			Name: v,
  418 			Op:   fsnotify.Create,
  419 		})
  420 	}
  421 
  422 	return events
  423 }
  424 
  425 func (s *IntegrationTestBuilder) readWorkingDir(t testing.TB, fs *hugofs.Fs, filename string) string {
  426 	t.Helper()
  427 	return s.readFileFromFs(t, fs.WorkingDirReadOnly, filename)
  428 }
  429 
  430 func (s *IntegrationTestBuilder) readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
  431 	t.Helper()
  432 	filename = filepath.Clean(filename)
  433 	b, err := afero.ReadFile(fs, filename)
  434 	if err != nil {
  435 		// Print some debug info
  436 		hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator)
  437 		start := 0
  438 		if hadSlash {
  439 			start = 1
  440 		}
  441 		end := start + 1
  442 
  443 		parts := strings.Split(filename, helpers.FilePathSeparator)
  444 		if parts[start] == "work" {
  445 			end++
  446 		}
  447 
  448 		s.Assert(err, qt.IsNil)
  449 
  450 	}
  451 	return string(b)
  452 }
  453 
  454 func (s *IntegrationTestBuilder) writeSource(filename, content string) {
  455 	s.Helper()
  456 	s.writeToFs(s.fs.Source, filename, content)
  457 }
  458 
  459 func (s *IntegrationTestBuilder) writeToFs(fs afero.Fs, filename, content string) {
  460 	s.Helper()
  461 	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
  462 		s.Fatalf("Failed to write file: %s", err)
  463 	}
  464 }
  465 
  466 type IntegrationTestConfig struct {
  467 	T testing.TB
  468 
  469 	// The files to use on txtar format, see
  470 	// https://pkg.go.dev/golang.org/x/exp/cmd/txtar
  471 	TxtarString string
  472 
  473 	// Whether to simulate server mode.
  474 	Running bool
  475 
  476 	// Will print the log buffer after the build
  477 	Verbose bool
  478 
  479 	LogLevel jww.Threshold
  480 
  481 	// Whether it needs the real file system (e.g. for js.Build tests).
  482 	NeedsOsFS bool
  483 
  484 	// Do not remove the temp dir after the test.
  485 	PrintAndKeepTempDir bool
  486 
  487 	// Whether to run npm install before Build.
  488 	NeedsNpmInstall bool
  489 
  490 	WorkingDir string
  491 }