integrationtest_builder.go (12227B)
1 package hugolib
2
3 import (
4 "bytes"
5 "encoding/base64"
6 "fmt"
7 "io"
8 "os"
9 "path/filepath"
10 "regexp"
11 "strings"
12 "sync"
13 "testing"
14
15 jww "github.com/spf13/jwalterweatherman"
16
17 qt "github.com/frankban/quicktest"
18 "github.com/fsnotify/fsnotify"
19 "github.com/gohugoio/hugo/common/herrors"
20 "github.com/gohugoio/hugo/common/hexec"
21 "github.com/gohugoio/hugo/common/loggers"
22 "github.com/gohugoio/hugo/config"
23 "github.com/gohugoio/hugo/config/security"
24 "github.com/gohugoio/hugo/deps"
25 "github.com/gohugoio/hugo/helpers"
26 "github.com/gohugoio/hugo/htesting"
27 "github.com/gohugoio/hugo/hugofs"
28 "github.com/spf13/afero"
29 "golang.org/x/tools/txtar"
30 )
31
32 func NewIntegrationTestBuilder(conf IntegrationTestConfig) *IntegrationTestBuilder {
33 // Code fences.
34 conf.TxtarString = strings.ReplaceAll(conf.TxtarString, "§§§", "```")
35
36 data := txtar.Parse([]byte(conf.TxtarString))
37
38 c, ok := conf.T.(*qt.C)
39 if !ok {
40 c = qt.New(conf.T)
41 }
42
43 if conf.NeedsOsFS {
44 if !filepath.IsAbs(conf.WorkingDir) {
45 tempDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-integration-test")
46 c.Assert(err, qt.IsNil)
47 conf.WorkingDir = filepath.Join(tempDir, conf.WorkingDir)
48 if !conf.PrintAndKeepTempDir {
49 c.Cleanup(clean)
50 } else {
51 fmt.Println("\nUsing WorkingDir dir:", conf.WorkingDir)
52 }
53 }
54 } else if conf.WorkingDir == "" {
55 conf.WorkingDir = helpers.FilePathSeparator
56 }
57
58 return &IntegrationTestBuilder{
59 Cfg: conf,
60 C: c,
61 data: data,
62 }
63 }
64
65 // IntegrationTestBuilder is a (partial) rewrite of sitesBuilder.
66 // The main problem with the "old" one was that it was that the test data was often a little hidden,
67 // so it became hard to look at a test and determine what it should do, especially coming back to the
68 // test after a year or so.
69 type IntegrationTestBuilder struct {
70 *qt.C
71
72 data *txtar.Archive
73
74 fs *hugofs.Fs
75 H *HugoSites
76
77 Cfg IntegrationTestConfig
78
79 changedFiles []string
80 createdFiles []string
81 removedFiles []string
82 renamedFiles []string
83
84 buildCount int
85 counters *testCounters
86 logBuff lockingBuffer
87
88 builderInit sync.Once
89 }
90
91 type lockingBuffer struct {
92 sync.Mutex
93 bytes.Buffer
94 }
95
96 func (b *lockingBuffer) Write(p []byte) (n int, err error) {
97 b.Lock()
98 n, err = b.Buffer.Write(p)
99 b.Unlock()
100 return
101 }
102
103 func (s *IntegrationTestBuilder) AssertLogContains(text string) {
104 s.Helper()
105 s.Assert(s.logBuff.String(), qt.Contains, text)
106 }
107
108 func (s *IntegrationTestBuilder) AssertLogMatches(expression string) {
109 s.Helper()
110 re := regexp.MustCompile(expression)
111 s.Assert(re.MatchString(s.logBuff.String()), qt.IsTrue, qt.Commentf(s.logBuff.String()))
112 }
113
114 func (s *IntegrationTestBuilder) AssertBuildCountData(count int) {
115 s.Helper()
116 s.Assert(s.H.init.data.InitCount(), qt.Equals, count)
117 }
118
119 func (s *IntegrationTestBuilder) AssertBuildCountGitInfo(count int) {
120 s.Helper()
121 s.Assert(s.H.init.gitInfo.InitCount(), qt.Equals, count)
122 }
123
124 func (s *IntegrationTestBuilder) AssertBuildCountLayouts(count int) {
125 s.Helper()
126 s.Assert(s.H.init.layouts.InitCount(), qt.Equals, count)
127 }
128
129 func (s *IntegrationTestBuilder) AssertBuildCountTranslations(count int) {
130 s.Helper()
131 s.Assert(s.H.init.translations.InitCount(), qt.Equals, count)
132 }
133
134 func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) {
135 s.Helper()
136 content := strings.TrimSpace(s.FileContent(filename))
137 for _, m := range matches {
138 lines := strings.Split(m, "\n")
139 for _, match := range lines {
140 match = strings.TrimSpace(match)
141 if match == "" || strings.HasPrefix(match, "#") {
142 continue
143 }
144 s.Assert(content, qt.Contains, match, qt.Commentf(m))
145 }
146 }
147 }
148
149 func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) {
150 s.Helper()
151 content := s.FileContent(filename)
152 for _, m := range matches {
153 s.Assert(content, qt.Contains, m, qt.Commentf(m))
154 }
155 }
156
157 func (s *IntegrationTestBuilder) AssertDestinationExists(filename string, b bool) {
158 checker := qt.IsTrue
159 if !b {
160 checker = qt.IsFalse
161 }
162 s.Assert(s.destinationExists(filepath.Clean(filename)), checker)
163 }
164
165 func (s *IntegrationTestBuilder) destinationExists(filename string) bool {
166 b, err := helpers.Exists(filename, s.fs.PublishDir)
167 if err != nil {
168 panic(err)
169 }
170 return b
171 }
172
173 func (s *IntegrationTestBuilder) AssertIsFileError(err error) herrors.FileError {
174 s.Assert(err, qt.ErrorAs, new(herrors.FileError))
175 return herrors.UnwrapFileError(err)
176 }
177
178 func (s *IntegrationTestBuilder) AssertRenderCountContent(count int) {
179 s.Helper()
180 s.Assert(s.counters.contentRenderCounter, qt.Equals, uint64(count))
181 }
182
183 func (s *IntegrationTestBuilder) AssertRenderCountPage(count int) {
184 s.Helper()
185 s.Assert(s.counters.pageRenderCounter, qt.Equals, uint64(count))
186 }
187
188 func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder {
189 s.Helper()
190 _, err := s.BuildE()
191 if s.Cfg.Verbose || err != nil {
192 fmt.Println(s.logBuff.String())
193 }
194 s.Assert(err, qt.IsNil)
195 return s
196 }
197
198 func (s *IntegrationTestBuilder) BuildE() (*IntegrationTestBuilder, error) {
199 s.Helper()
200 if err := s.initBuilder(); err != nil {
201 return s, err
202 }
203
204 err := s.build(BuildCfg{})
205 return s, err
206 }
207
208 type IntegrationTestDebugConfig struct {
209 Out io.Writer
210
211 PrintDestinationFs bool
212 PrintPagemap bool
213
214 PrefixDestinationFs string
215 PrefixPagemap string
216 }
217
218 func (s *IntegrationTestBuilder) EditFileReplace(filename string, replacementFunc func(s string) string) *IntegrationTestBuilder {
219 absFilename := s.absFilename(filename)
220 b, err := afero.ReadFile(s.fs.Source, absFilename)
221 s.Assert(err, qt.IsNil)
222 s.changedFiles = append(s.changedFiles, absFilename)
223 oldContent := string(b)
224 s.writeSource(absFilename, replacementFunc(oldContent))
225 return s
226 }
227
228 func (s *IntegrationTestBuilder) EditFiles(filenameContent ...string) *IntegrationTestBuilder {
229 for i := 0; i < len(filenameContent); i += 2 {
230 filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
231 absFilename := s.absFilename(filename)
232 s.changedFiles = append(s.changedFiles, absFilename)
233 s.writeSource(absFilename, content)
234 }
235 return s
236 }
237
238 func (s *IntegrationTestBuilder) AddFiles(filenameContent ...string) *IntegrationTestBuilder {
239 for i := 0; i < len(filenameContent); i += 2 {
240 filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
241 absFilename := s.absFilename(filename)
242 s.createdFiles = append(s.createdFiles, absFilename)
243 s.writeSource(absFilename, content)
244 }
245 return s
246 }
247
248 func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTestBuilder {
249 for _, filename := range filenames {
250 absFilename := s.absFilename(filename)
251 s.removedFiles = append(s.removedFiles, absFilename)
252 s.Assert(s.fs.Source.Remove(absFilename), qt.IsNil)
253
254 }
255
256 return s
257 }
258
259 func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBuilder {
260 absOldFilename := s.absFilename(old)
261 absNewFilename := s.absFilename(new)
262 s.renamedFiles = append(s.renamedFiles, absOldFilename)
263 s.createdFiles = append(s.createdFiles, absNewFilename)
264 s.Assert(s.fs.Source.Rename(absOldFilename, absNewFilename), qt.IsNil)
265 return s
266 }
267
268 func (s *IntegrationTestBuilder) FileContent(filename string) string {
269 s.Helper()
270 return s.readWorkingDir(s, s.fs, filepath.FromSlash(filename))
271 }
272
273 func (s *IntegrationTestBuilder) initBuilder() error {
274 var initErr error
275 s.builderInit.Do(func() {
276 var afs afero.Fs
277 if s.Cfg.NeedsOsFS {
278 afs = afero.NewOsFs()
279 } else {
280 afs = afero.NewMemMapFs()
281 }
282
283 if s.Cfg.LogLevel == 0 {
284 s.Cfg.LogLevel = jww.LevelWarn
285 }
286
287 logger := loggers.NewBasicLoggerForWriter(s.Cfg.LogLevel, &s.logBuff)
288
289 isBinaryRe := regexp.MustCompile(`^(.*)(\.png|\.jpg)$`)
290
291 for _, f := range s.data.Files {
292 filename := filepath.Join(s.Cfg.WorkingDir, f.Name)
293 data := bytes.TrimSuffix(f.Data, []byte("\n"))
294 if isBinaryRe.MatchString(filename) {
295 var err error
296 data, err = base64.StdEncoding.DecodeString(string(data))
297 s.Assert(err, qt.IsNil)
298
299 }
300 s.Assert(afs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil)
301 s.Assert(afero.WriteFile(afs, filename, data, 0666), qt.IsNil)
302 }
303
304 cfg, _, err := LoadConfig(
305 ConfigSourceDescriptor{
306 WorkingDir: s.Cfg.WorkingDir,
307 Fs: afs,
308 Logger: logger,
309 Environ: []string{},
310 Filename: "config.toml",
311 },
312 func(cfg config.Provider) error {
313 return nil
314 },
315 )
316
317 s.Assert(err, qt.IsNil)
318
319 cfg.Set("workingDir", s.Cfg.WorkingDir)
320
321 fs := hugofs.NewFrom(afs, cfg)
322
323 s.Assert(err, qt.IsNil)
324
325 depsCfg := deps.DepsCfg{Cfg: cfg, Fs: fs, Running: s.Cfg.Running, Logger: logger}
326 sites, err := NewHugoSites(depsCfg)
327 if err != nil {
328 initErr = err
329 return
330 }
331
332 s.H = sites
333 s.fs = fs
334
335 if s.Cfg.NeedsNpmInstall {
336 wd, _ := os.Getwd()
337 s.Assert(os.Chdir(s.Cfg.WorkingDir), qt.IsNil)
338 s.C.Cleanup(func() { os.Chdir(wd) })
339 sc := security.DefaultConfig
340 sc.Exec.Allow = security.NewWhitelist("npm")
341 ex := hexec.New(sc)
342 command, err := ex.New("npm", "install")
343 s.Assert(err, qt.IsNil)
344 s.Assert(command.Run(), qt.IsNil)
345
346 }
347 })
348
349 return initErr
350 }
351
352 func (s *IntegrationTestBuilder) absFilename(filename string) string {
353 filename = filepath.FromSlash(filename)
354 if filepath.IsAbs(filename) {
355 return filename
356 }
357 if s.Cfg.WorkingDir != "" && !strings.HasPrefix(filename, s.Cfg.WorkingDir) {
358 filename = filepath.Join(s.Cfg.WorkingDir, filename)
359 }
360 return filename
361 }
362
363 func (s *IntegrationTestBuilder) build(cfg BuildCfg) error {
364 s.Helper()
365 defer func() {
366 s.changedFiles = nil
367 s.createdFiles = nil
368 s.removedFiles = nil
369 s.renamedFiles = nil
370 }()
371
372 changeEvents := s.changeEvents()
373 s.logBuff.Reset()
374 s.counters = &testCounters{}
375 cfg.testCounters = s.counters
376
377 if s.buildCount > 0 && (len(changeEvents) == 0) {
378 return nil
379 }
380
381 s.buildCount++
382
383 err := s.H.Build(cfg, changeEvents...)
384 if err != nil {
385 return err
386 }
387 logErrorCount := s.H.NumLogErrors()
388 if logErrorCount > 0 {
389 return fmt.Errorf("logged %d error(s): %s", logErrorCount, s.logBuff.String())
390 }
391
392 return nil
393 }
394
395 func (s *IntegrationTestBuilder) changeEvents() []fsnotify.Event {
396 var events []fsnotify.Event
397 for _, v := range s.removedFiles {
398 events = append(events, fsnotify.Event{
399 Name: v,
400 Op: fsnotify.Remove,
401 })
402 }
403 for _, v := range s.renamedFiles {
404 events = append(events, fsnotify.Event{
405 Name: v,
406 Op: fsnotify.Rename,
407 })
408 }
409 for _, v := range s.changedFiles {
410 events = append(events, fsnotify.Event{
411 Name: v,
412 Op: fsnotify.Write,
413 })
414 }
415 for _, v := range s.createdFiles {
416 events = append(events, fsnotify.Event{
417 Name: v,
418 Op: fsnotify.Create,
419 })
420 }
421
422 return events
423 }
424
425 func (s *IntegrationTestBuilder) readWorkingDir(t testing.TB, fs *hugofs.Fs, filename string) string {
426 t.Helper()
427 return s.readFileFromFs(t, fs.WorkingDirReadOnly, filename)
428 }
429
430 func (s *IntegrationTestBuilder) readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
431 t.Helper()
432 filename = filepath.Clean(filename)
433 b, err := afero.ReadFile(fs, filename)
434 if err != nil {
435 // Print some debug info
436 hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator)
437 start := 0
438 if hadSlash {
439 start = 1
440 }
441 end := start + 1
442
443 parts := strings.Split(filename, helpers.FilePathSeparator)
444 if parts[start] == "work" {
445 end++
446 }
447
448 s.Assert(err, qt.IsNil)
449
450 }
451 return string(b)
452 }
453
454 func (s *IntegrationTestBuilder) writeSource(filename, content string) {
455 s.Helper()
456 s.writeToFs(s.fs.Source, filename, content)
457 }
458
459 func (s *IntegrationTestBuilder) writeToFs(fs afero.Fs, filename, content string) {
460 s.Helper()
461 if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
462 s.Fatalf("Failed to write file: %s", err)
463 }
464 }
465
466 type IntegrationTestConfig struct {
467 T testing.TB
468
469 // The files to use on txtar format, see
470 // https://pkg.go.dev/golang.org/x/exp/cmd/txtar
471 TxtarString string
472
473 // Whether to simulate server mode.
474 Running bool
475
476 // Will print the log buffer after the build
477 Verbose bool
478
479 LogLevel jww.Threshold
480
481 // Whether it needs the real file system (e.g. for js.Build tests).
482 NeedsOsFS bool
483
484 // Do not remove the temp dir after the test.
485 PrintAndKeepTempDir bool
486
487 // Whether to run npm install before Build.
488 NeedsNpmInstall bool
489
490 WorkingDir string
491 }