hugo_sites_build.go (11289B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "bytes"
18 "context"
19 "encoding/json"
20 "fmt"
21 "os"
22 "path/filepath"
23 "runtime/trace"
24 "strings"
25
26 "github.com/gohugoio/hugo/publisher"
27
28 "github.com/gohugoio/hugo/hugofs"
29
30 "github.com/gohugoio/hugo/common/para"
31 "github.com/gohugoio/hugo/config"
32 "github.com/gohugoio/hugo/resources/postpub"
33
34 "github.com/spf13/afero"
35
36 "github.com/gohugoio/hugo/output"
37
38 "errors"
39
40 "github.com/fsnotify/fsnotify"
41 "github.com/gohugoio/hugo/helpers"
42 )
43
44 // Build builds all sites. If filesystem events are provided,
45 // this is considered to be a potential partial rebuild.
46 func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
47 ctx, task := trace.NewTask(context.Background(), "Build")
48 defer task.End()
49
50 if !config.NoBuildLock {
51 unlock, err := h.BaseFs.LockBuild()
52 if err != nil {
53 return fmt.Errorf("failed to acquire a build lock: %w", err)
54 }
55 defer unlock()
56 }
57
58 errCollector := h.StartErrorCollector()
59 errs := make(chan error)
60
61 go func(from, to chan error) {
62 var errors []error
63 i := 0
64 for e := range from {
65 i++
66 if i > 50 {
67 break
68 }
69 errors = append(errors, e)
70 }
71 to <- h.pickOneAndLogTheRest(errors)
72
73 close(to)
74 }(errCollector, errs)
75
76 if h.Metrics != nil {
77 h.Metrics.Reset()
78 }
79
80 h.testCounters = config.testCounters
81
82 // Need a pointer as this may be modified.
83 conf := &config
84
85 if conf.whatChanged == nil {
86 // Assume everything has changed
87 conf.whatChanged = &whatChanged{source: true}
88 }
89
90 var prepareErr error
91
92 if !config.PartialReRender {
93 prepare := func() error {
94 init := func(conf *BuildCfg) error {
95 for _, s := range h.Sites {
96 s.Deps.BuildStartListeners.Notify()
97 }
98
99 if len(events) > 0 {
100 // Rebuild
101 if err := h.initRebuild(conf); err != nil {
102 return fmt.Errorf("initRebuild: %w", err)
103 }
104 } else {
105 if err := h.initSites(conf); err != nil {
106 return fmt.Errorf("initSites: %w", err)
107 }
108 }
109
110 return nil
111 }
112
113 var err error
114
115 f := func() {
116 err = h.process(conf, init, events...)
117 }
118 trace.WithRegion(ctx, "process", f)
119 if err != nil {
120 return fmt.Errorf("process: %w", err)
121 }
122
123 f = func() {
124 err = h.assemble(conf)
125 }
126 trace.WithRegion(ctx, "assemble", f)
127 if err != nil {
128 return err
129 }
130
131 return nil
132 }
133
134 f := func() {
135 prepareErr = prepare()
136 }
137 trace.WithRegion(ctx, "prepare", f)
138 if prepareErr != nil {
139 h.SendError(prepareErr)
140 }
141
142 }
143
144 if prepareErr == nil {
145 var err error
146 f := func() {
147 err = h.render(conf)
148 }
149 trace.WithRegion(ctx, "render", f)
150 if err != nil {
151 h.SendError(err)
152 }
153
154 if err = h.postProcess(); err != nil {
155 h.SendError(err)
156 }
157 }
158
159 if h.Metrics != nil {
160 var b bytes.Buffer
161 h.Metrics.WriteMetrics(&b)
162
163 h.Log.Printf("\nTemplate Metrics:\n\n")
164 h.Log.Println(b.String())
165 }
166
167 select {
168 // Make sure the channel always gets something.
169 case errCollector <- nil:
170 default:
171 }
172 close(errCollector)
173
174 err := <-errs
175 if err != nil {
176 return err
177 }
178
179 if err := h.fatalErrorHandler.getErr(); err != nil {
180 return err
181 }
182
183 errorCount := h.Log.LogCounters().ErrorCounter.Count()
184 if errorCount > 0 {
185 return fmt.Errorf("logged %d error(s)", errorCount)
186 }
187
188 return nil
189 }
190
191 // Build lifecycle methods below.
192 // The order listed matches the order of execution.
193
194 func (h *HugoSites) initSites(config *BuildCfg) error {
195 h.reset(config)
196
197 if config.NewConfig != nil {
198 if err := h.createSitesFromConfig(config.NewConfig); err != nil {
199 return err
200 }
201 }
202
203 return nil
204 }
205
206 func (h *HugoSites) initRebuild(config *BuildCfg) error {
207 if config.NewConfig != nil {
208 return errors.New("rebuild does not support 'NewConfig'")
209 }
210
211 if config.ResetState {
212 return errors.New("rebuild does not support 'ResetState'")
213 }
214
215 if !h.running {
216 return errors.New("rebuild called when not in watch mode")
217 }
218
219 for _, s := range h.Sites {
220 s.resetBuildState(config.whatChanged.source)
221 }
222
223 h.reset(config)
224 h.resetLogs()
225 helpers.InitLoggers()
226
227 return nil
228 }
229
230 func (h *HugoSites) process(config *BuildCfg, init func(config *BuildCfg) error, events ...fsnotify.Event) error {
231 // We should probably refactor the Site and pull up most of the logic from there to here,
232 // but that seems like a daunting task.
233 // So for now, if there are more than one site (language),
234 // we pre-process the first one, then configure all the sites based on that.
235
236 firstSite := h.Sites[0]
237
238 if len(events) > 0 {
239 // This is a rebuild
240 return firstSite.processPartial(config, init, events)
241 }
242
243 return firstSite.process(*config)
244 }
245
246 func (h *HugoSites) assemble(bcfg *BuildCfg) error {
247 if len(h.Sites) > 1 {
248 // The first is initialized during process; initialize the rest
249 for _, site := range h.Sites[1:] {
250 if err := site.initializeSiteInfo(); err != nil {
251 return err
252 }
253 }
254 }
255
256 if !bcfg.whatChanged.source {
257 return nil
258 }
259
260 if err := h.getContentMaps().AssemblePages(); err != nil {
261 return err
262 }
263
264 if err := h.createPageCollections(); err != nil {
265 return err
266 }
267
268 return nil
269 }
270
271 func (h *HugoSites) render(config *BuildCfg) error {
272 if _, err := h.init.layouts.Do(); err != nil {
273 return err
274 }
275
276 siteRenderContext := &siteRenderContext{cfg: config, multihost: h.multihost}
277
278 if !config.PartialReRender {
279 h.renderFormats = output.Formats{}
280 h.withSite(func(s *Site) error {
281 s.initRenderFormats()
282 return nil
283 })
284
285 for _, s := range h.Sites {
286 h.renderFormats = append(h.renderFormats, s.renderFormats...)
287 }
288 }
289
290 i := 0
291 for _, s := range h.Sites {
292 h.currentSite = s
293 for siteOutIdx, renderFormat := range s.renderFormats {
294 siteRenderContext.outIdx = siteOutIdx
295 siteRenderContext.sitesOutIdx = i
296 i++
297
298 select {
299 case <-h.Done():
300 return nil
301 default:
302 for _, s2 := range h.Sites {
303 // We render site by site, but since the content is lazily rendered
304 // and a site can "borrow" content from other sites, every site
305 // needs this set.
306 s2.rc = &siteRenderingContext{Format: renderFormat}
307
308 if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil {
309 return err
310 }
311 }
312
313 if !config.SkipRender {
314 if config.PartialReRender {
315 if err := s.renderPages(siteRenderContext); err != nil {
316 return err
317 }
318 } else {
319 if err := s.render(siteRenderContext); err != nil {
320 return err
321 }
322 }
323 }
324 }
325
326 }
327 }
328
329 if !config.SkipRender {
330 if err := h.renderCrossSitesSitemap(); err != nil {
331 return err
332 }
333 if err := h.renderCrossSitesRobotsTXT(); err != nil {
334 return err
335 }
336 }
337
338 return nil
339 }
340
341 func (h *HugoSites) postProcess() error {
342 // Make sure to write any build stats to disk first so it's available
343 // to the post processors.
344 if err := h.writeBuildStats(); err != nil {
345 return err
346 }
347
348 // This will only be set when js.Build have been triggered with
349 // imports that resolves to the project or a module.
350 // Write a jsconfig.json file to the project's /asset directory
351 // to help JS intellisense in VS Code etc.
352 if !h.ResourceSpec.BuildConfig.NoJSConfigInAssets && h.BaseFs.Assets.Dirs != nil {
353 fi, err := h.BaseFs.Assets.Fs.Stat("")
354 if err != nil {
355 h.Log.Warnf("Failed to resolve jsconfig.json dir: %s", err)
356 } else {
357 m := fi.(hugofs.FileMetaInfo).Meta()
358 assetsDir := m.SourceRoot
359 if strings.HasPrefix(assetsDir, h.ResourceSpec.WorkingDir) {
360 if jsConfig := h.ResourceSpec.JSConfigBuilder.Build(assetsDir); jsConfig != nil {
361
362 b, err := json.MarshalIndent(jsConfig, "", " ")
363 if err != nil {
364 h.Log.Warnf("Failed to create jsconfig.json: %s", err)
365 } else {
366 filename := filepath.Join(assetsDir, "jsconfig.json")
367 if h.running {
368 h.skipRebuildForFilenamesMu.Lock()
369 h.skipRebuildForFilenames[filename] = true
370 h.skipRebuildForFilenamesMu.Unlock()
371 }
372 // Make sure it's written to the OS fs as this is used by
373 // editors.
374 if err := afero.WriteFile(hugofs.Os, filename, b, 0666); err != nil {
375 h.Log.Warnf("Failed to write jsconfig.json: %s", err)
376 }
377 }
378 }
379 }
380
381 }
382 }
383
384 var toPostProcess []postpub.PostPublishedResource
385 for _, r := range h.ResourceSpec.PostProcessResources {
386 toPostProcess = append(toPostProcess, r)
387 }
388
389 if len(toPostProcess) == 0 {
390 // Nothing more to do.
391 return nil
392 }
393
394 workers := para.New(config.GetNumWorkerMultiplier())
395 g, _ := workers.Start(context.Background())
396
397 handleFile := func(filename string) error {
398 content, err := afero.ReadFile(h.BaseFs.PublishFs, filename)
399 if err != nil {
400 return err
401 }
402
403 k := 0
404 changed := false
405
406 for {
407 l := bytes.Index(content[k:], []byte(postpub.PostProcessPrefix))
408 if l == -1 {
409 break
410 }
411 m := bytes.Index(content[k+l:], []byte(postpub.PostProcessSuffix)) + len(postpub.PostProcessSuffix)
412
413 low, high := k+l, k+l+m
414
415 field := content[low:high]
416
417 forward := l + m
418
419 for i, r := range toPostProcess {
420 if r == nil {
421 panic(fmt.Sprintf("resource %d to post process is nil", i+1))
422 }
423 v, ok := r.GetFieldString(string(field))
424 if ok {
425 content = append(content[:low], append([]byte(v), content[high:]...)...)
426 changed = true
427 forward = len(v)
428 break
429 }
430 }
431
432 k += forward
433 }
434
435 if changed {
436 return afero.WriteFile(h.BaseFs.PublishFs, filename, content, 0666)
437 }
438
439 return nil
440 }
441
442 _ = afero.Walk(h.BaseFs.PublishFs, "", func(path string, info os.FileInfo, err error) error {
443 if info == nil || info.IsDir() {
444 return nil
445 }
446
447 if !strings.HasSuffix(path, "html") {
448 return nil
449 }
450
451 g.Run(func() error {
452 return handleFile(path)
453 })
454
455 return nil
456 })
457
458 // Prepare for a new build.
459 for _, s := range h.Sites {
460 s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource)
461 }
462
463 return g.Wait()
464 }
465
466 type publishStats struct {
467 CSSClasses string `json:"cssClasses"`
468 }
469
470 func (h *HugoSites) writeBuildStats() error {
471 if !h.ResourceSpec.BuildConfig.WriteStats {
472 return nil
473 }
474
475 htmlElements := &publisher.HTMLElements{}
476 for _, s := range h.Sites {
477 stats := s.publisher.PublishStats()
478 htmlElements.Merge(stats.HTMLElements)
479 }
480
481 htmlElements.Sort()
482
483 stats := publisher.PublishStats{
484 HTMLElements: *htmlElements,
485 }
486
487 js, err := json.MarshalIndent(stats, "", " ")
488 if err != nil {
489 return err
490 }
491
492 filename := filepath.Join(h.WorkingDir, "hugo_stats.json")
493
494 // Make sure it's always written to the OS fs.
495 if err := afero.WriteFile(hugofs.Os, filename, js, 0666); err != nil {
496 return err
497 }
498
499 // Write to the destination as well if it's a in-memory fs.
500 if !hugofs.IsOsFs(h.Fs.Source) {
501 if err := afero.WriteFile(h.Fs.WorkingDirWritable, filename, js, 0666); err != nil {
502 return err
503 }
504 }
505
506 return nil
507 }