walk.go (6899B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugofs
15
16 import (
17 "fmt"
18 "os"
19 "path/filepath"
20 "sort"
21 "strings"
22
23 "github.com/gohugoio/hugo/common/loggers"
24
25 "errors"
26
27 "github.com/spf13/afero"
28 )
29
30 type (
31 WalkFunc func(path string, info FileMetaInfo, err error) error
32 WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error)
33 )
34
35 type Walkway struct {
36 fs afero.Fs
37 root string
38 basePath string
39
40 logger loggers.Logger
41
42 // May be pre-set
43 fi FileMetaInfo
44 dirEntries []FileMetaInfo
45
46 walkFn WalkFunc
47 walked bool
48
49 // We may traverse symbolic links and bite ourself.
50 seen map[string]bool
51
52 // Optional hooks
53 hookPre WalkHook
54 hookPost WalkHook
55 }
56
57 type WalkwayConfig struct {
58 Fs afero.Fs
59 Root string
60 BasePath string
61
62 Logger loggers.Logger
63
64 // One or both of these may be pre-set.
65 Info FileMetaInfo
66 DirEntries []FileMetaInfo
67
68 WalkFn WalkFunc
69 HookPre WalkHook
70 HookPost WalkHook
71 }
72
73 func NewWalkway(cfg WalkwayConfig) *Walkway {
74 var fs afero.Fs
75 if cfg.Info != nil {
76 fs = cfg.Info.Meta().Fs
77 } else {
78 fs = cfg.Fs
79 }
80
81 basePath := cfg.BasePath
82 if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) {
83 basePath += filepathSeparator
84 }
85
86 logger := cfg.Logger
87 if logger == nil {
88 logger = loggers.NewWarningLogger()
89 }
90
91 return &Walkway{
92 fs: fs,
93 root: cfg.Root,
94 basePath: basePath,
95 fi: cfg.Info,
96 dirEntries: cfg.DirEntries,
97 walkFn: cfg.WalkFn,
98 hookPre: cfg.HookPre,
99 hookPost: cfg.HookPost,
100 logger: logger,
101 seen: make(map[string]bool),
102 }
103 }
104
105 func (w *Walkway) Walk() error {
106 if w.walked {
107 panic("this walkway is already walked")
108 }
109 w.walked = true
110
111 if w.fs == NoOpFs {
112 return nil
113 }
114
115 var fi FileMetaInfo
116 if w.fi != nil {
117 fi = w.fi
118 } else {
119 info, _, err := lstatIfPossible(w.fs, w.root)
120 if err != nil {
121 if os.IsNotExist(err) {
122 return nil
123 }
124
125 if w.checkErr(w.root, err) {
126 return nil
127 }
128 return w.walkFn(w.root, nil, fmt.Errorf("walk: %q: %w", w.root, err))
129 }
130 fi = info.(FileMetaInfo)
131 }
132
133 if !fi.IsDir() {
134 return w.walkFn(w.root, nil, errors.New("file to walk must be a directory"))
135 }
136
137 return w.walk(w.root, fi, w.dirEntries, w.walkFn)
138 }
139
140 // if the filesystem supports it, use Lstat, else use fs.Stat
141 func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) {
142 if lfs, ok := fs.(afero.Lstater); ok {
143 fi, b, err := lfs.LstatIfPossible(path)
144 return fi, b, err
145 }
146 fi, err := fs.Stat(path)
147 return fi, false, err
148 }
149
150 // checkErr returns true if the error is handled.
151 func (w *Walkway) checkErr(filename string, err error) bool {
152 if err == ErrPermissionSymlink {
153 logUnsupportedSymlink(filename, w.logger)
154 return true
155 }
156
157 if os.IsNotExist(err) {
158 // The file may be removed in process.
159 // This may be a ERROR situation, but it is not possible
160 // to determine as a general case.
161 w.logger.Warnf("File %q not found, skipping.", filename)
162 return true
163 }
164
165 return false
166 }
167
168 func logUnsupportedSymlink(filename string, logger loggers.Logger) {
169 logger.Warnf("Unsupported symlink found in %q, skipping.", filename)
170 }
171
172 // walk recursively descends path, calling walkFn.
173 // It follow symlinks if supported by the filesystem, but only the same path once.
174 func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error {
175 err := walkFn(path, info, nil)
176 if err != nil {
177 if info.IsDir() && err == filepath.SkipDir {
178 return nil
179 }
180 return err
181 }
182 if !info.IsDir() {
183 return nil
184 }
185
186 meta := info.Meta()
187 filename := meta.Filename
188
189 if dirEntries == nil {
190 f, err := w.fs.Open(path)
191 if err != nil {
192 if w.checkErr(path, err) {
193 return nil
194 }
195 return walkFn(path, info, fmt.Errorf("walk: open %q (%q): %w", path, w.root, err))
196 }
197
198 fis, err := f.Readdir(-1)
199 f.Close()
200 if err != nil {
201 if w.checkErr(filename, err) {
202 return nil
203 }
204 return walkFn(path, info, fmt.Errorf("walk: Readdir: %w", err))
205 }
206
207 dirEntries = fileInfosToFileMetaInfos(fis)
208
209 if !meta.IsOrdered {
210 sort.Slice(dirEntries, func(i, j int) bool {
211 fii := dirEntries[i]
212 fij := dirEntries[j]
213
214 fim, fjm := fii.Meta(), fij.Meta()
215
216 // Pull bundle headers to the top.
217 ficlass, fjclass := fim.Classifier, fjm.Classifier
218 if ficlass != fjclass {
219 return ficlass < fjclass
220 }
221
222 // With multiple content dirs with different languages,
223 // there can be duplicate files, and a weight will be added
224 // to the closest one.
225 fiw, fjw := fim.Weight, fjm.Weight
226 if fiw != fjw {
227
228 return fiw > fjw
229 }
230
231 // When we walk into a symlink, we keep the reference to
232 // the original name.
233 fin, fjn := fim.Name, fjm.Name
234 if fin != "" && fjn != "" {
235 return fin < fjn
236 }
237
238 return fii.Name() < fij.Name()
239 })
240 }
241 }
242
243 // First add some metadata to the dir entries
244 for _, fi := range dirEntries {
245 fim := fi.(FileMetaInfo)
246
247 meta := fim.Meta()
248
249 // Note that we use the original Name even if it's a symlink.
250 name := meta.Name
251 if name == "" {
252 name = fim.Name()
253 }
254
255 if name == "" {
256 panic(fmt.Sprintf("[%s] no name set in %v", path, meta))
257 }
258 pathn := filepath.Join(path, name)
259
260 pathMeta := pathn
261 if w.basePath != "" {
262 pathMeta = strings.TrimPrefix(pathn, w.basePath)
263 }
264
265 meta.Path = normalizeFilename(pathMeta)
266 meta.PathWalk = pathn
267
268 if fim.IsDir() && meta.IsSymlink && w.isSeen(meta.Filename) {
269 // Prevent infinite recursion
270 // Possible cyclic reference
271 meta.SkipDir = true
272 }
273 }
274
275 if w.hookPre != nil {
276 dirEntries, err = w.hookPre(info, path, dirEntries)
277 if err != nil {
278 if err == filepath.SkipDir {
279 return nil
280 }
281 return err
282 }
283 }
284
285 for _, fi := range dirEntries {
286 fim := fi.(FileMetaInfo)
287 meta := fim.Meta()
288
289 if meta.SkipDir {
290 continue
291 }
292
293 err := w.walk(meta.PathWalk, fim, nil, walkFn)
294 if err != nil {
295 if !fi.IsDir() || err != filepath.SkipDir {
296 return err
297 }
298 }
299 }
300
301 if w.hookPost != nil {
302 dirEntries, err = w.hookPost(info, path, dirEntries)
303 if err != nil {
304 if err == filepath.SkipDir {
305 return nil
306 }
307 return err
308 }
309 }
310 return nil
311 }
312
313 func (w *Walkway) isSeen(filename string) bool {
314 if filename == "" {
315 return false
316 }
317
318 if w.seen[filename] {
319 return true
320 }
321
322 w.seen[filename] = true
323 return false
324 }