filecache_test.go (8739B)
1 // Copyright 2018 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package filecache
15
16 import (
17 "errors"
18 "fmt"
19 "io"
20 "io/ioutil"
21 "os"
22 "path/filepath"
23 "strings"
24 "sync"
25 "testing"
26 "time"
27
28 "github.com/gobwas/glob"
29
30 "github.com/gohugoio/hugo/langs"
31 "github.com/gohugoio/hugo/modules"
32
33 "github.com/gohugoio/hugo/common/hugio"
34 "github.com/gohugoio/hugo/config"
35 "github.com/gohugoio/hugo/helpers"
36
37 "github.com/gohugoio/hugo/hugofs"
38 "github.com/spf13/afero"
39
40 qt "github.com/frankban/quicktest"
41 )
42
43 func TestFileCache(t *testing.T) {
44 t.Parallel()
45 c := qt.New(t)
46
47 tempWorkingDir, err := ioutil.TempDir("", "hugo_filecache_test_work")
48 c.Assert(err, qt.IsNil)
49 defer os.Remove(tempWorkingDir)
50
51 tempCacheDir, err := ioutil.TempDir("", "hugo_filecache_test_cache")
52 c.Assert(err, qt.IsNil)
53 defer os.Remove(tempCacheDir)
54
55 osfs := afero.NewOsFs()
56
57 for _, test := range []struct {
58 cacheDir string
59 workingDir string
60 }{
61 // Run with same dirs twice to make sure that works.
62 {tempCacheDir, tempWorkingDir},
63 {tempCacheDir, tempWorkingDir},
64 } {
65
66 configStr := `
67 workingDir = "WORKING_DIR"
68 resourceDir = "resources"
69 cacheDir = "CACHEDIR"
70 contentDir = "content"
71 dataDir = "data"
72 i18nDir = "i18n"
73 layoutDir = "layouts"
74 assetDir = "assets"
75 archeTypedir = "archetypes"
76
77 [caches]
78 [caches.getJSON]
79 maxAge = "10h"
80 dir = ":cacheDir/c"
81
82 `
83
84 winPathSep := "\\\\"
85
86 replacer := strings.NewReplacer("CACHEDIR", test.cacheDir, "WORKING_DIR", test.workingDir)
87
88 configStr = replacer.Replace(configStr)
89 configStr = strings.Replace(configStr, "\\", winPathSep, -1)
90
91 p := newPathsSpec(t, osfs, configStr)
92
93 caches, err := NewCaches(p)
94 c.Assert(err, qt.IsNil)
95
96 cache := caches.Get("GetJSON")
97 c.Assert(cache, qt.Not(qt.IsNil))
98 c.Assert(cache.maxAge.String(), qt.Equals, "10h0m0s")
99
100 bfs, ok := cache.Fs.(*afero.BasePathFs)
101 c.Assert(ok, qt.Equals, true)
102 filename, err := bfs.RealPath("key")
103 c.Assert(err, qt.IsNil)
104 if test.cacheDir != "" {
105 c.Assert(filename, qt.Equals, filepath.Join(test.cacheDir, "c/"+filecacheRootDirname+"/getjson/key"))
106 } else {
107 // Temp dir.
108 c.Assert(filename, qt.Matches, ".*hugo_cache.*"+filecacheRootDirname+".*key")
109 }
110
111 cache = caches.Get("Images")
112 c.Assert(cache, qt.Not(qt.IsNil))
113 c.Assert(cache.maxAge, qt.Equals, time.Duration(-1))
114 bfs, ok = cache.Fs.(*afero.BasePathFs)
115 c.Assert(ok, qt.Equals, true)
116 filename, _ = bfs.RealPath("key")
117 c.Assert(filename, qt.Equals, filepath.FromSlash("_gen/images/key"))
118
119 rf := func(s string) func() (io.ReadCloser, error) {
120 return func() (io.ReadCloser, error) {
121 return struct {
122 io.ReadSeeker
123 io.Closer
124 }{
125 strings.NewReader(s),
126 ioutil.NopCloser(nil),
127 }, nil
128 }
129 }
130
131 bf := func() ([]byte, error) {
132 return []byte("bcd"), nil
133 }
134
135 for _, ca := range []*Cache{caches.ImageCache(), caches.AssetsCache(), caches.GetJSONCache(), caches.GetCSVCache()} {
136 for i := 0; i < 2; i++ {
137 info, r, err := ca.GetOrCreate("a", rf("abc"))
138 c.Assert(err, qt.IsNil)
139 c.Assert(r, qt.Not(qt.IsNil))
140 c.Assert(info.Name, qt.Equals, "a")
141 b, _ := ioutil.ReadAll(r)
142 r.Close()
143 c.Assert(string(b), qt.Equals, "abc")
144
145 info, b, err = ca.GetOrCreateBytes("b", bf)
146 c.Assert(err, qt.IsNil)
147 c.Assert(r, qt.Not(qt.IsNil))
148 c.Assert(info.Name, qt.Equals, "b")
149 c.Assert(string(b), qt.Equals, "bcd")
150
151 _, b, err = ca.GetOrCreateBytes("a", bf)
152 c.Assert(err, qt.IsNil)
153 c.Assert(string(b), qt.Equals, "abc")
154
155 _, r, err = ca.GetOrCreate("a", rf("bcd"))
156 c.Assert(err, qt.IsNil)
157 b, _ = ioutil.ReadAll(r)
158 r.Close()
159 c.Assert(string(b), qt.Equals, "abc")
160 }
161 }
162
163 c.Assert(caches.Get("getJSON"), qt.Not(qt.IsNil))
164
165 info, w, err := caches.ImageCache().WriteCloser("mykey")
166 c.Assert(err, qt.IsNil)
167 c.Assert(info.Name, qt.Equals, "mykey")
168 io.WriteString(w, "Hugo is great!")
169 w.Close()
170 c.Assert(caches.ImageCache().getString("mykey"), qt.Equals, "Hugo is great!")
171
172 info, r, err := caches.ImageCache().Get("mykey")
173 c.Assert(err, qt.IsNil)
174 c.Assert(r, qt.Not(qt.IsNil))
175 c.Assert(info.Name, qt.Equals, "mykey")
176 b, _ := ioutil.ReadAll(r)
177 r.Close()
178 c.Assert(string(b), qt.Equals, "Hugo is great!")
179
180 info, b, err = caches.ImageCache().GetBytes("mykey")
181 c.Assert(err, qt.IsNil)
182 c.Assert(info.Name, qt.Equals, "mykey")
183 c.Assert(string(b), qt.Equals, "Hugo is great!")
184
185 }
186 }
187
188 func TestFileCacheConcurrent(t *testing.T) {
189 t.Parallel()
190
191 c := qt.New(t)
192
193 configStr := `
194 resourceDir = "myresources"
195 contentDir = "content"
196 dataDir = "data"
197 i18nDir = "i18n"
198 layoutDir = "layouts"
199 assetDir = "assets"
200 archeTypedir = "archetypes"
201
202 [caches]
203 [caches.getjson]
204 maxAge = "1s"
205 dir = "/cache/c"
206
207 `
208
209 p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
210
211 caches, err := NewCaches(p)
212 c.Assert(err, qt.IsNil)
213
214 const cacheName = "getjson"
215
216 filenameData := func(i int) (string, string) {
217 data := fmt.Sprintf("data: %d", i)
218 filename := fmt.Sprintf("file%d", i)
219 return filename, data
220 }
221
222 var wg sync.WaitGroup
223
224 for i := 0; i < 50; i++ {
225 wg.Add(1)
226 go func(i int) {
227 defer wg.Done()
228 for j := 0; j < 20; j++ {
229 ca := caches.Get(cacheName)
230 c.Assert(ca, qt.Not(qt.IsNil))
231 filename, data := filenameData(i)
232 _, r, err := ca.GetOrCreate(filename, func() (io.ReadCloser, error) {
233 return hugio.ToReadCloser(strings.NewReader(data)), nil
234 })
235 c.Assert(err, qt.IsNil)
236 b, _ := ioutil.ReadAll(r)
237 r.Close()
238 c.Assert(string(b), qt.Equals, data)
239 // Trigger some expiration.
240 time.Sleep(50 * time.Millisecond)
241 }
242 }(i)
243
244 }
245 wg.Wait()
246 }
247
248 func TestFileCacheReadOrCreateErrorInRead(t *testing.T) {
249 t.Parallel()
250 c := qt.New(t)
251
252 var result string
253
254 rf := func(failLevel int) func(info ItemInfo, r io.ReadSeeker) error {
255 return func(info ItemInfo, r io.ReadSeeker) error {
256 if failLevel > 0 {
257 if failLevel > 1 {
258 return ErrFatal
259 }
260 return errors.New("fail")
261 }
262
263 b, _ := ioutil.ReadAll(r)
264 result = string(b)
265
266 return nil
267 }
268 }
269
270 bf := func(s string) func(info ItemInfo, w io.WriteCloser) error {
271 return func(info ItemInfo, w io.WriteCloser) error {
272 defer w.Close()
273 result = s
274 _, err := w.Write([]byte(s))
275 return err
276 }
277 }
278
279 cache := NewCache(afero.NewMemMapFs(), 100*time.Hour, "")
280
281 const id = "a32"
282
283 _, err := cache.ReadOrCreate(id, rf(0), bf("v1"))
284 c.Assert(err, qt.IsNil)
285 c.Assert(result, qt.Equals, "v1")
286 _, err = cache.ReadOrCreate(id, rf(0), bf("v2"))
287 c.Assert(err, qt.IsNil)
288 c.Assert(result, qt.Equals, "v1")
289 _, err = cache.ReadOrCreate(id, rf(1), bf("v3"))
290 c.Assert(err, qt.IsNil)
291 c.Assert(result, qt.Equals, "v3")
292 _, err = cache.ReadOrCreate(id, rf(2), bf("v3"))
293 c.Assert(err, qt.Equals, ErrFatal)
294 }
295
296 func TestCleanID(t *testing.T) {
297 c := qt.New(t)
298 c.Assert(cleanID(filepath.FromSlash("/a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt"))
299 c.Assert(cleanID(filepath.FromSlash("a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt"))
300 }
301
302 func initConfig(fs afero.Fs, cfg config.Provider) error {
303 if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
304 return err
305 }
306
307 modConfig, err := modules.DecodeConfig(cfg)
308 if err != nil {
309 return err
310 }
311
312 workingDir := cfg.GetString("workingDir")
313 themesDir := cfg.GetString("themesDir")
314 if !filepath.IsAbs(themesDir) {
315 themesDir = filepath.Join(workingDir, themesDir)
316 }
317 globAll := glob.MustCompile("**", '/')
318 modulesClient := modules.NewClient(modules.ClientConfig{
319 Fs: fs,
320 WorkingDir: workingDir,
321 ThemesDir: themesDir,
322 ModuleConfig: modConfig,
323 IgnoreVendor: globAll,
324 })
325
326 moduleConfig, err := modulesClient.Collect()
327 if err != nil {
328 return err
329 }
330
331 if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {
332 return err
333 }
334
335 cfg.Set("allModules", moduleConfig.ActiveModules)
336
337 return nil
338 }
339
340 func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec {
341 c := qt.New(t)
342 cfg, err := config.FromConfigString(configStr, "toml")
343 c.Assert(err, qt.IsNil)
344 initConfig(fs, cfg)
345 config.SetBaseTestDefaults(cfg)
346 p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, nil)
347 c.Assert(err, qt.IsNil)
348 return p
349 }