deploy_test.go (30404B)
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 //go:build !nodeploy
15 // +build !nodeploy
16
17 package deploy
18
19 import (
20 "bytes"
21 "compress/gzip"
22 "context"
23 "crypto/md5"
24 "fmt"
25 "io"
26 "io/ioutil"
27 "os"
28 "path"
29 "path/filepath"
30 "regexp"
31 "sort"
32 "testing"
33
34 "github.com/gohugoio/hugo/media"
35 "github.com/google/go-cmp/cmp"
36 "github.com/google/go-cmp/cmp/cmpopts"
37 "github.com/spf13/afero"
38 "gocloud.dev/blob"
39 "gocloud.dev/blob/fileblob"
40 "gocloud.dev/blob/memblob"
41 )
42
43 func TestFindDiffs(t *testing.T) {
44 hash1 := []byte("hash 1")
45 hash2 := []byte("hash 2")
46 makeLocal := func(path string, size int64, hash []byte) *localFile {
47 return &localFile{NativePath: path, SlashPath: filepath.ToSlash(path), UploadSize: size, md5: hash}
48 }
49 makeRemote := func(path string, size int64, hash []byte) *blob.ListObject {
50 return &blob.ListObject{Key: path, Size: size, MD5: hash}
51 }
52
53 tests := []struct {
54 Description string
55 Local []*localFile
56 Remote []*blob.ListObject
57 Force bool
58 WantUpdates []*fileToUpload
59 WantDeletes []string
60 }{
61 {
62 Description: "empty -> no diffs",
63 },
64 {
65 Description: "local == remote -> no diffs",
66 Local: []*localFile{
67 makeLocal("aaa", 1, hash1),
68 makeLocal("bbb", 2, hash1),
69 makeLocal("ccc", 3, hash2),
70 },
71 Remote: []*blob.ListObject{
72 makeRemote("aaa", 1, hash1),
73 makeRemote("bbb", 2, hash1),
74 makeRemote("ccc", 3, hash2),
75 },
76 },
77 {
78 Description: "local w/ separators == remote -> no diffs",
79 Local: []*localFile{
80 makeLocal(filepath.Join("aaa", "aaa"), 1, hash1),
81 makeLocal(filepath.Join("bbb", "bbb"), 2, hash1),
82 makeLocal(filepath.Join("ccc", "ccc"), 3, hash2),
83 },
84 Remote: []*blob.ListObject{
85 makeRemote("aaa/aaa", 1, hash1),
86 makeRemote("bbb/bbb", 2, hash1),
87 makeRemote("ccc/ccc", 3, hash2),
88 },
89 },
90 {
91 Description: "local == remote with force flag true -> diffs",
92 Local: []*localFile{
93 makeLocal("aaa", 1, hash1),
94 makeLocal("bbb", 2, hash1),
95 makeLocal("ccc", 3, hash2),
96 },
97 Remote: []*blob.ListObject{
98 makeRemote("aaa", 1, hash1),
99 makeRemote("bbb", 2, hash1),
100 makeRemote("ccc", 3, hash2),
101 },
102 Force: true,
103 WantUpdates: []*fileToUpload{
104 {makeLocal("aaa", 1, nil), reasonForce},
105 {makeLocal("bbb", 2, nil), reasonForce},
106 {makeLocal("ccc", 3, nil), reasonForce},
107 },
108 },
109 {
110 Description: "local == remote with route.Force true -> diffs",
111 Local: []*localFile{
112 {NativePath: "aaa", SlashPath: "aaa", UploadSize: 1, matcher: &matcher{Force: true}, md5: hash1},
113 makeLocal("bbb", 2, hash1),
114 },
115 Remote: []*blob.ListObject{
116 makeRemote("aaa", 1, hash1),
117 makeRemote("bbb", 2, hash1),
118 },
119 WantUpdates: []*fileToUpload{
120 {makeLocal("aaa", 1, nil), reasonForce},
121 },
122 },
123 {
124 Description: "extra local file -> upload",
125 Local: []*localFile{
126 makeLocal("aaa", 1, hash1),
127 makeLocal("bbb", 2, hash2),
128 },
129 Remote: []*blob.ListObject{
130 makeRemote("aaa", 1, hash1),
131 },
132 WantUpdates: []*fileToUpload{
133 {makeLocal("bbb", 2, nil), reasonNotFound},
134 },
135 },
136 {
137 Description: "extra remote file -> delete",
138 Local: []*localFile{
139 makeLocal("aaa", 1, hash1),
140 },
141 Remote: []*blob.ListObject{
142 makeRemote("aaa", 1, hash1),
143 makeRemote("bbb", 2, hash2),
144 },
145 WantDeletes: []string{"bbb"},
146 },
147 {
148 Description: "diffs in size or md5 -> upload",
149 Local: []*localFile{
150 makeLocal("aaa", 1, hash1),
151 makeLocal("bbb", 2, hash1),
152 makeLocal("ccc", 1, hash2),
153 },
154 Remote: []*blob.ListObject{
155 makeRemote("aaa", 1, nil),
156 makeRemote("bbb", 1, hash1),
157 makeRemote("ccc", 1, hash1),
158 },
159 WantUpdates: []*fileToUpload{
160 {makeLocal("aaa", 1, nil), reasonMD5Missing},
161 {makeLocal("bbb", 2, nil), reasonSize},
162 {makeLocal("ccc", 1, nil), reasonMD5Differs},
163 },
164 },
165 {
166 Description: "mix of updates and deletes",
167 Local: []*localFile{
168 makeLocal("same", 1, hash1),
169 makeLocal("updated", 2, hash1),
170 makeLocal("updated2", 1, hash2),
171 makeLocal("new", 1, hash1),
172 makeLocal("new2", 2, hash2),
173 },
174 Remote: []*blob.ListObject{
175 makeRemote("same", 1, hash1),
176 makeRemote("updated", 1, hash1),
177 makeRemote("updated2", 1, hash1),
178 makeRemote("stale", 1, hash1),
179 makeRemote("stale2", 1, hash1),
180 },
181 WantUpdates: []*fileToUpload{
182 {makeLocal("new", 1, nil), reasonNotFound},
183 {makeLocal("new2", 2, nil), reasonNotFound},
184 {makeLocal("updated", 2, nil), reasonSize},
185 {makeLocal("updated2", 1, nil), reasonMD5Differs},
186 },
187 WantDeletes: []string{"stale", "stale2"},
188 },
189 }
190
191 for _, tc := range tests {
192 t.Run(tc.Description, func(t *testing.T) {
193 local := map[string]*localFile{}
194 for _, l := range tc.Local {
195 local[l.SlashPath] = l
196 }
197 remote := map[string]*blob.ListObject{}
198 for _, r := range tc.Remote {
199 remote[r.Key] = r
200 }
201 gotUpdates, gotDeletes := findDiffs(local, remote, tc.Force)
202 gotUpdates = applyOrdering(nil, gotUpdates)[0]
203 sort.Slice(gotDeletes, func(i, j int) bool { return gotDeletes[i] < gotDeletes[j] })
204 if diff := cmp.Diff(gotUpdates, tc.WantUpdates, cmpopts.IgnoreUnexported(localFile{})); diff != "" {
205 t.Errorf("updates differ:\n%s", diff)
206 }
207 if diff := cmp.Diff(gotDeletes, tc.WantDeletes); diff != "" {
208 t.Errorf("deletes differ:\n%s", diff)
209 }
210 })
211 }
212 }
213
214 func TestWalkLocal(t *testing.T) {
215 tests := map[string]struct {
216 Given []string
217 Expect []string
218 }{
219 "Empty": {
220 Given: []string{},
221 Expect: []string{},
222 },
223 "Normal": {
224 Given: []string{"file.txt", "normal_dir/file.txt"},
225 Expect: []string{"file.txt", "normal_dir/file.txt"},
226 },
227 "Hidden": {
228 Given: []string{"file.txt", ".hidden_dir/file.txt", "normal_dir/file.txt"},
229 Expect: []string{"file.txt", "normal_dir/file.txt"},
230 },
231 "Well Known": {
232 Given: []string{"file.txt", ".hidden_dir/file.txt", ".well-known/file.txt"},
233 Expect: []string{"file.txt", ".well-known/file.txt"},
234 },
235 }
236
237 for desc, tc := range tests {
238 t.Run(desc, func(t *testing.T) {
239 fs := afero.NewMemMapFs()
240 for _, name := range tc.Given {
241 dir, _ := path.Split(name)
242 if dir != "" {
243 if err := fs.MkdirAll(dir, 0755); err != nil {
244 t.Fatal(err)
245 }
246 }
247 if fd, err := fs.Create(name); err != nil {
248 t.Fatal(err)
249 } else {
250 fd.Close()
251 }
252 }
253 if got, err := walkLocal(fs, nil, nil, nil, media.DefaultTypes); err != nil {
254 t.Fatal(err)
255 } else {
256 expect := map[string]any{}
257 for _, path := range tc.Expect {
258 if _, ok := got[path]; !ok {
259 t.Errorf("expected %q in results, but was not found", path)
260 }
261 expect[path] = nil
262 }
263 for path := range got {
264 if _, ok := expect[path]; !ok {
265 t.Errorf("got %q in results unexpectedly", path)
266 }
267 }
268 }
269 })
270 }
271 }
272
273 func TestLocalFile(t *testing.T) {
274 const (
275 content = "hello world!"
276 )
277 contentBytes := []byte(content)
278 contentLen := int64(len(contentBytes))
279 contentMD5 := md5.Sum(contentBytes)
280 var buf bytes.Buffer
281 gz := gzip.NewWriter(&buf)
282 if _, err := gz.Write(contentBytes); err != nil {
283 t.Fatal(err)
284 }
285 gz.Close()
286 gzBytes := buf.Bytes()
287 gzLen := int64(len(gzBytes))
288 gzMD5 := md5.Sum(gzBytes)
289
290 tests := []struct {
291 Description string
292 Path string
293 Matcher *matcher
294 MediaTypesConfig []map[string]any
295 WantContent []byte
296 WantSize int64
297 WantMD5 []byte
298 WantContentType string // empty string is always OK, since content type detection is OS-specific
299 WantCacheControl string
300 WantContentEncoding string
301 }{
302 {
303 Description: "file with no suffix",
304 Path: "foo",
305 WantContent: contentBytes,
306 WantSize: contentLen,
307 WantMD5: contentMD5[:],
308 },
309 {
310 Description: "file with .txt suffix",
311 Path: "foo.txt",
312 WantContent: contentBytes,
313 WantSize: contentLen,
314 WantMD5: contentMD5[:],
315 },
316 {
317 Description: "CacheControl from matcher",
318 Path: "foo.txt",
319 Matcher: &matcher{CacheControl: "max-age=630720000"},
320 WantContent: contentBytes,
321 WantSize: contentLen,
322 WantMD5: contentMD5[:],
323 WantCacheControl: "max-age=630720000",
324 },
325 {
326 Description: "ContentEncoding from matcher",
327 Path: "foo.txt",
328 Matcher: &matcher{ContentEncoding: "foobar"},
329 WantContent: contentBytes,
330 WantSize: contentLen,
331 WantMD5: contentMD5[:],
332 WantContentEncoding: "foobar",
333 },
334 {
335 Description: "ContentType from matcher",
336 Path: "foo.txt",
337 Matcher: &matcher{ContentType: "foo/bar"},
338 WantContent: contentBytes,
339 WantSize: contentLen,
340 WantMD5: contentMD5[:],
341 WantContentType: "foo/bar",
342 },
343 {
344 Description: "gzipped content",
345 Path: "foo.txt",
346 Matcher: &matcher{Gzip: true},
347 WantContent: gzBytes,
348 WantSize: gzLen,
349 WantMD5: gzMD5[:],
350 WantContentEncoding: "gzip",
351 },
352 {
353 Description: "Custom MediaType",
354 Path: "foo.hugo",
355 MediaTypesConfig: []map[string]any{
356 {
357 "hugo/custom": map[string]any{
358 "suffixes": []string{"hugo"},
359 },
360 },
361 },
362 WantContent: contentBytes,
363 WantSize: contentLen,
364 WantMD5: contentMD5[:],
365 WantContentType: "hugo/custom",
366 },
367 }
368
369 for _, tc := range tests {
370 t.Run(tc.Description, func(t *testing.T) {
371 fs := new(afero.MemMapFs)
372 if err := afero.WriteFile(fs, tc.Path, []byte(content), os.ModePerm); err != nil {
373 t.Fatal(err)
374 }
375 mediaTypes := media.DefaultTypes
376 if len(tc.MediaTypesConfig) > 0 {
377 mt, err := media.DecodeTypes(tc.MediaTypesConfig...)
378 if err != nil {
379 t.Fatal(err)
380 }
381 mediaTypes = mt
382 }
383 lf, err := newLocalFile(fs, tc.Path, filepath.ToSlash(tc.Path), tc.Matcher, mediaTypes)
384 if err != nil {
385 t.Fatal(err)
386 }
387 if got := lf.UploadSize; got != tc.WantSize {
388 t.Errorf("got size %d want %d", got, tc.WantSize)
389 }
390 if got := lf.MD5(); !bytes.Equal(got, tc.WantMD5) {
391 t.Errorf("got MD5 %x want %x", got, tc.WantMD5)
392 }
393 if got := lf.CacheControl(); got != tc.WantCacheControl {
394 t.Errorf("got CacheControl %q want %q", got, tc.WantCacheControl)
395 }
396 if got := lf.ContentEncoding(); got != tc.WantContentEncoding {
397 t.Errorf("got ContentEncoding %q want %q", got, tc.WantContentEncoding)
398 }
399 if tc.WantContentType != "" {
400 if got := lf.ContentType(); got != tc.WantContentType {
401 t.Errorf("got ContentType %q want %q", got, tc.WantContentType)
402 }
403 }
404 // Verify the reader last to ensure the previous operations don't
405 // interfere with it.
406 r, err := lf.Reader()
407 if err != nil {
408 t.Fatal(err)
409 }
410 gotContent, err := ioutil.ReadAll(r)
411 if err != nil {
412 t.Fatal(err)
413 }
414 if !bytes.Equal(gotContent, tc.WantContent) {
415 t.Errorf("got content %q want %q", string(gotContent), string(tc.WantContent))
416 }
417 r.Close()
418 // Verify we can read again.
419 r, err = lf.Reader()
420 if err != nil {
421 t.Fatal(err)
422 }
423 gotContent, err = ioutil.ReadAll(r)
424 if err != nil {
425 t.Fatal(err)
426 }
427 r.Close()
428 if !bytes.Equal(gotContent, tc.WantContent) {
429 t.Errorf("got content %q want %q", string(gotContent), string(tc.WantContent))
430 }
431 })
432 }
433 }
434
435 func TestOrdering(t *testing.T) {
436 tests := []struct {
437 Description string
438 Uploads []string
439 Ordering []*regexp.Regexp
440 Want [][]string
441 }{
442 {
443 Description: "empty",
444 Want: [][]string{nil},
445 },
446 {
447 Description: "no ordering",
448 Uploads: []string{"c", "b", "a", "d"},
449 Want: [][]string{{"a", "b", "c", "d"}},
450 },
451 {
452 Description: "one ordering",
453 Uploads: []string{"db", "c", "b", "a", "da"},
454 Ordering: []*regexp.Regexp{regexp.MustCompile("^d")},
455 Want: [][]string{{"da", "db"}, {"a", "b", "c"}},
456 },
457 {
458 Description: "two orderings",
459 Uploads: []string{"db", "c", "b", "a", "da"},
460 Ordering: []*regexp.Regexp{
461 regexp.MustCompile("^d"),
462 regexp.MustCompile("^b"),
463 },
464 Want: [][]string{{"da", "db"}, {"b"}, {"a", "c"}},
465 },
466 }
467
468 for _, tc := range tests {
469 t.Run(tc.Description, func(t *testing.T) {
470 uploads := make([]*fileToUpload, len(tc.Uploads))
471 for i, u := range tc.Uploads {
472 uploads[i] = &fileToUpload{Local: &localFile{SlashPath: u}}
473 }
474 gotUploads := applyOrdering(tc.Ordering, uploads)
475 var got [][]string
476 for _, subslice := range gotUploads {
477 var gotsubslice []string
478 for _, u := range subslice {
479 gotsubslice = append(gotsubslice, u.Local.SlashPath)
480 }
481 got = append(got, gotsubslice)
482 }
483 if diff := cmp.Diff(got, tc.Want); diff != "" {
484 t.Error(diff)
485 }
486 })
487 }
488 }
489
490 type fileData struct {
491 Name string // name of the file
492 Contents string // contents of the file
493 }
494
495 // initLocalFs initializes fs with some test files.
496 func initLocalFs(ctx context.Context, fs afero.Fs) ([]*fileData, error) {
497 // The initial local filesystem.
498 local := []*fileData{
499 {"aaa", "aaa"},
500 {"bbb", "bbb"},
501 {"subdir/aaa", "subdir-aaa"},
502 {"subdir/nested/aaa", "subdir-nested-aaa"},
503 {"subdir2/bbb", "subdir2-bbb"},
504 }
505 if err := writeFiles(fs, local); err != nil {
506 return nil, err
507 }
508 return local, nil
509 }
510
511 // fsTest represents an (afero.FS, Go CDK blob.Bucket) against which end-to-end
512 // tests can be run.
513 type fsTest struct {
514 name string
515 fs afero.Fs
516 bucket *blob.Bucket
517 }
518
519 // initFsTests initializes a pair of tests for end-to-end test:
520 // 1. An in-memory afero.Fs paired with an in-memory Go CDK bucket.
521 // 2. A filesystem-based afero.Fs paired with an filesystem-based Go CDK bucket.
522 // It returns the pair of tests and a cleanup function.
523 func initFsTests() ([]*fsTest, func(), error) {
524 tmpfsdir, err := ioutil.TempDir("", "fs")
525 if err != nil {
526 return nil, nil, err
527 }
528 tmpbucketdir, err := ioutil.TempDir("", "bucket")
529 if err != nil {
530 return nil, nil, err
531 }
532
533 memfs := afero.NewMemMapFs()
534 membucket := memblob.OpenBucket(nil)
535
536 filefs := afero.NewBasePathFs(afero.NewOsFs(), tmpfsdir)
537 filebucket, err := fileblob.OpenBucket(tmpbucketdir, nil)
538 if err != nil {
539 return nil, nil, err
540 }
541
542 tests := []*fsTest{
543 {"mem", memfs, membucket},
544 {"file", filefs, filebucket},
545 }
546 cleanup := func() {
547 membucket.Close()
548 filebucket.Close()
549 os.RemoveAll(tmpfsdir)
550 os.RemoveAll(tmpbucketdir)
551 }
552 return tests, cleanup, nil
553 }
554
555 // TestEndToEndSync verifies that basic adds, updates, and deletes are working
556 // correctly.
557 func TestEndToEndSync(t *testing.T) {
558 ctx := context.Background()
559 tests, cleanup, err := initFsTests()
560 if err != nil {
561 t.Fatal(err)
562 }
563 defer cleanup()
564 for _, test := range tests {
565 t.Run(test.name, func(t *testing.T) {
566 local, err := initLocalFs(ctx, test.fs)
567 if err != nil {
568 t.Fatal(err)
569 }
570 deployer := &Deployer{
571 localFs: test.fs,
572 maxDeletes: -1,
573 bucket: test.bucket,
574 mediaTypes: media.DefaultTypes,
575 }
576
577 // Initial deployment should sync remote with local.
578 if err := deployer.Deploy(ctx); err != nil {
579 t.Errorf("initial deploy: failed: %v", err)
580 }
581 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
582 if !cmp.Equal(deployer.summary, wantSummary) {
583 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
584 }
585 if diff, err := verifyRemote(ctx, deployer.bucket, local); err != nil {
586 t.Errorf("initial deploy: failed to verify remote: %v", err)
587 } else if diff != "" {
588 t.Errorf("initial deploy: remote snapshot doesn't match expected:\n%v", diff)
589 }
590
591 // A repeat deployment shouldn't change anything.
592 if err := deployer.Deploy(ctx); err != nil {
593 t.Errorf("no-op deploy: %v", err)
594 }
595 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
596 if !cmp.Equal(deployer.summary, wantSummary) {
597 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
598 }
599
600 // Make some changes to the local filesystem:
601 // 1. Modify file [0].
602 // 2. Delete file [1].
603 // 3. Add a new file (sorted last).
604 updatefd := local[0]
605 updatefd.Contents = "new contents"
606 deletefd := local[1]
607 local = append(local[:1], local[2:]...) // removing deleted [1]
608 newfd := &fileData{"zzz", "zzz"}
609 local = append(local, newfd)
610 if err := writeFiles(test.fs, []*fileData{updatefd, newfd}); err != nil {
611 t.Fatal(err)
612 }
613 if err := test.fs.Remove(deletefd.Name); err != nil {
614 t.Fatal(err)
615 }
616
617 // A deployment should apply those 3 changes.
618 if err := deployer.Deploy(ctx); err != nil {
619 t.Errorf("deploy after changes: failed: %v", err)
620 }
621 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 2, NumDeletes: 1}
622 if !cmp.Equal(deployer.summary, wantSummary) {
623 t.Errorf("deploy after changes: got %v, want %v", deployer.summary, wantSummary)
624 }
625 if diff, err := verifyRemote(ctx, deployer.bucket, local); err != nil {
626 t.Errorf("deploy after changes: failed to verify remote: %v", err)
627 } else if diff != "" {
628 t.Errorf("deploy after changes: remote snapshot doesn't match expected:\n%v", diff)
629 }
630
631 // Again, a repeat deployment shouldn't change anything.
632 if err := deployer.Deploy(ctx); err != nil {
633 t.Errorf("no-op deploy: %v", err)
634 }
635 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
636 if !cmp.Equal(deployer.summary, wantSummary) {
637 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
638 }
639 })
640 }
641 }
642
643 // TestMaxDeletes verifies that the "maxDeletes" flag is working correctly.
644 func TestMaxDeletes(t *testing.T) {
645 ctx := context.Background()
646 tests, cleanup, err := initFsTests()
647 if err != nil {
648 t.Fatal(err)
649 }
650 defer cleanup()
651 for _, test := range tests {
652 t.Run(test.name, func(t *testing.T) {
653 local, err := initLocalFs(ctx, test.fs)
654 if err != nil {
655 t.Fatal(err)
656 }
657 deployer := &Deployer{
658 localFs: test.fs,
659 maxDeletes: -1,
660 bucket: test.bucket,
661 mediaTypes: media.DefaultTypes,
662 }
663
664 // Sync remote with local.
665 if err := deployer.Deploy(ctx); err != nil {
666 t.Errorf("initial deploy: failed: %v", err)
667 }
668 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
669 if !cmp.Equal(deployer.summary, wantSummary) {
670 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
671 }
672
673 // Delete two files, [1] and [2].
674 if err := test.fs.Remove(local[1].Name); err != nil {
675 t.Fatal(err)
676 }
677 if err := test.fs.Remove(local[2].Name); err != nil {
678 t.Fatal(err)
679 }
680
681 // A deployment with maxDeletes=0 shouldn't change anything.
682 deployer.maxDeletes = 0
683 if err := deployer.Deploy(ctx); err != nil {
684 t.Errorf("deploy failed: %v", err)
685 }
686 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
687 if !cmp.Equal(deployer.summary, wantSummary) {
688 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
689 }
690
691 // A deployment with maxDeletes=1 shouldn't change anything either.
692 deployer.maxDeletes = 1
693 if err := deployer.Deploy(ctx); err != nil {
694 t.Errorf("deploy failed: %v", err)
695 }
696 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
697 if !cmp.Equal(deployer.summary, wantSummary) {
698 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
699 }
700
701 // A deployment with maxDeletes=2 should make the changes.
702 deployer.maxDeletes = 2
703 if err := deployer.Deploy(ctx); err != nil {
704 t.Errorf("deploy failed: %v", err)
705 }
706 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2}
707 if !cmp.Equal(deployer.summary, wantSummary) {
708 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
709 }
710
711 // Delete two more files, [0] and [3].
712 if err := test.fs.Remove(local[0].Name); err != nil {
713 t.Fatal(err)
714 }
715 if err := test.fs.Remove(local[3].Name); err != nil {
716 t.Fatal(err)
717 }
718
719 // A deployment with maxDeletes=-1 should make the changes.
720 deployer.maxDeletes = -1
721 if err := deployer.Deploy(ctx); err != nil {
722 t.Errorf("deploy failed: %v", err)
723 }
724 wantSummary = deploySummary{NumLocal: 1, NumRemote: 3, NumUploads: 0, NumDeletes: 2}
725 if !cmp.Equal(deployer.summary, wantSummary) {
726 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
727 }
728 })
729 }
730 }
731
732 // TestIncludeExclude verifies that the include/exclude options for targets work.
733 func TestIncludeExclude(t *testing.T) {
734 ctx := context.Background()
735 tests := []struct {
736 Include string
737 Exclude string
738 Want deploySummary
739 }{
740 {
741 Want: deploySummary{NumLocal: 5, NumUploads: 5},
742 },
743 {
744 Include: "**aaa",
745 Want: deploySummary{NumLocal: 3, NumUploads: 3},
746 },
747 {
748 Include: "**bbb",
749 Want: deploySummary{NumLocal: 2, NumUploads: 2},
750 },
751 {
752 Include: "aaa",
753 Want: deploySummary{NumLocal: 1, NumUploads: 1},
754 },
755 {
756 Exclude: "**aaa",
757 Want: deploySummary{NumLocal: 2, NumUploads: 2},
758 },
759 {
760 Exclude: "**bbb",
761 Want: deploySummary{NumLocal: 3, NumUploads: 3},
762 },
763 {
764 Exclude: "aaa",
765 Want: deploySummary{NumLocal: 4, NumUploads: 4},
766 },
767 {
768 Include: "**aaa",
769 Exclude: "**nested**",
770 Want: deploySummary{NumLocal: 2, NumUploads: 2},
771 },
772 }
773 for _, test := range tests {
774 t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
775 fsTests, cleanup, err := initFsTests()
776 if err != nil {
777 t.Fatal(err)
778 }
779 defer cleanup()
780 fsTest := fsTests[1] // just do file-based test
781
782 _, err = initLocalFs(ctx, fsTest.fs)
783 if err != nil {
784 t.Fatal(err)
785 }
786 tgt := &target{
787 Include: test.Include,
788 Exclude: test.Exclude,
789 }
790 if err := tgt.parseIncludeExclude(); err != nil {
791 t.Error(err)
792 }
793 deployer := &Deployer{
794 localFs: fsTest.fs,
795 maxDeletes: -1,
796 bucket: fsTest.bucket,
797 target: tgt,
798 mediaTypes: media.DefaultTypes,
799 }
800
801 // Sync remote with local.
802 if err := deployer.Deploy(ctx); err != nil {
803 t.Errorf("deploy: failed: %v", err)
804 }
805 if !cmp.Equal(deployer.summary, test.Want) {
806 t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
807 }
808 })
809 }
810 }
811
812 // TestIncludeExcludeRemoteDelete verifies deleted local files that don't match include/exclude patterns
813 // are not deleted on the remote.
814 func TestIncludeExcludeRemoteDelete(t *testing.T) {
815 ctx := context.Background()
816
817 tests := []struct {
818 Include string
819 Exclude string
820 Want deploySummary
821 }{
822 {
823 Want: deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2},
824 },
825 {
826 Include: "**aaa",
827 Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
828 },
829 {
830 Include: "subdir/**",
831 Want: deploySummary{NumLocal: 1, NumRemote: 2, NumUploads: 0, NumDeletes: 1},
832 },
833 {
834 Exclude: "**bbb",
835 Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
836 },
837 {
838 Exclude: "bbb",
839 Want: deploySummary{NumLocal: 3, NumRemote: 4, NumUploads: 0, NumDeletes: 1},
840 },
841 }
842 for _, test := range tests {
843 t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
844 fsTests, cleanup, err := initFsTests()
845 if err != nil {
846 t.Fatal(err)
847 }
848 defer cleanup()
849 fsTest := fsTests[1] // just do file-based test
850
851 local, err := initLocalFs(ctx, fsTest.fs)
852 if err != nil {
853 t.Fatal(err)
854 }
855 deployer := &Deployer{
856 localFs: fsTest.fs,
857 maxDeletes: -1,
858 bucket: fsTest.bucket,
859 mediaTypes: media.DefaultTypes,
860 }
861
862 // Initial sync to get the files on the remote
863 if err := deployer.Deploy(ctx); err != nil {
864 t.Errorf("deploy: failed: %v", err)
865 }
866
867 // Delete two files, [1] and [2].
868 if err := fsTest.fs.Remove(local[1].Name); err != nil {
869 t.Fatal(err)
870 }
871 if err := fsTest.fs.Remove(local[2].Name); err != nil {
872 t.Fatal(err)
873 }
874
875 // Second sync
876 tgt := &target{
877 Include: test.Include,
878 Exclude: test.Exclude,
879 }
880 if err := tgt.parseIncludeExclude(); err != nil {
881 t.Error(err)
882 }
883 deployer.target = tgt
884 if err := deployer.Deploy(ctx); err != nil {
885 t.Errorf("deploy: failed: %v", err)
886 }
887
888 if !cmp.Equal(deployer.summary, test.Want) {
889 t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
890 }
891 })
892 }
893 }
894
895 // TestCompression verifies that gzip compression works correctly.
896 // In particular, MD5 hashes must be of the compressed content.
897 func TestCompression(t *testing.T) {
898 ctx := context.Background()
899
900 tests, cleanup, err := initFsTests()
901 if err != nil {
902 t.Fatal(err)
903 }
904 defer cleanup()
905 for _, test := range tests {
906 t.Run(test.name, func(t *testing.T) {
907 local, err := initLocalFs(ctx, test.fs)
908 if err != nil {
909 t.Fatal(err)
910 }
911 deployer := &Deployer{
912 localFs: test.fs,
913 bucket: test.bucket,
914 matchers: []*matcher{{Pattern: ".*", Gzip: true, re: regexp.MustCompile(".*")}},
915 mediaTypes: media.DefaultTypes,
916 }
917
918 // Initial deployment should sync remote with local.
919 if err := deployer.Deploy(ctx); err != nil {
920 t.Errorf("initial deploy: failed: %v", err)
921 }
922 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
923 if !cmp.Equal(deployer.summary, wantSummary) {
924 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
925 }
926
927 // A repeat deployment shouldn't change anything.
928 if err := deployer.Deploy(ctx); err != nil {
929 t.Errorf("no-op deploy: %v", err)
930 }
931 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
932 if !cmp.Equal(deployer.summary, wantSummary) {
933 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
934 }
935
936 // Make an update to the local filesystem, on [1].
937 updatefd := local[1]
938 updatefd.Contents = "new contents"
939 if err := writeFiles(test.fs, []*fileData{updatefd}); err != nil {
940 t.Fatal(err)
941 }
942
943 // A deployment should apply the changes.
944 if err := deployer.Deploy(ctx); err != nil {
945 t.Errorf("deploy after changes: failed: %v", err)
946 }
947 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 1, NumDeletes: 0}
948 if !cmp.Equal(deployer.summary, wantSummary) {
949 t.Errorf("deploy after changes: got %v, want %v", deployer.summary, wantSummary)
950 }
951 })
952 }
953 }
954
955 // TestMatching verifies that matchers match correctly, and that the Force
956 // attribute for matcher works.
957 func TestMatching(t *testing.T) {
958 ctx := context.Background()
959 tests, cleanup, err := initFsTests()
960 if err != nil {
961 t.Fatal(err)
962 }
963 defer cleanup()
964 for _, test := range tests {
965 t.Run(test.name, func(t *testing.T) {
966 _, err := initLocalFs(ctx, test.fs)
967 if err != nil {
968 t.Fatal(err)
969 }
970 deployer := &Deployer{
971 localFs: test.fs,
972 bucket: test.bucket,
973 matchers: []*matcher{{Pattern: "^subdir/aaa$", Force: true, re: regexp.MustCompile("^subdir/aaa$")}},
974 mediaTypes: media.DefaultTypes,
975 }
976
977 // Initial deployment to sync remote with local.
978 if err := deployer.Deploy(ctx); err != nil {
979 t.Errorf("initial deploy: failed: %v", err)
980 }
981 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
982 if !cmp.Equal(deployer.summary, wantSummary) {
983 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
984 }
985
986 // A repeat deployment should upload a single file, the one that matched the Force matcher.
987 // Note that matching happens based on the ToSlash form, so this matches
988 // even on Windows.
989 if err := deployer.Deploy(ctx); err != nil {
990 t.Errorf("no-op deploy with single force matcher: %v", err)
991 }
992 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 1, NumDeletes: 0}
993 if !cmp.Equal(deployer.summary, wantSummary) {
994 t.Errorf("no-op deploy with single force matcher: got %v, want %v", deployer.summary, wantSummary)
995 }
996
997 // Repeat with a matcher that should now match 3 files.
998 deployer.matchers = []*matcher{{Pattern: "aaa", Force: true, re: regexp.MustCompile("aaa")}}
999 if err := deployer.Deploy(ctx); err != nil {
1000 t.Errorf("no-op deploy with triple force matcher: %v", err)
1001 }
1002 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 3, NumDeletes: 0}
1003 if !cmp.Equal(deployer.summary, wantSummary) {
1004 t.Errorf("no-op deploy with triple force matcher: got %v, want %v", deployer.summary, wantSummary)
1005 }
1006 })
1007 }
1008 }
1009
1010 // writeFiles writes the files in fds to fd.
1011 func writeFiles(fs afero.Fs, fds []*fileData) error {
1012 for _, fd := range fds {
1013 dir := path.Dir(fd.Name)
1014 if dir != "." {
1015 err := fs.MkdirAll(dir, os.ModePerm)
1016 if err != nil {
1017 return err
1018 }
1019 }
1020 f, err := fs.Create(fd.Name)
1021 if err != nil {
1022 return err
1023 }
1024 defer f.Close()
1025 _, err = f.WriteString(fd.Contents)
1026 if err != nil {
1027 return err
1028 }
1029 }
1030 return nil
1031 }
1032
1033 // verifyRemote that the current contents of bucket matches local.
1034 // It returns an empty string if the contents matched, and a non-empty string
1035 // capturing the diff if they didn't.
1036 func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (string, error) {
1037 var cur []*fileData
1038 iter := bucket.List(nil)
1039 for {
1040 obj, err := iter.Next(ctx)
1041 if err == io.EOF {
1042 break
1043 }
1044 if err != nil {
1045 return "", err
1046 }
1047 contents, err := bucket.ReadAll(ctx, obj.Key)
1048 if err != nil {
1049 return "", err
1050 }
1051 cur = append(cur, &fileData{obj.Key, string(contents)})
1052 }
1053 if cmp.Equal(cur, local) {
1054 return "", nil
1055 }
1056 diff := "got: \n"
1057 for _, f := range cur {
1058 diff += fmt.Sprintf(" %s: %s\n", f.Name, f.Contents)
1059 }
1060 diff += "want: \n"
1061 for _, f := range local {
1062 diff += fmt.Sprintf(" %s: %s\n", f.Name, f.Contents)
1063 }
1064 return diff, nil
1065 }