commit 05a74eaec0d944a4b29445c878a431cd6ae12277
parent 33ae62108325f703f1eaeabef1e8a80950229415
Author: Robert van Gent <rvangent@google.com>
Date: Wed, 26 Feb 2020 22:26:05 -0800
deploy: Implement include/exclude filters for deploy
Fixes #6922
Diffstat:
5 files changed, 151 insertions(+), 5 deletions(-)
diff --git a/deploy/deploy.go b/deploy/deploy.go
@@ -31,6 +31,7 @@ import (
"sync"
"github.com/dustin/go-humanize"
+ "github.com/gobwas/glob"
"github.com/gohugoio/hugo/config"
"github.com/pkg/errors"
"github.com/spf13/afero"
@@ -125,7 +126,11 @@ func (d *Deployer) Deploy(ctx context.Context) error {
}
// Load local files from the source directory.
- local, err := walkLocal(d.localFs, d.matchers)
+ var include, exclude glob.Glob
+ if d.target != nil {
+ include, exclude = d.target.includeGlob, d.target.excludeGlob
+ }
+ local, err := walkLocal(d.localFs, d.matchers, include, exclude)
if err != nil {
return err
}
@@ -437,7 +442,7 @@ func (lf *localFile) MD5() []byte {
// walkLocal walks the source directory and returns a flat list of files,
// using localFile.SlashPath as the map keys.
-func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error) {
+func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (map[string]*localFile, error) {
retval := map[string]*localFile{}
err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
if err != nil {
@@ -461,8 +466,18 @@ func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error)
path = norm.NFC.String(path)
}
- // Find the first matching matcher (if any).
+ // Check include/exclude matchers.
slashpath := filepath.ToSlash(path)
+ if include != nil && !include.Match(slashpath) {
+ jww.INFO.Printf(" dropping %q due to include\n", slashpath)
+ return nil
+ }
+ if exclude != nil && exclude.Match(slashpath) {
+ jww.INFO.Printf(" dropping %q due to exclude\n", slashpath)
+ return nil
+ }
+
+ // Find the first matching matcher (if any).
var m *matcher
for _, cur := range matchers {
if cur.Matches(slashpath) {
diff --git a/deploy/deployConfig.go b/deploy/deployConfig.go
@@ -17,7 +17,9 @@ import (
"fmt"
"regexp"
+ "github.com/gobwas/glob"
"github.com/gohugoio/hugo/config"
+ hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/mitchellh/mapstructure"
)
@@ -41,6 +43,32 @@ type target struct {
// GoogleCloudCDNOrigin specifies the Google Cloud project and CDN origin to
// invalidate when deploying this target. It is specified as <project>/<origin>.
GoogleCloudCDNOrigin string
+
+ // Optional patterns of files to include/exclude for this target.
+ // Parsed using github.com/gobwas/glob.
+ Include string
+ Exclude string
+
+ // Parsed versions of Include/Exclude.
+ includeGlob glob.Glob
+ excludeGlob glob.Glob
+}
+
+func (tgt *target) parseIncludeExclude() error {
+ var err error
+ if tgt.Include != "" {
+ tgt.includeGlob, err = hglob.GetGlob(tgt.Include)
+ if err != nil {
+ return fmt.Errorf("invalid deployment.target.include %q: %v", tgt.Include, err)
+ }
+ }
+ if tgt.Exclude != "" {
+ tgt.excludeGlob, err = hglob.GetGlob(tgt.Exclude)
+ if err != nil {
+ return fmt.Errorf("invalid deployment.target.exclude %q: %v", tgt.Exclude, err)
+ }
+ }
+ return nil
}
// matcher represents configuration to be applied to files whose paths match
@@ -87,6 +115,11 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) {
if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil {
return dcfg, err
}
+ for _, tgt := range dcfg.Targets {
+ if err := tgt.parseIncludeExclude(); err != nil {
+ return dcfg, err
+ }
+ }
var err error
for _, m := range dcfg.Matchers {
m.re, err = regexp.Compile(m.Pattern)
diff --git a/deploy/deployConfig_test.go b/deploy/deployConfig_test.go
@@ -38,18 +38,21 @@ order = ["o1", "o2"]
name = "name0"
url = "url0"
cloudfrontdistributionid = "cdn0"
+include = "*.html"
# All uppercase.
[[deployment.targets]]
NAME = "name1"
URL = "url1"
CLOUDFRONTDISTRIBUTIONID = "cdn1"
+INCLUDE = "*.jpg"
# Camelcase.
[[deployment.targets]]
name = "name2"
url = "url2"
cloudFrontDistributionID = "cdn2"
+exclude = "*.png"
# All lowercase.
[[deployment.matchers]]
@@ -90,11 +93,21 @@ force = true
// Targets.
c.Assert(len(dcfg.Targets), qt.Equals, 3)
+ wantInclude := []string{"*.html", "*.jpg", ""}
+ wantExclude := []string{"", "", "*.png"}
for i := 0; i < 3; i++ {
tgt := dcfg.Targets[i]
c.Assert(tgt.Name, qt.Equals, fmt.Sprintf("name%d", i))
c.Assert(tgt.URL, qt.Equals, fmt.Sprintf("url%d", i))
c.Assert(tgt.CloudFrontDistributionID, qt.Equals, fmt.Sprintf("cdn%d", i))
+ c.Assert(tgt.Include, qt.Equals, wantInclude[i])
+ if wantInclude[i] != "" {
+ c.Assert(tgt.includeGlob, qt.Not(qt.IsNil))
+ }
+ c.Assert(tgt.Exclude, qt.Equals, wantExclude[i])
+ if wantExclude[i] != "" {
+ c.Assert(tgt.excludeGlob, qt.Not(qt.IsNil))
+ }
}
// Matchers.
diff --git a/deploy/deploy_test.go b/deploy/deploy_test.go
@@ -640,6 +640,86 @@ func TestMaxDeletes(t *testing.T) {
}
}
+// TestIncludeExclude verifies that the include/exclude options for targets work.
+func TestIncludeExclude(t *testing.T) {
+ ctx := context.Background()
+
+ tests := []struct {
+ Include string
+ Exclude string
+ Want deploySummary
+ }{
+ {
+ Want: deploySummary{NumLocal: 5, NumUploads: 5},
+ },
+ {
+ Include: "**aaa",
+ Want: deploySummary{NumLocal: 3, NumUploads: 3},
+ },
+ {
+ Include: "**bbb",
+ Want: deploySummary{NumLocal: 2, NumUploads: 2},
+ },
+ {
+ Include: "aaa",
+ Want: deploySummary{NumLocal: 1, NumUploads: 1},
+ },
+ {
+ Exclude: "**aaa",
+ Want: deploySummary{NumLocal: 2, NumUploads: 2},
+ },
+ {
+ Exclude: "**bbb",
+ Want: deploySummary{NumLocal: 3, NumUploads: 3},
+ },
+ {
+ Exclude: "aaa",
+ Want: deploySummary{NumLocal: 4, NumUploads: 4},
+ },
+ {
+ Include: "**aaa",
+ Exclude: "**nested**",
+ Want: deploySummary{NumLocal: 2, NumUploads: 2},
+ },
+ }
+ for _, test := range tests {
+ t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
+ fsTests, cleanup, err := initFsTests()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cleanup()
+ fsTest := fsTests[1] // just do file-based test
+
+ _, err = initLocalFs(ctx, fsTest.fs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tgt := &target{
+ Include: test.Include,
+ Exclude: test.Exclude,
+ }
+ if err := tgt.parseIncludeExclude(); err != nil {
+ t.Error(err)
+ }
+ deployer := &Deployer{
+ localFs: fsTest.fs,
+ maxDeletes: -1,
+ bucket: fsTest.bucket,
+ target: tgt,
+ }
+
+ // Sync remote with local.
+ if err := deployer.Deploy(ctx); err != nil {
+ t.Errorf("deploy: failed: %v", err)
+ }
+ if !cmp.Equal(deployer.summary, test.Want) {
+ t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
+ }
+ })
+ }
+}
+
// TestCompression verifies that gzip compression works correctly.
// In particular, MD5 hashes must be of the compressed content.
func TestCompression(t *testing.T) {
diff --git a/docs/content/en/hosting-and-deployment/hugo-deploy.md b/docs/content/en/hosting-and-deployment/hugo-deploy.md
@@ -82,8 +82,13 @@ name = "mydeployment"
# If you are using a CloudFront CDN, deploy will invalidate the cache as needed.
cloudFrontDistributionID = <ID>
-
-# ... add more [[deployment.targets]] sections ...
+# Optionally, you can include or exclude specific files.
+# See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax.
+# If non-empty, the pattern is matched against the local path.
+# If exclude is non-empty, and a file's path matches it, that file is dropped.
+# If include is non-empty, and a file's path does not match it, that file is dropped.
+# include = "**.html" # would only include files with ".html" suffix
+# exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix
# [[deployment.matchers]] configure behavior for files that match the Pattern.