commit 54750b078059734c3f2047ae3f56cac64a9f0666
parent 43b5dfabb5fb36bb4574289912c66a46ef20ffce
Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Tue, 5 Apr 2016 22:20:39 +0200
Do not create robots.txt by default
Meny people, including me, have a custom robots.txt in static.
Also remove that option from the command line; it doesn't feel
important enough.
Fixes ##2049
Diffstat:
5 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/commands/hugo.go b/commands/hugo.go
@@ -126,7 +126,7 @@ var (
buildWatch bool
canonifyURLs bool
cleanDestination bool
- disableRobotsTXT bool
+ enableRobotsTXT bool
disableRSS bool
disableSitemap bool
draft bool
@@ -216,7 +216,6 @@ func initHugoBuildCommonFlags(cmd *cobra.Command) {
cmd.Flags().BoolVarP(&future, "buildFuture", "F", false, "include content with publishdate in the future")
cmd.Flags().BoolVar(&disableRSS, "disableRSS", false, "Do not build RSS files")
cmd.Flags().BoolVar(&disableSitemap, "disableSitemap", false, "Do not build Sitemap file")
- cmd.Flags().BoolVar(&disableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")
cmd.Flags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
cmd.Flags().StringVarP(&contentDir, "contentDir", "c", "", "filesystem path to content directory")
cmd.Flags().StringVarP(&layoutDir, "layoutDir", "l", "", "filesystem path to layout directory")
@@ -364,8 +363,8 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
if flagChanged(cmdV.Flags(), "disableSitemap") {
viper.Set("DisableSitemap", disableSitemap)
}
- if flagChanged(cmdV.Flags(), "disableRobotsTXT") {
- viper.Set("DisableRobotsTXT", disableRobotsTXT)
+ if flagChanged(cmdV.Flags(), "enableRobotsTXT") {
+ viper.Set("EnableRobotsTXT", enableRobotsTXT)
}
if flagChanged(cmdV.Flags(), "pluralizeListTitles") {
viper.Set("PluralizeListTitles", pluralizeListTitles)
diff --git a/docs/content/extras/robots-txt.md b/docs/content/extras/robots-txt.md
@@ -13,13 +13,12 @@ weight: 120
Hugo can generated a customized [robots.txt](http://www.robotstxt.org/) in the
[same way as any other templates]({{< ref "templates/go-templates.md" >}}).
-By default, it generates a robots.txt, which allows everything, with the following content:
+To enable it, just set `enableRobotsTXT` option to `true` in the [configuration file]({{< ref "overview/configuration.md" >}}). By default, it generates a robots.txt, which allows everything, with the following content:
```http
User-agent: *
```
-To disable it, just set `disableRobotsTXT` option to `false` in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
Hugo will use the template `robots.txt` according to the following list in descending precedence:
diff --git a/docs/content/overview/configuration.md b/docs/content/overview/configuration.md
@@ -95,8 +95,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
disableRSS: false
# Do not build Sitemap file
disableSitemap: false
- # Do not build robots.txt file
- disableRobotsTXT: false
+ # Build robots.txt file
+ enableRobotsTXT: false
# edit new content with this editor, if provided
editor: ""
# Enable Emoji emoticons support for page content.
diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go
@@ -36,6 +36,7 @@ func TestRobotsTXTOutput(t *testing.T) {
hugofs.InitMemFs()
viper.Set("baseurl", "http://auth/bub/")
+ viper.Set("enableRobotsTXT", true)
s := &Site{
Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},
diff --git a/hugolib/site.go b/hugolib/site.go
@@ -1864,7 +1864,7 @@ func (s *Site) RenderSitemap() error {
}
func (s *Site) RenderRobotsTXT() error {
- if viper.GetBool("DisableRobotsTXT") {
+ if !viper.GetBool("EnableRobotsTXT") {
return nil
}