hugo

Fork of github.com/gohugoio/hugo with reverse pagination support

git clone git://git.shimmy1996.com/hugo.git

lex_test.go (13688B)

    1 // Copyright 2011 The Go Authors. All rights reserved.
    2 // Use of this source code is governed by a BSD-style
    3 // license that can be found in the LICENSE file.
    4 
    5 //go:build go1.13
    6 // +build go1.13
    7 
    8 package parse
    9 
   10 import (
   11 	"fmt"
   12 	"testing"
   13 )
   14 
   15 // Make the types prettyprint.
   16 var itemName = map[itemType]string{
   17 	itemError:        "error",
   18 	itemBool:         "bool",
   19 	itemChar:         "char",
   20 	itemCharConstant: "charconst",
   21 	itemComment:      "comment",
   22 	itemComplex:      "complex",
   23 	itemDeclare:      ":=",
   24 	itemEOF:          "EOF",
   25 	itemField:        "field",
   26 	itemIdentifier:   "identifier",
   27 	itemLeftDelim:    "left delim",
   28 	itemLeftParen:    "(",
   29 	itemNumber:       "number",
   30 	itemPipe:         "pipe",
   31 	itemRawString:    "raw string",
   32 	itemRightDelim:   "right delim",
   33 	itemRightParen:   ")",
   34 	itemSpace:        "space",
   35 	itemString:       "string",
   36 	itemVariable:     "variable",
   37 
   38 	// keywords
   39 	itemDot:      ".",
   40 	itemBlock:    "block",
   41 	itemBreak:    "break",
   42 	itemContinue: "continue",
   43 	itemDefine:   "define",
   44 	itemElse:     "else",
   45 	itemIf:       "if",
   46 	itemEnd:      "end",
   47 	itemNil:      "nil",
   48 	itemRange:    "range",
   49 	itemTemplate: "template",
   50 	itemWith:     "with",
   51 }
   52 
   53 func (i itemType) String() string {
   54 	s := itemName[i]
   55 	if s == "" {
   56 		return fmt.Sprintf("item%d", int(i))
   57 	}
   58 	return s
   59 }
   60 
   61 type lexTest struct {
   62 	name  string
   63 	input string
   64 	items []item
   65 }
   66 
   67 func mkItem(typ itemType, text string) item {
   68 	return item{
   69 		typ: typ,
   70 		val: text,
   71 	}
   72 }
   73 
   74 var (
   75 	tDot        = mkItem(itemDot, ".")
   76 	tBlock      = mkItem(itemBlock, "block")
   77 	tEOF        = mkItem(itemEOF, "")
   78 	tFor        = mkItem(itemIdentifier, "for")
   79 	tLeft       = mkItem(itemLeftDelim, "{{")
   80 	tLpar       = mkItem(itemLeftParen, "(")
   81 	tPipe       = mkItem(itemPipe, "|")
   82 	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
   83 	tRange      = mkItem(itemRange, "range")
   84 	tRight      = mkItem(itemRightDelim, "}}")
   85 	tRpar       = mkItem(itemRightParen, ")")
   86 	tSpace      = mkItem(itemSpace, " ")
   87 	raw         = "`" + `abc\n\t\" ` + "`"
   88 	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
   89 	tRawQuote   = mkItem(itemRawString, raw)
   90 	tRawQuoteNL = mkItem(itemRawString, rawNL)
   91 )
   92 
   93 var lexTests = []lexTest{
   94 	{"empty", "", []item{tEOF}},
   95 	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
   96 	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
   97 	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
   98 		mkItem(itemText, "hello-"),
   99 		mkItem(itemComment, "/* this is a comment */"),
  100 		mkItem(itemText, "-world"),
  101 		tEOF,
  102 	}},
  103 	{"punctuation", "{{,@% }}", []item{
  104 		tLeft,
  105 		mkItem(itemChar, ","),
  106 		mkItem(itemChar, "@"),
  107 		mkItem(itemChar, "%"),
  108 		tSpace,
  109 		tRight,
  110 		tEOF,
  111 	}},
  112 	{"parens", "{{((3))}}", []item{
  113 		tLeft,
  114 		tLpar,
  115 		tLpar,
  116 		mkItem(itemNumber, "3"),
  117 		tRpar,
  118 		tRpar,
  119 		tRight,
  120 		tEOF,
  121 	}},
  122 	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
  123 	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
  124 	{"block", `{{block "foo" .}}`, []item{
  125 		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
  126 	}},
  127 	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
  128 	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
  129 	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
  130 	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
  131 		tLeft,
  132 		mkItem(itemNumber, "1"),
  133 		tSpace,
  134 		mkItem(itemNumber, "02"),
  135 		tSpace,
  136 		mkItem(itemNumber, "0x14"),
  137 		tSpace,
  138 		mkItem(itemNumber, "0X14"),
  139 		tSpace,
  140 		mkItem(itemNumber, "-7.2i"),
  141 		tSpace,
  142 		mkItem(itemNumber, "1e3"),
  143 		tSpace,
  144 		mkItem(itemNumber, "1E3"),
  145 		tSpace,
  146 		mkItem(itemNumber, "+1.2e-4"),
  147 		tSpace,
  148 		mkItem(itemNumber, "4.2i"),
  149 		tSpace,
  150 		mkItem(itemComplex, "1+2i"),
  151 		tSpace,
  152 		mkItem(itemNumber, "1_2"),
  153 		tSpace,
  154 		mkItem(itemNumber, "0x1.e_fp4"),
  155 		tSpace,
  156 		mkItem(itemNumber, "0X1.E_FP4"),
  157 		tRight,
  158 		tEOF,
  159 	}},
  160 	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
  161 		tLeft,
  162 		mkItem(itemCharConstant, `'a'`),
  163 		tSpace,
  164 		mkItem(itemCharConstant, `'\n'`),
  165 		tSpace,
  166 		mkItem(itemCharConstant, `'\''`),
  167 		tSpace,
  168 		mkItem(itemCharConstant, `'\\'`),
  169 		tSpace,
  170 		mkItem(itemCharConstant, `'\u00FF'`),
  171 		tSpace,
  172 		mkItem(itemCharConstant, `'\xFF'`),
  173 		tSpace,
  174 		mkItem(itemCharConstant, `'本'`),
  175 		tRight,
  176 		tEOF,
  177 	}},
  178 	{"bools", "{{true false}}", []item{
  179 		tLeft,
  180 		mkItem(itemBool, "true"),
  181 		tSpace,
  182 		mkItem(itemBool, "false"),
  183 		tRight,
  184 		tEOF,
  185 	}},
  186 	{"dot", "{{.}}", []item{
  187 		tLeft,
  188 		tDot,
  189 		tRight,
  190 		tEOF,
  191 	}},
  192 	{"nil", "{{nil}}", []item{
  193 		tLeft,
  194 		mkItem(itemNil, "nil"),
  195 		tRight,
  196 		tEOF,
  197 	}},
  198 	{"dots", "{{.x . .2 .x.y.z}}", []item{
  199 		tLeft,
  200 		mkItem(itemField, ".x"),
  201 		tSpace,
  202 		tDot,
  203 		tSpace,
  204 		mkItem(itemNumber, ".2"),
  205 		tSpace,
  206 		mkItem(itemField, ".x"),
  207 		mkItem(itemField, ".y"),
  208 		mkItem(itemField, ".z"),
  209 		tRight,
  210 		tEOF,
  211 	}},
  212 	{"keywords", "{{range if else end with}}", []item{
  213 		tLeft,
  214 		mkItem(itemRange, "range"),
  215 		tSpace,
  216 		mkItem(itemIf, "if"),
  217 		tSpace,
  218 		mkItem(itemElse, "else"),
  219 		tSpace,
  220 		mkItem(itemEnd, "end"),
  221 		tSpace,
  222 		mkItem(itemWith, "with"),
  223 		tRight,
  224 		tEOF,
  225 	}},
  226 	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
  227 		tLeft,
  228 		mkItem(itemVariable, "$c"),
  229 		tSpace,
  230 		mkItem(itemDeclare, ":="),
  231 		tSpace,
  232 		mkItem(itemIdentifier, "printf"),
  233 		tSpace,
  234 		mkItem(itemVariable, "$"),
  235 		tSpace,
  236 		mkItem(itemVariable, "$hello"),
  237 		tSpace,
  238 		mkItem(itemVariable, "$23"),
  239 		tSpace,
  240 		mkItem(itemVariable, "$"),
  241 		tSpace,
  242 		mkItem(itemVariable, "$var"),
  243 		mkItem(itemField, ".Field"),
  244 		tSpace,
  245 		mkItem(itemField, ".Method"),
  246 		tRight,
  247 		tEOF,
  248 	}},
  249 	{"variable invocation", "{{$x 23}}", []item{
  250 		tLeft,
  251 		mkItem(itemVariable, "$x"),
  252 		tSpace,
  253 		mkItem(itemNumber, "23"),
  254 		tRight,
  255 		tEOF,
  256 	}},
  257 	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
  258 		mkItem(itemText, "intro "),
  259 		tLeft,
  260 		mkItem(itemIdentifier, "echo"),
  261 		tSpace,
  262 		mkItem(itemIdentifier, "hi"),
  263 		tSpace,
  264 		mkItem(itemNumber, "1.2"),
  265 		tSpace,
  266 		tPipe,
  267 		mkItem(itemIdentifier, "noargs"),
  268 		tPipe,
  269 		mkItem(itemIdentifier, "args"),
  270 		tSpace,
  271 		mkItem(itemNumber, "1"),
  272 		tSpace,
  273 		mkItem(itemString, `"hi"`),
  274 		tRight,
  275 		mkItem(itemText, " outro"),
  276 		tEOF,
  277 	}},
  278 	{"declaration", "{{$v := 3}}", []item{
  279 		tLeft,
  280 		mkItem(itemVariable, "$v"),
  281 		tSpace,
  282 		mkItem(itemDeclare, ":="),
  283 		tSpace,
  284 		mkItem(itemNumber, "3"),
  285 		tRight,
  286 		tEOF,
  287 	}},
  288 	{"2 declarations", "{{$v , $w := 3}}", []item{
  289 		tLeft,
  290 		mkItem(itemVariable, "$v"),
  291 		tSpace,
  292 		mkItem(itemChar, ","),
  293 		tSpace,
  294 		mkItem(itemVariable, "$w"),
  295 		tSpace,
  296 		mkItem(itemDeclare, ":="),
  297 		tSpace,
  298 		mkItem(itemNumber, "3"),
  299 		tRight,
  300 		tEOF,
  301 	}},
  302 	{"field of parenthesized expression", "{{(.X).Y}}", []item{
  303 		tLeft,
  304 		tLpar,
  305 		mkItem(itemField, ".X"),
  306 		tRpar,
  307 		mkItem(itemField, ".Y"),
  308 		tRight,
  309 		tEOF,
  310 	}},
  311 	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
  312 		mkItem(itemText, "hello-"),
  313 		tLeft,
  314 		mkItem(itemNumber, "3"),
  315 		tRight,
  316 		mkItem(itemText, "-world"),
  317 		tEOF,
  318 	}},
  319 	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
  320 		mkItem(itemText, "hello-"),
  321 		mkItem(itemComment, "/* hello */"),
  322 		mkItem(itemText, "-world"),
  323 		tEOF,
  324 	}},
  325 	// errors
  326 	{"badchar", "#{{\x01}}", []item{
  327 		mkItem(itemText, "#"),
  328 		tLeft,
  329 		mkItem(itemError, "unrecognized character in action: U+0001"),
  330 	}},
  331 	{"unclosed action", "{{", []item{
  332 		tLeft,
  333 		mkItem(itemError, "unclosed action"),
  334 	}},
  335 	{"EOF in action", "{{range", []item{
  336 		tLeft,
  337 		tRange,
  338 		mkItem(itemError, "unclosed action"),
  339 	}},
  340 	{"unclosed quote", "{{\"\n\"}}", []item{
  341 		tLeft,
  342 		mkItem(itemError, "unterminated quoted string"),
  343 	}},
  344 	{"unclosed raw quote", "{{`xx}}", []item{
  345 		tLeft,
  346 		mkItem(itemError, "unterminated raw quoted string"),
  347 	}},
  348 	{"unclosed char constant", "{{'\n}}", []item{
  349 		tLeft,
  350 		mkItem(itemError, "unterminated character constant"),
  351 	}},
  352 	{"bad number", "{{3k}}", []item{
  353 		tLeft,
  354 		mkItem(itemError, `bad number syntax: "3k"`),
  355 	}},
  356 	{"unclosed paren", "{{(3}}", []item{
  357 		tLeft,
  358 		tLpar,
  359 		mkItem(itemNumber, "3"),
  360 		mkItem(itemError, `unclosed left paren`),
  361 	}},
  362 	{"extra right paren", "{{3)}}", []item{
  363 		tLeft,
  364 		mkItem(itemNumber, "3"),
  365 		tRpar,
  366 		mkItem(itemError, `unexpected right paren U+0029 ')'`),
  367 	}},
  368 
  369 	// Fixed bugs
  370 	// Many elements in an action blew the lookahead until
  371 	// we made lexInsideAction not loop.
  372 	{"long pipeline deadlock", "{{|||||}}", []item{
  373 		tLeft,
  374 		tPipe,
  375 		tPipe,
  376 		tPipe,
  377 		tPipe,
  378 		tPipe,
  379 		tRight,
  380 		tEOF,
  381 	}},
  382 	{"text with bad comment", "hello-{{/*/}}-world", []item{
  383 		mkItem(itemText, "hello-"),
  384 		mkItem(itemError, `unclosed comment`),
  385 	}},
  386 	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
  387 		mkItem(itemText, "hello-"),
  388 		mkItem(itemError, `comment ends before closing delimiter`),
  389 	}},
  390 	// This one is an error that we can't catch because it breaks templates with
  391 	// minimized JavaScript. Should have fixed it before Go 1.1.
  392 	{"unmatched right delimiter", "hello-{.}}-world", []item{
  393 		mkItem(itemText, "hello-{.}}-world"),
  394 		tEOF,
  395 	}},
  396 }
  397 
  398 // collect gathers the emitted items into a slice.
  399 func collect(t *lexTest, left, right string) (items []item) {
  400 	l := lex(t.name, t.input, left, right, true)
  401 	for {
  402 		item := l.nextItem()
  403 		items = append(items, item)
  404 		if item.typ == itemEOF || item.typ == itemError {
  405 			break
  406 		}
  407 	}
  408 	return
  409 }
  410 
  411 func equal(i1, i2 []item, checkPos bool) bool {
  412 	if len(i1) != len(i2) {
  413 		return false
  414 	}
  415 	for k := range i1 {
  416 		if i1[k].typ != i2[k].typ {
  417 			return false
  418 		}
  419 		if i1[k].val != i2[k].val {
  420 			return false
  421 		}
  422 		if checkPos && i1[k].pos != i2[k].pos {
  423 			return false
  424 		}
  425 		if checkPos && i1[k].line != i2[k].line {
  426 			return false
  427 		}
  428 	}
  429 	return true
  430 }
  431 
  432 func TestLex(t *testing.T) {
  433 	for _, test := range lexTests {
  434 		items := collect(&test, "", "")
  435 		if !equal(items, test.items, false) {
  436 			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
  437 		}
  438 	}
  439 }
  440 
  441 // Some easy cases from above, but with delimiters $$ and @@
  442 var lexDelimTests = []lexTest{
  443 	{"punctuation", "$$,@%{{}}@@", []item{
  444 		tLeftDelim,
  445 		mkItem(itemChar, ","),
  446 		mkItem(itemChar, "@"),
  447 		mkItem(itemChar, "%"),
  448 		mkItem(itemChar, "{"),
  449 		mkItem(itemChar, "{"),
  450 		mkItem(itemChar, "}"),
  451 		mkItem(itemChar, "}"),
  452 		tRightDelim,
  453 		tEOF,
  454 	}},
  455 	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
  456 	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
  457 	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
  458 	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
  459 }
  460 
  461 var (
  462 	tLeftDelim  = mkItem(itemLeftDelim, "$$")
  463 	tRightDelim = mkItem(itemRightDelim, "@@")
  464 )
  465 
  466 func TestDelims(t *testing.T) {
  467 	for _, test := range lexDelimTests {
  468 		items := collect(&test, "$$", "@@")
  469 		if !equal(items, test.items, false) {
  470 			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
  471 		}
  472 	}
  473 }
  474 
  475 var lexPosTests = []lexTest{
  476 	{"empty", "", []item{{itemEOF, 0, "", 1}}},
  477 	{"punctuation", "{{,@%#}}", []item{
  478 		{itemLeftDelim, 0, "{{", 1},
  479 		{itemChar, 2, ",", 1},
  480 		{itemChar, 3, "@", 1},
  481 		{itemChar, 4, "%", 1},
  482 		{itemChar, 5, "#", 1},
  483 		{itemRightDelim, 6, "}}", 1},
  484 		{itemEOF, 8, "", 1},
  485 	}},
  486 	{"sample", "0123{{hello}}xyz", []item{
  487 		{itemText, 0, "0123", 1},
  488 		{itemLeftDelim, 4, "{{", 1},
  489 		{itemIdentifier, 6, "hello", 1},
  490 		{itemRightDelim, 11, "}}", 1},
  491 		{itemText, 13, "xyz", 1},
  492 		{itemEOF, 16, "", 1},
  493 	}},
  494 	{"trimafter", "{{x -}}\n{{y}}", []item{
  495 		{itemLeftDelim, 0, "{{", 1},
  496 		{itemIdentifier, 2, "x", 1},
  497 		{itemRightDelim, 5, "}}", 1},
  498 		{itemLeftDelim, 8, "{{", 2},
  499 		{itemIdentifier, 10, "y", 2},
  500 		{itemRightDelim, 11, "}}", 2},
  501 		{itemEOF, 13, "", 2},
  502 	}},
  503 	{"trimbefore", "{{x}}\n{{- y}}", []item{
  504 		{itemLeftDelim, 0, "{{", 1},
  505 		{itemIdentifier, 2, "x", 1},
  506 		{itemRightDelim, 3, "}}", 1},
  507 		{itemLeftDelim, 6, "{{", 2},
  508 		{itemIdentifier, 10, "y", 2},
  509 		{itemRightDelim, 11, "}}", 2},
  510 		{itemEOF, 13, "", 2},
  511 	}},
  512 }
  513 
  514 // The other tests don't check position, to make the test cases easier to construct.
  515 // This one does.
  516 func TestPos(t *testing.T) {
  517 	for _, test := range lexPosTests {
  518 		items := collect(&test, "", "")
  519 		if !equal(items, test.items, true) {
  520 			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
  521 			if len(items) == len(test.items) {
  522 				// Detailed print; avoid item.String() to expose the position value.
  523 				for i := range items {
  524 					if !equal(items[i:i+1], test.items[i:i+1], true) {
  525 						i1 := items[i]
  526 						i2 := test.items[i]
  527 						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
  528 							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
  529 					}
  530 				}
  531 			}
  532 		}
  533 	}
  534 }
  535 
  536 // Test that an error shuts down the lexing goroutine.
  537 func TestShutdown(t *testing.T) {
  538 	// We need to duplicate template.Parse here to hold on to the lexer.
  539 	const text = "erroneous{{define}}{{else}}1234"
  540 	lexer := lex("foo", text, "{{", "}}", false)
  541 	_, err := New("root").parseLexer(lexer)
  542 	if err == nil {
  543 		t.Fatalf("expected error")
  544 	}
  545 	// The error should have drained the input. Therefore, the lexer should be shut down.
  546 	token, ok := <-lexer.items
  547 	if ok {
  548 		t.Fatalf("input was not drained; got %v", token)
  549 	}
  550 }
  551 
  552 // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
  553 // We expect an error, so the tree set and funcs list are explicitly nil.
  554 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
  555 	defer t.recover(&err)
  556 	t.ParseName = t.Name
  557 	t.startParse(nil, lex, map[string]*Tree{})
  558 	t.parse()
  559 	t.add()
  560 	t.stopParse()
  561 	return t, nil
  562 }