lex_test.go (13688B)
1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build go1.13 6 // +build go1.13 7 8 package parse 9 10 import ( 11 "fmt" 12 "testing" 13 ) 14 15 // Make the types prettyprint. 16 var itemName = map[itemType]string{ 17 itemError: "error", 18 itemBool: "bool", 19 itemChar: "char", 20 itemCharConstant: "charconst", 21 itemComment: "comment", 22 itemComplex: "complex", 23 itemDeclare: ":=", 24 itemEOF: "EOF", 25 itemField: "field", 26 itemIdentifier: "identifier", 27 itemLeftDelim: "left delim", 28 itemLeftParen: "(", 29 itemNumber: "number", 30 itemPipe: "pipe", 31 itemRawString: "raw string", 32 itemRightDelim: "right delim", 33 itemRightParen: ")", 34 itemSpace: "space", 35 itemString: "string", 36 itemVariable: "variable", 37 38 // keywords 39 itemDot: ".", 40 itemBlock: "block", 41 itemBreak: "break", 42 itemContinue: "continue", 43 itemDefine: "define", 44 itemElse: "else", 45 itemIf: "if", 46 itemEnd: "end", 47 itemNil: "nil", 48 itemRange: "range", 49 itemTemplate: "template", 50 itemWith: "with", 51 } 52 53 func (i itemType) String() string { 54 s := itemName[i] 55 if s == "" { 56 return fmt.Sprintf("item%d", int(i)) 57 } 58 return s 59 } 60 61 type lexTest struct { 62 name string 63 input string 64 items []item 65 } 66 67 func mkItem(typ itemType, text string) item { 68 return item{ 69 typ: typ, 70 val: text, 71 } 72 } 73 74 var ( 75 tDot = mkItem(itemDot, ".") 76 tBlock = mkItem(itemBlock, "block") 77 tEOF = mkItem(itemEOF, "") 78 tFor = mkItem(itemIdentifier, "for") 79 tLeft = mkItem(itemLeftDelim, "{{") 80 tLpar = mkItem(itemLeftParen, "(") 81 tPipe = mkItem(itemPipe, "|") 82 tQuote = mkItem(itemString, `"abc \n\t\" "`) 83 tRange = mkItem(itemRange, "range") 84 tRight = mkItem(itemRightDelim, "}}") 85 tRpar = mkItem(itemRightParen, ")") 86 tSpace = mkItem(itemSpace, " ") 87 raw = "`" + `abc\n\t\" ` + "`" 88 rawNL = "`now is{{\n}}the time`" // Contains newline inside raw quote. 89 tRawQuote = mkItem(itemRawString, raw) 90 tRawQuoteNL = mkItem(itemRawString, rawNL) 91 ) 92 93 var lexTests = []lexTest{ 94 {"empty", "", []item{tEOF}}, 95 {"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}}, 96 {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}}, 97 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{ 98 mkItem(itemText, "hello-"), 99 mkItem(itemComment, "/* this is a comment */"), 100 mkItem(itemText, "-world"), 101 tEOF, 102 }}, 103 {"punctuation", "{{,@% }}", []item{ 104 tLeft, 105 mkItem(itemChar, ","), 106 mkItem(itemChar, "@"), 107 mkItem(itemChar, "%"), 108 tSpace, 109 tRight, 110 tEOF, 111 }}, 112 {"parens", "{{((3))}}", []item{ 113 tLeft, 114 tLpar, 115 tLpar, 116 mkItem(itemNumber, "3"), 117 tRpar, 118 tRpar, 119 tRight, 120 tEOF, 121 }}, 122 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}}, 123 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}}, 124 {"block", `{{block "foo" .}}`, []item{ 125 tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF, 126 }}, 127 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}}, 128 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}}, 129 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}}, 130 {"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{ 131 tLeft, 132 mkItem(itemNumber, "1"), 133 tSpace, 134 mkItem(itemNumber, "02"), 135 tSpace, 136 mkItem(itemNumber, "0x14"), 137 tSpace, 138 mkItem(itemNumber, "0X14"), 139 tSpace, 140 mkItem(itemNumber, "-7.2i"), 141 tSpace, 142 mkItem(itemNumber, "1e3"), 143 tSpace, 144 mkItem(itemNumber, "1E3"), 145 tSpace, 146 mkItem(itemNumber, "+1.2e-4"), 147 tSpace, 148 mkItem(itemNumber, "4.2i"), 149 tSpace, 150 mkItem(itemComplex, "1+2i"), 151 tSpace, 152 mkItem(itemNumber, "1_2"), 153 tSpace, 154 mkItem(itemNumber, "0x1.e_fp4"), 155 tSpace, 156 mkItem(itemNumber, "0X1.E_FP4"), 157 tRight, 158 tEOF, 159 }}, 160 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{ 161 tLeft, 162 mkItem(itemCharConstant, `'a'`), 163 tSpace, 164 mkItem(itemCharConstant, `'\n'`), 165 tSpace, 166 mkItem(itemCharConstant, `'\''`), 167 tSpace, 168 mkItem(itemCharConstant, `'\\'`), 169 tSpace, 170 mkItem(itemCharConstant, `'\u00FF'`), 171 tSpace, 172 mkItem(itemCharConstant, `'\xFF'`), 173 tSpace, 174 mkItem(itemCharConstant, `'本'`), 175 tRight, 176 tEOF, 177 }}, 178 {"bools", "{{true false}}", []item{ 179 tLeft, 180 mkItem(itemBool, "true"), 181 tSpace, 182 mkItem(itemBool, "false"), 183 tRight, 184 tEOF, 185 }}, 186 {"dot", "{{.}}", []item{ 187 tLeft, 188 tDot, 189 tRight, 190 tEOF, 191 }}, 192 {"nil", "{{nil}}", []item{ 193 tLeft, 194 mkItem(itemNil, "nil"), 195 tRight, 196 tEOF, 197 }}, 198 {"dots", "{{.x . .2 .x.y.z}}", []item{ 199 tLeft, 200 mkItem(itemField, ".x"), 201 tSpace, 202 tDot, 203 tSpace, 204 mkItem(itemNumber, ".2"), 205 tSpace, 206 mkItem(itemField, ".x"), 207 mkItem(itemField, ".y"), 208 mkItem(itemField, ".z"), 209 tRight, 210 tEOF, 211 }}, 212 {"keywords", "{{range if else end with}}", []item{ 213 tLeft, 214 mkItem(itemRange, "range"), 215 tSpace, 216 mkItem(itemIf, "if"), 217 tSpace, 218 mkItem(itemElse, "else"), 219 tSpace, 220 mkItem(itemEnd, "end"), 221 tSpace, 222 mkItem(itemWith, "with"), 223 tRight, 224 tEOF, 225 }}, 226 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{ 227 tLeft, 228 mkItem(itemVariable, "$c"), 229 tSpace, 230 mkItem(itemDeclare, ":="), 231 tSpace, 232 mkItem(itemIdentifier, "printf"), 233 tSpace, 234 mkItem(itemVariable, "$"), 235 tSpace, 236 mkItem(itemVariable, "$hello"), 237 tSpace, 238 mkItem(itemVariable, "$23"), 239 tSpace, 240 mkItem(itemVariable, "$"), 241 tSpace, 242 mkItem(itemVariable, "$var"), 243 mkItem(itemField, ".Field"), 244 tSpace, 245 mkItem(itemField, ".Method"), 246 tRight, 247 tEOF, 248 }}, 249 {"variable invocation", "{{$x 23}}", []item{ 250 tLeft, 251 mkItem(itemVariable, "$x"), 252 tSpace, 253 mkItem(itemNumber, "23"), 254 tRight, 255 tEOF, 256 }}, 257 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{ 258 mkItem(itemText, "intro "), 259 tLeft, 260 mkItem(itemIdentifier, "echo"), 261 tSpace, 262 mkItem(itemIdentifier, "hi"), 263 tSpace, 264 mkItem(itemNumber, "1.2"), 265 tSpace, 266 tPipe, 267 mkItem(itemIdentifier, "noargs"), 268 tPipe, 269 mkItem(itemIdentifier, "args"), 270 tSpace, 271 mkItem(itemNumber, "1"), 272 tSpace, 273 mkItem(itemString, `"hi"`), 274 tRight, 275 mkItem(itemText, " outro"), 276 tEOF, 277 }}, 278 {"declaration", "{{$v := 3}}", []item{ 279 tLeft, 280 mkItem(itemVariable, "$v"), 281 tSpace, 282 mkItem(itemDeclare, ":="), 283 tSpace, 284 mkItem(itemNumber, "3"), 285 tRight, 286 tEOF, 287 }}, 288 {"2 declarations", "{{$v , $w := 3}}", []item{ 289 tLeft, 290 mkItem(itemVariable, "$v"), 291 tSpace, 292 mkItem(itemChar, ","), 293 tSpace, 294 mkItem(itemVariable, "$w"), 295 tSpace, 296 mkItem(itemDeclare, ":="), 297 tSpace, 298 mkItem(itemNumber, "3"), 299 tRight, 300 tEOF, 301 }}, 302 {"field of parenthesized expression", "{{(.X).Y}}", []item{ 303 tLeft, 304 tLpar, 305 mkItem(itemField, ".X"), 306 tRpar, 307 mkItem(itemField, ".Y"), 308 tRight, 309 tEOF, 310 }}, 311 {"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{ 312 mkItem(itemText, "hello-"), 313 tLeft, 314 mkItem(itemNumber, "3"), 315 tRight, 316 mkItem(itemText, "-world"), 317 tEOF, 318 }}, 319 {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{ 320 mkItem(itemText, "hello-"), 321 mkItem(itemComment, "/* hello */"), 322 mkItem(itemText, "-world"), 323 tEOF, 324 }}, 325 // errors 326 {"badchar", "#{{\x01}}", []item{ 327 mkItem(itemText, "#"), 328 tLeft, 329 mkItem(itemError, "unrecognized character in action: U+0001"), 330 }}, 331 {"unclosed action", "{{", []item{ 332 tLeft, 333 mkItem(itemError, "unclosed action"), 334 }}, 335 {"EOF in action", "{{range", []item{ 336 tLeft, 337 tRange, 338 mkItem(itemError, "unclosed action"), 339 }}, 340 {"unclosed quote", "{{\"\n\"}}", []item{ 341 tLeft, 342 mkItem(itemError, "unterminated quoted string"), 343 }}, 344 {"unclosed raw quote", "{{`xx}}", []item{ 345 tLeft, 346 mkItem(itemError, "unterminated raw quoted string"), 347 }}, 348 {"unclosed char constant", "{{'\n}}", []item{ 349 tLeft, 350 mkItem(itemError, "unterminated character constant"), 351 }}, 352 {"bad number", "{{3k}}", []item{ 353 tLeft, 354 mkItem(itemError, `bad number syntax: "3k"`), 355 }}, 356 {"unclosed paren", "{{(3}}", []item{ 357 tLeft, 358 tLpar, 359 mkItem(itemNumber, "3"), 360 mkItem(itemError, `unclosed left paren`), 361 }}, 362 {"extra right paren", "{{3)}}", []item{ 363 tLeft, 364 mkItem(itemNumber, "3"), 365 tRpar, 366 mkItem(itemError, `unexpected right paren U+0029 ')'`), 367 }}, 368 369 // Fixed bugs 370 // Many elements in an action blew the lookahead until 371 // we made lexInsideAction not loop. 372 {"long pipeline deadlock", "{{|||||}}", []item{ 373 tLeft, 374 tPipe, 375 tPipe, 376 tPipe, 377 tPipe, 378 tPipe, 379 tRight, 380 tEOF, 381 }}, 382 {"text with bad comment", "hello-{{/*/}}-world", []item{ 383 mkItem(itemText, "hello-"), 384 mkItem(itemError, `unclosed comment`), 385 }}, 386 {"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{ 387 mkItem(itemText, "hello-"), 388 mkItem(itemError, `comment ends before closing delimiter`), 389 }}, 390 // This one is an error that we can't catch because it breaks templates with 391 // minimized JavaScript. Should have fixed it before Go 1.1. 392 {"unmatched right delimiter", "hello-{.}}-world", []item{ 393 mkItem(itemText, "hello-{.}}-world"), 394 tEOF, 395 }}, 396 } 397 398 // collect gathers the emitted items into a slice. 399 func collect(t *lexTest, left, right string) (items []item) { 400 l := lex(t.name, t.input, left, right, true) 401 for { 402 item := l.nextItem() 403 items = append(items, item) 404 if item.typ == itemEOF || item.typ == itemError { 405 break 406 } 407 } 408 return 409 } 410 411 func equal(i1, i2 []item, checkPos bool) bool { 412 if len(i1) != len(i2) { 413 return false 414 } 415 for k := range i1 { 416 if i1[k].typ != i2[k].typ { 417 return false 418 } 419 if i1[k].val != i2[k].val { 420 return false 421 } 422 if checkPos && i1[k].pos != i2[k].pos { 423 return false 424 } 425 if checkPos && i1[k].line != i2[k].line { 426 return false 427 } 428 } 429 return true 430 } 431 432 func TestLex(t *testing.T) { 433 for _, test := range lexTests { 434 items := collect(&test, "", "") 435 if !equal(items, test.items, false) { 436 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items) 437 } 438 } 439 } 440 441 // Some easy cases from above, but with delimiters $$ and @@ 442 var lexDelimTests = []lexTest{ 443 {"punctuation", "$$,@%{{}}@@", []item{ 444 tLeftDelim, 445 mkItem(itemChar, ","), 446 mkItem(itemChar, "@"), 447 mkItem(itemChar, "%"), 448 mkItem(itemChar, "{"), 449 mkItem(itemChar, "{"), 450 mkItem(itemChar, "}"), 451 mkItem(itemChar, "}"), 452 tRightDelim, 453 tEOF, 454 }}, 455 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}}, 456 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}}, 457 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}}, 458 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}}, 459 } 460 461 var ( 462 tLeftDelim = mkItem(itemLeftDelim, "$$") 463 tRightDelim = mkItem(itemRightDelim, "@@") 464 ) 465 466 func TestDelims(t *testing.T) { 467 for _, test := range lexDelimTests { 468 items := collect(&test, "$$", "@@") 469 if !equal(items, test.items, false) { 470 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 471 } 472 } 473 } 474 475 var lexPosTests = []lexTest{ 476 {"empty", "", []item{{itemEOF, 0, "", 1}}}, 477 {"punctuation", "{{,@%#}}", []item{ 478 {itemLeftDelim, 0, "{{", 1}, 479 {itemChar, 2, ",", 1}, 480 {itemChar, 3, "@", 1}, 481 {itemChar, 4, "%", 1}, 482 {itemChar, 5, "#", 1}, 483 {itemRightDelim, 6, "}}", 1}, 484 {itemEOF, 8, "", 1}, 485 }}, 486 {"sample", "0123{{hello}}xyz", []item{ 487 {itemText, 0, "0123", 1}, 488 {itemLeftDelim, 4, "{{", 1}, 489 {itemIdentifier, 6, "hello", 1}, 490 {itemRightDelim, 11, "}}", 1}, 491 {itemText, 13, "xyz", 1}, 492 {itemEOF, 16, "", 1}, 493 }}, 494 {"trimafter", "{{x -}}\n{{y}}", []item{ 495 {itemLeftDelim, 0, "{{", 1}, 496 {itemIdentifier, 2, "x", 1}, 497 {itemRightDelim, 5, "}}", 1}, 498 {itemLeftDelim, 8, "{{", 2}, 499 {itemIdentifier, 10, "y", 2}, 500 {itemRightDelim, 11, "}}", 2}, 501 {itemEOF, 13, "", 2}, 502 }}, 503 {"trimbefore", "{{x}}\n{{- y}}", []item{ 504 {itemLeftDelim, 0, "{{", 1}, 505 {itemIdentifier, 2, "x", 1}, 506 {itemRightDelim, 3, "}}", 1}, 507 {itemLeftDelim, 6, "{{", 2}, 508 {itemIdentifier, 10, "y", 2}, 509 {itemRightDelim, 11, "}}", 2}, 510 {itemEOF, 13, "", 2}, 511 }}, 512 } 513 514 // The other tests don't check position, to make the test cases easier to construct. 515 // This one does. 516 func TestPos(t *testing.T) { 517 for _, test := range lexPosTests { 518 items := collect(&test, "", "") 519 if !equal(items, test.items, true) { 520 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 521 if len(items) == len(test.items) { 522 // Detailed print; avoid item.String() to expose the position value. 523 for i := range items { 524 if !equal(items[i:i+1], test.items[i:i+1], true) { 525 i1 := items[i] 526 i2 := test.items[i] 527 t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}", 528 i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line) 529 } 530 } 531 } 532 } 533 } 534 } 535 536 // Test that an error shuts down the lexing goroutine. 537 func TestShutdown(t *testing.T) { 538 // We need to duplicate template.Parse here to hold on to the lexer. 539 const text = "erroneous{{define}}{{else}}1234" 540 lexer := lex("foo", text, "{{", "}}", false) 541 _, err := New("root").parseLexer(lexer) 542 if err == nil { 543 t.Fatalf("expected error") 544 } 545 // The error should have drained the input. Therefore, the lexer should be shut down. 546 token, ok := <-lexer.items 547 if ok { 548 t.Fatalf("input was not drained; got %v", token) 549 } 550 } 551 552 // parseLexer is a local version of parse that lets us pass in the lexer instead of building it. 553 // We expect an error, so the tree set and funcs list are explicitly nil. 554 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) { 555 defer t.recover(&err) 556 t.ParseName = t.Name 557 t.startParse(nil, lex, map[string]*Tree{}) 558 t.parse() 559 t.add() 560 t.stopParse() 561 return t, nil 562 }