...

Source file src/text/template/parse/lex_test.go

Documentation: text/template/parse

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package parse
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  )
    11  
    12  // Make the types prettyprint.
    13  var itemName = map[itemType]string{
    14  	itemError:        "error",
    15  	itemBool:         "bool",
    16  	itemChar:         "char",
    17  	itemCharConstant: "charconst",
    18  	itemComment:      "comment",
    19  	itemComplex:      "complex",
    20  	itemDeclare:      ":=",
    21  	itemEOF:          "EOF",
    22  	itemField:        "field",
    23  	itemIdentifier:   "identifier",
    24  	itemLeftDelim:    "left delim",
    25  	itemLeftParen:    "(",
    26  	itemNumber:       "number",
    27  	itemPipe:         "pipe",
    28  	itemRawString:    "raw string",
    29  	itemRightDelim:   "right delim",
    30  	itemRightParen:   ")",
    31  	itemSpace:        "space",
    32  	itemString:       "string",
    33  	itemVariable:     "variable",
    34  
    35  	// keywords
    36  	itemDot:      ".",
    37  	itemBlock:    "block",
    38  	itemBreak:    "break",
    39  	itemContinue: "continue",
    40  	itemDefine:   "define",
    41  	itemElse:     "else",
    42  	itemIf:       "if",
    43  	itemEnd:      "end",
    44  	itemNil:      "nil",
    45  	itemRange:    "range",
    46  	itemTemplate: "template",
    47  	itemWith:     "with",
    48  }
    49  
    50  func (i itemType) String() string {
    51  	s := itemName[i]
    52  	if s == "" {
    53  		return fmt.Sprintf("item%d", int(i))
    54  	}
    55  	return s
    56  }
    57  
    58  type lexTest struct {
    59  	name  string
    60  	input string
    61  	items []item
    62  }
    63  
    64  func mkItem(typ itemType, text string) item {
    65  	return item{
    66  		typ: typ,
    67  		val: text,
    68  	}
    69  }
    70  
    71  var (
    72  	tDot        = mkItem(itemDot, ".")
    73  	tBlock      = mkItem(itemBlock, "block")
    74  	tEOF        = mkItem(itemEOF, "")
    75  	tFor        = mkItem(itemIdentifier, "for")
    76  	tLeft       = mkItem(itemLeftDelim, "{{")
    77  	tLpar       = mkItem(itemLeftParen, "(")
    78  	tPipe       = mkItem(itemPipe, "|")
    79  	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
    80  	tRange      = mkItem(itemRange, "range")
    81  	tRight      = mkItem(itemRightDelim, "}}")
    82  	tRpar       = mkItem(itemRightParen, ")")
    83  	tSpace      = mkItem(itemSpace, " ")
    84  	raw         = "`" + `abc\n\t\" ` + "`"
    85  	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
    86  	tRawQuote   = mkItem(itemRawString, raw)
    87  	tRawQuoteNL = mkItem(itemRawString, rawNL)
    88  )
    89  
    90  var lexTests = []lexTest{
    91  	{"empty", "", []item{tEOF}},
    92  	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
    93  	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
    94  	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
    95  		mkItem(itemText, "hello-"),
    96  		mkItem(itemComment, "/* this is a comment */"),
    97  		mkItem(itemText, "-world"),
    98  		tEOF,
    99  	}},
   100  	{"punctuation", "{{,@% }}", []item{
   101  		tLeft,
   102  		mkItem(itemChar, ","),
   103  		mkItem(itemChar, "@"),
   104  		mkItem(itemChar, "%"),
   105  		tSpace,
   106  		tRight,
   107  		tEOF,
   108  	}},
   109  	{"parens", "{{((3))}}", []item{
   110  		tLeft,
   111  		tLpar,
   112  		tLpar,
   113  		mkItem(itemNumber, "3"),
   114  		tRpar,
   115  		tRpar,
   116  		tRight,
   117  		tEOF,
   118  	}},
   119  	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
   120  	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
   121  	{"block", `{{block "foo" .}}`, []item{
   122  		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
   123  	}},
   124  	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
   125  	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
   126  	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
   127  	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
   128  		tLeft,
   129  		mkItem(itemNumber, "1"),
   130  		tSpace,
   131  		mkItem(itemNumber, "02"),
   132  		tSpace,
   133  		mkItem(itemNumber, "0x14"),
   134  		tSpace,
   135  		mkItem(itemNumber, "0X14"),
   136  		tSpace,
   137  		mkItem(itemNumber, "-7.2i"),
   138  		tSpace,
   139  		mkItem(itemNumber, "1e3"),
   140  		tSpace,
   141  		mkItem(itemNumber, "1E3"),
   142  		tSpace,
   143  		mkItem(itemNumber, "+1.2e-4"),
   144  		tSpace,
   145  		mkItem(itemNumber, "4.2i"),
   146  		tSpace,
   147  		mkItem(itemComplex, "1+2i"),
   148  		tSpace,
   149  		mkItem(itemNumber, "1_2"),
   150  		tSpace,
   151  		mkItem(itemNumber, "0x1.e_fp4"),
   152  		tSpace,
   153  		mkItem(itemNumber, "0X1.E_FP4"),
   154  		tRight,
   155  		tEOF,
   156  	}},
   157  	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
   158  		tLeft,
   159  		mkItem(itemCharConstant, `'a'`),
   160  		tSpace,
   161  		mkItem(itemCharConstant, `'\n'`),
   162  		tSpace,
   163  		mkItem(itemCharConstant, `'\''`),
   164  		tSpace,
   165  		mkItem(itemCharConstant, `'\\'`),
   166  		tSpace,
   167  		mkItem(itemCharConstant, `'\u00FF'`),
   168  		tSpace,
   169  		mkItem(itemCharConstant, `'\xFF'`),
   170  		tSpace,
   171  		mkItem(itemCharConstant, `'本'`),
   172  		tRight,
   173  		tEOF,
   174  	}},
   175  	{"bools", "{{true false}}", []item{
   176  		tLeft,
   177  		mkItem(itemBool, "true"),
   178  		tSpace,
   179  		mkItem(itemBool, "false"),
   180  		tRight,
   181  		tEOF,
   182  	}},
   183  	{"dot", "{{.}}", []item{
   184  		tLeft,
   185  		tDot,
   186  		tRight,
   187  		tEOF,
   188  	}},
   189  	{"nil", "{{nil}}", []item{
   190  		tLeft,
   191  		mkItem(itemNil, "nil"),
   192  		tRight,
   193  		tEOF,
   194  	}},
   195  	{"dots", "{{.x . .2 .x.y.z}}", []item{
   196  		tLeft,
   197  		mkItem(itemField, ".x"),
   198  		tSpace,
   199  		tDot,
   200  		tSpace,
   201  		mkItem(itemNumber, ".2"),
   202  		tSpace,
   203  		mkItem(itemField, ".x"),
   204  		mkItem(itemField, ".y"),
   205  		mkItem(itemField, ".z"),
   206  		tRight,
   207  		tEOF,
   208  	}},
   209  	{"keywords", "{{range if else end with}}", []item{
   210  		tLeft,
   211  		mkItem(itemRange, "range"),
   212  		tSpace,
   213  		mkItem(itemIf, "if"),
   214  		tSpace,
   215  		mkItem(itemElse, "else"),
   216  		tSpace,
   217  		mkItem(itemEnd, "end"),
   218  		tSpace,
   219  		mkItem(itemWith, "with"),
   220  		tRight,
   221  		tEOF,
   222  	}},
   223  	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
   224  		tLeft,
   225  		mkItem(itemVariable, "$c"),
   226  		tSpace,
   227  		mkItem(itemDeclare, ":="),
   228  		tSpace,
   229  		mkItem(itemIdentifier, "printf"),
   230  		tSpace,
   231  		mkItem(itemVariable, "$"),
   232  		tSpace,
   233  		mkItem(itemVariable, "$hello"),
   234  		tSpace,
   235  		mkItem(itemVariable, "$23"),
   236  		tSpace,
   237  		mkItem(itemVariable, "$"),
   238  		tSpace,
   239  		mkItem(itemVariable, "$var"),
   240  		mkItem(itemField, ".Field"),
   241  		tSpace,
   242  		mkItem(itemField, ".Method"),
   243  		tRight,
   244  		tEOF,
   245  	}},
   246  	{"variable invocation", "{{$x 23}}", []item{
   247  		tLeft,
   248  		mkItem(itemVariable, "$x"),
   249  		tSpace,
   250  		mkItem(itemNumber, "23"),
   251  		tRight,
   252  		tEOF,
   253  	}},
   254  	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
   255  		mkItem(itemText, "intro "),
   256  		tLeft,
   257  		mkItem(itemIdentifier, "echo"),
   258  		tSpace,
   259  		mkItem(itemIdentifier, "hi"),
   260  		tSpace,
   261  		mkItem(itemNumber, "1.2"),
   262  		tSpace,
   263  		tPipe,
   264  		mkItem(itemIdentifier, "noargs"),
   265  		tPipe,
   266  		mkItem(itemIdentifier, "args"),
   267  		tSpace,
   268  		mkItem(itemNumber, "1"),
   269  		tSpace,
   270  		mkItem(itemString, `"hi"`),
   271  		tRight,
   272  		mkItem(itemText, " outro"),
   273  		tEOF,
   274  	}},
   275  	{"declaration", "{{$v := 3}}", []item{
   276  		tLeft,
   277  		mkItem(itemVariable, "$v"),
   278  		tSpace,
   279  		mkItem(itemDeclare, ":="),
   280  		tSpace,
   281  		mkItem(itemNumber, "3"),
   282  		tRight,
   283  		tEOF,
   284  	}},
   285  	{"2 declarations", "{{$v , $w := 3}}", []item{
   286  		tLeft,
   287  		mkItem(itemVariable, "$v"),
   288  		tSpace,
   289  		mkItem(itemChar, ","),
   290  		tSpace,
   291  		mkItem(itemVariable, "$w"),
   292  		tSpace,
   293  		mkItem(itemDeclare, ":="),
   294  		tSpace,
   295  		mkItem(itemNumber, "3"),
   296  		tRight,
   297  		tEOF,
   298  	}},
   299  	{"field of parenthesized expression", "{{(.X).Y}}", []item{
   300  		tLeft,
   301  		tLpar,
   302  		mkItem(itemField, ".X"),
   303  		tRpar,
   304  		mkItem(itemField, ".Y"),
   305  		tRight,
   306  		tEOF,
   307  	}},
   308  	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
   309  		mkItem(itemText, "hello-"),
   310  		tLeft,
   311  		mkItem(itemNumber, "3"),
   312  		tRight,
   313  		mkItem(itemText, "-world"),
   314  		tEOF,
   315  	}},
   316  	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
   317  		mkItem(itemText, "hello-"),
   318  		mkItem(itemComment, "/* hello */"),
   319  		mkItem(itemText, "-world"),
   320  		tEOF,
   321  	}},
   322  	// errors
   323  	{"badchar", "#{{\x01}}", []item{
   324  		mkItem(itemText, "#"),
   325  		tLeft,
   326  		mkItem(itemError, "unrecognized character in action: U+0001"),
   327  	}},
   328  	{"unclosed action", "{{", []item{
   329  		tLeft,
   330  		mkItem(itemError, "unclosed action"),
   331  	}},
   332  	{"EOF in action", "{{range", []item{
   333  		tLeft,
   334  		tRange,
   335  		mkItem(itemError, "unclosed action"),
   336  	}},
   337  	{"unclosed quote", "{{\"\n\"}}", []item{
   338  		tLeft,
   339  		mkItem(itemError, "unterminated quoted string"),
   340  	}},
   341  	{"unclosed raw quote", "{{`xx}}", []item{
   342  		tLeft,
   343  		mkItem(itemError, "unterminated raw quoted string"),
   344  	}},
   345  	{"unclosed char constant", "{{'\n}}", []item{
   346  		tLeft,
   347  		mkItem(itemError, "unterminated character constant"),
   348  	}},
   349  	{"bad number", "{{3k}}", []item{
   350  		tLeft,
   351  		mkItem(itemError, `bad number syntax: "3k"`),
   352  	}},
   353  	{"unclosed paren", "{{(3}}", []item{
   354  		tLeft,
   355  		tLpar,
   356  		mkItem(itemNumber, "3"),
   357  		mkItem(itemError, `unclosed left paren`),
   358  	}},
   359  	{"extra right paren", "{{3)}}", []item{
   360  		tLeft,
   361  		mkItem(itemNumber, "3"),
   362  		mkItem(itemError, "unexpected right paren"),
   363  	}},
   364  
   365  	// Fixed bugs
   366  	// Many elements in an action blew the lookahead until
   367  	// we made lexInsideAction not loop.
   368  	{"long pipeline deadlock", "{{|||||}}", []item{
   369  		tLeft,
   370  		tPipe,
   371  		tPipe,
   372  		tPipe,
   373  		tPipe,
   374  		tPipe,
   375  		tRight,
   376  		tEOF,
   377  	}},
   378  	{"text with bad comment", "hello-{{/*/}}-world", []item{
   379  		mkItem(itemText, "hello-"),
   380  		mkItem(itemError, `unclosed comment`),
   381  	}},
   382  	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
   383  		mkItem(itemText, "hello-"),
   384  		mkItem(itemError, `comment ends before closing delimiter`),
   385  	}},
   386  	// This one is an error that we can't catch because it breaks templates with
   387  	// minimized JavaScript. Should have fixed it before Go 1.1.
   388  	{"unmatched right delimiter", "hello-{.}}-world", []item{
   389  		mkItem(itemText, "hello-{.}}-world"),
   390  		tEOF,
   391  	}},
   392  }
   393  
   394  // collect gathers the emitted items into a slice.
   395  func collect(t *lexTest, left, right string) (items []item) {
   396  	l := lex(t.name, t.input, left, right)
   397  	l.options = lexOptions{
   398  		emitComment: true,
   399  		breakOK:     true,
   400  		continueOK:  true,
   401  	}
   402  	for {
   403  		item := l.nextItem()
   404  		items = append(items, item)
   405  		if item.typ == itemEOF || item.typ == itemError {
   406  			break
   407  		}
   408  	}
   409  	return
   410  }
   411  
   412  func equal(i1, i2 []item, checkPos bool) bool {
   413  	if len(i1) != len(i2) {
   414  		return false
   415  	}
   416  	for k := range i1 {
   417  		if i1[k].typ != i2[k].typ {
   418  			return false
   419  		}
   420  		if i1[k].val != i2[k].val {
   421  			return false
   422  		}
   423  		if checkPos && i1[k].pos != i2[k].pos {
   424  			return false
   425  		}
   426  		if checkPos && i1[k].line != i2[k].line {
   427  			return false
   428  		}
   429  	}
   430  	return true
   431  }
   432  
   433  func TestLex(t *testing.T) {
   434  	for _, test := range lexTests {
   435  		items := collect(&test, "", "")
   436  		if !equal(items, test.items, false) {
   437  			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
   438  			return // TODO
   439  		}
   440  		t.Log(test.name, "OK")
   441  	}
   442  }
   443  
   444  // Some easy cases from above, but with delimiters $$ and @@
   445  var lexDelimTests = []lexTest{
   446  	{"punctuation", "$$,@%{{}}@@", []item{
   447  		tLeftDelim,
   448  		mkItem(itemChar, ","),
   449  		mkItem(itemChar, "@"),
   450  		mkItem(itemChar, "%"),
   451  		mkItem(itemChar, "{"),
   452  		mkItem(itemChar, "{"),
   453  		mkItem(itemChar, "}"),
   454  		mkItem(itemChar, "}"),
   455  		tRightDelim,
   456  		tEOF,
   457  	}},
   458  	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
   459  	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
   460  	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
   461  	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
   462  }
   463  
   464  var (
   465  	tLeftDelim  = mkItem(itemLeftDelim, "$$")
   466  	tRightDelim = mkItem(itemRightDelim, "@@")
   467  )
   468  
   469  func TestDelims(t *testing.T) {
   470  	for _, test := range lexDelimTests {
   471  		items := collect(&test, "$$", "@@")
   472  		if !equal(items, test.items, false) {
   473  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   474  		}
   475  	}
   476  }
   477  
   478  func TestDelimsAlphaNumeric(t *testing.T) {
   479  	test := lexTest{"right delimiter with alphanumeric start", "{{hub .host hub}}", []item{
   480  		mkItem(itemLeftDelim, "{{hub"),
   481  		mkItem(itemSpace, " "),
   482  		mkItem(itemField, ".host"),
   483  		mkItem(itemSpace, " "),
   484  		mkItem(itemRightDelim, "hub}}"),
   485  		tEOF,
   486  	}}
   487  	items := collect(&test, "{{hub", "hub}}")
   488  
   489  	if !equal(items, test.items, false) {
   490  		t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   491  	}
   492  }
   493  
   494  func TestDelimsAndMarkers(t *testing.T) {
   495  	test := lexTest{"delims that look like markers", "{{- .x -}} {{- - .x - -}}", []item{
   496  		mkItem(itemLeftDelim, "{{- "),
   497  		mkItem(itemField, ".x"),
   498  		mkItem(itemRightDelim, " -}}"),
   499  		mkItem(itemLeftDelim, "{{- "),
   500  		mkItem(itemField, ".x"),
   501  		mkItem(itemRightDelim, " -}}"),
   502  		tEOF,
   503  	}}
   504  	items := collect(&test, "{{- ", " -}}")
   505  
   506  	if !equal(items, test.items, false) {
   507  		t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   508  	}
   509  }
   510  
   511  var lexPosTests = []lexTest{
   512  	{"empty", "", []item{{itemEOF, 0, "", 1}}},
   513  	{"punctuation", "{{,@%#}}", []item{
   514  		{itemLeftDelim, 0, "{{", 1},
   515  		{itemChar, 2, ",", 1},
   516  		{itemChar, 3, "@", 1},
   517  		{itemChar, 4, "%", 1},
   518  		{itemChar, 5, "#", 1},
   519  		{itemRightDelim, 6, "}}", 1},
   520  		{itemEOF, 8, "", 1},
   521  	}},
   522  	{"sample", "0123{{hello}}xyz", []item{
   523  		{itemText, 0, "0123", 1},
   524  		{itemLeftDelim, 4, "{{", 1},
   525  		{itemIdentifier, 6, "hello", 1},
   526  		{itemRightDelim, 11, "}}", 1},
   527  		{itemText, 13, "xyz", 1},
   528  		{itemEOF, 16, "", 1},
   529  	}},
   530  	{"trimafter", "{{x -}}\n{{y}}", []item{
   531  		{itemLeftDelim, 0, "{{", 1},
   532  		{itemIdentifier, 2, "x", 1},
   533  		{itemRightDelim, 5, "}}", 1},
   534  		{itemLeftDelim, 8, "{{", 2},
   535  		{itemIdentifier, 10, "y", 2},
   536  		{itemRightDelim, 11, "}}", 2},
   537  		{itemEOF, 13, "", 2},
   538  	}},
   539  	{"trimbefore", "{{x}}\n{{- y}}", []item{
   540  		{itemLeftDelim, 0, "{{", 1},
   541  		{itemIdentifier, 2, "x", 1},
   542  		{itemRightDelim, 3, "}}", 1},
   543  		{itemLeftDelim, 6, "{{", 2},
   544  		{itemIdentifier, 10, "y", 2},
   545  		{itemRightDelim, 11, "}}", 2},
   546  		{itemEOF, 13, "", 2},
   547  	}},
   548  }
   549  
   550  // The other tests don't check position, to make the test cases easier to construct.
   551  // This one does.
   552  func TestPos(t *testing.T) {
   553  	for _, test := range lexPosTests {
   554  		items := collect(&test, "", "")
   555  		if !equal(items, test.items, true) {
   556  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   557  			if len(items) == len(test.items) {
   558  				// Detailed print; avoid item.String() to expose the position value.
   559  				for i := range items {
   560  					if !equal(items[i:i+1], test.items[i:i+1], true) {
   561  						i1 := items[i]
   562  						i2 := test.items[i]
   563  						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
   564  							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
   565  					}
   566  				}
   567  			}
   568  		}
   569  	}
   570  }
   571  
   572  // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
   573  // We expect an error, so the tree set and funcs list are explicitly nil.
   574  func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
   575  	defer t.recover(&err)
   576  	t.ParseName = t.Name
   577  	t.startParse(nil, lex, map[string]*Tree{})
   578  	t.parse()
   579  	t.add()
   580  	t.stopParse()
   581  	return t, nil
   582  }
   583  

View as plain text