Rollup merge of rust-lang#25137 - carols10cents:fix-token-declaration…

…, r=alexcrichton

I know this is the most trivial thing since it's *just* the model lexer, not the real lexer, but [it is simpler to read](rust-lang#15883 (comment)) and it'd be great if it was up to date but [it's been rotting](rust-lang#22379) and this is a tiny bit of that.

Thanks!!!!!!

src/grammar/RustLexer.g4

@@ -8,14 +8,14 @@ lexer grammar RustLexer;
 
 
 tokens {
-    EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT,
+    EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUS,
     MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
     BINOPEQ, AT, DOT, DOTDOT, DOTDOTDOT, COMMA, SEMI, COLON,
     MOD_SEP, RARROW, FAT_ARROW, LPAREN, RPAREN, LBRACKET, RBRACKET,
-    LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
+    LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR, LIT_BYTE,
     LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
-    LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
-    COMMENT, SHEBANG
+    LIT_BINARY_RAW, QUESTION, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
+    COMMENT, SHEBANG, UTF8_BOM
 }
 
 import xidstart , xidcontinue;

src/grammar/verify.rs

@@ -111,7 +111,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
             "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0), None),
             "QUESTION"          => token::Question,
             "SHEBANG"           => token::Shebang(Name(0)),
-            _                   => continue,
+            _                   => panic!("Bad token str `{}`", val),
         };
 
         res.insert(num.to_string(), tok);