Simplify token names and values before changes to allow pattern match…

…ing.
dconneely · Mar 7, 2024 · 47b5e83 · 47b5e83
1 parent 13eb971
commit 47b5e83
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 36 deletions.
diff --git a/src/main/java/com/davidconneely/looplang/parser/DefaultParser.java b/src/main/java/com/davidconneely/looplang/parser/DefaultParser.java
@@ -28,13 +28,13 @@ public Statement next() throws IOException {
             switch (token.kind()) {
                 case IDENTIFIER:
                     return nextAssign(token);
-                case KW_PRINT:
+                case PRINT:
                     lexer.pushback(token);
                     return StatementFactory.newPrint(context, lexer);
-                case KW_LOOP:
+                case LOOP:
                     lexer.pushback(token);
                     return StatementFactory.newLoop(context, lexer);
-                case KW_PROGRAM:
+                case PROGRAM:
                     lexer.pushback(token);
                     return StatementFactory.newDefinition(context, lexer);
                 case SEMICOLON:
@@ -58,11 +58,11 @@ private Statement nextAssign(final Token identifier) throws IOException {
         if (arg1.kind() == NUMBER) {
             lexer.pushback(arg1); lexer.pushback(assign); lexer.pushback(identifier);
             return StatementFactory.newAssignNumber(context, lexer);
-        } else if (arg1.kind() == KW_INPUT) {
+        } else if (arg1.kind() == INPUT) {
             lexer.pushback(arg1); lexer.pushback(assign); lexer.pushback(identifier);
             return StatementFactory.newAssignInput(context, lexer);
         } else if (arg1.kind() != IDENTIFIER) {
-            throwUnexpectedParserException(NUMBER, KW_INPUT, IDENTIFIER, "after `:=` in assignment", arg1);
+            throwUnexpectedParserException(NUMBER, INPUT, IDENTIFIER, "after `:=` in assignment", arg1);
         }
         Token arg2 = lexer.next();
         if (arg2.kind() == PLUS) {

diff --git a/src/main/java/com/davidconneely/looplang/statement/AssignInput.java b/src/main/java/com/davidconneely/looplang/statement/AssignInput.java
@@ -16,7 +16,7 @@ record AssignInput(String variable, List<Token> printTokens) implements Statemen
     static AssignInput parse(final ParserContext context, final Lexer lexer) throws IOException {
         final String variable = nextTokenWithKind(lexer, IDENTIFIER, "as lvalue variable name in input").value();
         nextTokenWithKind(lexer, ASSIGN, "after lvalue in input");
-        nextTokenWithKind(lexer, KW_INPUT, "in input").value();
+        nextTokenWithKind(lexer, INPUT, "in input");
         final List<Token> printTokens = Print.nextPrintTokens(lexer, "in input arguments");
         return new AssignInput(variable, printTokens);
     }

diff --git a/src/main/java/com/davidconneely/looplang/statement/Definition.java b/src/main/java/com/davidconneely/looplang/statement/Definition.java
@@ -18,11 +18,11 @@
 
 record Definition(String program, List<String> params, List<Statement> body) implements Statement {
     static Definition parse(final ParserContext context, final Lexer lexer) throws IOException {
-        nextTokenWithKind(lexer, KW_PROGRAM, "in definition");
+        nextTokenWithKind(lexer, PROGRAM, "in definition");
         final String program = nextTokenWithKind(lexer, IDENTIFIER, "as program in definition").value();
         final List<String> params = nextTokensAsCSVNames(lexer, "in params list in definition");
         Token token = lexer.next();
-        if (token.kind() != KW_DO) {
+        if (token.kind() != DO) {
             lexer.pushback(token); // `DO` is optional.
         }
         final List<Statement> body = parseBody(lexer, context);
@@ -48,7 +48,7 @@ static List<String> nextTokensAsCSVNames(final Lexer lexer, final String role) t
 
     static List<Statement> parseBody(final Lexer lexer, final ParserContext context) throws IOException {
         List<Statement> body = new ArrayList<>();
-        final Parser parser = ParserFactory.newParser(lexer, context, KW_END);
+        final Parser parser = ParserFactory.newParser(lexer, context, END);
         Statement statement = parser.next();
         while (statement != null) {
             body.add(statement);

diff --git a/src/main/java/com/davidconneely/looplang/statement/Loop.java b/src/main/java/com/davidconneely/looplang/statement/Loop.java
@@ -16,10 +16,10 @@
 
 record Loop(String variable, List<Statement> body) implements Statement {
     static Loop parse(final ParserContext context, final Lexer lexer) throws IOException {
-        nextTokenWithKind(lexer, KW_LOOP, "in loop");
+        nextTokenWithKind(lexer, LOOP, "in loop");
         final String variable = nextTokenWithKind(lexer, IDENTIFIER, "as count variable in loop").value();
         Token token = lexer.next();
-        if (token.kind() != KW_DO) {
+        if (token.kind() != DO) {
             lexer.pushback(token); // `DO` is optional.
         }
         final List<Statement> body = Definition.parseBody(lexer, context);

diff --git a/src/main/java/com/davidconneely/looplang/statement/Print.java b/src/main/java/com/davidconneely/looplang/statement/Print.java
@@ -16,7 +16,7 @@
 
 record Print(List<Token> printTokens) implements Statement {
     static Print parse(final ParserContext context, final Lexer lexer) throws IOException {
-        nextTokenWithKind(lexer, KW_PRINT, "in print");
+        nextTokenWithKind(lexer, PRINT, "in print");
         List<Token> printTokens = nextPrintTokens(lexer, "in print arguments");
         return new Print(printTokens);
     }
@@ -56,9 +56,10 @@ static String printTokensToText(final List<Token> printTokens, final Interpreter
                 sb.append(' ');
             }
             sb.append(switch (token.kind()) {
+                case STRING -> token.value();
                 case NUMBER -> Integer.toString(token.valueInt());
-                case IDENTIFIER -> context.getVariable(token.value()).stream().mapToObj(Integer::toString).findFirst().orElse("undefined");
-                default -> token.value();
+                case IDENTIFIER -> context.getVariable(token.value()).stream().mapToObj(Integer::toString).findFirst().orElse("(undefined)");
+                default -> "(unexpected " + token.kind().name() + ")";
             });
             wasLastTokenString = isThisTokenString;
         }
@@ -75,7 +76,7 @@ static String printTokensToString(final List<Token> printTokens) {
             case STRING -> Token.escaped(token.value());
             case NUMBER -> Integer.toString(token.valueInt());
             case IDENTIFIER -> token.value().toLowerCase(Locale.ROOT);
-            default -> token.value();
+            default -> "(unexpected " + token.kind().name() + ")";
         }).collect(Collectors.joining(", "));
     }
 }
diff --git a/src/main/java/com/davidconneely/looplang/token/SimpleToken.java b/src/main/java/com/davidconneely/looplang/token/SimpleToken.java
@@ -2,10 +2,12 @@
 
 import com.davidconneely.looplang.lexer.LexerException;
 
+import static com.davidconneely.looplang.token.Token.Kind.NUMBER;
+
 record SimpleToken(Kind kind, String value) implements Token {
     @Override
     public int valueInt() {
-        if (kind() != Kind.NUMBER) {
+        if (kind() != NUMBER) {
             throwNotNumericLexerException("kind", this);
         }
         int number = -1;
@@ -21,8 +23,9 @@ public int valueInt() {
     public String toString() {
         return switch (kind()) {
             case STRING -> kind().name() + "[" + Token.escaped(value()) + "]";
-            case NUMBER, IDENTIFIER, EOF, ASSIGN, PLUS, LPAREN, RPAREN, COMMA, SEMICOLON, KW_PROGRAM, KW_LOOP, KW_DO, KW_END, KW_INPUT, KW_PRINT ->
-                    kind().name() + "[" + value() + "]";
+            case NUMBER, IDENTIFIER -> kind().name() + "[" + value() + "]";
+            case EOF, ASSIGN, PLUS, LPAREN, RPAREN, COMMA, SEMICOLON, PROGRAM, LOOP, DO, END, INPUT, PRINT ->
+                    kind().name() + "[]";
         };
     }
 

diff --git a/src/main/java/com/davidconneely/looplang/token/Token.java b/src/main/java/com/davidconneely/looplang/token/Token.java
@@ -14,12 +14,12 @@ enum Kind {
         STRING,      // quoted string literal
         NUMBER,      // non-negative integer numeric literal
         IDENTIFIER,  // an identifier
-        KW_PROGRAM,  // keyword `PROGRAM`
-        KW_LOOP,     // keyword `LOOP`
-        KW_DO,       // keyword `DO`
-        KW_END,      // keyword `END`
-        KW_INPUT,    // keyword `INPUT`
-        KW_PRINT     // keyword `PRINT`
+        PROGRAM,     // keyword `PROGRAM`
+        LOOP,        // keyword `LOOP`
+        DO,          // keyword `DO`
+        END,         // keyword `END`
+        INPUT,       // keyword `INPUT`
+        PRINT        // keyword `PRINT`
     }
 
     Kind kind();

diff --git a/src/main/java/com/davidconneely/looplang/token/TokenFactory.java b/src/main/java/com/davidconneely/looplang/token/TokenFactory.java
@@ -2,37 +2,39 @@
 
 import java.util.Map;
 
+import static com.davidconneely.looplang.token.Token.Kind.*;
+
 public final class TokenFactory {
     private static final Map<String, Token.Kind> keywords;
 
     static {
-        keywords = Map.of("PROGRAM", Token.Kind.KW_PROGRAM, "LOOP", Token.Kind.KW_LOOP, "DO", Token.Kind.KW_DO, "END", Token.Kind.KW_END, "INPUT", Token.Kind.KW_INPUT, "PRINT", Token.Kind.KW_PRINT);
+        keywords = Map.of("PROGRAM", PROGRAM, "LOOP", LOOP, "DO", DO, "END", END, "INPUT", INPUT, "PRINT", PRINT);
     }
 
     private TokenFactory() {
     }
 
-    public static Token TOK_EOF = new SimpleToken(Token.Kind.EOF, "");
-    public static Token TOK_ASSIGN = new SimpleToken(Token.Kind.ASSIGN, ":=");
-    public static Token TOK_PLUS = new SimpleToken(Token.Kind.PLUS, "+");
-    public static Token TOK_LPAREN = new SimpleToken(Token.Kind.LPAREN, "(");
-    public static Token TOK_RPAREN = new SimpleToken(Token.Kind.RPAREN, ")");
-    public static Token TOK_COMMA = new SimpleToken(Token.Kind.COMMA, ",");
-    public static Token TOK_SEMICOLON = new SimpleToken(Token.Kind.SEMICOLON, ";");
+    public static Token TOK_EOF = new SimpleToken(EOF, null);
+    public static Token TOK_ASSIGN = new SimpleToken(ASSIGN, null);
+    public static Token TOK_PLUS = new SimpleToken(PLUS, null);
+    public static Token TOK_LPAREN = new SimpleToken(LPAREN, null);
+    public static Token TOK_RPAREN = new SimpleToken(RPAREN, null);
+    public static Token TOK_COMMA = new SimpleToken(COMMA, null);
+    public static Token TOK_SEMICOLON = new SimpleToken(SEMICOLON, null);
 
     public static Token newString(final String string) {
-        return new SimpleToken(Token.Kind.STRING, string);
+        return new SimpleToken(STRING, string);
     }
 
     public static Token newNumber(final String number) {
-        return new SimpleToken(Token.Kind.NUMBER, number);
+        return new SimpleToken(NUMBER, number);
     }
 
     /**
      * Identifier or keyword (based on value).
      */
     public static Token newIdentifierOrKeyword(final String value) {
-        final Token.Kind kind = keywords.getOrDefault(value, Token.Kind.IDENTIFIER);
-        return new SimpleToken(kind, value);
+        final Token.Kind kind = keywords.getOrDefault(value, IDENTIFIER);
+        return new SimpleToken(kind, kind == IDENTIFIER ? value : null);
     }
 }