Fix off-by-one issues in TokenTree and TreeCursor

smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturingTokenizer.java

@@ -132,10 +132,18 @@ public IdlToken next() {
         if (getToken().getIdlToken() == IdlToken.EOF) {
             throw new NoSuchElementException();
         }
-        trees.getFirst().appendChild(TokenTree.of(CapturedToken.from(this, this.stringTable)));
+        appendCurrentTokenToFirstTree();
         return tokens.get(++cursor).getIdlToken();
     }
 
+    void eof() {
+        appendCurrentTokenToFirstTree();
+    }
+
+    private void appendCurrentTokenToFirstTree() {
+        trees.getFirst().appendChild(TokenTree.of(CapturedToken.from(this, this.stringTable)));
+    }
+
     CapturedToken peekPastSpaces() {
         return peekWhile(0, token -> token == IdlToken.SPACE);
     }

smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeCursor.java

@@ -304,7 +304,7 @@ public TreeCursor findAt(int line, int column) {
                     isMatch = column >= startColumn && column < endColumn;
                 } else if (line == startLine && column >= startColumn) {
                     isMatch = true;
-                } else if (line == endLine && column <= endColumn) {
+                } else if (line == endLine && column < endColumn) {
                     isMatch = true;
                 } else if (line > startLine && line < endLine) {
                     isMatch = true;

smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeType.java

@@ -1012,6 +1012,8 @@ void parse(CapturingTokenizer tokenizer) {
                         optionalWs(tokenizer);
                         break;
                     case EOF:
+                        tokenizer.eof();
+                        break;
                     default:
                         break;
                 }

smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeCursorTest.java

@@ -17,7 +17,7 @@
 public class TreeCursorTest {
     @Test
     public void hasChildren() {
-        TokenTree tree = createTree();
+        TokenTree tree = createTree("simple-model.smithy");
         TreeCursor cursor = tree.zipper();
         List<TreeCursor> children = cursor.getChildren();
 
@@ -36,7 +36,7 @@ public void hasChildren() {
 
     @Test
     public void hasParentAndSiblings() {
-        TokenTree tree = createTree();
+        TokenTree tree = createTree("simple-model.smithy");
         TreeCursor cursor = tree.zipper();
         List<TreeCursor> children = cursor.getChildren();
 
@@ -55,7 +55,7 @@ public void hasParentAndSiblings() {
 
     @Test
     public void findsNodeAtPosition() {
-        TokenTree tree = createTree();
+        TokenTree tree = createTree("simple-model.smithy");
         TreeCursor cursor = tree.zipper();
         TreeCursor click = cursor.findAt(3, 17);
 
@@ -65,8 +65,32 @@ public void findsNodeAtPosition() {
         assertThat(click.getRoot(), equalTo(cursor));
     }
 
-    private TokenTree createTree() {
-        String model = IoUtils.readUtf8Url(getClass().getResource("formatter/simple-model.smithy"));
+    @Test
+    public void findsNodeAtPositionBetweenTokens() {
+        TokenTree tree = createTree("incorrect-indentation.smithy");
+        TreeCursor cursor = tree.zipper();
+        TreeCursor click = cursor.findAt(10, 5);
+
+        assertThat(click, notNullValue());
+        assertThat(click.getTree().getType(), is(TreeType.TOKEN));
+        assertThat(click.getTree().tokens().iterator().next().getLexeme().toString(), equalTo("@"));
+        assertThat(click.getRoot(), equalTo(cursor));
+    }
+
+    @Test
+    public void findsNodeAtLastLineOfFile() {
+        TokenTree tree = createTree("missing-trailing-newline.smithy");
+        TreeCursor cursor = tree.zipper();
+        TreeCursor click = cursor.findAt(8, 4);
+
+        assertThat(click, notNullValue());
+        assertThat(click.getTree().getType(), is(TreeType.TOKEN));
+        assertThat(click.getTree().tokens().iterator().next().getLexeme().toString(), equalTo("foo"));
+        assertThat(click.getRoot(), equalTo(cursor));
+    }
+
+    private TokenTree createTree(String filename) {
+        String model = IoUtils.readUtf8Url(getClass().getResource("formatter/" + filename));
         IdlTokenizer tokenizer = IdlTokenizer.create(model);
         return TokenTree.of(tokenizer);
     }

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/incorrect-indentation.formatted.smithy

@@ -0,0 +1,11 @@
+$version: "2.0"
+
+namespace smithy.example
+
+structure Foo {}
+
+structure Bar {}
+
+// Comment
+@input
+structure Baz {}

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/incorrect-indentation.smithy

@@ -0,0 +1,11 @@
+$version: "2.0"
+
+namespace smithy.example
+
+structure Foo {}
+
+    structure Bar {}
+
+// Comment
+    @input
+structure Baz {}

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/missing-trailing-newline.formatted.smithy

@@ -0,0 +1,9 @@
+$version: "2.0"
+
+namespace smithy.example
+
+@trait
+structure foo {}
+
+@foo
+structure Bar {}

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/missing-trailing-newline.smithy

@@ -0,0 +1,9 @@
+$version: "2.0"
+
+namespace smithy.example
+
+@trait
+structure foo {}
+
+@foo
+structure Bar {}