diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java index b5290ab10..01c617776 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java @@ -387,14 +387,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept final boolean isNumbered; // Is this tok numbered? (tokens and comments) String extraNewline = null; // Extra newline at end? List strings = new ArrayList<>(); - if (tokText.startsWith("'") - || tokText.startsWith("\"") - || JavacTokens.isStringFragment(t.kind())) { - // Perform this check first, STRINGFRAGMENT tokens can start with arbitrary characters. - isToken = true; - isNumbered = true; - strings.add(originalTokText); - } else if (Character.isWhitespace(tokText0)) { + if (Character.isWhitespace(tokText0)) { isToken = false; isNumbered = false; Iterator it = Newlines.lineIterator(originalTokText); @@ -411,6 +404,10 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept strings.add(line); } } + } else if (tokText.startsWith("'") || tokText.startsWith("\"")) { + isToken = true; + isNumbered = true; + strings.add(originalTokText); } else if (tokText.startsWith("//") || tokText.startsWith("/*")) { // For compatibility with an earlier lexer, the newline after a // comment is its own tok. if (tokText.startsWith("//") diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java index 986db3912..6daac33b9 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java @@ -15,8 +15,6 @@ package com.google.googlejavaformat.java; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkElementIndex; -import static java.util.Arrays.stream; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -29,15 +27,7 @@ import com.sun.tools.javac.parser.Tokens.TokenKind; import com.sun.tools.javac.parser.UnicodeReader; import com.sun.tools.javac.util.Context; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; import java.util.Set; -import org.jspecify.annotations.Nullable; /** A wrapper around javac's lexer. */ final class JavacTokens { @@ -54,8 +44,6 @@ static class RawTok { private final int endPos; RawTok(String stringVal, TokenKind kind, int pos, int endPos) { - checkElementIndex(pos, endPos, "pos"); - checkArgument(pos < endPos, "expected pos (%s) < endPos (%s)", pos, endPos); this.stringVal = stringVal; this.kind = kind; this.pos = pos; @@ -83,18 +71,9 @@ public String stringVal() { } } - private static final TokenKind STRINGFRAGMENT = - stream(TokenKind.values()) - .filter(t -> t.name().contentEquals("STRINGFRAGMENT")) - .findFirst() - .orElse(null); - - static boolean isStringFragment(TokenKind kind) { - return STRINGFRAGMENT != null && Objects.equals(kind, STRINGFRAGMENT); - } - - private static ImmutableList readAllTokens( - String source, Context context, Set nonTerminalStringFragments) { + /** Lex the input and return a list of {@link RawTok}s. */ + public static ImmutableList getTokens( + String source, Context context, Set stopTokens) { if (source == null) { return ImmutableList.of(); } @@ -102,69 +81,20 @@ private static ImmutableList readAllTokens( char[] buffer = (source + EOF_COMMENT).toCharArray(); Scanner scanner = new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length)); - List tokens = new ArrayList<>(); - do { - scanner.nextToken(); - tokens.add(scanner.token()); - } while (scanner.token().kind != TokenKind.EOF); - for (int i = 0; i < tokens.size(); i++) { - if (isStringFragment(tokens.get(i).kind)) { - int start = i; - while (isStringFragment(tokens.get(i).kind)) { - i++; - } - for (int j = start; j < i - 1; j++) { - nonTerminalStringFragments.add(tokens.get(j).pos); - } - } - } - // A string template is tokenized as a series of STRINGFRAGMENT tokens containing the string - // literal values, followed by the tokens for the template arguments. For the formatter, we - // want the stream of tokens to appear in order by their start position. - if (Runtime.version().feature() >= 21) { - Collections.sort(tokens, Comparator.comparingInt(t -> t.pos)); - } - return ImmutableList.copyOf(tokens); - } - - /** Lex the input and return a list of {@link RawTok}s. */ - public static ImmutableList getTokens( - String source, Context context, Set stopTokens) { - if (source == null) { - return ImmutableList.of(); - } - Set nonTerminalStringFragments = new HashSet<>(); - ImmutableList javacTokens = readAllTokens(source, context, nonTerminalStringFragments); - ImmutableList.Builder tokens = ImmutableList.builder(); int end = source.length(); int last = 0; - for (Token t : javacTokens) { + do { + scanner.nextToken(); + Token t = scanner.token(); if (t.comments != null) { - // javac accumulates comments in reverse order for (Comment c : Lists.reverse(t.comments)) { - int pos = c.getSourcePos(0); - int length; - if (pos == -1) { - // We've found a comment whose position hasn't been recorded. Deduce its position as the - // first `/` character after the end of the previous token. - // - // javac creates a new JavaTokenizer to process string template arguments, so - // CommentSavingTokenizer doesn't get a chance to preprocess those comments and save - // their text and positions. - // - // TODO: consider always using this approach once the minimum supported JDK is 16 and - // we can assume BasicComment#getRawCharacters is always available. - pos = source.indexOf('/', last); - length = CommentSavingTokenizer.commentLength(c); - } else { - length = c.getText().length(); + if (last < c.getSourcePos(0)) { + tokens.add(new RawTok(null, null, last, c.getSourcePos(0))); } - if (last < pos) { - tokens.add(new RawTok(null, null, last, pos)); - } - tokens.add(new RawTok(null, null, pos, pos + length)); - last = pos + length; + tokens.add( + new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length())); + last = c.getSourcePos(0) + c.getText().length(); } } if (stopTokens.contains(t.kind)) { @@ -176,25 +106,14 @@ public static ImmutableList getTokens( if (last < t.pos) { tokens.add(new RawTok(null, null, last, t.pos)); } - if (isStringFragment(t.kind)) { - int endPos = t.endPos; - int pos = t.pos; - if (nonTerminalStringFragments.contains(t.pos)) { - // Include the \ escape from \{...} in the preceding string fragment - endPos++; - } - tokens.add(new RawTok(source.substring(pos, endPos), t.kind, pos, endPos)); - last = endPos; - } else { - tokens.add( - new RawTok( - t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, - t.kind, - t.pos, - t.endPos)); - last = t.endPos; - } - } + tokens.add( + new RawTok( + t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, + t.kind, + t.pos, + t.endPos)); + last = t.endPos; + } while (scanner.token().kind != TokenKind.EOF); if (last < end) { tokens.add(new RawTok(null, null, last, end)); } @@ -203,32 +122,6 @@ public static ImmutableList getTokens( /** A {@link JavaTokenizer} that saves comments. */ static class CommentSavingTokenizer extends JavaTokenizer { - - private static final Method GET_RAW_CHARACTERS_METHOD = getRawCharactersMethod(); - - private static @Nullable Method getRawCharactersMethod() { - try { - // This is a method in PositionTrackingReader, but that class is not public. - return BasicComment.class.getMethod("getRawCharacters"); - } catch (NoSuchMethodException e) { - return null; - } - } - - static int commentLength(Comment comment) { - if (comment instanceof BasicComment && GET_RAW_CHARACTERS_METHOD != null) { - // If we've seen a BasicComment instead of a CommentWithTextAndPosition, getText() will - // be null, so we deduce the length using getRawCharacters. See also the comment at the - // usage of this method in getTokens. - try { - return ((char[]) GET_RAW_CHARACTERS_METHOD.invoke(((BasicComment) comment))).length; - } catch (ReflectiveOperationException e) { - throw new LinkageError(e.getMessage(), e); - } - } - return comment.getText().length(); - } - CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) { super(fac, buffer, length); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java b/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java index 2192a32a9..859c9c0cf 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java +++ b/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java @@ -23,7 +23,6 @@ import com.sun.source.tree.ExpressionTree; import com.sun.source.tree.PatternCaseLabelTree; import com.sun.source.tree.PatternTree; -import com.sun.source.tree.StringTemplateTree; import com.sun.source.tree.Tree; import com.sun.tools.javac.tree.JCTree; import javax.lang.model.element.Name; @@ -63,7 +62,6 @@ public Void visitConstantCaseLabel(ConstantCaseLabelTree node, Void aVoid) { @Override public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unused) { - sync(node); scan(node.getDeconstructor(), null); builder.open(plusFour); token("("); @@ -82,25 +80,6 @@ public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unus return null; } - @SuppressWarnings("preview") - @Override - public Void visitStringTemplate(StringTemplateTree node, Void unused) { - sync(node); - builder.open(plusFour); - scan(node.getProcessor(), null); - token("."); - token(builder.peekToken().get()); - for (int i = 0; i < node.getFragments().size() - 1; i++) { - token("{"); - builder.breakOp(); - scan(node.getExpressions().get(i), null); - token("}"); - token(builder.peekToken().get()); - } - builder.close(); - return null; - } - @Override protected void variableName(Name name) { if (name.isEmpty()) { @@ -118,7 +97,7 @@ public Void scan(Tree tree, Void unused) { visitJcAnyPattern((JCTree.JCAnyPattern) tree); return null; } else { - return super.scan(tree, unused); + return super.scan(tree, null); } } diff --git a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java index d31218e28..4a4cedc82 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java @@ -61,10 +61,8 @@ public class FormatterIntegrationTest { "I880", "Unnamed", "I981", - "StringTemplate", "I1020", - "I1037", - "TextBlockTemplates") + "I1037") .build(); @Parameters(name = "{index}: {0}") diff --git a/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java b/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java index b0d7b4030..3835673d9 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java @@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertWithMessage; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; import com.google.common.base.Joiner; import com.google.common.io.CharStreams; @@ -511,27 +510,4 @@ public void removeTrailingTabsInComments() throws Exception { + " }\n" + "}\n"); } - - @Test - public void stringTemplateTests() throws Exception { - assumeTrue(Runtime.version().feature() >= 21); - assertThat( - new Formatter() - .formatSource( - "public class Foo {\n" - + " String test(){\n" - + " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n" - + " var nested = STR.\"template \\{example. foo()+" - + " STR.\"templateInner\\{ example}\"}xxx }\";\n" - + " }\n" - + "}\n")) - .isEqualTo( - "public class Foo {\n" - + " String test() {\n" - + " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n" - + " var nested = STR.\"template \\{example.foo() +" - + " STR.\"templateInner\\{example}\"}xxx }\";\n" - + " }\n" - + "}\n"); - } } diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.input b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.input deleted file mode 100644 index bba0b7267..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.input +++ /dev/null @@ -1,12 +0,0 @@ -class Foo { - private static final int X = 42; - private static final String A = STR."\{X} = \{X}"; - private static final String B = STR.""; - private static final String C = STR."\{X}"; - private static final String D = STR."\{X}\{X}"; - private static final String E = STR."\{X}\{X}\{X}"; - private static final String F = STR." \{X}"; - private static final String G = STR."\{X} "; - private static final String H = STR."\{X} one long incredibly unbroken sentence moving from "+"topic to topic so that no-one had a chance to interrupt"; - private static final String I = STR."\{X} \uD83D\uDCA9 "; -} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.output b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.output deleted file mode 100644 index ff173fb6a..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.output +++ /dev/null @@ -1,14 +0,0 @@ -class Foo { - private static final int X = 42; - private static final String A = STR."\{X} = \{X}"; - private static final String B = STR.""; - private static final String C = STR."\{X}"; - private static final String D = STR."\{X}\{X}"; - private static final String E = STR."\{X}\{X}\{X}"; - private static final String F = STR." \{X}"; - private static final String G = STR."\{X} "; - private static final String H = - STR."\{X} one long incredibly unbroken sentence moving from " - + "topic to topic so that no-one had a chance to interrupt"; - private static final String I = STR."\{X} \uD83D\uDCA9 "; -} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input deleted file mode 100644 index 98a19bba7..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input +++ /dev/null @@ -1,10 +0,0 @@ -public class StringTemplates { - void test(){ - var m = STR."template \{example}xxx"; - var nested = STR."template \{example.foo()+ STR."templateInner\{example}"}xxx }"; - var nestNested = STR."template \{example0. - foo() + - STR."templateInner\{example1.test(STR."\{example2 - }")}"}xxx }"; - } -} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output deleted file mode 100644 index e60bab70b..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output +++ /dev/null @@ -1,9 +0,0 @@ -public class StringTemplates { - void test() { - var m = STR."template \{example}xxx"; - var nested = STR."template \{example.foo() + STR."templateInner\{example}"}xxx }"; - var nestNested = - STR."template \{ - example0.foo() + STR."templateInner\{example1.test(STR."\{example2}")}"}xxx }"; - } -} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.input b/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.input deleted file mode 100644 index 2d00d3300..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.input +++ /dev/null @@ -1,11 +0,0 @@ -abstract class RSLs { - abstract String f(); - - String a = STR.""" - lorem - foo\{ /** a method */ f( // line comment - /* TODO */ - )}bar - ipsum - """; -} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.output b/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.output deleted file mode 100644 index 36736724f..000000000 --- a/core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.output +++ /dev/null @@ -1,14 +0,0 @@ -abstract class RSLs { - abstract String f(); - - String a = - STR.""" - lorem - foo\{ - /** a method */ - f( // line comment - /* TODO */ - )}bar - ipsum - """; -}