From f0e83fc3479136c0b2ed6357d2be1b167a074150 Mon Sep 17 00:00:00 2001 From: 0dinD Date: Mon, 28 Sep 2020 19:47:13 +0200 Subject: [PATCH] Improve semantic token modifiers Signed-off-by: 0dinD --- .../internal/JDTDelegateCommandHandler.java | 5 +- .../commands/SemanticTokensCommand.java | 14 +- .../semantictokens/SemanticTokenManager.java | 53 --- .../semantictokens/SemanticTokens.java | 12 +- .../semantictokens/SemanticTokensLegend.java | 28 +- .../semantictokens/SemanticTokensVisitor.java | 188 +++++---- .../semantictokens/TokenModifier.java | 197 ++++++---- .../internal/semantictokens/TokenType.java | 3 +- .../semantic-tokens/src/foo/Methods.java | 15 +- .../semantic-tokens/src/foo/Packages.java | 2 +- .../semantic-tokens/src/foo/Properties.java | 4 +- .../semantic-tokens/src/foo/Types.java | 10 +- .../semantic-tokens/src/foo/Variables.java | 2 +- .../commands/SemanticTokensCommandTest.java | 369 ++++++++++++------ 14 files changed, 535 insertions(+), 367 deletions(-) delete mode 100644 org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokenManager.java diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JDTDelegateCommandHandler.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JDTDelegateCommandHandler.java index 9b10545e4b..989cdadcae 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JDTDelegateCommandHandler.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JDTDelegateCommandHandler.java @@ -21,9 +21,10 @@ import org.eclipse.jdt.ls.core.internal.commands.DiagnosticsCommand; import org.eclipse.jdt.ls.core.internal.commands.OrganizeImportsCommand; import org.eclipse.jdt.ls.core.internal.commands.ProjectCommand; +import org.eclipse.jdt.ls.core.internal.commands.ProjectCommand.ClasspathOptions; import org.eclipse.jdt.ls.core.internal.commands.SemanticTokensCommand; import org.eclipse.jdt.ls.core.internal.commands.SourceAttachmentCommand; -import org.eclipse.jdt.ls.core.internal.commands.ProjectCommand.ClasspathOptions; +import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensLegend; import org.eclipse.lsp4j.WorkspaceEdit; public class JDTDelegateCommandHandler implements IDelegateCommandHandler { @@ -74,7 +75,7 @@ public Object executeCommand(String commandId, List arguments, IProgress case "java.project.provideSemanticTokens": return SemanticTokensCommand.provide((String) arguments.get(0)); case "java.project.getSemanticTokensLegend": - return SemanticTokensCommand.getLegend(); + return new SemanticTokensLegend(); case "java.project.import": ProjectCommand.importProject(monitor); return null; diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommand.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommand.java index faf3316e62..0b344cf838 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommand.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommand.java @@ -13,8 +13,6 @@ package org.eclipse.jdt.ls.core.internal.commands; -import java.util.Collections; - import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.jdt.core.ITypeRoot; import org.eclipse.jdt.core.dom.CompilationUnit; @@ -22,9 +20,7 @@ import org.eclipse.jdt.ls.core.internal.JDTUtils; import org.eclipse.jdt.ls.core.internal.JobHelpers; import org.eclipse.jdt.ls.core.internal.handlers.DocumentLifeCycleHandler; -import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokenManager; import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokens; -import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensLegend; import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensVisitor; public class SemanticTokensCommand { @@ -36,20 +32,16 @@ public static SemanticTokens provide(String uri) { private static SemanticTokens doProvide(String uri) { ITypeRoot typeRoot = JDTUtils.resolveTypeRoot(uri); if (typeRoot == null) { - return new SemanticTokens(Collections.emptyList()); + return new SemanticTokens(new int[0]); } CompilationUnit root = CoreASTProvider.getInstance().getAST(typeRoot, CoreASTProvider.WAIT_YES, new NullProgressMonitor()); if (root == null) { - return new SemanticTokens(Collections.emptyList()); + return new SemanticTokens(new int[0]); } - SemanticTokensVisitor collector = new SemanticTokensVisitor(root, SemanticTokenManager.getInstance()); + SemanticTokensVisitor collector = new SemanticTokensVisitor(root); root.accept(collector); return collector.getSemanticTokens(); } - - public static SemanticTokensLegend getLegend() { - return SemanticTokenManager.getInstance().getLegend(); - } } diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokenManager.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokenManager.java deleted file mode 100644 index 83115f8839..0000000000 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokenManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Microsoft Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License 2.0 - * which accompanies this distribution, and is available at - * https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * 0dinD - Semantic highlighting improvements - https://github.com/eclipse/eclipse.jdt.ls/pull/1501 - *******************************************************************************/ -package org.eclipse.jdt.ls.core.internal.semantictokens; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class SemanticTokenManager { - private List tokenTypes; - private List tokenModifiers; - private SemanticTokensLegend legend; - - private SemanticTokenManager() { - this.tokenTypes = Arrays.asList(TokenType.values()); - this.tokenModifiers = Arrays.asList(TokenModifier.values()); - List types = tokenTypes.stream().map(TokenType::toString).collect(Collectors.toList()); - List modifiers = tokenModifiers.stream().map(TokenModifier::toString).collect(Collectors.toList()); - this.legend = new SemanticTokensLegend(types, modifiers); - } - - private static class SingletonHelper{ - private static final SemanticTokenManager INSTANCE = new SemanticTokenManager(); - } - - public static SemanticTokenManager getInstance(){ - return SingletonHelper.INSTANCE; - } - - public SemanticTokensLegend getLegend() { - return this.legend; - } - - public List getTokenTypes() { - return tokenTypes; - } - - public List getTokenModifiers() { - return tokenModifiers; - } - -} diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokens.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokens.java index ab0ccfa347..98b234ed2c 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokens.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokens.java @@ -13,8 +13,6 @@ *******************************************************************************/ package org.eclipse.jdt.ls.core.internal.semantictokens; -import java.util.List; - import org.eclipse.lsp4j.util.Preconditions; public class SemanticTokens { @@ -74,7 +72,7 @@ public class SemanticTokens { * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * ``` */ - private final List data; + private final int[] data; /** * The result id of the tokens (optional). @@ -83,12 +81,12 @@ public class SemanticTokens { */ private final String resultId; - public SemanticTokens(List data) { + public SemanticTokens(int[] data) { this(data, null); } - public SemanticTokens(List data, String resultId) { - this.data = Preconditions.>checkNotNull(data, "data"); + public SemanticTokens(int[] data, String resultId) { + this.data = Preconditions.checkNotNull(data, "data"); this.resultId = resultId; } @@ -96,7 +94,7 @@ public String getResultId() { return resultId; } - public List getData() { + public int[] getData() { return data; } } diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensLegend.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensLegend.java index 1346330875..d1eb0ead48 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensLegend.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensLegend.java @@ -13,22 +13,26 @@ *******************************************************************************/ package org.eclipse.jdt.ls.core.internal.semantictokens; -import java.util.List; +import java.util.Arrays; -public class SemanticTokensLegend { - private final List tokenTypes; - private final List tokenModifiers; +public final class SemanticTokensLegend { + private final String[] tokenTypes; + private final String[] tokenModifiers; - public SemanticTokensLegend(List tokenTypes, List tokenModifiers){ - this.tokenTypes = tokenTypes; - this.tokenModifiers = tokenModifiers; - }; + public SemanticTokensLegend() { + tokenTypes = Arrays.stream(TokenType.values()) + .map(TokenType::toString) + .toArray(String[]::new); + tokenModifiers = Arrays.stream(TokenModifier.values()) + .map(TokenModifier::toString) + .toArray(String[]::new); + } - public List getTokenTypes() { - return this.tokenTypes; + public String[] getTokenTypes() { + return tokenTypes; } - public List getTokenModifiers() { - return this.tokenModifiers; + public String[] getTokenModifiers() { + return tokenModifiers; } } diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensVisitor.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensVisitor.java index 591df8ecdf..54b91e38fd 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensVisitor.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensVisitor.java @@ -21,9 +21,11 @@ import org.eclipse.jdt.core.dom.ClassInstanceCreation; import org.eclipse.jdt.core.dom.CompilationUnit; import org.eclipse.jdt.core.dom.IBinding; +import org.eclipse.jdt.core.dom.IMethodBinding; import org.eclipse.jdt.core.dom.IPackageBinding; import org.eclipse.jdt.core.dom.ITypeBinding; import org.eclipse.jdt.core.dom.ImportDeclaration; +import org.eclipse.jdt.core.dom.Modifier; import org.eclipse.jdt.core.dom.Name; import org.eclipse.jdt.core.dom.NameQualifiedType; import org.eclipse.jdt.core.dom.ParameterizedType; @@ -35,22 +37,20 @@ public class SemanticTokensVisitor extends ASTVisitor { private CompilationUnit cu; - private SemanticTokenManager manager; private List tokens; - public SemanticTokensVisitor(CompilationUnit cu, SemanticTokenManager manager) { - this.manager = manager; + public SemanticTokensVisitor(CompilationUnit cu) { this.cu = cu; this.tokens = new ArrayList<>(); } private class SemanticToken { private final TokenType tokenType; - private final TokenModifier[] tokenModifiers; + private final int tokenModifiers; private final int offset; private final int length; - public SemanticToken(int offset, int length, TokenType tokenType, TokenModifier[] tokenModifiers) { + public SemanticToken(int offset, int length, TokenType tokenType, int tokenModifiers) { this.offset = offset; this.length = length; this.tokenType = tokenType; @@ -61,7 +61,7 @@ public TokenType getTokenType() { return tokenType; } - public TokenModifier[] getTokenModifiers() { + public int getTokenModifiers() { return tokenModifiers; } @@ -75,15 +75,16 @@ public int getLength() { } public SemanticTokens getSemanticTokens() { - List data = encodedTokens(); - return new SemanticTokens(data); + return new SemanticTokens(encodedTokens()); } - private List encodedTokens() { - List data = new ArrayList<>(); + private int[] encodedTokens() { + int numTokens = tokens.size(); + int[] data = new int[numTokens * 5]; int currentLine = 0; int currentColumn = 0; - for (SemanticToken token : this.tokens) { + for (int i = 0; i < numTokens; i++) { + SemanticToken token = tokens.get(i); int line = cu.getLineNumber(token.getOffset()) - 1; int column = cu.getColumnNumber(token.getOffset()); int deltaLine = line - currentLine; @@ -95,18 +96,15 @@ private List encodedTokens() { currentColumn = column; // Disallow duplicate/conflict token (if exists) if (deltaLine != 0 || deltaColumn != 0) { - int tokenTypeIndex = manager.getTokenTypes().indexOf(token.getTokenType()); - TokenModifier[] modifiers = token.getTokenModifiers(); - int encodedModifiers = 0; - for (TokenModifier modifier : modifiers) { - int bit = manager.getTokenModifiers().indexOf(modifier); - encodedModifiers = encodedModifiers | (0b00000001 << bit); - } - data.add(deltaLine); - data.add(deltaColumn); - data.add(token.getLength()); - data.add(tokenTypeIndex); - data.add(encodedModifiers); + int tokenTypeIndex = token.getTokenType().ordinal(); + int tokenModifiers = token.getTokenModifiers(); + + int offset = i * 5; + data[offset] = deltaLine; + data[offset + 1] = deltaColumn; + data[offset + 2] = token.getLength(); + data[offset + 3] = tokenTypeIndex; + data[offset + 4] = tokenModifiers; } } return data; @@ -118,13 +116,13 @@ private List encodedTokens() { * * @param node The AST node representing the location of the semantic token. * @param tokenType The type of the semantic token. - * @param modifiers The modifiers of the semantic token. + * @param modifiers The bitwise OR of the semantic token modifiers, see {@link TokenModifier#bitmask}. * * @apiNote This method is order-dependent because of {@link #encodedTokens()}. * If semantic tokens are not added in the order they appear in the document, * the encoding algorithm might discard them. */ - private void addToken(ASTNode node, TokenType tokenType, TokenModifier[] modifiers) { + private void addToken(ASTNode node, TokenType tokenType, int modifiers) { int offset = node.getStartPosition(); int length = node.getLength(); SemanticToken token = new SemanticToken(offset, length, tokenType, modifiers); @@ -132,23 +130,14 @@ private void addToken(ASTNode node, TokenType tokenType, TokenModifier[] modifie } /** - * Adds a semantic token to the list of tokens being collected by this - * semantic token provider. Overload for {@link #addToken(ASTNode, TokenType, TokenModifier[])} - * that adds no modifiers to the semantic token. - * - * @param node The AST node representing the location of the semantic token. - * @param tokenType The type of the semantic token. - * - * @apiNote This method is order-dependent because of {@link #encodedTokens()}. - * If semantic tokens are not added in the order they appear in the document, - * the encoding algorithm might discard them. + * Indicates that the visitor is currently inside an {@link ImportDeclaration} node. */ - private void addToken(ASTNode node, TokenType tokenType) { - addToken(node, tokenType, new TokenModifier[0]); - } + private boolean isInsideImportDeclaration = false; @Override public boolean visit(ImportDeclaration node) { + isInsideImportDeclaration = true; + IBinding binding = node.resolveBinding(); if (binding == null || binding instanceof IPackageBinding) { visitPackageName(node.getName()); @@ -157,6 +146,7 @@ public boolean visit(ImportDeclaration node) { visitNonPackageName(node.getName()); } + isInsideImportDeclaration = false; return false; } @@ -169,13 +159,14 @@ public boolean visit(ImportDeclaration node) { * @param packageName The package name node to recursively visit. */ private void visitPackageName(Name packageName) { + int modifiers = isInsideImportDeclaration ? TokenModifier.IMPORT_DECLARATION.bitmask : 0; if (packageName instanceof SimpleName) { - addToken(packageName, TokenType.PACKAGE); + addToken(packageName, TokenType.PACKAGE, modifiers); } else { QualifiedName qualifiedName = (QualifiedName) packageName; visitPackageName(qualifiedName.getQualifier()); - addToken(qualifiedName.getName(), TokenType.PACKAGE); + addToken(qualifiedName.getName(), TokenType.PACKAGE, modifiers); } } @@ -228,16 +219,42 @@ private boolean hasPackageQualifier(QualifiedName qualifiedName) { } } + @Override + public boolean visit(Modifier node) { + addToken(node, TokenType.MODIFIER, 0); + return super.visit(node); + } + @Override public boolean visit(SimpleName node) { - TokenType tokenType = TokenType.getApplicableType(node.resolveBinding()); + IBinding binding = node.resolveBinding(); + TokenType tokenType = TokenType.getApplicableType(binding); if (tokenType != null) { - addToken(node, tokenType, TokenModifier.getApplicableModifiers(node)); + int modifiers = TokenModifier.getApplicableModifiers(binding); + if (TokenModifier.isGeneric(binding)) { + modifiers |= TokenModifier.GENERIC.bitmask; + } + if (isInsideImportDeclaration) { + modifiers |= TokenModifier.IMPORT_DECLARATION.bitmask; + } + else if (TokenModifier.isDeclaration(node)) { + modifiers |= TokenModifier.DECLARATION.bitmask; + } + addToken(node, tokenType, modifiers); } return super.visit(node); } + @Override + public boolean visit(ParameterizedType node) { + node.getType().accept(this); + for (Object typeArgument : node.typeArguments()) { + visitTypeArgument((Type) typeArgument); + } + return false; + } + @Override public boolean visit(ClassInstanceCreation node) { if (node.getExpression() != null) { @@ -248,7 +265,14 @@ public boolean visit(ClassInstanceCreation node) { ((ASTNode) typeArgument).accept(this); } - visitClassInstanceCreationType(node, node.getType()); + visitSimpleNameOfType(node.getType(), (simpleName) -> { + IMethodBinding constructorBinding = node.resolveConstructorBinding(); + int modifiers = TokenModifier.getApplicableModifiers(constructorBinding); + if (TokenModifier.isGeneric(constructorBinding) || node.getType().isParameterizedType()) { + modifiers |= TokenModifier.GENERIC.bitmask; + } + addToken(simpleName, TokenType.METHOD, modifiers); + }); for (Object argument : node.arguments()) { ((ASTNode) argument).accept(this); @@ -262,68 +286,88 @@ public boolean visit(ClassInstanceCreation node) { } /** - * Visits the type node of a class instance creation, and recursively tries to find - * a simple name which represents the constructor method binding. If it does, - * a {@link TokenType#METHOD} is added to the simple name, with the modifiers - * of the constructor method binding. + * Visits the given type argument, making sure to add the + * {@link TokenModifier#TYPE_ARGUMENT} modifier to its simple name. + * + * @param typeArgument A type node that is known to be a type argument. + */ + private void visitTypeArgument(Type typeArgument) { + visitSimpleNameOfType(typeArgument, (simpleName) -> { + IBinding binding = simpleName.resolveBinding(); + TokenType tokenType = TokenType.getApplicableType(binding); + if (tokenType != null) { + int modifiers = TokenModifier.getApplicableModifiers(binding); + if (TokenModifier.isGeneric((ITypeBinding) binding)) { + modifiers |= TokenModifier.GENERIC.bitmask; + } + modifiers |= TokenModifier.TYPE_ARGUMENT.bitmask; + addToken(simpleName, tokenType, modifiers); + } + }); + } + + /** + * Visits the given type node, making sure to exaustively visit + * its child nodes, until the simple name of the type is found or + * until there are no more child nodes. If and when the simple name + * is found, the given {@link NodeVisitor} is called with the simple + * name as its argument. * - * @param classInstanceCreation The class instance creation which parents the type node. - * @param type The type node to visit. + * @param type A type node. + * @param visitor A node visitor which might get called if the simple + * name of the given type node is found. */ - private void visitClassInstanceCreationType(ClassInstanceCreation classInstanceCreation, Type type) { - if (type instanceof SimpleType) { + private void visitSimpleNameOfType(Type type, NodeVisitor visitor) { + if (type == null) { + return; + } + else if (type instanceof SimpleType) { SimpleType simpleType = (SimpleType) type; for (Object annotation : simpleType.annotations()) { ((ASTNode) annotation).accept(this); } - if (simpleType.getName() instanceof SimpleName) { - addToken(simpleType.getName(), TokenType.METHOD, - TokenModifier.getApplicableModifiers(classInstanceCreation.resolveConstructorBinding())); + Name simpleTypeName = simpleType.getName(); + if (simpleTypeName instanceof SimpleName) { + visitor.visit((SimpleName) simpleTypeName); } else { - QualifiedName qualifiedName = (QualifiedName) simpleType.getName(); + QualifiedName qualifiedName = (QualifiedName) simpleTypeName; qualifiedName.getQualifier().accept(this); - addToken(qualifiedName.getName(), TokenType.METHOD, - TokenModifier.getApplicableModifiers(classInstanceCreation.resolveConstructorBinding())); + visitor.visit(qualifiedName.getName()); } } else if (type instanceof QualifiedType) { QualifiedType qualifiedType = (QualifiedType) type; - qualifiedType.getQualifier().accept(this); - for (Object annotation : qualifiedType.annotations()) { ((ASTNode) annotation).accept(this); } - - addToken(qualifiedType.getName(), TokenType.METHOD, - TokenModifier.getApplicableModifiers(classInstanceCreation.resolveConstructorBinding())); + visitor.visit(qualifiedType.getName()); } else if (type instanceof NameQualifiedType) { NameQualifiedType nameQualifiedType = (NameQualifiedType) type; - nameQualifiedType.getQualifier().accept(this); - for (Object annotation : nameQualifiedType.annotations()) { ((ASTNode) annotation).accept(this); } - - addToken(nameQualifiedType.getName(), TokenType.METHOD, - TokenModifier.getApplicableModifiers(classInstanceCreation.resolveConstructorBinding())); + visitor.visit(nameQualifiedType.getName()); } else if (type instanceof ParameterizedType) { ParameterizedType parameterizedType = (ParameterizedType) type; - - visitClassInstanceCreationType(classInstanceCreation, parameterizedType.getType()); - - for (Object typeParameter : parameterizedType.typeArguments()) { - ((ASTNode) typeParameter).accept(this); + visitSimpleNameOfType(parameterizedType.getType(), visitor); + for (Object typeArgument : parameterizedType.typeArguments()) { + visitTypeArgument((Type) typeArgument); } } else { type.accept(this); } } + + @FunctionalInterface + private interface NodeVisitor { + public void visit(T node); + } } diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenModifier.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenModifier.java index 6ba00672c9..b147399e0d 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenModifier.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenModifier.java @@ -13,14 +13,14 @@ *******************************************************************************/ package org.eclipse.jdt.ls.core.internal.semantictokens; -import java.util.Arrays; - import org.eclipse.jdt.core.dom.ASTNode; import org.eclipse.jdt.core.dom.AnnotationTypeDeclaration; import org.eclipse.jdt.core.dom.AnnotationTypeMemberDeclaration; import org.eclipse.jdt.core.dom.EnumConstantDeclaration; import org.eclipse.jdt.core.dom.EnumDeclaration; import org.eclipse.jdt.core.dom.IBinding; +import org.eclipse.jdt.core.dom.IMethodBinding; +import org.eclipse.jdt.core.dom.ITypeBinding; import org.eclipse.jdt.core.dom.MethodDeclaration; import org.eclipse.jdt.core.dom.Modifier; import org.eclipse.jdt.core.dom.SimpleName; @@ -31,54 +31,21 @@ import org.eclipse.jdt.core.dom.VariableDeclarationFragment; public enum TokenModifier { - STATIC("static") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isStatic(binding.getModifiers()); - } - }, - FINAL("readonly") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isFinal(binding.getModifiers()); - } - }, - DEPRECATED("deprecated") { - @Override - protected boolean applies(IBinding binding) { - return binding.isDeprecated(); - } - }, - PUBLIC("public") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isPublic(binding.getModifiers()); - } - }, - PRIVATE("private") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isPrivate(binding.getModifiers()); - } - }, - PROTECTED("protected") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isProtected(binding.getModifiers()); - } - }, - ABSTRACT("abstract") { - @Override - protected boolean applies(IBinding binding) { - return Modifier.isAbstract(binding.getModifiers()); - } - }, - DECLARATION("declaration") { - @Override - protected boolean applies(SimpleName simpleName) { - return isDeclaration(simpleName); - } - }; + // Standard Java modifiers + PUBLIC("public"), + PRIVATE("private"), + PROTECTED("protected"), + ABSTRACT("abstract"), + STATIC("static"), + FINAL("readonly"), + NATIVE("native"), + + // Additional semantic modifiers + DEPRECATED("deprecated"), + GENERIC("generic"), + TYPE_ARGUMENT("typeArgument"), + DECLARATION("declaration"), + IMPORT_DECLARATION("importDeclaration"); /** * This is the name of the token modifier given to the client, so it @@ -90,7 +57,13 @@ protected boolean applies(SimpleName simpleName) { * * @see https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide#semantic-token-classification */ - private String genericName; + private final String genericName; + + /** + * The bitmask for this semantic token modifier. + * Use bitwise OR to combine with other token modifiers. + */ + public final int bitmask = 1 << ordinal(); TokenModifier(String genericName) { this.genericName = genericName; @@ -102,45 +75,105 @@ public String toString() { } /** - * Returns an array of all the semantic token modifiers that apply to a binding. - * Used when the desired binding can't be found by calling - * {@link SimpleName#resolveBinding()}, as is the case - * for the simple name of a constructor invocation. - * - * @param binding A binding. - * @return An array of all the applicable modifiers for the binding. - * - * @apiNote The declaration modifier never applies to a binding, use - * {@link #getApplicableModifiers(SimpleName)} to check for declarations. - */ - public static TokenModifier[] getApplicableModifiers(IBinding binding) { - if (binding == null) return new TokenModifier[0]; + * Returns the bitwise OR of all the semantic token modifiers that can + * be easily figured out from a given binding. No modifiers (the value 0) + * are returned if the binding is {@code null}. + * + * @param binding A binding. + * @return The bitwise OR of the applicable modifiers for the binding. + */ + public static int getApplicableModifiers(IBinding binding) { + if (binding == null) { + return 0; + } - return Arrays.stream(TokenModifier.values()) - .filter(tm -> tm.applies(binding)) - .toArray(size -> new TokenModifier[size]); + int modifiers = 0; + int bindingModifiers = binding.getModifiers(); + if (Modifier.isPublic(bindingModifiers)) { + modifiers |= PUBLIC.bitmask; + } + if (Modifier.isPrivate(bindingModifiers)) { + modifiers |= PRIVATE.bitmask; + } + if (Modifier.isProtected(bindingModifiers)) { + modifiers |= PROTECTED.bitmask; + } + if (Modifier.isAbstract(bindingModifiers)) { + modifiers |= ABSTRACT.bitmask; + } + if (Modifier.isStatic(bindingModifiers)) { + modifiers |= STATIC.bitmask; + } + if (Modifier.isFinal(bindingModifiers)) { + modifiers |= FINAL.bitmask; + } + if (Modifier.isNative(bindingModifiers)) { + modifiers |= NATIVE.bitmask; + } + if (binding.isDeprecated()) { + modifiers |= DEPRECATED.bitmask; + } + return modifiers; } /** - * Returns an array of all the semantic token modifiers that apply to a simple name. - * - * @param simpleName A simple name. - * @return An array of all the applicable modifiers for the simple name. - */ - public static TokenModifier[] getApplicableModifiers(SimpleName simpleName) { - if (simpleName == null) return new TokenModifier[0]; + * Returns whether or not a given binding corresponds to + * a generic (or parameterized) type or method. {@code false} + * is returned if the binding is {@code null}. + * + * @param binding A binding. + * @return {@code true} if the binding corresponds to a generic + * type or method, {@code false} otherwise. + */ + public static boolean isGeneric(IBinding binding) { + if (binding == null) { + return false; + } - return Arrays.stream(TokenModifier.values()) - .filter(tm -> tm.applies(simpleName)) - .toArray(size -> new TokenModifier[size]); + switch (binding.getKind()) { + case IBinding.TYPE: { + return isGeneric((ITypeBinding) binding); + } + case IBinding.METHOD: { + return isGeneric((IMethodBinding) binding); + } + default: + return false; + } } - protected boolean applies(IBinding binding) { - return false; + /** + * Returns whether or not a given type binding corresponds to + * a generic (or parameterized) type. {@code false} + * is returned if the type binding is {@code null}. + * + * @param typeBinding A type binding. + * @return {@code true} if the type binding corresponds to a generic + * type, {@code false} otherwise. + */ + public static boolean isGeneric(ITypeBinding typeBinding) { + if (typeBinding == null) { + return false; + } + + return typeBinding.isGenericType() || typeBinding.isParameterizedType(); } - protected boolean applies(SimpleName simpleName) { - return applies(simpleName.resolveBinding()); + /** + * Returns whether or not a given method binding corresponds to + * a generic (or parameterized) method. {@code false} + * is returned if the method binding is {@code null}. + * + * @param methodBinding A method binding. + * @return {@code true} if the method binding corresponds to a generic + * method, {@code false} otherwise. + */ + public static boolean isGeneric(IMethodBinding methodBinding) { + if (methodBinding == null) { + return false; + } + + return methodBinding.isGenericMethod() || methodBinding.isParameterizedMethod(); } /** @@ -154,7 +187,7 @@ protected boolean applies(SimpleName simpleName) { * * @see SimpleName#isDeclaration() */ - private static boolean isDeclaration(SimpleName simpleName) { + public static boolean isDeclaration(SimpleName simpleName) { StructuralPropertyDescriptor d = simpleName.getLocationInParent(); if (d == null) { return false; diff --git a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenType.java b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenType.java index caafc07c58..d087c1ed86 100644 --- a/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenType.java +++ b/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/TokenType.java @@ -31,7 +31,8 @@ public enum TokenType { METHOD("function"), PROPERTY("property"), VARIABLE("variable"), - PARAMETER("parameter"); + PARAMETER("parameter"), + MODIFIER("modifier"); /** * This is the name of the token type given to the client, so it diff --git a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Methods.java b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Methods.java index 2372359364..4a4fef04f8 100644 --- a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Methods.java +++ b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Methods.java @@ -2,12 +2,21 @@ public class Methods { - public void foo1() {} + public void foo1() {} private void foo2() {} protected void foo3() {} static void foo4() {} + native void foo5(); @Deprecated - void foo5() {} - public static void main(String args[]) {} + void foo6() {} + public static void main(String args[]) { + Methods m = new Methods(); + m.foo1(); + m.foo2(); + m.foo3(); + foo4(); + m.foo5(); + m.foo6(); + } } diff --git a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Packages.java b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Packages.java index 411d6b0752..ff86574943 100644 --- a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Packages.java +++ b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Packages.java @@ -9,5 +9,5 @@ import static java.lang.Math.*; public class Packages { - + public java.lang.String string = new java.lang.String(); } diff --git a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Properties.java b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Properties.java index d194e6cbd9..6551ed7334 100644 --- a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Properties.java +++ b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Properties.java @@ -3,8 +3,8 @@ public class Properties { public String bar1; - private int bar2; - protected boolean bar3; + private int bar2 = ""; + protected boolean bar3 = bar2; final String bar4; static int bar5; public static final int bar6 = 1; diff --git a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Types.java b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Types.java index 71edcaebdf..530ae66516 100644 --- a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Types.java +++ b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Types.java @@ -3,9 +3,15 @@ public class Types { public String // comments - s1 = happy, + s1 = "happy", s2, s3; - class SomeClass {} + + public SomeClass> someClass; + + class SomeClass { + T1 t1; + T2 t2; + } interface SomeInterface {} enum SomeEnum {} @interface SomeAnnotation {} diff --git a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Variables.java b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Variables.java index 0f077924d6..608c0f85ea 100644 --- a/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Variables.java +++ b/org.eclipse.jdt.ls.tests/projects/eclipse/semantic-tokens/src/foo/Variables.java @@ -4,7 +4,7 @@ public class Variables { public static void foo(String string) { String bar1 = string; - String bar2; + String bar2 = bar1; final String bar3 = "test"; } diff --git a/org.eclipse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommandTest.java b/org.eclipse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommandTest.java index 2d3c7ca85a..c9c3d03090 100644 --- a/org.eclipse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommandTest.java +++ b/org.eclipse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommandTest.java @@ -13,15 +13,14 @@ package org.eclipse.jdt.ls.core.internal.commands; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.eclipse.jdt.core.ICompilationUnit; +import org.eclipse.jdt.core.IBuffer; import org.eclipse.jdt.core.IJavaProject; import org.eclipse.jdt.core.IPackageFragment; import org.eclipse.jdt.core.JavaCore; @@ -29,6 +28,7 @@ import org.eclipse.jdt.ls.core.internal.JDTUtils; import org.eclipse.jdt.ls.core.internal.WorkspaceHelper; import org.eclipse.jdt.ls.core.internal.correction.TestOptions; +import org.eclipse.jdt.ls.core.internal.handlers.JsonRpcHelpers; import org.eclipse.jdt.ls.core.internal.managers.AbstractProjectsManagerBasedTest; import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokens; import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensLegend; @@ -41,7 +41,6 @@ public class SemanticTokensCommandTest extends AbstractProjectsManagerBasedTest { private IJavaProject semanticTokensProject; private IPackageFragment fooPackage; - private SemanticTokensLegend legend = SemanticTokensCommand.getLegend(); private String classFileUri = "jdt://contents/foo.jar/foo/bar.class?%3Dsemantic-tokens%2Ffoo.jar%3Cfoo%28bar.class"; @Before @@ -55,158 +54,292 @@ public void setup() throws Exception { } @Test - public void testSemanticTokens_SourceAttachment() { - SemanticTokens tokens = SemanticTokensCommand.provide(classFileUri); - Map> decodedTokens = decode(tokens); - - assertToken(decodedTokens, 0, 8, 3, "namespace"); - assertToken(decodedTokens, 2, 13, 3, "class", "public", "declaration"); - assertToken(decodedTokens, 3, 22, 3, "function", "public", "static", "declaration"); - assertToken(decodedTokens, 3, 33, 1, "parameter", "declaration"); - assertToken(decodedTokens, 4, 12, 3, "variable", "declaration"); + public void testSemanticTokens_SourceAttachment() throws JavaModelException { + TokenAssertionHelper.beginAssertion(classFileUri) + .assertNextToken("foo", "namespace") + .assertNextToken("public", "modifier") + .assertNextToken("bar", "class", "public", "declaration") + .assertNextToken("public", "modifier") + .assertNextToken("static", "modifier") + .assertNextToken("add", "function", "public", "static", "declaration") + .assertNextToken("a", "parameter", "declaration") + .assertNextToken("sum", "variable", "declaration") + .assertNextToken("element", "variable", "declaration") + .assertNextToken("a", "parameter") + .assertNextToken("sum", "variable") + .assertNextToken("element", "variable") + .assertNextToken("sum", "variable") + .endAssertion(); } @Test public void testSemanticTokens_Methods() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Methods.java"); - - assertToken(decodedTokens, 4, 13, 4, "function", "public", "declaration"); - assertToken(decodedTokens, 5, 14, 4, "function", "private", "declaration"); - assertToken(decodedTokens, 6, 16, 4, "function", "protected", "declaration"); - assertToken(decodedTokens, 7, 13, 4, "function", "static", "declaration"); - assertToken(decodedTokens, 9, 6, 4, "function", "deprecated", "declaration"); - assertToken(decodedTokens, 10, 20, 4, "function", "public", "static", "declaration"); + TokenAssertionHelper.beginAssertion(getURI("Methods.java"), "function") + .assertNextToken("foo1", "function", "public", "generic", "declaration") + .assertNextToken("foo2", "function", "private", "declaration") + .assertNextToken("foo3", "function", "protected", "declaration") + .assertNextToken("foo4", "function", "static", "declaration") + .assertNextToken("foo5", "function", "native", "declaration") + .assertNextToken("foo6", "function", "deprecated", "declaration") + + .assertNextToken("main", "function", "public", "static", "declaration") + .assertNextToken("Methods", "function", "public") + + .assertNextToken("foo1", "function", "public", "generic") + .assertNextToken("foo2", "function", "private") + .assertNextToken("foo3", "function", "protected") + .assertNextToken("foo4", "function", "static") + .assertNextToken("foo5", "function", "native") + .assertNextToken("foo6", "function", "deprecated") + .endAssertion(); } @Test public void testSemanticTokens_Constructors() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Constructors.java"); - - assertToken(decodedTokens, 4, 9, 12, "function", "private", "declaration"); - assertToken(decodedTokens, 5, 23, 12, "function", "private"); - assertToken(decodedTokens, 6, 48, 10, "function", "protected"); - assertToken(decodedTokens, 7, 64, 10, "function", "protected"); - assertToken(decodedTokens, 8, 56, 10, "function", "protected"); + TokenAssertionHelper.beginAssertion(getURI("Constructors.java"), "function") + .assertNextToken("Constructors", "function", "private", "declaration") + .assertNextToken("Constructors", "function", "private") + .assertNextToken("InnerClass", "function", "protected") + .assertNextToken("InnerClass", "function", "protected") + .assertNextToken("InnerClass", "function", "protected", "generic") + .endAssertion(); } @Test public void testSemanticTokens_Properties() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Properties.java"); - - assertToken(decodedTokens, 4, 15, 4, "property", "public", "declaration"); - assertToken(decodedTokens, 5, 13, 4, "property", "private", "declaration"); - assertToken(decodedTokens, 6, 19, 4, "property", "protected", "declaration"); - assertToken(decodedTokens, 7, 14, 4, "property", "readonly", "declaration"); - assertToken(decodedTokens, 8, 12, 4, "property", "static", "declaration"); - assertToken(decodedTokens, 9, 25, 4, "property", "static", "public", "readonly", "declaration"); - assertToken(decodedTokens, 12, 2, 5, "enumMember", "static", "public", "readonly", "declaration"); - assertToken(decodedTokens, 13, 2, 6, "enumMember", "static", "public", "readonly", "declaration"); + TokenAssertionHelper.beginAssertion(getURI("Properties.java"), "property", "enumMember") + .assertNextToken("bar1", "property", "public", "declaration") + .assertNextToken("bar2", "property", "private", "declaration") + .assertNextToken("bar3", "property", "protected", "declaration") + .assertNextToken("bar2", "property", "private") + .assertNextToken("bar4", "property", "readonly", "declaration") + .assertNextToken("bar5", "property", "static", "declaration") + .assertNextToken("bar6", "property", "public", "static", "readonly", "declaration") + .assertNextToken("FIRST", "enumMember", "public", "static", "readonly", "declaration") + .assertNextToken("SECOND", "enumMember", "public", "static", "readonly", "declaration") + .endAssertion(); } @Test public void testSemanticTokens_Variables() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Variables.java"); - - assertToken(decodedTokens, 4, 31, 6, "parameter", "declaration"); - assertToken(decodedTokens, 5, 16, 6, "parameter"); - assertToken(decodedTokens, 6, 9, 4, "variable", "declaration"); - assertToken(decodedTokens, 7, 15, 4, "variable", "readonly", "declaration"); + TokenAssertionHelper.beginAssertion(getURI("Variables.java"), "variable", "parameter") + .assertNextToken("string", "parameter", "declaration") + .assertNextToken("bar1", "variable", "declaration") + .assertNextToken("string", "parameter") + .assertNextToken("bar2", "variable", "declaration") + .assertNextToken("bar1", "variable") + .assertNextToken("bar3", "variable", "readonly", "declaration") + .endAssertion(); } @Test public void testSemanticTokens_Types() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Types.java"); - - assertToken(decodedTokens, 4, 8, 6, "class", "public", "readonly"); - assertToken(decodedTokens, 7, 7, 9, "class", "declaration"); - assertToken(decodedTokens, 7, 17, 1, "typeParameter", "declaration"); - assertToken(decodedTokens, 8, 11, 13, "interface", "declaration", "static"); - assertToken(decodedTokens, 9, 6, 8, "enum", "declaration", "static", "readonly"); - assertToken(decodedTokens, 10, 12, 14, "annotation", "declaration", "static"); + TokenAssertionHelper.beginAssertion(getURI("Types.java"), "class", "interface", "enum", "annotation", "typeParameter") + .assertNextToken("Types", "class", "public", "declaration") + .assertNextToken("String", "class", "public", "readonly") + + .assertNextToken("SomeClass", "class", "generic") + .assertNextToken("String", "class", "public", "readonly", "typeArgument") + .assertNextToken("SomeClass", "class", "generic", "typeArgument") + .assertNextToken("String", "class", "public", "readonly", "typeArgument") + .assertNextToken("Integer", "class", "public", "readonly", "typeArgument") + + .assertNextToken("SomeClass", "class", "generic", "declaration") + .assertNextToken("T1", "typeParameter", "declaration") + .assertNextToken("T2", "typeParameter", "declaration") + .assertNextToken("T1", "typeParameter") + .assertNextToken("T2", "typeParameter") + + .assertNextToken("SomeInterface", "interface", "static", "declaration") + .assertNextToken("SomeEnum", "enum", "static", "readonly", "declaration") + .assertNextToken("SomeAnnotation", "annotation", "static", "declaration") + .endAssertion(); } @Test public void testSemanticTokens_Packages() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Packages.java"); - - assertToken(decodedTokens, 2, 14, 4, "namespace"); - assertToken(decodedTokens, 2, 19, 4, "namespace"); - assertToken(decodedTokens, 3, 7, 4, "namespace"); - assertToken(decodedTokens, 3, 12, 4, "namespace"); - assertToken(decodedTokens, 4, 7, 4, "namespace"); - assertToken(decodedTokens, 5, 7, 4, "namespace"); - assertToken(decodedTokens, 5, 12, 3, "namespace"); - assertToken(decodedTokens, 6, 7, 4, "namespace"); - assertToken(decodedTokens, 7, 7, 4, "namespace"); - assertToken(decodedTokens, 7, 12, 4, "namespace"); - assertToken(decodedTokens, 7, 17, 4, "class", "public", "readonly"); - assertToken(decodedTokens, 8, 14, 4, "namespace"); - assertToken(decodedTokens, 8, 19, 4, "namespace"); - assertToken(decodedTokens, 8, 24, 4, "class", "public", "readonly"); + TokenAssertionHelper.beginAssertion(getURI("Packages.java"), "namespace", "class", "property") + .assertNextToken("foo", "namespace") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("lang", "namespace", "importDeclaration") + .assertNextToken("Math", "class", "public", "readonly", "importDeclaration") + .assertNextToken("PI", "property", "public", "static", "readonly", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("util", "namespace", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("NonExistentClass", "class", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("nio", "namespace", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("lang", "namespace", "importDeclaration") + .assertNextToken("Math", "class", "public", "readonly", "importDeclaration") + + .assertNextToken("java", "namespace", "importDeclaration") + .assertNextToken("lang", "namespace", "importDeclaration") + .assertNextToken("Math", "class", "public", "readonly", "importDeclaration") + + .assertNextToken("Packages", "class", "public", "declaration") + + .assertNextToken("java", "namespace") + .assertNextToken("lang", "namespace") + .assertNextToken("String", "class", "public", "readonly") + .assertNextToken("string", "property", "public", "declaration") + .assertNextToken("java", "namespace") + .assertNextToken("lang", "namespace") + .endAssertion(); } @Test public void testSemanticTokens_Annotations() throws JavaModelException { - Map> decodedTokens = decodeSourceFile("Annotations.java"); + TokenAssertionHelper.beginAssertion(getURI("Annotations.java"), "annotation", "annotationMember") + .assertNextToken("SuppressWarnings", "annotation", "public") + .assertNextToken("SuppressWarnings", "annotation", "public") + .assertNextToken("SuppressWarnings", "annotation", "public") + .assertNextToken("value", "annotationMember", "public", "abstract") + .endAssertion(); + } - assertToken(decodedTokens, 4, 2, 16, "annotation", "public"); - assertToken(decodedTokens, 7, 2, 16, "annotation", "public"); - assertToken(decodedTokens, 10, 2, 16, "annotation", "public"); - assertToken(decodedTokens, 10, 19, 5, "annotationMember", "public", "abstract"); + private String getURI(String compilationUnitName) { + return JDTUtils.toURI(fooPackage.getCompilationUnit(compilationUnitName)); } - private void assertModifiers(List tokenModifiers, int encodedModifiers, List modifierStrings) { - int cnt = 0; - for (int i=0; i tokenTypeFilter; + + private TokenAssertionHelper(IBuffer buffer, int[] semanticTokensData, List tokenTypeFilter) { + this.buffer = buffer; + this.semanticTokensData = semanticTokensData; + this.tokenTypeFilter = tokenTypeFilter; } - assertEquals(cnt, modifierStrings.size()); - } - private void assertToken(Map> decodedTokens, int line, int column, int length, String tokenTypeString, String... modifierStrings) { - Map tokensOfTheLine = decodedTokens.get(line); - assertNotNull(tokensOfTheLine); - int[] token = tokensOfTheLine.get(column); // 0: length, 1: typeIndex, 2: encodedModifiers - assertNotNull(token); - assertEquals(length, token[0]); - assertEquals(tokenTypeString, legend.getTokenTypes().get(token[1])); - assertModifiers(legend.getTokenModifiers(), token[2], Arrays.asList(modifierStrings)); - } + /** + * Begins an assertion for semantic tokens (calling {@link SemanticTokensCommand#provide(String)}), + * optionally providing a filter describing which token types to assert. + * + * @param uri The URI to assert provided semantic tokens for. + * @param tokenTypeFilter Specifies the type of semantic tokens to assert. Only token types + * matching the filter will be asserted. If the filter is empty, all tokens will be asserted. + * @return A new instace of {@link TokenAssertionHelper}. + * @throws JavaModelException + */ + public static TokenAssertionHelper beginAssertion(String uri, String... tokenTypeFilter) throws JavaModelException { + SemanticTokens semanticTokens = SemanticTokensCommand.provide(uri); + assertNotNull("Provided semantic tokens should not be null", semanticTokens); + assertNotNull("Semantic tokens data should not be null", semanticTokens.getData()); + assertTrue("Semantic tokens data should contain 5 integers per token", semanticTokens.getData().length % 5 == 0); + return new TokenAssertionHelper(JDTUtils.resolveTypeRoot(uri).getBuffer(), semanticTokens.getData(), Arrays.asList(tokenTypeFilter)); + } - private Map> decodeSourceFile(String name) { - ICompilationUnit cu = fooPackage.getCompilationUnit(name); - SemanticTokens tokens = SemanticTokensCommand.provide(JDTUtils.toURI(cu)); - return decode(tokens); - } + /** + * Asserts the next semantic token in the data provided by {@link SemanticTokensCommand}. + * + * @param expectedText The expected text at the location of the next semantic token. + * @param expectedType The expected type of the next semantic token. + * @param expectedModifiers The expected modifiers of the next semantic token. + * @return Itself. + */ + public TokenAssertionHelper assertNextToken(String expectedText, String expectedType, String... expectedModifiers) { + assertTrue("Token of type '" + expectedType + "' should be present in the semantic tokens data", + currentDataIndex < semanticTokensData.length); + + int deltaLine = semanticTokensData[currentDataIndex]; + int deltaColumn = semanticTokensData[currentDataIndex + 1]; + int length = semanticTokensData[currentDataIndex + 2]; + int typeIndex = semanticTokensData[currentDataIndex + 3]; + int encodedModifiers = semanticTokensData[currentDataIndex + 4]; - private Map> decode(SemanticTokens tokens) { - List data = tokens.getData(); - int total = data.size() / 5; - Map> decodedTokens = new HashMap<>(); - int currentLine = 0; - int currentColumn = 0; - for(int i = 0; i 0) { + assertTrue("Token deltaLine should not be negative", deltaLine >= 0); + assertTrue("Token deltaColumn should not be negative", deltaColumn >= 0); + assertTrue("Token length should be greater than zero", length > 0); + + if (deltaLine == 0) { + currentColumn += deltaColumn; + } + else { currentLine += deltaLine; currentColumn = deltaColumn; - } else { - currentColumn += deltaColumn; } - Map tokensOfTheLine = decodedTokens.getOrDefault(currentLine, new HashMap<>()); - tokensOfTheLine.put(currentColumn, new int[] {length, typeIndex, encodedModifiers}); - decodedTokens.putIfAbsent(currentLine, tokensOfTheLine); + + currentDataIndex += 5; + + if (tokenTypeFilter.isEmpty() || tokenTypeFilter.contains(LEGEND.getTokenTypes()[typeIndex])) { + assertTextMatchInBuffer(length, expectedText); + assertTokenType(typeIndex, expectedType); + assertTokenModifiers(encodedModifiers, Arrays.asList(expectedModifiers)); + + return this; + } + else { + return assertNextToken(expectedText, expectedType, expectedModifiers); + } + } + + /** + * Asserts that there are no more unexpected semantic tokens present in the data + * provided by {@link SemanticTokensCommand}. + */ + public void endAssertion() { + if (tokenTypeFilter.isEmpty()) { + assertTrue("There should be no more tokens", currentDataIndex == semanticTokensData.length); + } + else { + while (currentDataIndex < semanticTokensData.length) { + int currentTypeIndex = semanticTokensData[currentDataIndex + 3]; + String currentType = LEGEND.getTokenTypes()[currentTypeIndex]; + assertFalse( + "There should be no more tokens matching the filter, but found '" + currentType + "' token", + tokenTypeFilter.contains(currentType) + ); + currentDataIndex += 5; + } + } + } + + private void assertTextMatchInBuffer(int length, String expectedText) { + String tokenTextInBuffer = buffer.getText(JsonRpcHelpers.toOffset(buffer, currentLine, currentColumn), length); + assertEquals("Token text should match the token text range in the buffer.", expectedText, tokenTextInBuffer); + } + + private void assertTokenType(int typeIndex, String expectedType) { + assertEquals("Token type should be correct.", expectedType, LEGEND.getTokenTypes()[typeIndex]); + } + + private void assertTokenModifiers(int encodedModifiers, List expectedModifiers) { + for (int i = 0; i < LEGEND.getTokenModifiers().length; i++) { + String modifier = LEGEND.getTokenModifiers()[i]; + boolean modifierIsEncoded = ((encodedModifiers >>> i) & 1) == 1; + boolean modifierIsExpected = expectedModifiers.contains(modifier); + + assertTrue( + modifierIsExpected + ? "Expected modifier '" + modifier + "' to be encoded" + : "Did not expect modifier '" + modifier + "' to be encoded", + modifierIsExpected == modifierIsEncoded + ); + } } - return decodedTokens; } }