Skip to content

Commit

Permalink
Merge pull request #93378 from microsoft/aeschli/tokenLanguage
Browse files Browse the repository at this point in the history 
language in token
  • Loading branch information
@aeschli
aeschli authored Mar 26, 2020
2 parents 2c4c755 + 0cf78e1 commit 55646f7
Show file tree
Hide file tree
Showing 10 changed files with 171 additions and 82 deletions.
33 changes: 19 additions & 14 deletions src/vs/editor/common/services/modelServiceImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,12 +661,14 @@ const enum Constants {
class HashTableEntry {
public readonly tokenTypeIndex: number;
public readonly tokenModifierSet: number;
public readonly languageId: number;
public readonly metadata: number;
public next: HashTableEntry | null;

constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
this.tokenTypeIndex = tokenTypeIndex;
this.tokenModifierSet = tokenModifierSet;
this.languageId = languageId;
this.metadata = metadata;
this.next = null;
}
Expand Down Expand Up @@ -697,16 +699,17 @@ class HashTable {
}
}

private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
const hash = (n1: number, n2: number) => (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
return hash(hash(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
}

public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);

let p = this._elements[hash];
while (p) {
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
return p;
}
p = p.next;
Expand All @@ -715,7 +718,7 @@ class HashTable {
return null;
}

public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
this._elementsCount++;
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
// expand!
Expand All @@ -737,11 +740,11 @@ class HashTable {
}
}
}
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
}

private _add(element: HashTableEntry): void {
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
element.next = this._elements[hash];
this._elements[hash] = element;
}
Expand All @@ -759,8 +762,8 @@ class SemanticColoringProviderStyling {
this._hashTable = new HashTable();
}

public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: LanguageIdentifier): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, languageId.id);
let metadata: number;
if (entry) {
metadata = entry.metadata;
Expand All @@ -775,7 +778,7 @@ class SemanticColoringProviderStyling {
modifierSet = modifierSet >> 1;
}

const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
if (typeof tokenStyle === 'undefined') {
metadata = Constants.NO_STYLING;
} else {
Expand All @@ -801,7 +804,7 @@ class SemanticColoringProviderStyling {
metadata = Constants.NO_STYLING;
}
}
this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
}
if (this._logService.getLevel() === LogLevel.Trace) {
const type = this._legend.tokenTypes[tokenTypeIndex];
Expand Down Expand Up @@ -1042,6 +1045,8 @@ class ModelSemanticColoring extends Disposable {

const result: MultilineTokens2[] = [];

const languageId = this._model.getLanguageIdentifier();

let tokenIndex = 0;
let lastLineNumber = 1;
let lastStartCharacter = 0;
Expand Down Expand Up @@ -1081,7 +1086,7 @@ class ModelSemanticColoring extends Disposable {
const length = srcData[srcOffset + 2];
const tokenTypeIndex = srcData[srcOffset + 3];
const tokenModifierSet = srcData[srcOffset + 4];
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);

if (metadata !== Constants.NO_STYLING) {
if (areaLine === 0) {
Expand Down
2 changes: 1 addition & 1 deletion src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ class StandaloneTheme implements IStandaloneTheme {
return this._tokenTheme;
}

public getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
public getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
return undefined;
}

Expand Down
2 changes: 1 addition & 1 deletion src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ suite('TokenizationSupport2Adapter', () => {
throw new Error('Not implemented');
},

getTokenStyleMetadata: (type: string, modifiers: string[]): ITokenStyle | undefined => {
getTokenStyleMetadata: (type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined => {
return undefined;
},

Expand Down
2 changes: 1 addition & 1 deletion src/vs/platform/theme/common/themeService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ export interface IColorTheme {
/**
* Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
*/
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined;
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined;

/**
* List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.
Expand Down
90 changes: 65 additions & 25 deletions src/vs/platform/theme/common/tokenClassificationRegistry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,21 @@ import { Event, Emitter } from 'vs/base/common/event';
import { IJSONSchema, IJSONSchemaMap } from 'vs/base/common/jsonSchema';

export const TOKEN_TYPE_WILDCARD = '*';
export const TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR = ':';
export const CLASSIFIER_MODIFIER_SEPARATOR = '.';

// qualified string [type|*](.modifier)*
// qualified string [type|*](.modifier)*(/language)!
export type TokenClassificationString = string;

export const typeAndModifierIdPattern = '^\\w+[-_\\w+]*$';
export const idPattern = '\\w+[-_\\w+]*';
export const typeAndModifierIdPattern = `^${idPattern}$`;

export const selectorPattern = `^(${idPattern}|\\*)(\\${CLASSIFIER_MODIFIER_SEPARATOR}${idPattern})*(\\${TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR}${idPattern})?$`;

export const fontStylePattern = '^(\\s*(-?italic|-?bold|-?underline))*\\s*$';

export interface TokenSelector {
match(type: string, modifiers: string[]): number;
match(type: string, modifiers: string[], language: string): number;
readonly selectorString: string;
}

Expand Down Expand Up @@ -269,33 +275,39 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
}

public parseTokenSelector(selectorString: string): TokenSelector {
const [selectorType, ...selectorModifiers] = selectorString.split('.');
const selector = parseClassifierString(selectorString);

if (!selectorType) {
if (!selector.type) {
return {
match: () => -1,
selectorString
};
}

return {
match: (type: string, modifiers: string[]) => {
match: (type: string, modifiers: string[], language: string) => {
let score = 0;
if (selectorType !== TOKEN_TYPE_WILDCARD) {
if (selector.language !== undefined) {
if (selector.language !== language) {
return -1;
}
score += 100;
}
if (selector.type !== TOKEN_TYPE_WILDCARD) {
const hierarchy = this.getTypeHierarchy(type);
const level = hierarchy.indexOf(selectorType);
const level = hierarchy.indexOf(selector.type);
if (level === -1) {
return -1;
}
score = 100 - level;
score += (100 - level);
}
// all selector modifiers must be present
for (const selectorModifier of selectorModifiers) {
for (const selectorModifier of selector.modifiers) {
if (modifiers.indexOf(selectorModifier) === -1) {
return -1;
}
}
return score + selectorModifiers.length * 100;
return score + selector.modifiers.length * 100;
},
selectorString
};
Expand Down Expand Up @@ -366,15 +378,41 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {

}

const CHAR_LANGUAGE = TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR.charCodeAt(0);
const CHAR_MODIFIER = CLASSIFIER_MODIFIER_SEPARATOR.charCodeAt(0);

export function parseClassifierString(s: string): { type: string, modifiers: string[], language: string | undefined; } {
let k = s.length;
let language: string | undefined = undefined;
const modifiers = [];

for (let i = k - 1; i >= 0; i--) {
const ch = s.charCodeAt(i);
if (ch === CHAR_LANGUAGE || ch === CHAR_MODIFIER) {
const segment = s.substring(i + 1, k);
k = i;
if (ch === CHAR_LANGUAGE) {
language = segment;
} else {
modifiers.push(segment);
}
}
}
const type = s.substring(0, k);
return { type, modifiers, language };
}


const tokenClassificationRegistry = new TokenClassificationRegistry();
let tokenClassificationRegistry = createDefaultTokenClassificationRegistry();
platform.Registry.add(Extensions.TokenClassificationContribution, tokenClassificationRegistry);

registerDefaultClassifications();

function registerDefaultClassifications(): void {
function createDefaultTokenClassificationRegistry(): TokenClassificationRegistry {

const registry = new TokenClassificationRegistry();

function registerTokenType(id: string, description: string, scopesToProbe: ProbeScope[] = [], superType?: string, deprecationMessage?: string): string {
tokenClassificationRegistry.registerTokenType(id, description, superType, deprecationMessage);
registry.registerTokenType(id, description, superType, deprecationMessage);
if (scopesToProbe) {
registerTokenStyleDefault(id, scopesToProbe);
}
Expand All @@ -383,8 +421,8 @@ function registerDefaultClassifications(): void {

function registerTokenStyleDefault(selectorString: string, scopesToProbe: ProbeScope[]) {
try {
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString);
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe });
const selector = registry.parseTokenSelector(selectorString);
registry.registerTokenStyleDefault(selector, { scopesToProbe });
} catch (e) {
console.log(e);
}
Expand Down Expand Up @@ -422,18 +460,20 @@ function registerDefaultClassifications(): void {

// default token modifiers

tokenClassificationRegistry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
tokenClassificationRegistry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
tokenClassificationRegistry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
tokenClassificationRegistry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
tokenClassificationRegistry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
tokenClassificationRegistry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
tokenClassificationRegistry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
tokenClassificationRegistry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
registry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
registry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
registry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
registry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
registry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
registry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
registry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
registry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);


registerTokenStyleDefault('variable.readonly', [['variable.other.constant']]);
registerTokenStyleDefault('property.readonly', [['variable.other.constant.property']]);

return registry;
}

export function getTokenClassificationRegistry(): ITokenClassificationRegistry {
Expand Down
2 changes: 1 addition & 1 deletion src/vs/platform/theme/test/common/testThemeService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export class TestColorTheme implements IColorTheme {
throw new Error('Method not implemented.');
}

getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
return undefined;
}

Expand Down
14 changes: 4 additions & 10 deletions src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {

private _getSemanticTokenAtPosition(semanticTokens: SemanticTokensResult, pos: Position): ISemanticTokenInfo | null {
const tokenData = semanticTokens.tokens.data;
const defaultLanguage = this._model.getLanguageIdentifier().language;
let lastLine = 0;
let lastCharacter = 0;
const posLine = pos.lineNumber - 1, posCharacter = pos.column - 1; // to 0-based position
Expand All @@ -511,7 +512,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
const definitions = {};
const colorMap = this._themeService.getColorTheme().tokenColorMap;
const theme = this._themeService.getColorTheme() as ColorThemeData;
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, defaultLanguage, true, definitions);

let metadata: IDecodedMetadata | undefined = undefined;
if (tokenStyle) {
Expand Down Expand Up @@ -556,16 +557,9 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
return `Color theme: ${definition.selector.selectorString} - ${this._renderStyleProperty(definition.style, property)}`;
}
return '';
} else if (typeof definition === 'string') {
const [type, ...modifiers] = definition.split('.');
const definitions: TokenStyleDefinitions = {};
const m = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
if (m && definitions.foreground) {
return this._renderTokenStyleDefinition(definitions[property], property);
}
return '';
} else {
return this._renderStyleProperty(definition, property);
const style = theme.resolveTokenStyleValue(definition);
return `Default: ${style ? this._renderStyleProperty(style, property) : ''}`;
}
}

Expand Down
Loading

0 comments on commit 55646f7

Please sign in to comment.