Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

language in token #93378

Merged
merged 1 commit into from
Mar 26, 2020
Merged

language in token #93378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions src/vs/editor/common/services/modelServiceImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,12 +550,14 @@ const enum Constants {
class HashTableEntry {
public readonly tokenTypeIndex: number;
public readonly tokenModifierSet: number;
public readonly languageId: number;
public readonly metadata: number;
public next: HashTableEntry | null;

constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
this.tokenTypeIndex = tokenTypeIndex;
this.tokenModifierSet = tokenModifierSet;
this.languageId = languageId;
this.metadata = metadata;
this.next = null;
}
Expand Down Expand Up @@ -586,16 +588,17 @@ class HashTable {
}
}

private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
const hash = (n1: number, n2: number) => (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
return hash(hash(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
}

public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);

let p = this._elements[hash];
while (p) {
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
return p;
}
p = p.next;
Expand All @@ -604,7 +607,7 @@ class HashTable {
return null;
}

public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
this._elementsCount++;
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
// expand!
Expand All @@ -626,11 +629,11 @@ class HashTable {
}
}
}
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
}

private _add(element: HashTableEntry): void {
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
element.next = this._elements[hash];
this._elements[hash] = element;
}
Expand All @@ -648,8 +651,8 @@ class SemanticColoringProviderStyling {
this._hashTable = new HashTable();
}

public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: LanguageIdentifier): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, languageId.id);
let metadata: number;
if (entry) {
metadata = entry.metadata;
Expand All @@ -664,7 +667,7 @@ class SemanticColoringProviderStyling {
modifierSet = modifierSet >> 1;
}

const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
if (typeof tokenStyle === 'undefined') {
metadata = Constants.NO_STYLING;
} else {
Expand All @@ -690,7 +693,7 @@ class SemanticColoringProviderStyling {
metadata = Constants.NO_STYLING;
}
}
this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
}
if (this._logService.getLevel() === LogLevel.Trace) {
const type = this._legend.tokenTypes[tokenTypeIndex];
Expand Down Expand Up @@ -931,6 +934,8 @@ class ModelSemanticColoring extends Disposable {

const result: MultilineTokens2[] = [];

const languageId = this._model.getLanguageIdentifier();

let tokenIndex = 0;
let lastLineNumber = 1;
let lastStartCharacter = 0;
Expand Down Expand Up @@ -970,7 +975,7 @@ class ModelSemanticColoring extends Disposable {
const length = srcData[srcOffset + 2];
const tokenTypeIndex = srcData[srcOffset + 3];
const tokenModifierSet = srcData[srcOffset + 4];
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);

if (metadata !== Constants.NO_STYLING) {
if (areaLine === 0) {
Expand Down
2 changes: 1 addition & 1 deletion src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ class StandaloneTheme implements IStandaloneTheme {
return this._tokenTheme;
}

public getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
public getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
return undefined;
}

Expand Down
2 changes: 1 addition & 1 deletion src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ suite('TokenizationSupport2Adapter', () => {
throw new Error('Not implemented');
},

getTokenStyleMetadata: (type: string, modifiers: string[]): ITokenStyle | undefined => {
getTokenStyleMetadata: (type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined => {
return undefined;
},

Expand Down
2 changes: 1 addition & 1 deletion src/vs/platform/theme/common/themeService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ export interface IColorTheme {
/**
* Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
*/
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined;
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined;

/**
* List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.
Expand Down
90 changes: 65 additions & 25 deletions src/vs/platform/theme/common/tokenClassificationRegistry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,21 @@ import { Event, Emitter } from 'vs/base/common/event';
import { IJSONSchema, IJSONSchemaMap } from 'vs/base/common/jsonSchema';

export const TOKEN_TYPE_WILDCARD = '*';
export const TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR = ':';
export const CLASSIFIER_MODIFIER_SEPARATOR = '.';

// qualified string [type|*](.modifier)*
// qualified string [type|*](.modifier)*(/language)!
export type TokenClassificationString = string;

export const typeAndModifierIdPattern = '^\\w+[-_\\w+]*$';
export const idPattern = '\\w+[-_\\w+]*';
export const typeAndModifierIdPattern = `^${idPattern}$`;

export const selectorPattern = `^(${idPattern}|\\*)(\\${CLASSIFIER_MODIFIER_SEPARATOR}${idPattern})*(\\${TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR}${idPattern})?$`;

export const fontStylePattern = '^(\\s*(-?italic|-?bold|-?underline))*\\s*$';

export interface TokenSelector {
match(type: string, modifiers: string[]): number;
match(type: string, modifiers: string[], language: string): number;
readonly selectorString: string;
}

Expand Down Expand Up @@ -269,33 +275,39 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
}

public parseTokenSelector(selectorString: string): TokenSelector {
const [selectorType, ...selectorModifiers] = selectorString.split('.');
const selector = parseClassifierString(selectorString);

if (!selectorType) {
if (!selector.type) {
return {
match: () => -1,
selectorString
};
}

return {
match: (type: string, modifiers: string[]) => {
match: (type: string, modifiers: string[], language: string) => {
let score = 0;
if (selectorType !== TOKEN_TYPE_WILDCARD) {
if (selector.language !== undefined) {
if (selector.language !== language) {
return -1;
}
score += 100;
}
if (selector.type !== TOKEN_TYPE_WILDCARD) {
const hierarchy = this.getTypeHierarchy(type);
const level = hierarchy.indexOf(selectorType);
const level = hierarchy.indexOf(selector.type);
if (level === -1) {
return -1;
}
score = 100 - level;
score += (100 - level);
}
// all selector modifiers must be present
for (const selectorModifier of selectorModifiers) {
for (const selectorModifier of selector.modifiers) {
if (modifiers.indexOf(selectorModifier) === -1) {
return -1;
}
}
return score + selectorModifiers.length * 100;
return score + selector.modifiers.length * 100;
},
selectorString
};
Expand Down Expand Up @@ -366,15 +378,41 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {

}

const CHAR_LANGUAGE = TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR.charCodeAt(0);
const CHAR_MODIFIER = CLASSIFIER_MODIFIER_SEPARATOR.charCodeAt(0);

export function parseClassifierString(s: string): { type: string, modifiers: string[], language: string | undefined; } {
let k = s.length;
let language: string | undefined = undefined;
const modifiers = [];

for (let i = k - 1; i >= 0; i--) {
const ch = s.charCodeAt(i);
if (ch === CHAR_LANGUAGE || ch === CHAR_MODIFIER) {
const segment = s.substring(i + 1, k);
k = i;
if (ch === CHAR_LANGUAGE) {
language = segment;
} else {
modifiers.push(segment);
}
}
}
const type = s.substring(0, k);
return { type, modifiers, language };
}


const tokenClassificationRegistry = new TokenClassificationRegistry();
let tokenClassificationRegistry = createDefaultTokenClassificationRegistry();
platform.Registry.add(Extensions.TokenClassificationContribution, tokenClassificationRegistry);

registerDefaultClassifications();

function registerDefaultClassifications(): void {
function createDefaultTokenClassificationRegistry(): TokenClassificationRegistry {

const registry = new TokenClassificationRegistry();

function registerTokenType(id: string, description: string, scopesToProbe: ProbeScope[] = [], superType?: string, deprecationMessage?: string): string {
tokenClassificationRegistry.registerTokenType(id, description, superType, deprecationMessage);
registry.registerTokenType(id, description, superType, deprecationMessage);
if (scopesToProbe) {
registerTokenStyleDefault(id, scopesToProbe);
}
Expand All @@ -383,8 +421,8 @@ function registerDefaultClassifications(): void {

function registerTokenStyleDefault(selectorString: string, scopesToProbe: ProbeScope[]) {
try {
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString);
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe });
const selector = registry.parseTokenSelector(selectorString);
registry.registerTokenStyleDefault(selector, { scopesToProbe });
} catch (e) {
console.log(e);
}
Expand Down Expand Up @@ -422,18 +460,20 @@ function registerDefaultClassifications(): void {

// default token modifiers

tokenClassificationRegistry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
tokenClassificationRegistry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
tokenClassificationRegistry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
tokenClassificationRegistry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
tokenClassificationRegistry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
tokenClassificationRegistry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
tokenClassificationRegistry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
tokenClassificationRegistry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
registry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
registry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
registry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
registry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
registry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
registry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
registry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
registry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);


registerTokenStyleDefault('variable.readonly', [['variable.other.constant']]);
registerTokenStyleDefault('property.readonly', [['variable.other.constant.property']]);

return registry;
}

export function getTokenClassificationRegistry(): ITokenClassificationRegistry {
Expand Down
2 changes: 1 addition & 1 deletion src/vs/platform/theme/test/common/testThemeService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export class TestColorTheme implements IColorTheme {
throw new Error('Method not implemented.');
}

getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
return undefined;
}

Expand Down
14 changes: 4 additions & 10 deletions src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {

private _getSemanticTokenAtPosition(semanticTokens: SemanticTokensResult, pos: Position): ISemanticTokenInfo | null {
const tokenData = semanticTokens.tokens.data;
const defaultLanguage = this._model.getLanguageIdentifier().language;
let lastLine = 0;
let lastCharacter = 0;
const posLine = pos.lineNumber - 1, posCharacter = pos.column - 1; // to 0-based position
Expand All @@ -505,7 +506,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
const definitions = {};
const colorMap = this._themeService.getColorTheme().tokenColorMap;
const theme = this._themeService.getColorTheme() as ColorThemeData;
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, defaultLanguage, true, definitions);

let metadata: IDecodedMetadata | undefined = undefined;
if (tokenStyle) {
Expand Down Expand Up @@ -550,16 +551,9 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
return `Color theme: ${definition.selector.selectorString} - ${this._renderStyleProperty(definition.style, property)}`;
}
return '';
} else if (typeof definition === 'string') {
const [type, ...modifiers] = definition.split('.');
const definitions: TokenStyleDefinitions = {};
const m = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
if (m && definitions.foreground) {
return this._renderTokenStyleDefinition(definitions[property], property);
}
return '';
} else {
return this._renderStyleProperty(definition, property);
const style = theme.resolveTokenStyleValue(definition);
return `Default: ${style ? this._renderStyleProperty(style, property) : ''}`;
}
}

Expand Down
Loading