diff --git a/packages/monaco/src/browser/monaco-loader.ts b/packages/monaco/src/browser/monaco-loader.ts index d21e65b223992..1be82b0f75944 100644 --- a/packages/monaco/src/browser/monaco-loader.ts +++ b/packages/monaco/src/browser/monaco-loader.ts @@ -57,6 +57,7 @@ export function loadMonaco(vsRequire: any): Promise { 'vs/editor/browser/editorExtensions', 'vs/editor/standalone/browser/simpleServices', 'vs/editor/standalone/browser/standaloneServices', + 'vs/editor/standalone/browser/standaloneLanguages', 'vs/base/parts/quickopen/browser/quickOpenWidget', 'vs/base/parts/quickopen/browser/quickOpenModel', 'vs/base/common/filters', @@ -77,36 +78,36 @@ export function loadMonaco(vsRequire: any): Promise { 'vs/base/common/errors' ], (css: any, html: any, commands: any, actions: any, keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any, - keyCodes: any, mime: any, editorExtensions: any, simpleServices: any, standaloneServices: any, quickOpenWidget: any, quickOpenModel: any, + keyCodes: any, mime: any, editorExtensions: any, simpleServices: any, + standaloneServices: any, standaloneLanguages: any, quickOpenWidget: any, quickOpenModel: any, filters: any, styler: any, colorRegistry: any, color: any, platform: any, modes: any, suggest: any, snippetParser: any, configuration: any, configurationModels: any, codeEditorService: any, codeEditorServiceImpl: any, markerService: any, contextKey: any, contextKeyService: any, - error: any - ) => { - const global: any = self; - global.monaco.commands = commands; - global.monaco.actions = actions; - global.monaco.keybindings = Object.assign({}, keybindingsRegistry, keybindingResolver, resolvedKeybinding, keybindingLabels, keyCodes); - global.monaco.services = Object.assign({}, simpleServices, standaloneServices, configuration, configurationModels, - codeEditorService, codeEditorServiceImpl, markerService); - global.monaco.quickOpen = Object.assign({}, quickOpenWidget, quickOpenModel); - global.monaco.filters = filters; - global.monaco.theme = styler; - global.monaco.color = Object.assign({}, colorRegistry, color); - global.monaco.platform = platform; - global.monaco.editorExtensions = editorExtensions; - global.monaco.modes = modes; - global.monaco.suggest = suggest; - global.monaco.snippetParser = snippetParser; - global.monaco.contextkey = contextKey; - global.monaco.contextKeyService = contextKeyService; - global.monaco.mime = mime; - global.monaco.error = error; - resolve(); - }); + error: any) => { + const global: any = self; + global.monaco.commands = commands; + global.monaco.actions = actions; + global.monaco.keybindings = Object.assign({}, keybindingsRegistry, keybindingResolver, resolvedKeybinding, keybindingLabels, keyCodes); + global.monaco.services = Object.assign({}, simpleServices, standaloneServices, standaloneLanguages, configuration, configurationModels, + codeEditorService, codeEditorServiceImpl, markerService); + global.monaco.quickOpen = Object.assign({}, quickOpenWidget, quickOpenModel); + global.monaco.filters = filters; + global.monaco.theme = styler; + global.monaco.color = Object.assign({}, colorRegistry, color); + global.monaco.platform = platform; + global.monaco.editorExtensions = editorExtensions; + global.monaco.modes = modes; + global.monaco.suggest = suggest; + global.monaco.snippetParser = snippetParser; + global.monaco.contextkey = contextKey; + global.monaco.contextKeyService = contextKeyService; + global.monaco.mime = mime; + global.monaco.error = error; + resolve(); + }); }); }); } diff --git a/packages/monaco/src/browser/textmate/monaco-textmate-service.ts b/packages/monaco/src/browser/textmate/monaco-textmate-service.ts index 29839b76abd2d..a48e50fa32b41 100644 --- a/packages/monaco/src/browser/textmate/monaco-textmate-service.ts +++ b/packages/monaco/src/browser/textmate/monaco-textmate-service.ts @@ -154,7 +154,13 @@ export class MonacoTextmateService implements FrontendApplicationContribution { throw new Error(`no grammar for ${scopeName}, ${initialLanguage}, ${JSON.stringify(configuration)}`); } const options = configuration.tokenizerOption ? configuration.tokenizerOption : TokenizerOption.DEFAULT; - monaco.languages.setTokensProvider(languageId, createTextmateTokenizer(grammar, options)); + const tokenizer = createTextmateTokenizer(grammar, options); + monaco.languages.setTokensProvider(languageId, tokenizer); + const support = monaco.modes.TokenizationRegistry.get(languageId); + const themeService = monaco.services.StaticServices.standaloneThemeService.get(); + const languageIdentifier = monaco.services.StaticServices.modeService.get().getLanguageIdentifier(languageId); + const adapter = new monaco.services.TokenizationSupport2Adapter(themeService, languageIdentifier!, tokenizer); + support!.tokenize = adapter.tokenize.bind(adapter); } catch (error) { this.logger.warn('No grammar for this language id', languageId, error); } diff --git a/packages/monaco/src/browser/textmate/textmate-tokenizer.ts b/packages/monaco/src/browser/textmate/textmate-tokenizer.ts index 62b5cc048bec0..1784e05cdf492 100644 --- a/packages/monaco/src/browser/textmate/textmate-tokenizer.ts +++ b/packages/monaco/src/browser/textmate/textmate-tokenizer.ts @@ -57,7 +57,7 @@ export namespace TokenizerOption { }; } -export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider { +export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider { if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) { throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`); } @@ -74,6 +74,21 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt endState: new TokenizerState(result.ruleStack), tokens: result.tokens }; + }, + tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens { + let processedLine = line; + if (options.lineLimit !== undefined && line.length > options.lineLimit) { + // Line is too long to be tokenized + processedLine = line.substr(0, options.lineLimit); + } + const result = grammar.tokenizeLine(processedLine, state.ruleStack); + return { + endState: new TokenizerState(result.ruleStack), + tokens: result.tokens.map(t => ({ + startIndex: t.startIndex, + scopes: t.scopes.reverse().join(' ') + })) + }; } }; } diff --git a/packages/monaco/src/typings/monaco/index.d.ts b/packages/monaco/src/typings/monaco/index.d.ts index f95ba7f3f79a5..a75214c879044 100644 --- a/packages/monaco/src/typings/monaco/index.d.ts +++ b/packages/monaco/src/typings/monaco/index.d.ts @@ -438,6 +438,11 @@ declare module monaco.keybindings { declare module monaco.services { + export class TokenizationSupport2Adapter implements monaco.modes.ITokenizationSupport { + constructor(standaloneThemeService: IStandaloneThemeService, languageIdentifier: LanguageIdentifier, actual: monaco.languages.TokensProvider) + tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any; + } + export const ICodeEditorService: any; export const IConfigurationService: any; @@ -511,7 +516,8 @@ declare module monaco.services { // https://github.com/TypeFox/vscode/blob/70b8db24a37fafc77247de7f7cb5bb0195120ed0/src/vs/editor/common/services/modeService.ts#L30 export interface IModeService { - createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection + createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection; + getLanguageIdentifier(modeId: string): LanguageIdentifier | null; } export interface ILanguageSelection { @@ -847,6 +853,14 @@ declare module monaco.editorExtensions { } declare module monaco.modes { + export interface ITokenizationSupport { + tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any; + } + export interface TokenizationRegistry { + get(language: string): ITokenizationSupport | null; + } + export const TokenizationRegistry: TokenizationRegistry; + export class TokenMetadata { public static getLanguageId(metadata: number): number;