[vscode] register grammars only when all languages are registered

.gitpod.yml

@@ -19,3 +19,6 @@ tasks:
 github:
   prebuilds:
     pullRequestsFromForks: true
+vscode:
+  extensions:
+    - [email protected]:CwAMx4wYz1Kq39+1Aul4VQ==

configs/errors.eslintrc.json

@@ -2,7 +2,7 @@
   "rules": {
     "@typescript-eslint/class-name-casing": "error",
     "@typescript-eslint/consistent-type-definitions": "error",
-    "@typescript-eslint/indent": "error",
+    "@typescript-eslint/indent": "off",
     "@typescript-eslint/no-explicit-any": "error",
     "@typescript-eslint/quotes": [
       "error",

examples/api-tests/src/monaco-api.spec.js

@@ -25,12 +25,13 @@ describe('Monaco API', async function () {
     const { WorkspaceService } = require('@theia/workspace/lib/browser/workspace-service');
     const { MonacoEditor } = require('@theia/monaco/lib/browser/monaco-editor');
     const { MonacoResolvedKeybinding } = require('@theia/monaco/lib/browser/monaco-resolved-keybinding');
+    const { MonacoTextmateService } = require('@theia/monaco/lib/browser/textmate/monaco-textmate-service');
 
     /** @type {import('inversify').Container} */
     const container = window['theia'].container;
     const editorManager = container.get(EditorManager);
     const workspaceService = container.get(WorkspaceService);
-
+    const textmateService = container.get(MonacoTextmateService);
     /** @type {MonacoEditor} */
     let monacoEditor;
 
@@ -87,4 +88,24 @@ describe('Monaco API', async function () {
         }
     });
 
+    it('TokenizationRegistry.getColorMap', async () => {
+        if (textmateService['monacoThemeRegistry'].getThemeData().base !== 'vs') {
+            const didChangeColorMap = new Promise(resolve => {
+                const toDispose = monaco.modes.TokenizationRegistry.onDidChange(() => {
+                    toDispose.dispose();
+                    resolve();
+                })
+            });
+            textmateService['themeService'].setCurrentTheme('light');
+            await didChangeColorMap;
+        }
+
+        const textMateColorMap = textmateService['grammarRegistry'].getColorMap();
+        assert.notEqual(textMateColorMap.indexOf('#795E26'), -1, 'Expected custom toke colors for the ligth theme to be enabled.')
+
+        const monacoColorMap = monaco.modes.TokenizationRegistry.getColorMap().
+            splice(0, textMateColorMap.length).map(c => c.toString().toUpperCase());
+        assert.deepStrictEqual(monacoColorMap, textMateColorMap, 'Expected textmate colors to have the same index in the monaco color map.');
+    });
+
 });

packages/monaco/src/browser/monaco-loader.ts

@@ -57,6 +57,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 'vs/editor/browser/editorExtensions',
                 'vs/editor/standalone/browser/simpleServices',
                 'vs/editor/standalone/browser/standaloneServices',
+                'vs/editor/standalone/browser/standaloneLanguages',
                 'vs/base/parts/quickopen/browser/quickOpenWidget',
                 'vs/base/parts/quickopen/browser/quickOpenModel',
                 'vs/base/common/filters',
@@ -77,36 +78,36 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 'vs/base/common/errors'
             ], (css: any, html: any, commands: any, actions: any,
                 keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any,
-                keyCodes: any, mime: any, editorExtensions: any, simpleServices: any, standaloneServices: any, quickOpenWidget: any, quickOpenModel: any,
+                keyCodes: any, mime: any, editorExtensions: any, simpleServices: any,
+                standaloneServices: any, standaloneLanguages: any, quickOpenWidget: any, quickOpenModel: any,
                 filters: any, styler: any, colorRegistry: any, color: any,
                 platform: any, modes: any, suggest: any, snippetParser: any,
                 configuration: any, configurationModels: any,
                 codeEditorService: any, codeEditorServiceImpl: any,
                 markerService: any,
                 contextKey: any, contextKeyService: any,
-                error: any
-            ) => {
-                const global: any = self;
-                global.monaco.commands = commands;
-                global.monaco.actions = actions;
-                global.monaco.keybindings = Object.assign({}, keybindingsRegistry, keybindingResolver, resolvedKeybinding, keybindingLabels, keyCodes);
-                global.monaco.services = Object.assign({}, simpleServices, standaloneServices, configuration, configurationModels,
-                    codeEditorService, codeEditorServiceImpl, markerService);
-                global.monaco.quickOpen = Object.assign({}, quickOpenWidget, quickOpenModel);
-                global.monaco.filters = filters;
-                global.monaco.theme = styler;
-                global.monaco.color = Object.assign({}, colorRegistry, color);
-                global.monaco.platform = platform;
-                global.monaco.editorExtensions = editorExtensions;
-                global.monaco.modes = modes;
-                global.monaco.suggest = suggest;
-                global.monaco.snippetParser = snippetParser;
-                global.monaco.contextkey = contextKey;
-                global.monaco.contextKeyService = contextKeyService;
-                global.monaco.mime = mime;
-                global.monaco.error = error;
-                resolve();
-            });
+                error: any) => {
+                    const global: any = self;
+                    global.monaco.commands = commands;
+                    global.monaco.actions = actions;
+                    global.monaco.keybindings = Object.assign({}, keybindingsRegistry, keybindingResolver, resolvedKeybinding, keybindingLabels, keyCodes);
+                    global.monaco.services = Object.assign({}, simpleServices, standaloneServices, standaloneLanguages, configuration, configurationModels,
+                        codeEditorService, codeEditorServiceImpl, markerService);
+                    global.monaco.quickOpen = Object.assign({}, quickOpenWidget, quickOpenModel);
+                    global.monaco.filters = filters;
+                    global.monaco.theme = styler;
+                    global.monaco.color = Object.assign({}, colorRegistry, color);
+                    global.monaco.platform = platform;
+                    global.monaco.editorExtensions = editorExtensions;
+                    global.monaco.modes = modes;
+                    global.monaco.suggest = suggest;
+                    global.monaco.snippetParser = snippetParser;
+                    global.monaco.contextkey = contextKey;
+                    global.monaco.contextKeyService = contextKeyService;
+                    global.monaco.mime = mime;
+                    global.monaco.error = error;
+                    resolve();
+                });
         });
     });
 }

packages/monaco/src/browser/textmate/monaco-textmate-service.ts

@@ -74,7 +74,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
 
         this.grammarRegistry = new Registry({
             getOnigLib: () => this.onigasmPromise,
-            theme: this.monacoThemeRegistry.getTheme(this.currentEditorTheme),
+            theme: this.monacoThemeRegistry.getThemeData(this.currentEditorTheme),
             loadGrammar: async (scopeName: string) => {
                 const provider = this.textmateRegistry.getProvider(scopeName);
                 if (provider) {
@@ -116,7 +116,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
         this.toDisposeOnUpdateTheme.push(Disposable.create(() => document.body.classList.remove(currentEditorTheme)));
 
         // first update registry to run tokenization with the proper theme
-        const theme = this.monacoThemeRegistry.getTheme(currentEditorTheme);
+        const theme = this.monacoThemeRegistry.getThemeData(currentEditorTheme);
         if (theme) {
             this.grammarRegistry.setTheme(theme);
         }
@@ -154,7 +154,13 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
                     throw new Error(`no grammar for ${scopeName}, ${initialLanguage}, ${JSON.stringify(configuration)}`);
                 }
                 const options = configuration.tokenizerOption ? configuration.tokenizerOption : TokenizerOption.DEFAULT;
-                monaco.languages.setTokensProvider(languageId, createTextmateTokenizer(grammar, options));
+                const tokenizer = createTextmateTokenizer(grammar, options);
+                monaco.languages.setTokensProvider(languageId, tokenizer);
+                const support = monaco.modes.TokenizationRegistry.get(languageId);
+                const themeService = monaco.services.StaticServices.standaloneThemeService.get();
+                const languageIdentifier = monaco.services.StaticServices.modeService.get().getLanguageIdentifier(languageId);
+                const adapter = new monaco.services.TokenizationSupport2Adapter(themeService, languageIdentifier!, tokenizer);
+                support!.tokenize = adapter.tokenize.bind(adapter);
             } catch (error) {
                 this.logger.warn('No grammar for this language id', languageId, error);
             }

packages/monaco/src/browser/textmate/monaco-theme-registry.ts

@@ -17,22 +17,37 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { IRawTheme, Registry } from 'vscode-textmate';
+import { IRawTheme, Registry, IRawThemeSetting } from 'vscode-textmate';
 
 export interface ThemeMix extends IRawTheme, monaco.editor.IStandaloneThemeData { }
+export interface MixStandaloneTheme extends monaco.services.IStandaloneTheme {
+    themeData: ThemeMix
+}
 
 export class MonacoThemeRegistry {
 
-    protected themes = new Map<string, ThemeMix>();
+    getThemeData(): ThemeMix;
+    getThemeData(name: string): ThemeMix | undefined;
+    getThemeData(name?: string): ThemeMix | undefined {
+        const theme = this.doGetTheme(name);
+        return theme && theme.themeData;
+    }
+
+    getTheme(): MixStandaloneTheme;
+    getTheme(name: string): MixStandaloneTheme | undefined;
+    getTheme(name?: string): MixStandaloneTheme | undefined {
+        return this.doGetTheme(name);
+    }
 
-    public getTheme(name: string): IRawTheme | undefined {
-        return this.themes.get(name);
+    protected doGetTheme(name: string | undefined): MixStandaloneTheme | undefined {
+        const standaloneThemeService = monaco.services.StaticServices.standaloneThemeService.get();
+        const theme = !name ? standaloneThemeService.getTheme() : standaloneThemeService._knownThemes.get(name);
+        return theme as MixStandaloneTheme | undefined;
     }
 
     setTheme(name: string, data: ThemeMix): void {
         // monaco auto refrehes a theme with new data
         monaco.editor.defineTheme(name, data);
-        this.themes.set(name, data);
     }
 
     /**
@@ -58,8 +73,20 @@ export class MonacoThemeRegistry {
                 result.settings.push(...parentTheme.settings);
             }
         }
-        if (json.tokenColors) {
-            result.settings.push(...json.tokenColors);
+        const tokenColors: Array<IRawThemeSetting> = json.tokenColors;
+        if (Array.isArray(tokenColors)) {
+            for (const tokenColor of tokenColors) {
+                if (tokenColor.scope && tokenColor.settings) {
+                    result.settings.push({
+                        scope: tokenColor.scope,
+                        settings: {
+                            foreground: this.normalizeColor(tokenColor.settings.foreground),
+                            background: this.normalizeColor(tokenColor.settings.background),
+                            fontStyle: tokenColor.settings.fontStyle
+                        }
+                    });
+                }
+            }
         }
         if (json.colors) {
             Object.assign(result.colors, json.colors);
@@ -95,22 +122,25 @@ export class MonacoThemeRegistry {
         }
 
         for (const scope of tokenColor.scope) {
-
-            // Converting numbers into a format that monaco understands
-            const settings = Object.keys(tokenColor.settings).reduce((previous: { [key: string]: string }, current) => {
-                let value: string = tokenColor.settings[current];
-                if (typeof value === typeof '') {
-                    value = value.replace(/^\#/, '').slice(0, 6);
-                }
-                previous[current] = value;
-                return previous;
-            }, {});
-
             acceptor({
-                ...settings, token: scope
+                ...tokenColor.settings, token: scope
             });
         }
     }
+
+    protected normalizeColor(color: string | undefined): string | undefined {
+        if (!color) {
+            return undefined;
+        }
+        color = color.replace(/^\#/, '').slice(0, 6);
+        if (color.length < 6) {
+            // ignoring not normalized colors to avoid breaking token color indexes between monaco and vscode-textmate
+            console.error(`Color '${color}' is NOT normalized, it must have 6 positions.`);
+            return undefined;
+        }
+        return '#' + color;
+    }
+
 }
 
 export namespace MonacoThemeRegistry {

packages/monaco/src/browser/textmate/textmate-tokenizer.ts

@@ -57,7 +57,7 @@ export namespace TokenizerOption {
     };
 }
 
-export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider {
+export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider {
     if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
         throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
     }
@@ -74,6 +74,21 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
                 endState: new TokenizerState(result.ruleStack),
                 tokens: result.tokens
             };
+        },
+        tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens {
+            let processedLine = line;
+            if (options.lineLimit !== undefined && line.length > options.lineLimit) {
+                // Line is too long to be tokenized
+                processedLine = line.substr(0, options.lineLimit);
+            }
+            const result = grammar.tokenizeLine(processedLine, state.ruleStack);
+            return {
+                endState: new TokenizerState(result.ruleStack),
+                tokens: result.tokens.map(t => ({
+                    startIndex: t.startIndex,
+                    scopes: t.scopes.reverse().join(' ')
+                }))
+            };
         }
     };
 }

packages/monaco/src/typings/monaco/index.d.ts

@@ -438,6 +438,11 @@ declare module monaco.keybindings {
 
 declare module monaco.services {
 
+    export class TokenizationSupport2Adapter implements monaco.modes.ITokenizationSupport {
+        constructor(standaloneThemeService: IStandaloneThemeService, languageIdentifier: LanguageIdentifier, actual: monaco.languages.TokensProvider)
+        tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any;
+    }
+
     export const ICodeEditorService: any;
     export const IConfigurationService: any;
 
@@ -481,10 +486,12 @@ declare module monaco.services {
     }
 
     export interface IStandaloneThemeService extends monaco.theme.IThemeService {
+        readonly _knownThemes: Map<string, IStandaloneTheme>;
         getTheme(): IStandaloneTheme;
     }
 
     export interface IStandaloneTheme {
+        themeData: monaco.editor.IStandaloneThemeData
         tokenTheme: TokenTheme;
         getColor(color: string): Color | undefined;
     }
@@ -511,7 +518,8 @@ declare module monaco.services {
 
     // https://github.com/TypeFox/vscode/blob/70b8db24a37fafc77247de7f7cb5bb0195120ed0/src/vs/editor/common/services/modeService.ts#L30
     export interface IModeService {
-        createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection
+        createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection;
+        getLanguageIdentifier(modeId: string): LanguageIdentifier | null;
     }
 
     export interface ILanguageSelection {
@@ -847,6 +855,16 @@ declare module monaco.editorExtensions {
 }
 declare module monaco.modes {
 
+    export interface ITokenizationSupport {
+        tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any;
+    }
+    export interface TokenizationRegistry {
+        get(language: string): ITokenizationSupport | null;
+        getColorMap(): monaco.color.Color[] | null;
+        readonly onDidChange: monaco.IEvent<any>;
+    }
+    export const TokenizationRegistry: TokenizationRegistry;
+
     export class TokenMetadata {
 
         public static getLanguageId(metadata: number): number;