Skip to content

Commit

Permalink
fix(monaco): options for tokenize limit (#657)
Browse files Browse the repository at this point in the history
Co-authored-by: Anthony Fu <[email protected]>
Co-authored-by: Anthony Fu <[email protected]>
  • Loading branch information
3 people authored Apr 30, 2024
1 parent 86d5271 commit a606d44
Showing 1 changed file with 23 additions and 5 deletions.
28 changes: 23 additions & 5 deletions packages/monaco/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,21 @@ import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate'

export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {}

export interface ShikiToMonacoOptions {
/**
* The maximum length of a line to tokenize.
*
* @default 20000
*/
tokenizeMaxLineLength?: number
/**
* The time limit in milliseconds for tokenizing a line.
*
* @default 500
*/
tokenizeTimeLimit?: number
}

export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): MonacoTheme {
let rules = 'rules' in theme
? theme.rules as MonacoTheme['rules']
Expand Down Expand Up @@ -42,6 +57,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo
export function shikiToMonaco(
highlighter: ShikiInternal<any, any>,
monaco: typeof monacoNs,
options: ShikiToMonacoOptions = {},
) {
// Convert themes to Monaco themes and register them
const themeMap = new Map<string, MonacoTheme>()
Expand Down Expand Up @@ -82,6 +98,13 @@ export function shikiToMonaco(
return colorToScopeMap.get(color)
}

// Do not attempt to tokenize if a line is too long
// default to 20000 (as in monaco-editor-core defaults)
const {
tokenizeMaxLineLength = 20000,
tokenizeTimeLimit = 500,
} = options

const monacoLanguageIds = new Set(monaco.languages.getLanguages().map(l => l.id))
for (const lang of highlighter.getLoadedLanguages()) {
if (monacoLanguageIds.has(lang)) {
Expand All @@ -90,11 +113,6 @@ export function shikiToMonaco(
return new TokenizerState(INITIAL)
},
tokenize(line, state: TokenizerState) {
// Do not attempt to tokenize if a line is too long
// default to 20000 (as in monaco-editor-core defaults)
const tokenizeMaxLineLength = 20000
const tokenizeTimeLimit = 500

if (line.length >= tokenizeMaxLineLength) {
return {
endState: state,
Expand Down

0 comments on commit a606d44

Please sign in to comment.