Colorization tests for tree sitter (#229517)

* Colorization tests for tree sitter
Part of #210475

* Fix tests

* Actually fix tests

* Fix line endings

* Register preferTreeSitter setting

.eslintrc.json

@@ -953,6 +953,7 @@
 							"vs/workbench/contrib/*/~",
 							"vs/workbench/contrib/terminal/terminalContribExports*",
 							"vscode-notebook-renderer", // Type only import
+							"@vscode/tree-sitter-wasm", // type import
 							{
 								"when": "hasBrowser",
 								"pattern": "@xterm/xterm"

extensions/vscode-colorize-tests/package.json

@@ -14,7 +14,9 @@
   },
   "icon": "media/icon.png",
   "scripts": {
-    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-tests ./tsconfig.json"
+    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-tests ./tsconfig.json",
+    "watch": "gulp watch-extension:vscode-colorize-tests",
+    "compile": "gulp compile-extension:vscode-colorize-tests"
   },
   "dependencies": {
     "jsonc-parser": "^3.2.0"

extensions/vscode-colorize-tests/src/colorizer.test.ts

@@ -7,17 +7,20 @@ import * as assert from 'assert';
 import * as fs from 'fs';
 import 'mocha';
 import { join, normalize } from 'path';
-import { commands, Uri } from 'vscode';
+import { commands, Uri, workspace, ConfigurationTarget } from 'vscode';
 
-function assertUnchangedTokens(fixturesPath: string, resultsPath: string, fixture: string, done: any) {
+async function assertUnchangedTokens(fixturesPath: string, resultsPath: string, treeSitterResultsPath: string, fixture: string, done: any) {
 	const testFixurePath = join(fixturesPath, fixture);
+	const tokenizers = [{ command: '_workbench.captureSyntaxTokens', resultsPath }, { command: '_workbench.captureTreeSitterSyntaxTokens', resultsPath: treeSitterResultsPath }];
 
-	return commands.executeCommand('_workbench.captureSyntaxTokens', Uri.file(testFixurePath)).then(data => {
-		try {
-			if (!fs.existsSync(resultsPath)) {
-				fs.mkdirSync(resultsPath);
+	try {
+		await Promise.all(tokenizers.map(async (tokenizer) => {
+			const data = await commands.executeCommand(tokenizer.command, Uri.file(testFixurePath));
+
+			if (!fs.existsSync(tokenizer.resultsPath)) {
+				fs.mkdirSync(tokenizer.resultsPath);
 			}
-			const resultPath = join(resultsPath, fixture.replace('.', '_') + '.json');
+			const resultPath = join(tokenizer.resultsPath, fixture.replace('.', '_') + '.json');
 			if (fs.existsSync(resultPath)) {
 				const previousData = JSON.parse(fs.readFileSync(resultPath).toString());
 				try {
@@ -40,11 +43,11 @@ function assertUnchangedTokens(fixturesPath: string, resultsPath: string, fixtur
 			} else {
 				fs.writeFileSync(resultPath, JSON.stringify(data, null, '\t'));
 			}
-			done();
-		} catch (e) {
-			done(e);
-		}
-	}, done);
+		}));
+		done();
+	} catch (e) {
+		done(e);
+	}
 }
 
 function hasThemeChange(d: any, p: any): boolean {
@@ -61,11 +64,21 @@ suite('colorization', () => {
 	const testPath = normalize(join(__dirname, '../test'));
 	const fixturesPath = join(testPath, 'colorize-fixtures');
 	const resultsPath = join(testPath, 'colorize-results');
+	const treeSitterResultsPath = join(testPath, 'colorize-tree-sitter-results');
+	let originalSettingValue: any;
+
+	suiteSetup(async function () {
+		originalSettingValue = workspace.getConfiguration('editor').get('experimental.preferTreeSitter');
+		await workspace.getConfiguration('editor').update('experimental.preferTreeSitter', ["typescript"], ConfigurationTarget.Global);
+	});
+	suiteTeardown(async function () {
+		await workspace.getConfiguration('editor').update('experimental.preferTreeSitter', originalSettingValue, ConfigurationTarget.Global);
+	});
 
 	for (const fixture of fs.readdirSync(fixturesPath)) {
 		test(`colorize: ${fixture}`, function (done) {
 			commands.executeCommand('workbench.action.closeAllEditors').then(() => {
-				assertUnchangedTokens(fixturesPath, resultsPath, fixture, done);
+				assertUnchangedTokens(fixturesPath, resultsPath, treeSitterResultsPath, fixture, done);
 			});
 		});
 	}

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/12750_html.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/13448_html.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/14119_less.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/25920_html.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/COMMIT_EDITMSG.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/Dockerfile.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/basic_java.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/git-rebase-todo.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-1550_yaml.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-224862_yaml.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-28354_php.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-4008_yaml.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-6303_yaml.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/issue-76997_php.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/makefile.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/md-math_md.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-13777_go.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-166781_rs.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-173216_sh.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-173224_sh.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-173336_sh.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-23630_cpp.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-23850_cpp.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-33886_md.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-4287_pug.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-6611_rs.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-7115_xml.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-78769_cpp.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-80644_cpp.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-brackets_tsx.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-cssvariables_less.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-cssvariables_scss.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-embedding_html.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-freeze-56377_py.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-freeze-56476_ps1.json

@@ -0,0 +1 @@
+[]

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-function-inv_ts.json

@@ -0,0 +1,152 @@
+[
+	{
+		"c": "rowData",
+		"t": "variable",
+		"r": {
+			"dark_plus": "variable: #9CDCFE",
+			"light_plus": "variable: #001080",
+			"hc_black": "variable: #9CDCFE",
+			"dark_modern": "variable: #9CDCFE",
+			"hc_light": "variable: #001080",
+			"light_modern": "variable: #001080"
+		}
+	},
+	{
+		"c": ".",
+		"t": "punctuation.delimiter",
+		"r": {}
+	},
+	{
+		"c": "push",
+		"t": "variable entity.name.function",
+		"r": {
+			"dark_plus": "entity.name.function: #DCDCAA",
+			"light_plus": "entity.name.function: #795E26",
+			"hc_black": "entity.name.function: #DCDCAA",
+			"dark_modern": "entity.name.function: #DCDCAA",
+			"hc_light": "entity.name.function: #5E2CBC",
+			"light_modern": "entity.name.function: #795E26"
+		}
+	},
+	{
+		"c": "(",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": "callback",
+		"t": "variable entity.name.function",
+		"r": {
+			"dark_plus": "entity.name.function: #DCDCAA",
+			"light_plus": "entity.name.function: #795E26",
+			"hc_black": "entity.name.function: #DCDCAA",
+			"dark_modern": "entity.name.function: #DCDCAA",
+			"hc_light": "entity.name.function: #5E2CBC",
+			"light_modern": "entity.name.function: #795E26"
+		}
+	},
+	{
+		"c": "(",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": "new",
+		"t": "keyword.operator.new",
+		"r": {
+			"dark_plus": "keyword.operator.new: #569CD6",
+			"light_plus": "keyword.operator.new: #0000FF",
+			"dark_vs": "keyword.operator.new: #569CD6",
+			"light_vs": "keyword.operator.new: #0000FF",
+			"hc_black": "keyword.operator.new: #569CD6",
+			"dark_modern": "keyword.operator.new: #569CD6",
+			"hc_light": "keyword.operator.new: #0F4A85",
+			"light_modern": "keyword.operator.new: #0000FF"
+		}
+	},
+	{
+		"c": "Cell",
+		"t": "variable entity.name.function",
+		"r": {
+			"dark_plus": "entity.name.function: #DCDCAA",
+			"light_plus": "entity.name.function: #795E26",
+			"hc_black": "entity.name.function: #DCDCAA",
+			"dark_modern": "entity.name.function: #DCDCAA",
+			"hc_light": "entity.name.function: #5E2CBC",
+			"light_modern": "entity.name.function: #795E26"
+		}
+	},
+	{
+		"c": "(",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": "row",
+		"t": "variable",
+		"r": {
+			"dark_plus": "variable: #9CDCFE",
+			"light_plus": "variable: #001080",
+			"hc_black": "variable: #9CDCFE",
+			"dark_modern": "variable: #9CDCFE",
+			"hc_light": "variable: #001080",
+			"light_modern": "variable: #001080"
+		}
+	},
+	{
+		"c": ",",
+		"t": "punctuation.delimiter",
+		"r": {}
+	},
+	{
+		"c": "col",
+		"t": "variable",
+		"r": {
+			"dark_plus": "variable: #9CDCFE",
+			"light_plus": "variable: #001080",
+			"hc_black": "variable: #9CDCFE",
+			"dark_modern": "variable: #9CDCFE",
+			"hc_light": "variable: #001080",
+			"light_modern": "variable: #001080"
+		}
+	},
+	{
+		"c": ",",
+		"t": "punctuation.delimiter",
+		"r": {}
+	},
+	{
+		"c": "false",
+		"t": "constant.language",
+		"r": {
+			"dark_plus": "constant.language: #569CD6",
+			"light_plus": "constant.language: #0000FF",
+			"dark_vs": "constant.language: #569CD6",
+			"light_vs": "constant.language: #0000FF",
+			"hc_black": "constant.language: #569CD6",
+			"dark_modern": "constant.language: #569CD6",
+			"hc_light": "constant.language: #0F4A85",
+			"light_modern": "constant.language: #0000FF"
+		}
+	},
+	{
+		"c": ")",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": ")",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": ")",
+		"t": "",
+		"r": {}
+	},
+	{
+		"c": ";",
+		"t": "punctuation.delimiter",
+		"r": {}
+	}
+]