Implement test suite runner

package.json

@@ -4,7 +4,7 @@
   "description": "Lightweight, robust, elegant syntax highlighting. A spin-off project from Dabblet.",
   "main": "prism.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "mocha tests/run.js"
   },
   "repository": {
     "type": "git",
@@ -18,10 +18,12 @@
   "license": "MIT",
   "readmeFilename": "README.md",
   "devDependencies": {
+    "chai": "^2.3.0",
     "gulp": "^3.8.6",
     "gulp-concat": "^2.3.4",
     "gulp-header": "^1.0.5",
     "gulp-rename": "^1.2.0",
-    "gulp-uglify": "^0.3.1"
+    "gulp-uglify": "^0.3.1",
+    "mocha": "^2.2.5"
   }
 }

tests/helper/components.js

@@ -0,0 +1,11 @@
+"use strict";
+
+var fs = require("fs");
+var vm = require('vm');
+
+var fileContent = fs.readFileSync(__dirname + "/../../components.js", "utf8");
+var context = {};
+vm.createContext(context);
+vm.runInContext(fileContent, context);
+
+module.exports = context.components;

tests/helper/prism-loader.js

@@ -0,0 +1,116 @@
+"use strict";
+
+var fs = require("fs");
+var vm = require('vm');
+var components = require("./components");
+var languagesCatalog = components.languages;
+
+
+module.exports = {
+
+	/**
+	 * Creates a new Prism instance with the given language loaded
+	 *
+	 * @param {string|string[]} languages
+	 * @returns {Prism}
+	 */
+	createInstance: function (languages) {
+		var context = {
+			loadedLanguages: [],
+			Prism: this.createEmptyPrism()
+		};
+		languages = Array.isArray(languages) ? languages : [languages];
+
+		for (var i = 0, l = languages.length; i < l; i++) {
+			context = this.loadLanguage(languages[i], context);
+		}
+
+		return context.Prism;
+	},
+
+
+	/**
+	 * Loads the given language (including recursively loading the dependencies) and
+	 * appends the config to the given Prism object
+	 *
+	 * @private
+	 * @param {string} language
+	 * @param {{loadedLanguages: string[], Prism: Prism}} context
+	 * @returns {{loadedLanguages: string[], Prism: Prism}}
+	 */
+	loadLanguage: function (language, context) {
+		if (!languagesCatalog[language])
+		{
+			throw new Error("Language '" + language + "' not found.");
+		}
+
+		// the given language was already loaded
+		if (-1 < context.loadedLanguages.indexOf(language)) {
+			return context;
+		}
+
+		// if the language has a dependency -> load it first
+		if (languagesCatalog[language].require)
+		{
+			context = this.loadLanguage(languagesCatalog[language].require, context);
+		}
+
+		// load the language itself
+		var languageSource = this.loadFileSource(language);
+		context.Prism = this.runFileWithContext(languageSource, {Prism: context.Prism}).Prism;
+		context.loadedLanguages.push(language);
+
+		return context;
+	},
+
+
+	/**
+	 * Creates a new empty prism instance
+	 *
+	 * @private
+	 * @returns {Prism}
+	 */
+	createEmptyPrism: function () {
+		var coreSource = this.loadFileSource("core");
+		var context = this.runFileWithContext(coreSource);
+		return context.Prism;
+	},
+
+
+	/**
+	 * Cached file sources, to prevent massive HDD work
+	 *
+	 * @private
+	 * @type {Object.<string, string>}
+	 */
+	fileSourceCache: {},
+
+
+	/**
+	 * Loads the given file source as string
+	 *
+	 * @private
+	 * @param {string} name
+	 * @returns {string}
+	 */
+	loadFileSource: function (name) {
+		return this.fileSourceCache[name] = this.fileSourceCache[name] || fs.readFileSync(__dirname + "/../../components/prism-" + name + ".js", "utf8");
+	},
+
+
+	/**
+	 * Runs a VM for a given file source with the given context
+	 *
+	 * @private
+	 * @param {string} fileSource
+	 * @param {*} [context]
+	 *
+	 * @returns {*}
+	 */
+	runFileWithContext: function (fileSource, context) {
+		context = context || {};
+		vm.createContext(context);
+		vm.runInContext(fileSource, context);
+		return context;
+	}
+};

tests/helper/test-case.js

@@ -0,0 +1,122 @@
+"use strict";
+
+var fs = require("fs");
+var assert = require("chai").assert;
+var PrismLoader = require("./prism-loader");
+
+/**
+ * Handles parsing of a test case file.
+ *
+ *
+ * A test case file consists of at least two parts, separated by a line of dashes.
+ * This separation line must start at the beginning of the line and consist of at least three dashes.
+ *
+ * The test case file can either consist of two parts:
+ *
+ *     {source code}
+ *     ----
+ *     {expected token stream}
+ *
+ *
+ * or of three parts:
+ *
+ *     {source code}
+ *     ----
+ *     {expected token stream}
+ *     ----
+ *     {text comment explaining the test case}
+ *
+ * If the file contains more than three parts, the remaining parts are just ignored.
+ * If the file however does not contain at least two parts (so no expected token stream),
+ * the test case will later be marked as failed.
+ *
+ *
+ * @type {{runTestCase: Function, transformCompiledTokenStream: Function, parseTestCaseFile: Function}}
+ */
+module.exports = {
+
+	/**
+	 * Runs the given test case file and asserts the result
+	 *
+	 * The passed language identifier can either be a language like "css" or a composed language
+	 * identifier like "css+markup". Composed identifiers can be used for testing language inclusion.
+	 *
+	 * When testing language inclusion, the first given language is the main language which will be passed
+	 * to Prism for highlighting ("css+markup" will result in a call to Prism to highlight with the "css" grammar).
+	 * But it will be ensured, that the additional passed languages will be loaded too.
+	 *
+	 * The languages will be loaded in the order they were provided.
+	 *
+	 * @param {string} languageIdentifier
+	 * @param {string} filePath
+	 */
+	runTestCase: function (languageIdentifier, filePath) {
+		var testCase = this.parseTestCaseFile(filePath);
+		var languages = languageIdentifier.split("+");
+
+		if (null === testCase) {
+			throw new Error("Test case file has invalid format, please read the docs.");
+		}
+
+		var Prism = PrismLoader.createInstance(languages);
+		// the first language is the main language to highlight
+		var mainLanguageGrammar = Prism.languages[languages[0]];
+		var compiledTokenStream = Prism.tokenize(testCase.testSource, mainLanguageGrammar);
+		var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream);
+
+		assert.deepEqual(simplifiedTokenStream, testCase.expectedTokenStream, testCase.comment);
+	},
+
+
+	/**
+	 * Simplifies the token stream to ease the matching with the expected token stream
+	 *
+	 * @param {string} tokenStream
+	 * @returns {Array.<string[]>}
+	 */
+	transformCompiledTokenStream: function (tokenStream) {
+		return tokenStream.filter(
+			function (token) {
+				// only support objects
+				return (typeof token === "object");
+			}
+		).map(
+			function (entry)
+			{
+				return [entry.type, entry.content];
+			}
+		);
+	},
+
+
+	/**
+	 * Parses the test case from the given test case file
+	 *
+	 * @private
+	 * @param {string} filePath
+	 * @returns {{testSource: string, expectedTokenStream: Array.<Array.<string>>, comment:string?}|null}
+	 */
+	parseTestCaseFile: function (filePath) {
+		var testCaseSource = fs.readFileSync(filePath, "utf8");
+		var testCaseParts = testCaseSource.split(/^----*\w*$/m);
+
+		// No expected token stream found
+		if (2 > testCaseParts.length) {
+			return null;
+		}
+
+		var testCase = {
+			testSource: testCaseParts[0].trim(),
+			expectedTokenStream: JSON.parse(testCaseParts[1]),
+			comment: null
+		};
+
+		// if there are three parts, the third one is the comment
+		// explaining the test case
+		if (testCaseParts[2]) {
+			testCase.comment = testCaseParts[2].trim();
+		}
+
+		return testCase;
+	}
+};

tests/helper/test-discovery.js

@@ -0,0 +1,65 @@
+"use strict";
+
+var fs = require("fs");
+var path = require('path');
+
+
+module.exports = {
+
+	/**
+	 * Loads the list of all available tests
+	 *
+	 * @param {string} rootDir
+	 * @returns {Object.<string, string[]>}
+	 */
+	loadAllTests: function (rootDir) {
+		var testSuite = {};
+		var self = this;
+
+		this.getAllDirectories(rootDir).forEach(
+			function (language) {
+				testSuite[language] = self.getAllFiles(path.join(rootDir, language));
+			}
+		);
+
+		return testSuite;
+	},
+
+
+	/**
+	 * Returns a list of all (sub)directories (just the directory names, not full paths)
+	 * in the given src directory
+	 *
+	 * @param {string} src
+	 * @returns {Array.<string>}
+	 */
+	getAllDirectories: function (src) {
+		return fs.readdirSync(src).filter(
+			function (file) {
+				return fs.statSync(path.join(src, file)).isDirectory();
+			}
+		);
+	},
+
+
+	/**
+	 * Returns a list of all full file paths to all files in the given src directory
+	 *
+	 * @private
+	 * @param {string} src
+	 * @returns {Array.<string>}
+	 */
+	getAllFiles: function (src) {
+		return fs.readdirSync(src).filter(
+			function (fileName) {
+				// only find files that have the ".test" extension
+				return ".test" === path.extname(fileName) &&
+					fs.statSync(path.join(src, fileName)).isFile();
+			}
+		).map(
+			function (fileName) {
+				return path.join(src, fileName);
+			}
+		);
+	}
+};

tests/languages/javascript/testcase1.test

@@ -0,0 +1,14 @@
+var a = 5;
+
+----------------------------------------------------
+
+[
+	["keyword", "var"],
+	["operator", "="],
+	["number", "5"],
+	["punctuation", ";"]
+]
+
+----------------------------------------------------
+
+This is a comment explaining this test case.

tests/run.js

@@ -0,0 +1,32 @@
+"use strict";
+
+var TestDiscovery = require("./helper/test-discovery");
+var TestCase = require("./helper/test-case");
+var path = require("path");
+
+// load complete test suite
+var testSuite = TestDiscovery.loadAllTests(__dirname + "/languages");
+
+// define tests for all tests in all languages in the test suite
+for (var language in testSuite)
+{
+	if (!testSuite.hasOwnProperty(language)) {
+		continue;
+	}
+
+	(function (language, testFiles) {
+		describe("Testing language '" + language + "'", function() {
+			testFiles.forEach(
+				function (filePath) {
+					var fileName = path.basename(filePath, path.extname(filePath));
+
+					it("– should pass test case '" + fileName + "'",
+						function () {
+							TestCase.runTestCase(language, filePath);
+						}
+					);
+				}
+			);
+		});
+	})(language, testSuite[language]);
+}