Add support for comments in MDSL

dist/mdsl/MDSLUtilities.d.ts

@@ -1,9 +1,19 @@
 /**
- * A type defining an object that holds a reference to substitued string literals parsed from the definition.
+ * A type defining an object that holds a reference to substituted string literals parsed from the definition.
  */
 export type StringLiteralPlaceholders = {
     [key: string]: string;
 };
+/**
+ * An object representing the result of tokenising an MDSL definition.
+ */
+export type TokeniseResult = {
+    /**
+     * The array of tokens parsed from the definition.
+     */
+    tokens: string[];
+    placeholders: StringLiteralPlaceholders;
+};
 /**
  * Pop the next raw token from the specified array of tokens and throw an error if it wasn't the expected one.
  * @param tokens The array of tokens.
@@ -12,17 +22,8 @@ export type StringLiteralPlaceholders = {
  */
 export declare function popAndCheck(tokens: string[], expected?: string | string[]): string;
 /**
- * Swaps out any node/attribute argument string literals with placeholders.
- * @param definition The definition.
- * @returns An object containing a mapping of placeholders to original string values as well as the processed definition string.
- */
-export declare function substituteStringLiterals(definition: string): {
-    placeholders: StringLiteralPlaceholders;
-    processedDefinition: string;
-};
-/**
- * Parse the tree definition into an array of raw tokens.
- * @param definition The definition.
- * @returns An array of tokens parsed from the definition.
+ * Parse the MDSL definition into an array of raw tokens.
+ * @param definition The MDSL definition.
+ * @returns An object representing the result of tokenising the MDSL definition.
  */
-export declare function parseTokensFromDefinition(definition: string): string[];
+export declare function tokenise(definition: string): TokeniseResult;

src/mdsl/MDSLDefinitionParser.ts

@@ -27,25 +27,18 @@ import {
 } from "../BehaviourTreeDefinitionUtilities";
 import { parseArgumentTokens } from "./MDSLNodeArgumentParser";
 import { parseAttributeTokens } from "./MDSLNodeAttributeParser";
-import {
-    StringLiteralPlaceholders,
-    parseTokensFromDefinition,
-    popAndCheck,
-    substituteStringLiterals
-} from "./MDSLUtilities";
+import { StringLiteralPlaceholders, tokenise, popAndCheck } from "./MDSLUtilities";
 
 /**
  * Convert the MDSL tree definition string into an equivalent JSON definition.
  * @param definition The tree definition string as MDSL.
  * @returns The root node JSON definitions.
  */
 export function convertMDSLToJSON(definition: string): RootNodeDefinition[] {
-    // Swap out any node/attribute argument string literals with a placeholder and get a mapping of placeholders to original values as well as the processed definition.
-    const { placeholders, processedDefinition } = substituteStringLiterals(definition);
-
-    // Parse our definition definition string into an array of raw tokens.
-    const tokens = parseTokensFromDefinition(processedDefinition);
+    // Parse our definition string into a bunch of tokens.
+    const { tokens, placeholders } = tokenise(definition);
 
+    // Convert the tokens that we parsed from the MDSL definition into JSON and return it.
     return convertTokensToJSONDefinition(tokens, placeholders);
 }
 

src/mdsl/MDSLUtilities.ts

@@ -1,8 +1,22 @@
 /**
- * A type defining an object that holds a reference to substitued string literals parsed from the definition.
+ * A type defining an object that holds a reference to substituted string literals parsed from the definition.
  */
 export type StringLiteralPlaceholders = { [key: string]: string };
 
+/**
+ * An object representing the result of tokenising an MDSL definition.
+ */
+export type TokeniseResult = {
+    /**
+     * The array of tokens parsed from the definition.
+     */
+    tokens: string[];
+    /*
+     * An object that holds a reference to substituted string literals parsed from the definition.
+     */
+    placeholders: StringLiteralPlaceholders;
+};
+
 /**
  * Pop the next raw token from the specified array of tokens and throw an error if it wasn't the expected one.
  * @param tokens The array of tokens.
@@ -37,12 +51,41 @@ export function popAndCheck(tokens: string[], expected?: string | string[]): str
     return popped;
 }
 
+/**
+ * Parse the MDSL definition into an array of raw tokens.
+ * @param definition The MDSL definition.
+ * @returns An object representing the result of tokenising the MDSL definition.
+ */
+export function tokenise(definition: string): TokeniseResult {
+    // Clean the definition by removing any comments.
+    definition = definition.replace(/\/\*(.|\n)+?\*\//g, "");
+
+    // Swap out any node/attribute argument string literals with a placeholder and get a mapping of placeholders to original values as well as the processed definition.
+    const { placeholders, processedDefinition } = substituteStringLiterals(definition);
+
+    // Add some space around various important characters so that they can be plucked out easier as individual tokens.
+    definition = processedDefinition.replace(/\(/g, " ( ");
+    definition = definition.replace(/\)/g, " ) ");
+    definition = definition.replace(/\{/g, " { ");
+    definition = definition.replace(/\}/g, " } ");
+    definition = definition.replace(/\]/g, " ] ");
+    definition = definition.replace(/\[/g, " [ ");
+    definition = definition.replace(/,/g, " , ");
+
+    return {
+        // Split the definition into raw token form.
+        tokens: definition.replace(/\s+/g, " ").trim().split(" "),
+        // The placeholders for string literals that were found in the definition.
+        placeholders
+    };
+}
+
 /**
  * Swaps out any node/attribute argument string literals with placeholders.
  * @param definition The definition.
  * @returns An object containing a mapping of placeholders to original string values as well as the processed definition string.
  */
-export function substituteStringLiterals(definition: string): {
+function substituteStringLiterals(definition: string): {
     placeholders: StringLiteralPlaceholders;
     processedDefinition: string;
 } {
@@ -65,22 +108,3 @@ export function substituteStringLiterals(definition: string): {
 
     return { placeholders, processedDefinition };
 }
-
-/**
- * Parse the tree definition into an array of raw tokens.
- * @param definition The definition.
- * @returns An array of tokens parsed from the definition.
- */
-export function parseTokensFromDefinition(definition: string): string[] {
-    // Add some space around various important characters so that they can be plucked out easier as individual tokens.
-    definition = definition.replace(/\(/g, " ( ");
-    definition = definition.replace(/\)/g, " ) ");
-    definition = definition.replace(/\{/g, " { ");
-    definition = definition.replace(/\}/g, " } ");
-    definition = definition.replace(/\]/g, " ] ");
-    definition = definition.replace(/\[/g, " [ ");
-    definition = definition.replace(/,/g, " , ");
-
-    // Split the definition into raw token form and return it.
-    return definition.replace(/\s+/g, " ").trim().split(" ");
-}