Add source-with-inline-map. Fixes peggyjs#280.

docs/documentation.html

@@ -376,10 +376,13 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
   <code>null</code>).</dd>
 
   <dt><code>format</code></dt>
-  <dd>format of the generated parser (<code>"amd"</code>, <code>"bare"</code>,
-  <code>"commonjs"</code>, <code>"es"</code>, <code>"globals"</code>, or
-  <code>"umd"</code>); valid only when <code>output</code> is set to
-  <code>"source"</code> (default: <code>"bare"</code>).</dd>
+  <dd>
+    Format of the generated parser (<code>"amd"</code>, <code>"bare"</code>,
+    <code>"commonjs"</code>, <code>"es"</code>, <code>"globals"</code>, or
+    <code>"umd"</code>); valid only when <code>output</code> is set to
+    <code>"source"</code>, <code>"source-and-map"</code>, or
+    <code>"source-with-inline-map"</code>. (default: <code>"bare"</code>).
+  </dd>
 
   <dt><code>grammarSource</code></dt>
   <dd>any object that represent origin of the input grammar. The CLI will set
@@ -392,17 +395,33 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
   <dd>Callback for informational messages. See <a href="#error-reporting">Error Reporting</a></dd>
 
   <dt><code>output</code></dt>
-  <dd><p>If set to <code>"parser"</code>, the method will return generated parser
-  object; if set to <code>"source"</code>, it will return parser source code as
-  a string (default: <code>"parser"</code>).
-  If set to <code>"source-and-map"</code>, it will return a <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> object; you can
-  get source code by calling <code>toString()</code> method or source code and mapping by
-  calling <code>toStringWithSourceMap()</code> method, see the <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> documentation
-  (default: <code>"parser"</code>)</p>
-  <blockquote>
-  <p><strong>Note</strong>: because of bug <a href="https://github.com/mozilla/source-map/issues/444">source-map/444</a> you should also set <code>grammarSource</code> to
-  a not-empty string if you set this value to <code>"source-and-map"</code></p>
-  </blockquote></dd>
+  <dd><p>A string, one of:</p>
+    <ul>
+      <li><code>"parser"</code> - return generated parser object.</li>
+      <li><code>"source"</code> - return parser source code as a string.</li>
+      <li><code>"source-and-map"</code> - return a
+        <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a>
+        object; you can get source code by calling <code>toString()</code>
+        method or source code and mapping by calling
+        <code>toStringWithSourceMap()</code> method, see the
+        <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a>
+        documentation.
+      </li>
+      <li><code>"source-with-inline-map"</code> - return the parser source along
+        with an embedded source map as a <code>data:</code> URI.  This option
+        leads to a larger output string, but is the easiest to integrate with
+        developer tooling.</li>
+    </ul>
+    <p>(default: <code>"parser"</code>)</p>
+    <blockquote>
+      <p><strong>Note</strong>: because of bug <a
+      href="https://github.com/mozilla/source-map/issues/444">source-map/444</a>
+      you should also set <code>grammarSource</code> to a not-empty string if
+      you set this value to <code>"source-and-map"</code> or
+      <code>"source-with-inline-map"</code>.  The path should be relative to
+      the location where the generated parser code will be stored.</p>
+    </blockquote>
+  </dd>
 
   <dt><code>plugins</code></dt>
   <dd>Plugins to use. See the <a href="#plugins-api">Plugins API</a> section.</dd>

lib/compiler/index.js

@@ -12,6 +12,7 @@ const reportUndefinedRules = require("./passes/report-undefined-rules");
 const reportIncorrectPlucking = require("./passes/report-incorrect-plucking");
 const Session = require("./session");
 const visitor = require("./visitor");
+const { base64 } = require("./utils");
 
 function processOptions(options, defaults) {
   const processedOptions = {};
@@ -118,6 +119,20 @@ const compiler = {
       case "source-and-map":
         return ast.code;
 
+      case "source-with-inline-map": {
+        if (typeof TextEncoder === "undefined") {
+          throw new Error("TextEncoder is not supported by this platform");
+        }
+        const sourceMap = ast.code.toStringWithSourceMap();
+        const encoder = new TextEncoder();
+        const b64 = base64(
+          encoder.encode(JSON.stringify(sourceMap.map.toJSON()))
+        );
+        return sourceMap.code + `\
+//\x23 sourceMappingURL=data:application/json;charset=utf-8;base64,${b64}
+`;
+      }
+
       default:
         throw new Error("Invalid output format: " + options.output + ".");
     }

lib/compiler/utils.js

@@ -50,3 +50,41 @@ function regexpClassEscape(s) {
     .replace(/[\u1000-\uFFFF]/g,      ch => "\\u"  + hex(ch));
 }
 exports.regexpClassEscape = regexpClassEscape;
+
+/**
+ * Base64 encode a Uint8Array.  Needed for browser compatibility where
+ * the Buffer class is not available.
+ *
+ * @param {Uint8Array} u8 Bytes to encode
+ * @returns {string} Base64 encoded string
+ */
+function base64(u8) {
+  // Note: btoa has the worst API, and even mentioning Buffer here will
+  // cause rollup to suck it in.
+
+  // See RFC4648, sec. 4.
+  const A = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  const rem = u8.length % 3;
+  const len = u8.length - rem;
+  let res = "";
+
+  for (let i = 0; i < len; i += 3) {
+    res += A[u8[i] >> 2];
+    res += A[((u8[i] & 0x3) << 4) | (u8[i + 1] >> 4)];
+    res += A[((u8[i + 1] & 0xf) << 2) | (u8[i + 2] >> 6)];
+    res += A[u8[i + 2] & 0x3f];
+  }
+  if (rem === 1) {
+    res += A[u8[len] >> 2];
+    res += A[(u8[len] & 0x3) << 4];
+    res += "==";
+  } else if (rem === 2) {
+    res += A[u8[len] >> 2];
+    res += A[((u8[len] & 0x3) << 4) | (u8[len + 1] >> 4)];
+    res += A[(u8[len + 1] & 0xf) << 2];
+    res += "=";
+  }
+
+  return res;
+}
+exports.base64 = base64;

lib/peg.d.ts

@@ -765,6 +765,12 @@ export namespace compiler {
     options: SourceBuildOptions<"source">
   ): string;
 
+  function compile(
+    ast: ast.Grammar,
+    stages: Stages,
+    options: SourceBuildOptions<"source-with-inline-map">
+  ): string;
+
   /**
    * Generates a parser source and source map from a specified grammar AST.
    *
@@ -1087,14 +1093,20 @@ export interface ParserBuildOptions extends BuildOptionsBase {
    * If set to `"parser"`, the method will return generated parser object;
    * if set to `"source"`, it will return parser source code as a string;
    * if set to `"source-and-map"`, it will return a `SourceNode` object
-   * which can give a parser source code as a string and a source map;
+   *   which can give a parser source code as a string and a source map;
+   * if set to `"source-with-inline-map"`, it will return the parser source
+   *   along with an embedded source map as a `data:` URI;
    * (default: `"parser"`)
    */
   output?: "parser";
 }
 
 /** Possible kinds of source output generators. */
-export type SourceOutputs = "source" | "source-and-map";
+export type SourceOutputs =
+  "parser" |
+  "source" |
+  "source-and-map" |
+  "source-with-inline-map";
 
 /** Base options for all source-generating formats. */
 interface SourceOptionsBase<Output extends SourceOutputs>
@@ -1103,7 +1115,9 @@ interface SourceOptionsBase<Output extends SourceOutputs>
    * If set to `"parser"`, the method will return generated parser object;
    * if set to `"source"`, it will return parser source code as a string;
    * if set to `"source-and-map"`, it will return a `SourceNode` object
-   * which can give a parser source code as a string and a source map;
+   *   which can give a parser source code as a string and a source map;
+   * if set to `"source-with-inline-map"`, it will return the parser source
+   *   along with an embedded source map as a `data:` URI;
    * (default: `"parser"`)
    */
   output: Output;
@@ -1197,6 +1211,30 @@ export function generate(
   options: SourceBuildOptions<"source">
 ): string;
 
+/**
+ * Returns the generated source code as a string appended with a source map as
+ * a `data:` URI.
+ *
+ * Note, that `SourceNode.source`s of the generated source map will depend
+ * on the `options.grammarSource` value. Therefore, value `options.grammarSource`
+ * will propagate to the `sources` array of the source map. That array MUST
+ * contain a path relative to the source map location, as no further path
+ * processing will be performed.
+ *
+ * @param grammar String in the format described by the meta-grammar in the
+ *        `parser.pegjs` file
+ * @param options Options that allow you to customize returned parser object
+ *
+ * @throws {SyntaxError}  If the grammar contains a syntax error, for example,
+ *         an unclosed brace
+ * @throws {GrammarError} If the grammar contains a semantic error, for example,
+ *         duplicated labels
+ */
+export function generate(
+  grammar: string,
+  options: SourceBuildOptions<"source-with-inline-map">
+): string;
+
 /**
  * Returns the generated source code and its source map as a `SourceNode`
  * object. You can get the generated code and the source map by using a

test/types/peg.test-d.ts

@@ -81,6 +81,9 @@ describe("peg.d.ts", () => {
 
     const p3 = peggy.generate(src, { output: true as boolean ? "source-and-map" : "source" });
     expectType<string | SourceNode>(p3);
+
+    const p4 = peggy.generate(src, { output: "source-with-inline-map" });
+    expectType<string>(p4);
   });
 
   it("compiles with source map", () => {
@@ -107,6 +110,13 @@ describe("peg.d.ts", () => {
       { output: true as boolean ? "source-and-map" : "source" }
     );
     expectType<string | SourceNode>(p3);
+
+    const p4 = peggy.compiler.compile(
+      ast,
+      peggy.compiler.passes,
+      { output: "source-with-inline-map" }
+    );
+    expectType<string>(p4);
   });
 
   it("creates an AST", () => {

test/unit/compiler.spec.js

@@ -30,4 +30,31 @@ describe("Peggy compiler", () => {
       output: "INVALID OUTPUT TYPE",
     })).to.throw("Invalid output format: INVALID OUTPUT TYPE.");
   });
+
+  it("generates inline sourceMappingURL", () => {
+    const ast = parser.parse("foo='1'");
+    // Ensure there is an expect even if we don't run the good tests below.
+    expect(ast).to.be.an("object");
+
+    // Don't run on old IE
+    if (typeof TextEncoder === "function") {
+      expect(compiler.compile(ast, compiler.passes, {
+        output: "source-with-inline-map",
+      })).to.match(
+        /^\/\/# sourceMappingURL=data:application\/json;charset=utf-8;base64,/m
+      );
+      /* eslint-disable no-undef */
+      // I *think* everywhere that has TextEncoder also has globalThis, but
+      // I'm not positive.
+      if (typeof globalThis === "object") {
+        const TE = globalThis.TextEncoder;
+        delete globalThis.TextEncoder;
+        expect(() => compiler.compile(ast, compiler.passes, {
+          output: "source-with-inline-map",
+        })).to.throw("TextEncoder is not supported by this platform");
+        globalThis.TextEncoder = TE;
+      }
+      /* eslint-enable no-undef */
+    }
+  });
 });

test/unit/compiler/utils.spec.js

@@ -1,7 +1,12 @@
 "use strict";
 
 const chai = require("chai");
-const { hex, stringEscape, regexpClassEscape } = require("../../../lib/compiler/utils");
+const {
+  hex,
+  stringEscape,
+  regexpClassEscape,
+  base64,
+} = require("../../../lib/compiler/utils");
 
 const expect = chai.expect;
 
@@ -28,4 +33,13 @@ describe("utility functions", () => {
     expect(regexpClassEscape("\u0100\u0fff")).to.equal("\\u0100\\u0FFF");
     expect(regexpClassEscape("\u1000\uffff")).to.equal("\\u1000\\uFFFF");
   });
+  it("base64", () => {
+    expect(base64(new Uint8Array([]))).to.equal("");
+    expect(base64(new Uint8Array([97]))).to.equal("YQ==");
+    expect(base64(new Uint8Array([97, 98]))).to.equal("YWI=");
+    expect(base64(new Uint8Array([97, 98, 99]))).to.equal("YWJj");
+    expect(base64(new Uint8Array([97, 98, 99, 100]))).to.equal("YWJjZA==");
+    expect(base64(new Uint8Array([97, 98, 99, 100, 101]))).to.equal("YWJjZGU=");
+    expect(base64(new Uint8Array([97, 98, 99, 100, 101, 102]))).to.equal("YWJjZGVm");
+  });
 });