Fix converting template types to handle nested templates (babel#610)

Fixes babel#603 (and the fixture from babel#609 works).

Reworks our code that converts the format of Babylon template tokens to be a bit more robust, especially with things like nested templates with arrows.

(Adapted the logic from https://github.com/eslint/espree/blob/master/lib/token-translator.js)

.prettierignore

@@ -0,0 +1 @@
+*.json

lib/babylon-to-espree/convertTemplateType.js

@@ -1,99 +1,92 @@
 "use strict";
 
 module.exports = function(tokens, tt) {
-  var startingToken = 0;
-  var currentToken = 0;
-  var numBraces = 0; // track use of {}
-  var numBackQuotes = 0; // track number of nested templates
+  let curlyBrace = null;
+  let templateTokens = [];
+  const result = [];
 
-  function isBackQuote(token) {
-    return tokens[token].type === tt.backQuote;
-  }
-
-  function isTemplateStarter(token) {
-    return (
-      isBackQuote(token) ||
-      // only can be a template starter when in a template already
-      (tokens[token].type === tt.braceR && numBackQuotes > 0)
-    );
-  }
-
-  function isTemplateEnder(token) {
-    return isBackQuote(token) || tokens[token].type === tt.dollarBraceL;
-  }
+  function addTemplateType() {
+    const start = templateTokens[0];
+    const end = templateTokens[templateTokens.length - 1];
 
-  // append the values between start and end
-  function createTemplateValue(start, end) {
-    var value = "";
-    while (start <= end) {
-      if (tokens[start].value) {
-        value += tokens[start].value;
-      } else if (tokens[start].type !== tt.template) {
-        value += tokens[start].type.label;
+    const value = templateTokens.reduce((result, token) => {
+      if (token.value) {
+        result += token.value;
+      } else if (token.type !== tt.template) {
+        result += token.type.label;
       }
-      start++;
-    }
-    return value;
-  }
 
-  // create Template token
-  function replaceWithTemplateType(start, end) {
-    var templateToken = {
+      return result;
+    }, "");
+
+    result.push({
       type: "Template",
-      value: createTemplateValue(start, end),
-      start: tokens[start].start,
-      end: tokens[end].end,
+      value: value,
+      start: start.start,
+      end: end.end,
       loc: {
-        start: tokens[start].loc.start,
-        end: tokens[end].loc.end,
+        start: start.loc.start,
+        end: end.loc.end,
       },
-    };
+    });
 
-    // put new token in place of old tokens
-    tokens.splice(start, end - start + 1, templateToken);
+    templateTokens = [];
   }
 
-  function trackNumBraces(token) {
-    if (tokens[token].type === tt.braceL) {
-      numBraces++;
-    } else if (tokens[token].type === tt.braceR) {
-      numBraces--;
-    }
-  }
+  tokens.forEach(token => {
+    switch (token.type) {
+      case tt.backQuote:
+        if (curlyBrace) {
+          result.push(curlyBrace);
+          curlyBrace = null;
+        }
 
-  while (startingToken < tokens.length) {
-    // template start: check if ` or }
-    if (isTemplateStarter(startingToken) && numBraces === 0) {
-      if (isBackQuote(startingToken)) {
-        numBackQuotes++;
-      }
+        templateTokens.push(token);
 
-      currentToken = startingToken + 1;
+        if (templateTokens.length > 1) {
+          addTemplateType();
+        }
 
-      // check if token after template start is "template"
-      if (
-        currentToken >= tokens.length - 1 ||
-        tokens[currentToken].type !== tt.template
-      ) {
         break;
-      }
 
-      // template end: find ` or ${
-      while (!isTemplateEnder(currentToken)) {
-        if (currentToken >= tokens.length - 1) {
-          break;
+      case tt.dollarBraceL:
+        templateTokens.push(token);
+        addTemplateType();
+        break;
+
+      case tt.braceR:
+        if (curlyBrace) {
+          result.push(curlyBrace);
         }
-        currentToken++;
-      }
 
-      if (isBackQuote(currentToken)) {
-        numBackQuotes--;
-      }
-      // template start and end found: create new token
-      replaceWithTemplateType(startingToken, currentToken);
-    } else if (numBackQuotes > 0) {
-      trackNumBraces(startingToken);
+        curlyBrace = token;
+        break;
+
+      case tt.template:
+        if (curlyBrace) {
+          templateTokens.push(curlyBrace);
+          curlyBrace = null;
+        }
+
+        templateTokens.push(token);
+        break;
+
+      case tt.eof:
+        if (curlyBrace) {
+          result.push(curlyBrace);
+        }
+
+        break;
+
+      default:
+        if (curlyBrace) {
+          result.push(curlyBrace);
+          curlyBrace = null;
+        }
+
+        result.push(token);
     }
-    startingToken++;
-  }
+  });
+
+  return result;
 };

lib/babylon-to-espree/index.js

@@ -6,11 +6,6 @@ var toTokens = require("./toTokens");
 var toAST = require("./toAST");
 
 module.exports = function(ast, traverse, tt, code) {
-  // remove EOF token, eslint doesn't use this for anything and it interferes
-  // with some rules see https://github.com/babel/babel-eslint/issues/2
-  // todo: find a more elegant way to do this
-  ast.tokens.pop();
-
   // convert tokens
   ast.tokens = toTokens(ast.tokens, tt, code);
 

lib/babylon-to-espree/toTokens.js

@@ -4,16 +4,7 @@ var convertTemplateType = require("./convertTemplateType");
 var toToken = require("./toToken");
 
 module.exports = function(tokens, tt, code) {
-  // transform tokens to type "Template"
-  convertTemplateType(tokens, tt);
-
-  var transformedTokens = [];
-  for (var i = 0; i < tokens.length; i++) {
-    var token = tokens[i];
-    if (token.type !== "CommentLine" && token.type !== "CommentBlock") {
-      transformedTokens.push(toToken(token, tt, code));
-    }
-  }
-
-  return transformedTokens;
+  return convertTemplateType(tokens, tt)
+    .filter(t => t.type !== "CommentLine" && t.type !== "CommentBlock")
+    .map(t => toToken(t, tt, code));
 };

test/babel-eslint.js

@@ -62,7 +62,7 @@ function parseAndAssertSame(code) {
     `);
     throw err;
   }
-  // assert.equal(esAST, babylonAST);
+  //assert.equal(esAST, babylonAST);
 }
 
 describe("babylon-to-espree", () => {
@@ -158,6 +158,14 @@ describe("babylon-to-espree", () => {
         };
       `);
     });
+
+    it("template with arrow returning template #603", () => {
+      parseAndAssertSame(`
+        var a = \`\${() => {
+          \`\${''}\`
+        }}\`;
+      `);
+    });
   });
 
   it("simple expression", () => {

test/non-regression.js

@@ -1112,6 +1112,18 @@ describe("verify", () => {
     );
   });
 
+  it("template with arrow returning template #603", () => {
+    verifyAndAssertMessages(
+      `
+        var a = \`\${() => {
+            \`\${''}\`
+        }}\`;
+      `,
+      { indent: 1 },
+      []
+    );
+  });
+
   describe("decorators #72", () => {
     it("class declaration", () => {
       verifyAndAssertMessages(