babel · hzoo · May 26, 2017 · May 26, 2017 · May 26, 2017 · May 26, 2017
diff --git a/README.md b/README.md
@@ -134,6 +134,7 @@ require("babylon").parse("code", {
  - `functionBind`
  - `functionSent`
  - `dynamicImport`
+ - `numericSeparator` ([proposal](https://github.com/samuelgoto/proposal-numeric-separator))
 
 ### FAQ
 

diff --git a/src/tokenizer/index.js b/src/tokenizer/index.js
@@ -13,6 +13,23 @@ import { SourceLocation } from "../util/location";
 import { lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace } from "../util/whitespace";
 import State from "./state";
 
+
+// The following character codes are forbidden from being
+// an immediate sibling of NumericLiteralSeparator _
+
+const forbiddenNumericLiteralSeparatorSiblings = [
+  46,  // .
+  66,  // B
+  69,  // E
+  79,  // O
+  88,  // X
+  95,  // _ (multiple separators are not allowed)
+  98,  // b
+  101, // e
+  111, // o
+  120, // x
+];
+
 // Object type used to represent tokens. Note that normally, tokens
 // simply exist as properties on the parser object. This is only
 // used for the onToken callback and the external tokenizer.
@@ -555,6 +572,23 @@ export default class Tokenizer extends LocationParser {
     for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
       const code = this.input.charCodeAt(this.state.pos);
       let val;
+
+      if (this.hasPlugin("numericSeparator")) {
+        const prev = this.input.charCodeAt(this.state.pos - 1);
+        const next = this.input.charCodeAt(this.state.pos + 1);
+        if (code === 95) {
+          if ((forbiddenNumericLiteralSeparatorSiblings.indexOf(prev) > -1) ||
+              (forbiddenNumericLiteralSeparatorSiblings.indexOf(next) > -1) ||
+              Number.isNaN(next)) {
+            this.raise(this.state.pos, "Invalid NumericLiteralSeparator");
+          }
+
+          // Ignore this _ character
+          ++this.state.pos;
+          continue;
+        }
+      }
+
       if (code >= 97) {
         val = code - 97 + 10; // a
       } else if (code >= 65) {
@@ -608,7 +642,7 @@ export default class Tokenizer extends LocationParser {
 
     if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
 
-    const str = this.input.slice(start, this.state.pos);
+    const str = this.input.slice(start, this.state.pos).replace(/_/g, "");
     let val;
     if (isFloat) {
       val = parseFloat(str);

diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-0/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-0/actual.js
@@ -0,0 +1 @@
+1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-0/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-0/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:1)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-1/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-1/actual.js
@@ -0,0 +1 @@
+1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-1/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-1/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-10/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-10/actual.js
@@ -0,0 +1 @@
+0x1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-10/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-10/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-11/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-11/actual.js
@@ -0,0 +1 @@
+0xa_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-11/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-11/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-12/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-12/actual.js
@@ -0,0 +1 @@
+0x_a_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-12/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-12/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-13/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-13/actual.js
@@ -0,0 +1 @@
+0x__1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-13/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-13/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-14/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-14/actual.js
@@ -0,0 +1 @@
+0x_1__1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-14/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-14/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-15/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-15/actual.js
@@ -0,0 +1 @@
+0x_1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-15/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-15/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-16/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-16/actual.js
@@ -0,0 +1 @@
+0o_1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-16/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-16/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-17/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-17/actual.js
@@ -0,0 +1 @@
+0o_11
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-17/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-17/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-18/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-18/actual.js
@@ -0,0 +1 @@
+0o_01_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-18/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-18/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-19/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-19/actual.js
@@ -0,0 +1 @@
+0b_0_1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-19/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-19/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-2/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-2/actual.js
@@ -0,0 +1 @@
+1_1__
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-2/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-2/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-20/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-20/actual.js
@@ -0,0 +1 @@
+0b_01_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-20/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-20/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-21/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-21/actual.js
@@ -0,0 +1 @@
+0b01_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-21/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-21/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-22/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-22/actual.js
@@ -0,0 +1 @@
+0o1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-22/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-22/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-23/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-23/actual.js
@@ -0,0 +1 @@
+0o_1_1_
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-23/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-23/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-24/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-24/actual.js
@@ -0,0 +1 @@
+._1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-24/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-24/options.json
@@ -0,0 +1 @@
+{ "throws": "Unexpected token (1:0)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-25/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-25/actual.js
@@ -0,0 +1 @@
+0o01_8
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-25/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-25/options.json
@@ -0,0 +1 @@
+{ "throws": "Unexpected token, expected ; (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-26/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-26/actual.js
@@ -0,0 +1 @@
+0b2_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-26/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-26/options.json
@@ -0,0 +1 @@
+{ "throws": "Expected number in radix 2 (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-27/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-27/actual.js
@@ -0,0 +1 @@
+0xZ_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-27/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-27/options.json
@@ -0,0 +1 @@
+{ "throws": "Expected number in radix 16 (1:2)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-3/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-3/actual.js
@@ -0,0 +1 @@
+1__1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-3/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-3/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:1)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-4/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-4/actual.js
@@ -0,0 +1 @@
+1_1_.1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-4/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-4/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:3)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-5/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-5/actual.js
@@ -0,0 +1 @@
+1_1._1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-5/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-5/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:4)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-6/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-6/actual.js
@@ -0,0 +1 @@
+1_1.1_e1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-6/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-6/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-7/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-7/actual.js
@@ -0,0 +1 @@
+1_1.1_E1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-7/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-7/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:5)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-8/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-8/actual.js
@@ -0,0 +1 @@
+1_1.1e_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-8/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-8/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-9/actual.js b/test/fixtures/experimental/numeric-literal-separator/invalid-9/actual.js
@@ -0,0 +1 @@
+1_1.1E_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/invalid-9/options.json b/test/fixtures/experimental/numeric-literal-separator/invalid-9/options.json
@@ -0,0 +1 @@
+{ "throws": "Invalid NumericLiteralSeparator (1:6)" }
diff --git a/test/fixtures/experimental/numeric-literal-separator/options.json b/test/fixtures/experimental/numeric-literal-separator/options.json
@@ -0,0 +1,3 @@
+{
+  "plugins": ["numericSeparator"]
+}
diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-0/actual.js b/test/fixtures/experimental/numeric-literal-separator/valid-0/actual.js
@@ -0,0 +1 @@
+1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-0/expected.json b/test/fixtures/experimental/numeric-literal-separator/valid-0/expected.json
@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start": 0,
+  "end": 3,
+  "loc": {
+    "start": {
+      "line": 1,
+      "column": 0
+    },
+    "end": {
+      "line": 1,
+      "column": 3
+    }
+  },
+  "program": {
+    "type": "Program",
+    "start": 0,
+    "end": 3,
+    "loc": {
+      "start": {
+        "line": 1,
+        "column": 0
+      },
+      "end": {
+        "line": 1,
+        "column": 3
+      }
+    },
+    "sourceType": "script",
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 3,
+        "loc": {
+          "start": {
+            "line": 1,
+            "column": 0
+          },
+          "end": {
+            "line": 1,
+            "column": 3
+          }
+        },
+        "expression": {
+          "type": "NumericLiteral",
+          "start": 0,
+          "end": 3,
+          "loc": {
+            "start": {
+              "line": 1,
+              "column": 0
+            },
+            "end": {
+              "line": 1,
+              "column": 3
+            }
+          },
+          "extra": {
+            "rawValue": 11,
+            "raw": "1_1"
+          },
+          "value": 11
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-1/actual.js b/test/fixtures/experimental/numeric-literal-separator/valid-1/actual.js
@@ -0,0 +1 @@
+1_1.1_1
diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-1/expected.json b/test/fixtures/experimental/numeric-literal-separator/valid-1/expected.json
@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start": 0,
+  "end": 7,
+  "loc": {
+    "start": {
+      "line": 1,
+      "column": 0
+    },
+    "end": {
+      "line": 1,
+      "column": 7
+    }
+  },
+  "program": {
+    "type": "Program",
+    "start": 0,
+    "end": 7,
+    "loc": {
+      "start": {
+        "line": 1,
+        "column": 0
+      },
+      "end": {
+        "line": 1,
+        "column": 7
+      }
+    },
+    "sourceType": "script",
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 7,
+        "loc": {
+          "start": {
+            "line": 1,
+            "column": 0
+          },
+          "end": {
+            "line": 1,
+            "column": 7
+          }
+        },
+        "expression": {
+          "type": "NumericLiteral",
+          "start": 0,
+          "end": 7,
+          "loc": {
+            "start": {
+              "line": 1,
+              "column": 0
+            },
+            "end": {
+              "line": 1,
+              "column": 7
+            }
+          },
+          "extra": {
+            "rawValue": 11.11,
+            "raw": "1_1.1_1"
+          },
+          "value": 11.11
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-10/actual.js b/test/fixtures/experimental/numeric-literal-separator/valid-10/actual.js
@@ -0,0 +1 @@
+0o1_1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{ "throws": "Invalid NumericLiteralSeparator (1:1)" }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{ "throws": "Unexpected token, expected ; (1:5)" }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{ "throws": "Expected number in radix 2 (1:2)" }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{ "throws": "Expected number in radix 16 (1:2)" }