Implement generic number literals

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -109,6 +109,13 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   case class ContextBounds(bounds: TypeBoundsTree, cxBounds: List[Tree])(implicit @constructorOnly src: SourceFile) extends TypTree
   case class PatDef(mods: Modifiers, pats: List[Tree], tpt: Tree, rhs: Tree)(implicit @constructorOnly src: SourceFile) extends DefTree
   case class Export(impliedOnly: Boolean, expr: Tree, selectors: List[Tree])(implicit @constructorOnly src: SourceFile) extends Tree
+  case class Number(digits: String, kind: NumberKind)(implicit @constructorOnly src: SourceFile) extends TermTree
+
+  enum NumberKind {
+    case Whole(radix: Int)
+    case Decimal
+    case Floating
+  }
 
   /** Short-lived usage in typer, does not need copy/transform/fold infrastructure */
   case class DependentTypeTree(tp: List[Symbol] => Type)(implicit @constructorOnly src: SourceFile) extends Tree
@@ -558,6 +565,10 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
       case tree: Export if (impliedOnly == tree.impliedOnly) && (expr eq tree.expr) && (selectors eq tree.selectors) => tree
       case _ => finalize(tree, untpd.Export(impliedOnly, expr, selectors)(tree.source))
     }
+    def Number(tree: Tree)(digits: String, kind: NumberKind)(implicit ctx: Context): Tree = tree match {
+      case tree: Number if (digits == tree.digits) && (kind == tree.kind) => tree
+      case _ => finalize(tree, untpd.Number(digits, kind))
+    }
     def TypedSplice(tree: Tree)(splice: tpd.Tree)(implicit ctx: Context): ProxyTree = tree match {
       case tree: TypedSplice if splice `eq` tree.splice => tree
       case _ => finalize(tree, untpd.TypedSplice(splice)(ctx))
@@ -612,7 +623,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
         cpy.PatDef(tree)(mods, transform(pats), transform(tpt), transform(rhs))
       case Export(impliedOnly, expr, selectors) =>
         cpy.Export(tree)(impliedOnly, transform(expr), selectors)
-      case TypedSplice(_) =>
+      case Number(_, _) | TypedSplice(_) =>
         tree
       case _ =>
         super.transformMoreCases(tree)
@@ -667,6 +678,8 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
         this(this(this(x, pats), tpt), rhs)
       case Export(_, expr, _) =>
         this(x, expr)
+      case Number(_, _) =>
+        x
       case TypedSplice(splice) =>
         this(x, splice)
       case _ =>

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -669,6 +669,11 @@ class Definitions {
   @tu lazy val StatsModule: Symbol = ctx.requiredModule("dotty.tools.dotc.util.Stats")
     @tu lazy val Stats_doRecord: Symbol = StatsModule.requiredMethod("doRecord")
 
+  @tu lazy val FromDigitsClass: ClassSymbol           = ctx.requiredClass("scala.util.FromDigits")
+  @tu lazy val FromDigits_WithRadixClass: ClassSymbol = ctx.requiredClass("scala.util.FromDigits.WithRadix")
+  @tu lazy val FromDigits_DecimalClass: ClassSymbol   = ctx.requiredClass("scala.util.FromDigits.Decimal")
+  @tu lazy val FromDigits_FloatingClass: ClassSymbol  = ctx.requiredClass("scala.util.FromDigits.Floating")
+
   @tu lazy val XMLTopScopeModule: Symbol = ctx.requiredModule("scala.xml.TopScope")
 
   @tu lazy val CommandLineParserModule: Symbol = ctx.requiredModule("scala.util.CommandLineParser")

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -442,6 +442,7 @@ object StdNames {
     val flagsFromBits : N       = "flagsFromBits"
     val flatMap: N              = "flatMap"
     val foreach: N              = "foreach"
+    val fromDigits: N           = "fromDigits"
     val fromProduct: N          = "fromProduct"
     val genericArrayOps: N      = "genericArrayOps"
     val genericClass: N         = "genericClass"

compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala

@@ -519,20 +519,13 @@ object JavaScanners {
     // Errors -----------------------------------------------------------------
 
     override def toString(): String = token match {
-      case IDENTIFIER =>
-        "id(" + name + ")"
-      case CHARLIT =>
-        "char(" + intVal + ")"
-      case INTLIT =>
-        "int(" + intVal + ")"
-      case LONGLIT =>
-        "long(" + intVal + ")"
-      case FLOATLIT =>
-        "float(" + floatVal + ")"
-      case DOUBLELIT =>
-        "double(" + doubleVal + ")"
-      case STRINGLIT =>
-        "string(" + name + ")"
+      case IDENTIFIER => s"id($name)"
+      case CHARLIT => s"char($strVal)"
+      case INTLIT => s"int($strVal, $base)"
+      case LONGLIT => s"long($strVal, $base)"
+      case FLOATLIT => s"float($strVal)"
+      case DOUBLELIT => s"double($strVal)"
+      case STRINGLIT => s"string($strVal)"
       case SEMI =>
         ";"
       case COMMA =>

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -1016,24 +1016,37 @@ object Parsers {
      *  @param negOffset   The offset of a preceding `-' sign, if any.
      *                     If the literal is not negated, negOffset = in.offset.
      */
-    def literal(negOffset: Int = in.offset, inPattern: Boolean = false, inStringInterpolation: Boolean = false): Tree = {
-
-      def literalOf(token: Token): Literal = {
+    def literal(negOffset: Int = in.offset, inPattern: Boolean = false, inType: Boolean = false, inStringInterpolation: Boolean = false): Tree = {
+      def literalOf(token: Token): Tree = {
         val isNegated = negOffset < in.offset
-        val value = token match {
-          case CHARLIT                => in.charVal
-          case INTLIT                 => in.intVal(isNegated).toInt
-          case LONGLIT                => in.intVal(isNegated)
-          case FLOATLIT               => in.floatVal(isNegated).toFloat
-          case DOUBLELIT              => in.doubleVal(isNegated)
-          case STRINGLIT | STRINGPART => in.strVal
-          case TRUE                   => true
-          case FALSE                  => false
-          case NULL                   => null
-          case _                      =>
-            syntaxErrorOrIncomplete(IllegalLiteral())
-            null
-        }
+        def digits0 = in.removeNumberSeparators(in.strVal)
+        def digits = if (isNegated) "-" + digits0 else digits0
+        if (!inType)
+          token match {
+            case INTLIT  => return Number(digits, NumberKind.Whole(in.base))
+            case DECILIT => return Number(digits, NumberKind.Decimal)
+            case EXPOLIT => return Number(digits, NumberKind.Floating)
+            case _ =>
+          }
+        import scala.util.FromDigits._
+        val value =
+          try token match {
+            case INTLIT                        => intFromDigits(digits, in.base)
+            case LONGLIT                       => longFromDigits(digits, in.base)
+            case FLOATLIT                      => floatFromDigits(digits)
+            case DOUBLELIT | DECILIT | EXPOLIT => doubleFromDigits(digits)
+            case CHARLIT                       => in.strVal.head
+            case STRINGLIT | STRINGPART        => in.strVal
+            case TRUE                          => true
+            case FALSE                         => false
+            case NULL                          => null
+            case _                             =>
+              syntaxErrorOrIncomplete(IllegalLiteral())
+              null
+          }
+          catch {
+            case ex: FromDigitsException => syntaxErrorOrIncomplete(ex.getMessage)
+          }
         Literal(Constant(value))
       }
 
@@ -1163,6 +1176,7 @@ object Parsers {
     }
 
 /* ------------- TYPES ------------------------------------------------------ */
+
     /** Same as [[typ]], but if this results in a wildcard it emits a syntax error and
      *  returns a tree for type `Any` instead.
      */
@@ -1378,11 +1392,11 @@ object Parsers {
         }
       else if (in.token == LBRACE)
         atSpan(in.offset) { RefinedTypeTree(EmptyTree, refinement()) }
-      else if (isSimpleLiteral) { SingletonTypeTree(literal()) }
+      else if (isSimpleLiteral) { SingletonTypeTree(literal(inType = true)) }
       else if (isIdent(nme.raw.MINUS) && in.lookaheadIn(numericLitTokens)) {
         val start = in.offset
         in.nextToken()
-        SingletonTypeTree(literal(negOffset = start))
+        SingletonTypeTree(literal(negOffset = start, inType = true))
       }
       else if (in.token == USCORE) {
         if (ctx.settings.strict.value) {
@@ -2329,12 +2343,11 @@ object Parsers {
       if (isIdent(nme.raw.BAR)) { in.nextToken(); pattern1() :: patternAlts() }
       else Nil
 
-    /**  Pattern1          ::= PatVar Ascription
-     *                      |  Pattern2
+    /**  Pattern1          ::= Pattern2 [Ascription]
      */
     def pattern1(): Tree = {
       val p = pattern2()
-      if (isVarPattern(p) && in.token == COLON) {
+      if (in.token == COLON) {
         in.nextToken()
         ascription(p, Location.InPattern)
       }

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -123,99 +123,10 @@ object Scanners {
     def setStrVal(): Unit =
       strVal = flushBuf(litBuf)
 
-    /** Convert current strVal to char value
-      */
-    def charVal: Char = if (strVal.length > 0) strVal.charAt(0) else 0
-
-    /** Convert current strVal, base to long value
-      *  This is tricky because of max negative value.
-      */
-    def intVal(negated: Boolean): Long =
-      if (token == CHARLIT && !negated)
-        charVal
-      else {
-        var value: Long = 0
-        val divider = if (base == 10) 1 else 2
-        val limit: Long =
-          if (token == LONGLIT) Long.MaxValue else Int.MaxValue
-        var i = 0
-        val len = strVal.length
-        while (i < len) {
-          val c = strVal charAt i
-          if (! isNumberSeparator(c)) {
-            val d = digit2int(c, base)
-            if (d < 0) {
-              error(s"malformed integer number")
-              return 0
-            }
-            if (value < 0 ||
-              limit / (base / divider) < value ||
-              limit - (d / divider) < value * (base / divider) &&
-                !(negated && limit == value * base - 1 + d)) {
-              error("integer number too large")
-              return 0
-            }
-            value = value * base + d
-          }
-          i += 1
-        }
-        if (negated) -value else value
-      }
-
-    def intVal: Long = intVal(false)
-
-    private val zeroFloat = raw"[0.]+(?:[eE][+-]?[0-9]+)?[fFdD]?".r
-
-    /** Convert current strVal, base to double value
-      */
-    def floatVal(negated: Boolean): Float = {
-      assert(token == FLOATLIT)
-      val text = removeNumberSeparators(strVal)
-      try {
-        val value: Float = java.lang.Float.valueOf(text).floatValue()
-        if (value > Float.MaxValue)
-          errorButContinue("floating point number too large")
-
-        if (value == 0.0f && !zeroFloat.pattern.matcher(text).matches)
-          errorButContinue("floating point number too small")
-        if (negated) -value else value
-      }
-      catch {
-        case _: NumberFormatException =>
-          error("malformed floating point number")
-          0.0f
-      }
-    }
-
-    def floatVal: Float = floatVal(false)
-
-    /** Convert current strVal, base to double value
-      */
-    def doubleVal(negated: Boolean): Double = {
-      assert(token == DOUBLELIT)
-      val text = removeNumberSeparators(strVal)
-      try {
-        val value: Double = java.lang.Double.valueOf(text).doubleValue()
-        if (value > Double.MaxValue)
-          errorButContinue("double precision floating point number too large")
-
-        if (value == 0.0d && !zeroFloat.pattern.matcher(text).matches)
-          errorButContinue("double precision floating point number too small")
-        if (negated) -value else value
-      }
-      catch {
-        case _: NumberFormatException =>
-          error("malformed floating point number")
-          0.0
-      }
-    }
-
-    def doubleVal: Double = doubleVal(false)
-
     @inline def isNumberSeparator(c: Char): Boolean = c == '_'
 
     @inline def removeNumberSeparators(s: String): String =
-      if (s.indexOf('_') > 0) s.replaceAllLiterally("_", "") /*.replaceAll("'","")*/ else s
+      if (s.indexOf('_') > 0) s.replaceAllLiterally("_", "") else s
 
     // disallow trailing numeric separator char, but continue lexing
     def checkNoTrailingSeparator(): Unit =
@@ -1228,7 +1139,7 @@ object Scanners {
      *  if one is present.
      */
     protected def getFraction(): Unit = {
-      token = DOUBLELIT
+      token = DECILIT
       while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
         putChar(ch)
         nextChar()
@@ -1252,7 +1163,7 @@ object Scanners {
           }
           checkNoTrailingSeparator()
         }
-        token = DOUBLELIT
+        token = EXPOLIT
       }
       if (ch == 'd' || ch == 'D') {
         putChar(ch)
@@ -1329,11 +1240,11 @@ object Scanners {
 
     def show: String = token match {
       case IDENTIFIER | BACKQUOTED_IDENT => s"id($name)"
-      case CHARLIT => s"char($intVal)"
-      case INTLIT => s"int($intVal)"
-      case LONGLIT => s"long($intVal)"
-      case FLOATLIT => s"float($floatVal)"
-      case DOUBLELIT => s"double($doubleVal)"
+      case CHARLIT => s"char($strVal)"
+      case INTLIT => s"int($strVal, $base)"
+      case LONGLIT => s"long($strVal, $base)"
+      case FLOATLIT => s"float($strVal)"
+      case DOUBLELIT => s"double($strVal)"
       case STRINGLIT => s"string($strVal)"
       case STRINGPART => s"stringpart($strVal)"
       case INTERPOLATIONID => s"interpolationid($name)"

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -37,17 +37,19 @@ abstract class TokensCommon {
   /** literals */
   final val CHARLIT = 3;           enter(CHARLIT, "character literal")
   final val INTLIT = 4;            enter(INTLIT, "integer literal")
-  final val LONGLIT = 5;           enter(LONGLIT, "long literal")
-  final val FLOATLIT = 6;          enter(FLOATLIT, "float literal")
-  final val DOUBLELIT = 7;         enter(DOUBLELIT, "double literal")
-  final val STRINGLIT = 8;         enter(STRINGLIT, "string literal")
-  final val STRINGPART = 9;        enter(STRINGPART, "string literal", "string literal part")
-  //final val INTERPOLATIONID = 10;  enter(INTERPOLATIONID, "string interpolator")
-  //final val QUOTEID = 11;        enter(QUOTEID, "quoted identifier") // TODO: deprecate
+  final val DECILIT = 5;           enter(DECILIT, "number literal")  // with decimal point
+  final val EXPOLIT = 6;           enter(EXPOLIT, "number literal with exponent")
+  final val LONGLIT = 7;           enter(LONGLIT, "long literal")
+  final val FLOATLIT = 8;          enter(FLOATLIT, "float literal")
+  final val DOUBLELIT = 9;         enter(DOUBLELIT, "double literal")
+  final val STRINGLIT = 10;         enter(STRINGLIT, "string literal")
+  final val STRINGPART = 11;       enter(STRINGPART, "string literal", "string literal part")
+  //final val INTERPOLATIONID = 12;  enter(INTERPOLATIONID, "string interpolator")
+  //final val QUOTEID = 13;        enter(QUOTEID, "quoted identifier") // TODO: deprecate
 
   /** identifiers */
-  final val IDENTIFIER = 12;       enter(IDENTIFIER, "identifier")
-  //final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident")
+  final val IDENTIFIER = 14;       enter(IDENTIFIER, "identifier")
+  //final val BACKQUOTED_IDENT = 15; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident")
 
   /** alphabetic keywords */
   final val IF = 20;               enter(IF, "if")
@@ -150,10 +152,10 @@ object Tokens extends TokensCommon {
   final val minToken = EMPTY
   final def maxToken: Int = XMLSTART
 
-  final val INTERPOLATIONID = 10;  enter(INTERPOLATIONID, "string interpolator")
-  final val QUOTEID = 11;          enter(QUOTEID, "quoted identifier") // TODO: deprecate
+  final val INTERPOLATIONID = 12;  enter(INTERPOLATIONID, "string interpolator")
+  final val QUOTEID = 13;          enter(QUOTEID, "quoted identifier") // TODO: deprecate
 
-  final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident")
+  final val BACKQUOTED_IDENT = 15; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident")
 
   final val identifierTokens: TokenSet = BitSet(IDENTIFIER, BACKQUOTED_IDENT)
 
@@ -260,7 +262,7 @@ object Tokens extends TokensCommon {
   final val stopScanTokens: BitSet = mustStartStatTokens |
     BitSet(IF, ELSE, WHILE, DO, FOR, YIELD, NEW, TRY, CATCH, FINALLY, THROW, RETURN, MATCH, SEMI, EOF)
 
-  final val numericLitTokens: TokenSet = BitSet(INTLIT, LONGLIT, FLOATLIT, DOUBLELIT)
+  final val numericLitTokens: TokenSet = BitSet(INTLIT, DECILIT, EXPOLIT, LONGLIT, FLOATLIT, DOUBLELIT)
 
   final val statCtdTokens: BitSet = BitSet(THEN, ELSE, DO, CATCH, FINALLY, YIELD, MATCH)
 

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -604,6 +604,8 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         changePrec(GlobalPrec) {
           keywordStr("try ") ~ toText(expr) ~ " " ~ keywordStr("catch") ~ " {" ~ toText(handler) ~ "}" ~ optText(finalizer)(keywordStr(" finally ") ~ _)
         }
+      case Number(digits, kind) =>
+        digits
       case Quote(tree) =>
         if (tree.isType) keywordStr("'[") ~ toTextGlobal(dropBlock(tree)) ~ keywordStr("]")
         else keywordStr("'{") ~ toTextGlobal(dropBlock(tree)) ~ keywordStr("}")

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -591,16 +591,16 @@ private class ExtractAPICollector(implicit val ctx: Context) extends ThunkHolder
 
   def apiAnnotations(s: Symbol): List[api.Annotation] = {
     val annots = new mutable.ListBuffer[api.Annotation]
-
-    if (Inliner.hasBodyToInline(s)) {
+    val inlineBody = Inliner.bodyToInline(s)
+    if (!inlineBody.isEmpty) {
       // FIXME: If the body of an inlineable method changes, all the reverse
       // dependencies of this method need to be recompiled. sbt has no way
       // of tracking method bodies, so as a hack we include the pretty-printed
       // typed tree of the method as part of the signature we send to sbt.
       // To do this properly we would need a way to hash trees and types in
       // dotty itself.
       val printTypesCtx = ctx.fresh.setSetting(ctx.settings.XprintTypes, true)
-      annots += marker(Inliner.bodyToInline(s).show(printTypesCtx))
+      annots += marker(inlineBody.show(printTypesCtx))
     }
 
     // In the Scala2 ExtractAPI phase we only extract annotations that extend