Build Pipelining Support

src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala

@@ -54,19 +54,8 @@ trait MacroRuntimes extends JavaReflectionRuntimes {
   /** Macro classloader that is used to resolve and run macro implementations.
    *  Loads classes from from -cp (aka the library classpath).
    *  Is also capable of detecting REPL and reusing its classloader.
-   *
-   *  When -Xmacro-jit is enabled, we sometimes fallback to on-the-fly compilation of macro implementations,
-   *  which compiles implementations into a virtual directory (very much like REPL does) and then conjures
-   *  a classloader mapped to that virtual directory.
    */
-  private lazy val defaultMacroClassloaderCache = {
-    def attemptClose(loader: ClassLoader): Unit = loader match {
-      case u: URLClassLoader => debuglog("Closing macro runtime classloader"); u.close()
-      case afcl: AbstractFileClassLoader => attemptClose(afcl.getParent)
-      case _ => ???
-    }
-    perRunCaches.newGeneric(findMacroClassLoader, attemptClose _)
-  }
+  private lazy val defaultMacroClassloaderCache: () => ClassLoader = perRunCaches.newGeneric(findMacroClassLoader())
   def defaultMacroClassloader: ClassLoader = defaultMacroClassloaderCache()
 
   /** Abstracts away resolution of macro runtimes.

src/compiler/scala/tools/nsc/CloseableRegistry.scala

@@ -0,0 +1,34 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools.nsc
+
+import scala.util.control.NonFatal
+
+/** Registry for resources to close when `Global` is closed */
+final class CloseableRegistry {
+  private[this] var closeables: List[java.io.Closeable] = Nil
+  final def registerClosable(c: java.io.Closeable): Unit = {
+    closeables ::= c
+  }
+
+  def close(): Unit = {
+    for (c <- closeables) {
+      try {
+        c.close()
+      } catch {
+        case NonFatal(_) =>
+      }
+    }
+    closeables = Nil
+  }
+}

src/compiler/scala/tools/nsc/CompilationUnits.scala

@@ -21,7 +21,7 @@ trait CompilationUnits { global: Global =>
   /** An object representing a missing compilation unit.
    */
   object NoCompilationUnit extends CompilationUnit(NoSourceFile) {
-    override lazy val isJava = false
+    override val isJava = false
     override def exists = false
     override def toString() = "NoCompilationUnit"
   }
@@ -153,7 +153,7 @@ trait CompilationUnits { global: Global =>
     final def comment(pos: Position, msg: String): Unit = {}
 
     /** Is this about a .java source file? */
-    lazy val isJava = source.file.name.endsWith(".java")
+    val isJava = source.file.name.endsWith(".java")
 
     override def toString() = source.toString()
   }

src/compiler/scala/tools/nsc/GenericRunnerSettings.scala

@@ -16,7 +16,14 @@ import java.net.URL
 import scala.tools.util.PathResolver
 
 class GenericRunnerSettings(error: String => Unit) extends Settings(error) {
-  lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs
+  lazy val classpathURLs: Seq[URL] = {
+    val registry = new CloseableRegistry
+    try {
+      new PathResolver(this, registry).resultAsURLs
+    } finally {
+      registry.close()
+    }
+  }
 
   val howtorun =
     ChoiceSetting(

src/compiler/scala/tools/nsc/Global.scala

@@ -40,9 +40,11 @@ import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
 import scala.tools.nsc.classpath._
 import scala.tools.nsc.profile.Profiler
+import java.io.Closeable
 
 class Global(var currentSettings: Settings, reporter0: Reporter)
     extends SymbolTable
+    with Closeable
     with CompilationUnits
     with Plugins
     with PhaseAssembly
@@ -400,12 +402,16 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
     def apply(unit: CompilationUnit): Unit
 
+    // run only the phases needed
+    protected def shouldSkipThisPhaseForJava: Boolean = {
+      this.id > (if (createJavadoc) currentRun.typerPhase.id
+      else currentRun.namerPhase.id)
+    }
+
     /** Is current phase cancelled on this unit? */
     def cancelled(unit: CompilationUnit) = {
-      // run the typer only if in `createJavadoc` mode
-      val maxJavaPhase = if (createJavadoc) currentRun.typerPhase.id else currentRun.namerPhase.id
       if (Thread.interrupted()) reporter.cancelled = true
-      reporter.cancelled || unit.isJava && this.id > maxJavaPhase
+      reporter.cancelled || unit.isJava && shouldSkipThisPhaseForJava
     }
 
     private def beforeUnit(unit: CompilationUnit): Unit = {
@@ -817,7 +823,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
-    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings))
+    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry))
     val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*)
     platform.currentClassPath = Some(newClassPath)
     invalidateClassPathEntries(urls.map(_.getPath): _*)
@@ -879,7 +885,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
       }
       entries(classPath) find matchesCanonical match {
         case Some(oldEntry) =>
-          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings))
+          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry))
         case None =>
           error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
           None
@@ -1706,6 +1712,13 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
   }
 
   def createJavadoc    = false
+
+  final val closeableRegistry: CloseableRegistry = new CloseableRegistry
+
+  def close(): Unit = {
+    perRunCaches.clearAll()
+    closeableRegistry.close()
+  }
 }
 
 object Global {

src/compiler/scala/tools/nsc/PickleExtractor.scala

@@ -0,0 +1,133 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools.nsc
+
+import java.io.Closeable
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor, _}
+
+import scala.collection.JavaConverters.{asScalaBufferConverter, bufferAsJavaListConverter, collectionAsScalaIterableConverter}
+import scala.reflect.internal.pickling.ByteCodecs
+import scala.reflect.io.RootPath
+import scala.tools.asm.tree.ClassNode
+import scala.tools.asm.{ClassReader, ClassWriter, Opcodes}
+
+object PickleExtractor {
+
+  def main(args: Array[String]): Unit = {
+    args.toList match {
+      case input :: output :: Nil =>
+        process(Paths.get(input), Paths.get(output))
+      case _ =>
+    }
+  }
+  def process(input: Path, output: Path): Unit = {
+    val inputPath = RootPath(input, writable = false)
+    val outputPath = RootPath(output, writable = true)
+    try {
+      val root = inputPath.root
+      Files.createDirectories(outputPath.root)
+      val visitor = new SimpleFileVisitor[Path] {
+        override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = {
+          if (dir != root) {
+            val outputDir = outputPath.root.resolve(root.relativize(dir).toString)
+            Files.createDirectories(outputDir)
+          }
+          FileVisitResult.CONTINUE
+        }
+        override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+          if (file.getFileName.toString.endsWith(".class")) {
+            stripClassFile(Files.readAllBytes(file)) match {
+              case Class(out) =>
+                Files.write(outputPath.root.resolve(root.relativize(file).toString), out)
+              case Pickle(out) =>
+                Files.write(outputPath.root.resolve(root.relativize(file).toString.replaceAll(".class$", ".sig")), out)
+              case Skip =>
+            }
+          }
+          FileVisitResult.CONTINUE
+        }
+      }
+      Files.walkFileTree(root, visitor)
+    } finally {
+      inputPath.close()
+      outputPath.close()
+    }
+  }
+
+  def stripClassFile(classfile: Array[Byte]): OutputFile = {
+    val input = new ClassNode()
+    new ClassReader(classfile).accept(input, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES | ClassReader.SKIP_CODE)
+    var output = new ClassNode()
+    output.name = input.name
+    output.access = input.access
+    output.version = input.version
+
+    var foundScalaSig = false
+
+    def isScalaAnnotation(desc: String) = (desc == "Lscala/reflect/ScalaSignature;" || desc == "Lscala/reflect/ScalaLongSignature;") && {
+      foundScalaSig = true
+
+      true
+    }
+
+    var pickleData: Array[Byte] = null
+    if (input.visibleAnnotations != null) {
+      input.visibleAnnotations.asScala.foreach { node =>
+        if (node.desc == "Lscala/reflect/ScalaSignature;") {
+          val Array("bytes", data: String) = node.values.toArray()
+          val bytes = data.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+          val len = ByteCodecs.decode(bytes)
+          pickleData = bytes.take(len)
+        } else if (node.desc == "Lscala/reflect/ScalaLongSignature;") {
+          val Array("bytes", data: java.util.Collection[String @unchecked]) = node.values.toArray()
+          val encoded = data.asScala.toArray flatMap (_.getBytes(java.nio.charset.StandardCharsets.UTF_8))
+          val len = ByteCodecs.decode(encoded)
+          pickleData = encoded.take(len)
+        }
+      }
+      output.visibleAnnotations = input.visibleAnnotations.asScala.filter(node => isScalaAnnotation(node.desc) && {
+        true
+      }).asJava
+    }
+    var foundScalaAttr = false
+    if (input.attrs != null) {
+      output.attrs = input.attrs.asScala.filter(attr => (attr.`type` == "Scala" || attr.`type` == "ScalaSig") && {
+        foundScalaAttr = true;
+        true
+      }).asJava
+    }
+    val writer = new ClassWriter(Opcodes.ASM7)
+    val isScalaRaw = foundScalaAttr && !foundScalaSig
+    if (isScalaRaw) Skip
+    else {
+      if (pickleData == null) {
+        output = input
+        output.accept(writer)
+        Class(writer.toByteArray)
+      } else {
+        output.accept(writer)
+        Pickle(pickleData)
+      }
+    }
+  }
+
+  sealed abstract class OutputFile
+
+  case object Skip extends OutputFile
+
+  case class Class(content: Array[Byte]) extends OutputFile
+
+  case class Pickle(content: Array[Byte]) extends OutputFile
+
+}