improvement: Add option to watch unamanaged inputs

This allows us to rerun source generators if the scripts change.

build.sbt

@@ -4,7 +4,7 @@ import build.Dependencies
 import build.Dependencies.{Scala211Version, Scala212Version, SbtVersion}
 
 ThisBuild / dynverSeparator := "-"
-ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshot")
+ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots")
 
 // Add hook for scalafmt validation
 Global / onLoad ~= { old =>

frontend/src/main/scala/bloop/data/Project.scala

@@ -90,9 +90,6 @@ final case class Project(
     customWorkingDirectory.orElse(workspaceDirectory).getOrElse(baseDirectory)
   }
 
-  def allGeneratorInputs: Task[List[AbsolutePath]] =
-    Task.sequence(sourceGenerators.map(_.getSources)).map(_.flatten)
-
   /** Returns concatenated list of "sources" and expanded "sourcesGlobs". */
   def allUnmanagedSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task {
     val buf = mutable.ListBuffer.empty[AbsolutePath]

frontend/src/main/scala/bloop/engine/Interpreter.scala

@@ -120,7 +120,9 @@ object Interpreter {
     val projectsSourcesAndDirs = reachable.map { project =>
       for {
         unmanaged <- project.allUnmanagedSourceFilesAndDirectories
-        generatorSourceDirs = project.sourceGenerators.flatMap(_.sourcesGlobs.map(_.directory))
+        generatorSourceDirs = project.sourceGenerators.flatMap(gen =>
+          gen.sourcesGlobs.map(_.directory) ++ gen.unmangedInputs
+        )
       } yield unmanaged ++ generatorSourceDirs
     }
     val groupTasks =

frontend/src/main/scala/bloop/engine/SourceGenerator.scala

@@ -20,6 +20,7 @@ final case class SourceGenerator(
     cwd: AbsolutePath,
     sourcesGlobs: List[SourcesGlobs],
     outputDirectory: AbsolutePath,
+    unmangedInputs: List[AbsolutePath],
     command: List[String]
 ) {
 
@@ -40,12 +41,12 @@ final case class SourceGenerator(
     needsUpdate(previous).flatMap {
       case SourceGenerator.NoChanges =>
         Task.now(previous)
-      case SourceGenerator.InputChanges(newInputs) =>
+      case SourceGenerator.InputChanges(newInputs, newUnmanagedInputs) =>
         logger.debug("Changes detected to inputs of source generator")(DebugFilter.Compilation)
-        run(newInputs, logger, opts)
-      case SourceGenerator.OutputChanges(inputs) =>
+        run(newInputs, newUnmanagedInputs, logger, opts)
+      case SourceGenerator.OutputChanges(inputs, unmanagedInputs) =>
         logger.debug("Changes detected to outputs of source generator")(DebugFilter.Compilation)
-        run(inputs, logger, opts)
+        run(inputs, unmanagedInputs, logger, opts)
     }
 
   /**
@@ -61,6 +62,7 @@ final case class SourceGenerator(
 
   private def run(
       inputs: Map[AbsolutePath, Int],
+      unmangedInputs: Map[AbsolutePath, Int],
       logger: Logger,
       opts: CommonOptions
   ): Task[SourceGenerator.Run] = {
@@ -71,7 +73,7 @@ final case class SourceGenerator(
 
     Forker.run(cwd, cmd, logger, opts).flatMap {
       case 0 =>
-        hashOutputs.map(SourceGenerator.PreviousRun(inputs, _))
+        hashOutputs.map { SourceGenerator.PreviousRun(inputs, _, unmangedInputs) }
       case exitCode =>
         Task.raiseError(new SourceGenerator.SourceGeneratorException(exitCode))
     }
@@ -80,16 +82,23 @@ final case class SourceGenerator(
   private def needsUpdate(previous: SourceGenerator.Run): Task[SourceGenerator.Changes] = {
     previous match {
       case SourceGenerator.NoRun =>
-        hashInputs.map(SourceGenerator.InputChanges(_))
-      case SourceGenerator.PreviousRun(inputs, outputs) =>
-        hashInputs.flatMap { newInputs =>
-          if (newInputs != inputs) Task.now(SourceGenerator.InputChanges(newInputs))
-          else {
-            hashOutputs.map { newOutputs =>
-              if (newOutputs != outputs) SourceGenerator.OutputChanges(newInputs)
-              else SourceGenerator.NoChanges
+        Task.zip2(hashInputs, hashUnamanagedInputs).map {
+          case (inputs, unmanagedInputs) =>
+            SourceGenerator.InputChanges(inputs, unmanagedInputs)
+        }
+      case SourceGenerator.PreviousRun(inputs, outputs, unmanagedInputs) =>
+        Task.zip2(hashInputs, hashUnamanagedInputs).flatMap {
+          case (newInputs, newUnmanagedInputs) =>
+            if (newInputs != inputs || newUnmanagedInputs != unmanagedInputs)
+              Task.now(SourceGenerator.InputChanges(newInputs, newUnmanagedInputs))
+            else {
+              hashOutputs.map { newOutputs =>
+                if (newOutputs != outputs)
+                  SourceGenerator.OutputChanges(newInputs, newUnmanagedInputs)
+                else
+                  SourceGenerator.NoChanges
+              }
             }
-          }
         }
     }
   }
@@ -98,6 +107,10 @@ final case class SourceGenerator(
     for (inputs <- getSources) yield hashFiles(inputs)
   }
 
+  private def hashUnamanagedInputs: Task[Map[AbsolutePath, Int]] = Task {
+    hashFiles(unmangedInputs)
+  }
+
   private def hashOutputs: Task[Map[AbsolutePath, Int]] = Task {
     val outputs = Paths.pathFilesUnder(outputDirectory, "glob:**")
     hashFiles(outputs)
@@ -111,13 +124,22 @@ object SourceGenerator {
 
   sealed trait Run
   case object NoRun extends Run
-  case class PreviousRun(knownInputs: Map[AbsolutePath, Int], knownOutputs: Map[AbsolutePath, Int])
-      extends Run
+  case class PreviousRun(
+      knownInputs: Map[AbsolutePath, Int],
+      knownOutputs: Map[AbsolutePath, Int],
+      knownUnmanagedInputs: Map[AbsolutePath, Int]
+  ) extends Run
 
   private sealed trait Changes
   private case object NoChanges extends Changes
-  private case class InputChanges(newInputs: Map[AbsolutePath, Int]) extends Changes
-  private case class OutputChanges(inputs: Map[AbsolutePath, Int]) extends Changes
+  private case class InputChanges(
+      newInputs: Map[AbsolutePath, Int],
+      unamanagedInputs: Map[AbsolutePath, Int]
+  ) extends Changes
+  private case class OutputChanges(
+      inputs: Map[AbsolutePath, Int],
+      unamanagedInputs: Map[AbsolutePath, Int]
+  ) extends Changes
 
   def fromConfig(cwd: AbsolutePath, generator: Config.SourceGenerator): SourceGenerator = {
     val sourcesGlobs = generator.sourcesGlobs.map {
@@ -136,6 +158,7 @@ object SourceGenerator {
       cwd,
       sourcesGlobs,
       AbsolutePath(generator.outputDirectory),
+      generator.unmanagedInputs.map(AbsolutePath.apply),
       generator.command
     )
   }

frontend/src/main/scala/bloop/engine/caches/SourceGeneratorCache.scala

@@ -26,7 +26,7 @@ final class SourceGeneratorCache private (
       )
       .map {
         case SourceGenerator.NoRun => Nil
-        case SourceGenerator.PreviousRun(_, outputs) => outputs.keys.toList.sortBy(_.syntax)
+        case SourceGenerator.PreviousRun(_, outputs, _) => outputs.keys.toList.sortBy(_.syntax)
       }
   }
 }

frontend/src/test/resources/source-generator.py

@@ -58,3 +58,7 @@ def main(output_dir, args):
         args = sys.argv[2:]
         main(output_directory, args)
 
+
+def random():
+    return 123
+

frontend/src/test/scala/bloop/FileWatchingSpec.scala

@@ -10,25 +10,21 @@ import bloop.config.Config
 import bloop.data.Project
 import bloop.engine.Dag
 import bloop.engine.ExecutionContext
-import bloop.internal.build.BuildTestInfo
 import bloop.io.AbsolutePath
 import bloop.io.Environment.lineSeparator
 import bloop.logging.DebugFilter
 import bloop.logging.PublisherLogger
 import bloop.logging.RecordingLogger
 import bloop.task.Task
 import bloop.testing.BaseSuite
-import bloop.util.CrossPlatform
 import bloop.util.TestProject
 import bloop.util.TestUtil
 
 import monix.reactive.MulticastStrategy
 import monix.reactive.Observable
 
 object FileWatchingSpec extends BaseSuite {
-  private val generator: List[String] =
-    if (CrossPlatform.isWindows) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
-    else List(BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
+  private val generator: List[String] = TestUtil.generator
 
   System.setProperty("file-watcher-batch-window-ms", "100")
 

frontend/src/test/scala/bloop/SourceGeneratorBspSpec.scala

@@ -8,19 +8,16 @@ import ch.epfl.scala.bsp.Uri
 import bloop.bsp.BspBaseSuite
 import bloop.cli.BspProtocol
 import bloop.config.Config
-import bloop.internal.build.BuildTestInfo
 import bloop.logging.RecordingLogger
-import bloop.util.CrossPlatform
 import bloop.util.TestProject
 import bloop.util.TestUtil
 
 object TcpBspSourceGeneratorSpec extends BspSourceGeneratorSpec(BspProtocol.Tcp)
 object LocalBspSourceGeneratorSpec extends BspSourceGeneratorSpec(BspProtocol.Local)
 
 abstract class BspSourceGeneratorSpec(override val protocol: BspProtocol) extends BspBaseSuite {
-  private val generator =
-    if (CrossPlatform.isWindows) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
-    else List(BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
+
+  private val generator = TestUtil.generator
 
   test("sources request works") {
     TestUtil.withinWorkspace { workspace =>
@@ -53,6 +50,7 @@ abstract class BspSourceGeneratorSpec(override val protocol: BspProtocol) extend
         )
         assertEquals(obtained, expected :: Nil)
       }
+
     }
   }
 

frontend/src/test/scala/bloop/SourceGeneratorSpec.scala

@@ -1,8 +1,5 @@
 package bloop
 
-import scala.sys.process
-import scala.util.control.NonFatal
-
 import bloop.Compiler.Result.Success
 import bloop.cli.ExitStatus
 import bloop.config.Config
@@ -14,21 +11,9 @@ import bloop.util.TestUtil
 
 object SourceGeneratorSpec extends bloop.testing.BaseSuite {
 
-  lazy val hasPython3 = hasPythonNamed("python3")
-  lazy val hasPython2 = hasPythonNamed("python")
-
-  private val generator: List[String] =
-    if (hasPython3) List("python3", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
-    else if (hasPython2) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
-    else Nil
+  val generator = TestUtil.generator
 
-  private def hasPythonNamed(executable: String) = try {
-    process.Process(Seq(executable, "--version")).! == 0
-  } catch {
-    case NonFatal(_) => false
-  }
-
-  lazy val hasPython = hasPython2 || hasPython3
+  lazy val hasPython = generator.nonEmpty
 
   def testOnlyWithPython(name: String)(fun: => Any): Unit = {
     if (hasPython) test(name)(fun)
@@ -205,6 +190,32 @@ object SourceGeneratorSpec extends bloop.testing.BaseSuite {
     }
   }
 
+  testOnlyWithPython("source generator is re-run when generator file is modified") {
+    singleProjectWithSourceGenerator("glob:*.in" :: Nil) { (workspace, project, state) =>
+      writeFile(workspace.resolve("hello.in"), "hello")
+      writeFile(project.srcFor("test.scala", exists = false), assertNInputs(n = 1))
+      val compiledState1 = state.compile(project)
+      val origHash = sourceHashFor("NameLengths_1.scala", project, compiledState1)
+      assertExitStatus(compiledState1, ExitStatus.Ok)
+
+      val generatorFile = AbsolutePath(BuildTestInfo.sampleSourceGenerator.toPath())
+      writeFile(
+        generatorFile,
+        readFile(generatorFile) +
+          """|
+             |def random():
+             |    return 123
+             |
+             |""".stripMargin
+      )
+      val compiledState2 = compiledState1.compile(project)
+      assertExitStatus(compiledState2, ExitStatus.Ok)
+
+      val newHash = sourceHashFor("NameLengths_1.scala", project, compiledState2)
+      assertNotEquals(origHash, newHash)
+    }
+  }
+
   testOnlyWithPython("source generator is re-run when an output file is deleted") {
     singleProjectWithSourceGenerator("glob:*.in" :: Nil) { (workspace, project, state) =>
       val generatorOutput = project.config.sourceGenerators
@@ -257,7 +268,8 @@ object SourceGeneratorSpec extends bloop.testing.BaseSuite {
     val sourceGenerator = Config.SourceGenerator(
       sourcesGlobs = List(Config.SourcesGlobs(workspace.underlying, None, includeGlobs, Nil)),
       outputDirectory = workspace.underlying.resolve("source-generator-output"),
-      command = generator
+      command = generator,
+      unmanagedInputs = List(BuildTestInfo.sampleSourceGenerator.toPath())
     )
     val `A` = TestProject(workspace, "a", Nil, sourceGenerators = sourceGenerator :: Nil)
     val projects = List(`A`)

frontend/src/test/scala/bloop/util/TestUtil.scala

@@ -54,6 +54,7 @@ import _root_.monix.execution.Scheduler
 import org.junit.Assert
 import sbt.internal.inc.BloopComponentCompiler
 import xsbti.ComponentProvider
+import bloop.internal.build.BuildTestInfo
 
 object TestUtil {
   def projectDir(base: Path, name: String): Path = base.resolve(name)
@@ -639,4 +640,18 @@ object TestUtil {
     stacktraces.foreach(threadInfo => sb.append(threadInfo.toString()).append("\n"))
     sb.result()
   }
+
+  private def hasPythonNamed(executable: String) = try {
+    scala.sys.process.Process(Seq(executable, "--version")).! == 0
+  } catch {
+    case NonFatal(_) => false
+  }
+
+  private lazy val hasPython3 = hasPythonNamed("python3")
+  private lazy val hasPython2 = hasPythonNamed("python")
+
+  lazy val generator: List[String] =
+    if (hasPython3) List("python3", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
+    else if (hasPython2) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath)
+    else Nil
 }