diff --git a/build.sbt b/build.sbt index 89f538a39..6031c3469 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ import build.Dependencies import build.Dependencies.{Scala211Version, Scala212Version, SbtVersion} ThisBuild / dynverSeparator := "-" -ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshot") +ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots") // Add hook for scalafmt validation Global / onLoad ~= { old => diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index cc46af71b..da2107c7f 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -90,9 +90,6 @@ final case class Project( customWorkingDirectory.orElse(workspaceDirectory).getOrElse(baseDirectory) } - def allGeneratorInputs: Task[List[AbsolutePath]] = - Task.sequence(sourceGenerators.map(_.getSources)).map(_.flatten) - /** Returns concatenated list of "sources" and expanded "sourcesGlobs". */ def allUnmanagedSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task { val buf = mutable.ListBuffer.empty[AbsolutePath] diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index a647c7da7..2288e94c5 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -120,7 +120,9 @@ object Interpreter { val projectsSourcesAndDirs = reachable.map { project => for { unmanaged <- project.allUnmanagedSourceFilesAndDirectories - generatorSourceDirs = project.sourceGenerators.flatMap(_.sourcesGlobs.map(_.directory)) + generatorSourceDirs = project.sourceGenerators.flatMap(gen => + gen.sourcesGlobs.map(_.directory) ++ gen.unmangedInputs + ) } yield unmanaged ++ generatorSourceDirs } val groupTasks = diff --git a/frontend/src/main/scala/bloop/engine/SourceGenerator.scala b/frontend/src/main/scala/bloop/engine/SourceGenerator.scala index 245b80a72..83d7ebd25 100644 --- a/frontend/src/main/scala/bloop/engine/SourceGenerator.scala +++ b/frontend/src/main/scala/bloop/engine/SourceGenerator.scala @@ -20,6 +20,7 @@ final case class SourceGenerator( cwd: AbsolutePath, sourcesGlobs: List[SourcesGlobs], outputDirectory: AbsolutePath, + unmangedInputs: List[AbsolutePath], command: List[String] ) { @@ -40,12 +41,12 @@ final case class SourceGenerator( needsUpdate(previous).flatMap { case SourceGenerator.NoChanges => Task.now(previous) - case SourceGenerator.InputChanges(newInputs) => + case SourceGenerator.InputChanges(newInputs, newUnmanagedInputs) => logger.debug("Changes detected to inputs of source generator")(DebugFilter.Compilation) - run(newInputs, logger, opts) - case SourceGenerator.OutputChanges(inputs) => + run(newInputs, newUnmanagedInputs, logger, opts) + case SourceGenerator.OutputChanges(inputs, unmanagedInputs) => logger.debug("Changes detected to outputs of source generator")(DebugFilter.Compilation) - run(inputs, logger, opts) + run(inputs, unmanagedInputs, logger, opts) } /** @@ -61,6 +62,7 @@ final case class SourceGenerator( private def run( inputs: Map[AbsolutePath, Int], + unmangedInputs: Map[AbsolutePath, Int], logger: Logger, opts: CommonOptions ): Task[SourceGenerator.Run] = { @@ -71,7 +73,7 @@ final case class SourceGenerator( Forker.run(cwd, cmd, logger, opts).flatMap { case 0 => - hashOutputs.map(SourceGenerator.PreviousRun(inputs, _)) + hashOutputs.map { SourceGenerator.PreviousRun(inputs, _, unmangedInputs) } case exitCode => Task.raiseError(new SourceGenerator.SourceGeneratorException(exitCode)) } @@ -80,16 +82,23 @@ final case class SourceGenerator( private def needsUpdate(previous: SourceGenerator.Run): Task[SourceGenerator.Changes] = { previous match { case SourceGenerator.NoRun => - hashInputs.map(SourceGenerator.InputChanges(_)) - case SourceGenerator.PreviousRun(inputs, outputs) => - hashInputs.flatMap { newInputs => - if (newInputs != inputs) Task.now(SourceGenerator.InputChanges(newInputs)) - else { - hashOutputs.map { newOutputs => - if (newOutputs != outputs) SourceGenerator.OutputChanges(newInputs) - else SourceGenerator.NoChanges + Task.zip2(hashInputs, hashUnamanagedInputs).map { + case (inputs, unmanagedInputs) => + SourceGenerator.InputChanges(inputs, unmanagedInputs) + } + case SourceGenerator.PreviousRun(inputs, outputs, unmanagedInputs) => + Task.zip2(hashInputs, hashUnamanagedInputs).flatMap { + case (newInputs, newUnmanagedInputs) => + if (newInputs != inputs || newUnmanagedInputs != unmanagedInputs) + Task.now(SourceGenerator.InputChanges(newInputs, newUnmanagedInputs)) + else { + hashOutputs.map { newOutputs => + if (newOutputs != outputs) + SourceGenerator.OutputChanges(newInputs, newUnmanagedInputs) + else + SourceGenerator.NoChanges + } } - } } } } @@ -98,6 +107,10 @@ final case class SourceGenerator( for (inputs <- getSources) yield hashFiles(inputs) } + private def hashUnamanagedInputs: Task[Map[AbsolutePath, Int]] = Task { + hashFiles(unmangedInputs) + } + private def hashOutputs: Task[Map[AbsolutePath, Int]] = Task { val outputs = Paths.pathFilesUnder(outputDirectory, "glob:**") hashFiles(outputs) @@ -111,13 +124,22 @@ object SourceGenerator { sealed trait Run case object NoRun extends Run - case class PreviousRun(knownInputs: Map[AbsolutePath, Int], knownOutputs: Map[AbsolutePath, Int]) - extends Run + case class PreviousRun( + knownInputs: Map[AbsolutePath, Int], + knownOutputs: Map[AbsolutePath, Int], + knownUnmanagedInputs: Map[AbsolutePath, Int] + ) extends Run private sealed trait Changes private case object NoChanges extends Changes - private case class InputChanges(newInputs: Map[AbsolutePath, Int]) extends Changes - private case class OutputChanges(inputs: Map[AbsolutePath, Int]) extends Changes + private case class InputChanges( + newInputs: Map[AbsolutePath, Int], + unamanagedInputs: Map[AbsolutePath, Int] + ) extends Changes + private case class OutputChanges( + inputs: Map[AbsolutePath, Int], + unamanagedInputs: Map[AbsolutePath, Int] + ) extends Changes def fromConfig(cwd: AbsolutePath, generator: Config.SourceGenerator): SourceGenerator = { val sourcesGlobs = generator.sourcesGlobs.map { @@ -136,6 +158,7 @@ object SourceGenerator { cwd, sourcesGlobs, AbsolutePath(generator.outputDirectory), + generator.unmanagedInputs.map(AbsolutePath.apply), generator.command ) } diff --git a/frontend/src/main/scala/bloop/engine/caches/SourceGeneratorCache.scala b/frontend/src/main/scala/bloop/engine/caches/SourceGeneratorCache.scala index dc091c603..5ea8ede96 100644 --- a/frontend/src/main/scala/bloop/engine/caches/SourceGeneratorCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/SourceGeneratorCache.scala @@ -26,7 +26,7 @@ final class SourceGeneratorCache private ( ) .map { case SourceGenerator.NoRun => Nil - case SourceGenerator.PreviousRun(_, outputs) => outputs.keys.toList.sortBy(_.syntax) + case SourceGenerator.PreviousRun(_, outputs, _) => outputs.keys.toList.sortBy(_.syntax) } } } diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py old mode 100755 new mode 100644 index 20343bb2d..e7042e3db --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -58,3 +58,7 @@ def main(output_dir, args): args = sys.argv[2:] main(output_directory, args) + +def random(): + return 123 + diff --git a/frontend/src/test/scala/bloop/FileWatchingSpec.scala b/frontend/src/test/scala/bloop/FileWatchingSpec.scala index 64c3e40dc..493e438f4 100644 --- a/frontend/src/test/scala/bloop/FileWatchingSpec.scala +++ b/frontend/src/test/scala/bloop/FileWatchingSpec.scala @@ -10,7 +10,6 @@ import bloop.config.Config import bloop.data.Project import bloop.engine.Dag import bloop.engine.ExecutionContext -import bloop.internal.build.BuildTestInfo import bloop.io.AbsolutePath import bloop.io.Environment.lineSeparator import bloop.logging.DebugFilter @@ -18,7 +17,6 @@ import bloop.logging.PublisherLogger import bloop.logging.RecordingLogger import bloop.task.Task import bloop.testing.BaseSuite -import bloop.util.CrossPlatform import bloop.util.TestProject import bloop.util.TestUtil @@ -26,9 +24,7 @@ import monix.reactive.MulticastStrategy import monix.reactive.Observable object FileWatchingSpec extends BaseSuite { - private val generator: List[String] = - if (CrossPlatform.isWindows) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) - else List(BuildTestInfo.sampleSourceGenerator.getAbsolutePath) + private val generator: List[String] = TestUtil.generator System.setProperty("file-watcher-batch-window-ms", "100") diff --git a/frontend/src/test/scala/bloop/SourceGeneratorBspSpec.scala b/frontend/src/test/scala/bloop/SourceGeneratorBspSpec.scala index 84eca3bd4..0b83554b1 100644 --- a/frontend/src/test/scala/bloop/SourceGeneratorBspSpec.scala +++ b/frontend/src/test/scala/bloop/SourceGeneratorBspSpec.scala @@ -8,9 +8,7 @@ import ch.epfl.scala.bsp.Uri import bloop.bsp.BspBaseSuite import bloop.cli.BspProtocol import bloop.config.Config -import bloop.internal.build.BuildTestInfo import bloop.logging.RecordingLogger -import bloop.util.CrossPlatform import bloop.util.TestProject import bloop.util.TestUtil @@ -18,9 +16,8 @@ object TcpBspSourceGeneratorSpec extends BspSourceGeneratorSpec(BspProtocol.Tcp) object LocalBspSourceGeneratorSpec extends BspSourceGeneratorSpec(BspProtocol.Local) abstract class BspSourceGeneratorSpec(override val protocol: BspProtocol) extends BspBaseSuite { - private val generator = - if (CrossPlatform.isWindows) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) - else List(BuildTestInfo.sampleSourceGenerator.getAbsolutePath) + + private val generator = TestUtil.generator test("sources request works") { TestUtil.withinWorkspace { workspace => @@ -53,6 +50,7 @@ abstract class BspSourceGeneratorSpec(override val protocol: BspProtocol) extend ) assertEquals(obtained, expected :: Nil) } + } } diff --git a/frontend/src/test/scala/bloop/SourceGeneratorSpec.scala b/frontend/src/test/scala/bloop/SourceGeneratorSpec.scala index b23fec8f9..a0f542522 100644 --- a/frontend/src/test/scala/bloop/SourceGeneratorSpec.scala +++ b/frontend/src/test/scala/bloop/SourceGeneratorSpec.scala @@ -1,8 +1,5 @@ package bloop -import scala.sys.process -import scala.util.control.NonFatal - import bloop.Compiler.Result.Success import bloop.cli.ExitStatus import bloop.config.Config @@ -14,21 +11,9 @@ import bloop.util.TestUtil object SourceGeneratorSpec extends bloop.testing.BaseSuite { - lazy val hasPython3 = hasPythonNamed("python3") - lazy val hasPython2 = hasPythonNamed("python") - - private val generator: List[String] = - if (hasPython3) List("python3", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) - else if (hasPython2) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) - else Nil + val generator = TestUtil.generator - private def hasPythonNamed(executable: String) = try { - process.Process(Seq(executable, "--version")).! == 0 - } catch { - case NonFatal(_) => false - } - - lazy val hasPython = hasPython2 || hasPython3 + lazy val hasPython = generator.nonEmpty def testOnlyWithPython(name: String)(fun: => Any): Unit = { if (hasPython) test(name)(fun) @@ -205,6 +190,32 @@ object SourceGeneratorSpec extends bloop.testing.BaseSuite { } } + testOnlyWithPython("source generator is re-run when generator file is modified") { + singleProjectWithSourceGenerator("glob:*.in" :: Nil) { (workspace, project, state) => + writeFile(workspace.resolve("hello.in"), "hello") + writeFile(project.srcFor("test.scala", exists = false), assertNInputs(n = 1)) + val compiledState1 = state.compile(project) + val origHash = sourceHashFor("NameLengths_1.scala", project, compiledState1) + assertExitStatus(compiledState1, ExitStatus.Ok) + + val generatorFile = AbsolutePath(BuildTestInfo.sampleSourceGenerator.toPath()) + writeFile( + generatorFile, + readFile(generatorFile) + + """| + |def random(): + | return 123 + | + |""".stripMargin + ) + val compiledState2 = compiledState1.compile(project) + assertExitStatus(compiledState2, ExitStatus.Ok) + + val newHash = sourceHashFor("NameLengths_1.scala", project, compiledState2) + assertNotEquals(origHash, newHash) + } + } + testOnlyWithPython("source generator is re-run when an output file is deleted") { singleProjectWithSourceGenerator("glob:*.in" :: Nil) { (workspace, project, state) => val generatorOutput = project.config.sourceGenerators @@ -257,7 +268,8 @@ object SourceGeneratorSpec extends bloop.testing.BaseSuite { val sourceGenerator = Config.SourceGenerator( sourcesGlobs = List(Config.SourcesGlobs(workspace.underlying, None, includeGlobs, Nil)), outputDirectory = workspace.underlying.resolve("source-generator-output"), - command = generator + command = generator, + unmanagedInputs = List(BuildTestInfo.sampleSourceGenerator.toPath()) ) val `A` = TestProject(workspace, "a", Nil, sourceGenerators = sourceGenerator :: Nil) val projects = List(`A`) diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 406af67c7..d3c55969b 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -54,6 +54,7 @@ import _root_.monix.execution.Scheduler import org.junit.Assert import sbt.internal.inc.BloopComponentCompiler import xsbti.ComponentProvider +import bloop.internal.build.BuildTestInfo object TestUtil { def projectDir(base: Path, name: String): Path = base.resolve(name) @@ -639,4 +640,18 @@ object TestUtil { stacktraces.foreach(threadInfo => sb.append(threadInfo.toString()).append("\n")) sb.result() } + + private def hasPythonNamed(executable: String) = try { + scala.sys.process.Process(Seq(executable, "--version")).! == 0 + } catch { + case NonFatal(_) => false + } + + private lazy val hasPython3 = hasPythonNamed("python3") + private lazy val hasPython2 = hasPythonNamed("python") + + lazy val generator: List[String] = + if (hasPython3) List("python3", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) + else if (hasPython2) List("python", BuildTestInfo.sampleSourceGenerator.getAbsolutePath) + else Nil }