diff --git a/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt b/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt
index 28b9b0e2bee9..9299d94c060d 100644
--- a/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt
+++ b/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt
@@ -3,7 +3,7 @@ lazy val a = project.in(file("a"))
     scalacOptions += "-Yjava-tasty", // enable pickling of java signatures
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-annotation-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a-annotation-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-annotation-classes"), // send classfiles to a different directory
   )
 
 lazy val b = project.in(file("b"))
diff --git a/sbt-test/pipelining/Yjava-tasty-enum/build.sbt b/sbt-test/pipelining/Yjava-tasty-enum/build.sbt
index 1c416c65896f..0c95d8318913 100644
--- a/sbt-test/pipelining/Yjava-tasty-enum/build.sbt
+++ b/sbt-test/pipelining/Yjava-tasty-enum/build.sbt
@@ -4,7 +4,7 @@ lazy val a = project.in(file("a"))
     scalacOptions += "-Yjava-tasty", // enable pickling of java signatures
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a-enum-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-enum-classes"), // send classfiles to a different directory
   )
 
 
@@ -13,3 +13,7 @@ lazy val b = project.in(file("b"))
     Compile / unmanagedClasspath := Seq(Attributed.blank((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar")),
     scalacOptions += "-Ycheck:all",
   )
+  .settings(
+    fork := true, // we have to fork the JVM if we actually want to run the code with correct failure semantics
+    Runtime / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-enum-classes"), // make sure the java classes are visible at runtime
+  )
diff --git a/sbt-test/pipelining/Yjava-tasty-enum/test b/sbt-test/pipelining/Yjava-tasty-enum/test
index 68e3c170d3b5..fe04b1a7c7ea 100644
--- a/sbt-test/pipelining/Yjava-tasty-enum/test
+++ b/sbt-test/pipelining/Yjava-tasty-enum/test
@@ -1,3 +1,3 @@
 > a/compile
 # test depending on a java compiled enum through TASTy
-> b/compile
+> b/run
diff --git a/sbt-test/pipelining/Yjava-tasty-from-tasty/a/src/main/scala/a/A.java b/sbt-test/pipelining/Yjava-tasty-from-tasty/a/src/main/scala/a/A.java
index b8a278edc32f..381da612df90 100644
--- a/sbt-test/pipelining/Yjava-tasty-from-tasty/a/src/main/scala/a/A.java
+++ b/sbt-test/pipelining/Yjava-tasty-from-tasty/a/src/main/scala/a/A.java
@@ -1,5 +1,5 @@
 package a;
 
 public class A {
-  public static final String VALUE = "A";
+  public String VALUE = "A";
 }
diff --git a/sbt-test/pipelining/Yjava-tasty-from-tasty/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yjava-tasty-from-tasty/b/src/main/scala/b/B.scala
index 884bf1a927ff..43a45ae53ce2 100644
--- a/sbt-test/pipelining/Yjava-tasty-from-tasty/b/src/main/scala/b/B.scala
+++ b/sbt-test/pipelining/Yjava-tasty-from-tasty/b/src/main/scala/b/B.scala
@@ -1,5 +1,9 @@
 package b
 
 object B {
-  val A: "A" = a.A.VALUE
+  val A_VALUE = (new a.A).VALUE
+
+  @main def test = {
+    assert(A_VALUE == "A", s"actually was $A_VALUE")
+  }
 }
diff --git a/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt b/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt
index dc4950ec8379..570e72a40c1b 100644
--- a/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt
+++ b/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt
@@ -5,7 +5,7 @@ lazy val a = project.in(file("a"))
     scalacOptions += "-Yjava-tasty", // enable pickling of java signatures
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-pre-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a-pre-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-pre-classes"), // send classfiles to a different directory
   )
 
 // recompile `a` with `-from-tasty` flag to test idempotent read/write java signatures.
@@ -19,11 +19,17 @@ lazy val a_from_tasty = project.in(file("a_from_tasty"))
     scalacOptions += "-Yallow-outline-from-tasty", // allow outline signatures to be read with -from-tasty
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a_from_tasty-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a_from_tasty-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a_from_tasty-classes"), // send classfiles to a different directory
   )
 
 lazy val b = project.in(file("b"))
   .settings(
-    Compile / unmanagedClasspath := Seq(Attributed.blank((ThisBuild / baseDirectory).value / "a_from_tasty-java-tasty.jar")),
     scalacOptions += "-Ycheck:all",
+    Compile / unmanagedClasspath := Seq(Attributed.blank((ThisBuild / baseDirectory).value / "a_from_tasty-java-tasty.jar")),
+  )
+  .settings(
+    // we have to fork the JVM if we actually want to run the code with correct failure semantics
+    fork := true,
+    // make sure the java classes are visible at runtime
+    Runtime / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-pre-classes"),
   )
diff --git a/sbt-test/pipelining/Yjava-tasty-from-tasty/test b/sbt-test/pipelining/Yjava-tasty-from-tasty/test
index af7eced8e846..5c08ed4c4458 100644
--- a/sbt-test/pipelining/Yjava-tasty-from-tasty/test
+++ b/sbt-test/pipelining/Yjava-tasty-from-tasty/test
@@ -2,4 +2,4 @@
 # test reading java tasty with -from-tasty
 > a_from_tasty/compile
 # test java tasty is still written even with -from-tasty
-> b/compile
+> b/run
diff --git a/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala
index dcb4935860df..f132e012a5fc 100644
--- a/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala
+++ b/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala
@@ -7,6 +7,9 @@ class B[T] {
 }
 
 object B {
-  val derived: Int = (new B[Int]).inner.value
+  @main def test = {
+    val derived: Int = (new B[Int]).inner.value
+    assert(derived == 23, s"actually was $derived")
+  }
 }
 
diff --git a/sbt-test/pipelining/Yjava-tasty-generic/build.sbt b/sbt-test/pipelining/Yjava-tasty-generic/build.sbt
index aa5d3099e979..0c8a5c55fe7e 100644
--- a/sbt-test/pipelining/Yjava-tasty-generic/build.sbt
+++ b/sbt-test/pipelining/Yjava-tasty-generic/build.sbt
@@ -3,7 +3,7 @@ lazy val a = project.in(file("a"))
     scalacOptions += "-Yjava-tasty", // enable pickling of java signatures
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-generic-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a-generic-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-generic-classes"), // send classfiles to a different directory
   )
 
 lazy val b = project.in(file("b"))
@@ -11,3 +11,7 @@ lazy val b = project.in(file("b"))
     Compile / unmanagedClasspath := Seq(Attributed.blank((ThisBuild / baseDirectory).value / "a-generic-java-tasty.jar")),
     scalacOptions += "-Ycheck:all",
   )
+  .settings(
+    fork := true, // we have to fork the JVM if we actually want to run the code with correct failure semantics
+    Runtime / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-generic-classes"), // make sure the java classes are visible at runtime
+  )
diff --git a/sbt-test/pipelining/Yjava-tasty-generic/test b/sbt-test/pipelining/Yjava-tasty-generic/test
index 5abac4b5eae7..cbe3e14572a8 100644
--- a/sbt-test/pipelining/Yjava-tasty-generic/test
+++ b/sbt-test/pipelining/Yjava-tasty-generic/test
@@ -1,3 +1,3 @@
 > a/compile
 # Test depending on a java generic class through TASTy
-> b/compile
+> b/run
diff --git a/sbt-test/pipelining/Yjava-tasty-result-types/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yjava-tasty-result-types/b/src/main/scala/b/B.scala
index b67840f2f852..32d001075d40 100644
--- a/sbt-test/pipelining/Yjava-tasty-result-types/b/src/main/scala/b/B.scala
+++ b/sbt-test/pipelining/Yjava-tasty-result-types/b/src/main/scala/b/B.scala
@@ -9,9 +9,9 @@ object B {
   val a_true: String = (new A()).add(true)
 
   @main def test = {
-    assert(finalResult == "A")
-    assert(a_B == "AB")
-    assert(a_true == "Atrue")
+    assert(finalResult == "A", s"actually was $finalResult")
+    assert(a_B == "AB", s"actually was $a_B")
+    assert(a_true == "Atrue", s"actually was $a_true")
   }
 }
 
diff --git a/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt b/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt
index f9cf8082c731..8f9e782f8810 100644
--- a/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt
+++ b/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt
@@ -3,7 +3,7 @@ lazy val a = project.in(file("a"))
     scalacOptions += "-Yjava-tasty", // enable pickling of java signatures
     scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-result-types-java-tasty.jar").toString),
     scalacOptions += "-Ycheck:all",
-    classDirectory := ((ThisBuild / baseDirectory).value / "a-result-types-classes"), // send classfiles to a different directory
+    Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-result-types-classes"), // send classfiles to a different directory
   )
 
 lazy val b = project.in(file("b"))
@@ -11,3 +11,7 @@ lazy val b = project.in(file("b"))
     Compile / unmanagedClasspath := Seq(Attributed.blank((ThisBuild / baseDirectory).value / "a-result-types-java-tasty.jar")),
     scalacOptions += "-Ycheck:all",
   )
+  .settings(
+    fork := true, // we have to fork the JVM if we actually want to run the code with correct failure semantics
+    Runtime / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-result-types-classes"), // make sure the java classes are visible at runtime
+  )
diff --git a/sbt-test/pipelining/Yjava-tasty-result-types/test b/sbt-test/pipelining/Yjava-tasty-result-types/test
index f654cb06fc16..c1cbbb1f2fe5 100644
--- a/sbt-test/pipelining/Yjava-tasty-result-types/test
+++ b/sbt-test/pipelining/Yjava-tasty-result-types/test
@@ -1,3 +1,3 @@
 > a/compile
 # Test depending on a java static final result, and method result through TASTy
-> b/compile
+> b/run