WX-966 Add Cascades, remove Directory from Biscayne (#7105)

build.sbt

@@ -37,6 +37,11 @@ lazy val wdlModelBiscayne = (project in wdlModelRoot / "biscayne")
   .dependsOn(wdlModelDraft3)
   .dependsOn(common % "test->test")
 
+lazy val wdlModelCascades = (project in wdlModelRoot / "cascades")
+  .withLibrarySettings("cromwell-wdl-model-cascades")
+  .dependsOn(wdlSharedModel)
+  .dependsOn(common % "test->test")
+
 lazy val wdlTransformsRoot = wdlRoot / "transforms"
 
 lazy val wdlSharedTransforms = (project in wdlTransformsRoot / "shared")
@@ -74,6 +79,13 @@ lazy val wdlTransformsBiscayne = (project in wdlTransformsRoot / "biscayne")
   .dependsOn(common % "test->test")
   .dependsOn(wom % "test->test")
 
+lazy val wdlTransformsCascades = (project in wdlTransformsRoot / "cascades")
+  .withLibrarySettings("cromwell-wdl-transforms-cascades", wdlDependencies)
+  .dependsOn(wdlNewBaseTransforms)
+  .dependsOn(languageFactoryCore)
+  .dependsOn(common % "test->test")
+  .dependsOn(wom % "test->test")
+
 lazy val core = project
   .withLibrarySettings("cromwell-core", coreDependencies)
   .dependsOn(wom)
@@ -178,6 +190,7 @@ lazy val services = project
   .dependsOn(wdlDraft2LanguageFactory % "test->test") // because the WaaS tests init language config with all languages
   .dependsOn(wdlDraft3LanguageFactory % "test->test")
   .dependsOn(wdlBiscayneLanguageFactory % "test->test")
+  .dependsOn(wdlCascadesLanguageFactory % "test->test")
   .dependsOn(core % "test->test")
   .dependsOn(ftpFileSystem % "test->test")
   .dependsOn(common % "test->test")
@@ -263,6 +276,7 @@ lazy val engine = project
   .dependsOn(wdlDraft2LanguageFactory % "test->test")
   .dependsOn(wdlDraft3LanguageFactory % "test->test")
   .dependsOn(wdlBiscayneLanguageFactory % "test->test")
+  .dependsOn(wdlCascadesLanguageFactory % "test->test")
   .dependsOn(common % "test->test")
   .dependsOn(core % "test->test")
   .dependsOn(backend % "test->test")
@@ -279,6 +293,7 @@ lazy val womtool = project
   .dependsOn(wdlDraft2LanguageFactory)
   .dependsOn(wdlDraft3LanguageFactory)
   .dependsOn(wdlBiscayneLanguageFactory)
+  .dependsOn(wdlCascadesLanguageFactory)
   .dependsOn(wom % "test->test")
   .dependsOn(common % "test->test")
 
@@ -324,6 +339,13 @@ lazy val wdlBiscayneLanguageFactory = (project in languageFactoryRoot / "wdl-bis
   .dependsOn(wdlTransformsBiscayne)
   .dependsOn(common % "test->test")
 
+lazy val wdlCascadesLanguageFactory = (project in languageFactoryRoot / "wdl-cascades")
+  .withLibrarySettings("wdl-cascades")
+  .dependsOn(languageFactoryCore)
+  .dependsOn(wdlModelCascades)
+  .dependsOn(wdlTransformsCascades)
+  .dependsOn(common % "test->test")
+
 lazy val `cloud-nio-spi` = (project in cloudNio / "cloud-nio-spi")
   .withLibrarySettings(libraryName = "cloud-nio-spi", dependencies = spiDependencies)
   .dependsOn(common % "test->test")
@@ -366,6 +388,7 @@ lazy val server = project
   .dependsOn(wdlDraft2LanguageFactory)
   .dependsOn(wdlDraft3LanguageFactory)
   .dependsOn(wdlBiscayneLanguageFactory)
+  .dependsOn(wdlCascadesLanguageFactory)
   .dependsOn(engine % "test->test")
   .dependsOn(common % "test->test")
 
@@ -406,15 +429,18 @@ lazy val root = (project in file("."))
   .aggregate(sraFileSystem)
   .aggregate(tesBackend)
   .aggregate(wdlBiscayneLanguageFactory)
+  .aggregate(wdlCascadesLanguageFactory)
   .aggregate(wdlDraft2LanguageFactory)
   .aggregate(wdlDraft3LanguageFactory)
   .aggregate(wdlModelBiscayne)
+  .aggregate(wdlModelCascades)
   .aggregate(wdlModelDraft2)
   .aggregate(wdlModelDraft3)
   .aggregate(wdlNewBaseTransforms)
   .aggregate(wdlSharedModel)
   .aggregate(wdlSharedTransforms)
   .aggregate(wdlTransformsBiscayne)
+  .aggregate(wdlTransformsCascades)
   .aggregate(wdlTransformsDraft2)
   .aggregate(wdlTransformsDraft3)
   .aggregate(wes2cromwell)

centaur/src/main/resources/standardTestCases/biscayne_prohibits_directory.test

@@ -0,0 +1,13 @@
+name: directory_type_local_denied
+testFormat: workflowfailure
+tags: [localdockertest, "wdl_biscayne"]
+backends: [Local, LocalNoDocker]
+
+files {
+  workflow: wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl
+  inputs: wdl_biscayne/biscayne_prohibits_directory/directory_type_local_inputs.json
+}
+
+metadata {
+  status: Failed
+}

centaur/src/main/resources/standardTestCases/directory_type_local.test

@@ -1,11 +1,11 @@
 name: directory_type_local
 testFormat: workflowsuccess
-tags: [localdockertest, "wdl_biscayne"]
+tags: [localdockertest, "wdl_cascades"]
 backends: [Local, LocalNoDocker]
 
 files {
-  workflow: wdl_biscayne/directory_type/directory_type.wdl
-  inputs: wdl_biscayne/directory_type/directory_type_local_inputs.json
+  workflow: wdl_cascades/directory_type/directory_type.wdl
+  inputs: wdl_cascades/directory_type/directory_type_local_inputs.json
 }
 
 metadata {

centaur/src/main/resources/standardTestCases/directory_type_output_papi.test

@@ -1,10 +1,10 @@
 name: directory_type_output_papi
 testFormat: workflowsuccess
-tags: ["wdl_biscayne"]
+tags: ["wdl_cascades"]
 backends: [Papi]
 
 files {
-  workflow: wdl_biscayne/directory_type_output/directory_type_output.wdl
+  workflow: wdl_cascades/directory_type_output/directory_type_output.wdl
 }
 
 metadata {

centaur/src/main/resources/standardTestCases/directory_type_papi.test

@@ -1,10 +1,10 @@
 name: directory_type_papi
 testFormat: workflowsuccess
-tags: ["wdl_biscayne"]
+tags: ["wdl_cascades"]
 backends: [Papi]
 
 files {
-  workflow: wdl_biscayne/directory_type/directory_type.wdl
+  workflow: wdl_cascades/directory_type/directory_type.wdl
 }
 
 metadata {

centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_http_relative_imports/biscayne_http_relative_imports.wdl

@@ -1,7 +1,7 @@
-version development
+version development-1.1
 
 # Is there a better way to test this?
-import "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/biscayne/valid/relative_imports/sub_wfs/foo.wdl"
+import "https://raw.githubusercontent.com/broadinstitute/cromwell/aen_wx_966/womtool/src/test/resources/validate/biscayne/valid/relative_imports/sub_wfs/foo.wdl"
 
 workflow biscayne_http_relative_imports {
   call foo.foo_wf

centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl

@@ -1,4 +1,4 @@
-version development
+version development-1.1
 
 workflow biscayne_new_engine_functions {
 

centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl

@@ -0,0 +1,49 @@
+version development-1.1
+
+workflow directory_type {
+  input {
+    String text2loc = "text2"
+  }
+  call make_directory { input: text2loc = text2loc }
+  call read_from_directory { input: d = make_directory.d, text2loc = text2loc }
+
+  output {
+    Array[String] out = read_from_directory.contents
+  }
+}
+
+task make_directory {
+  input {
+    String text2loc
+  }
+  String text2dir = sub("foo/~{text2loc}", "/[^/]*$", "")
+  command {
+    mkdir foo
+    mkdir -p ~{text2dir}
+    echo "foo text" > foo/text
+    echo "foo text2" > foo/~{text2loc}
+  }
+  runtime {
+    docker: "ubuntu:latest"
+  }
+  output {
+    Directory d = "foo/"
+  }
+}
+
+task read_from_directory {
+  input {
+    String text2loc
+    Directory d
+  }
+  command {
+    cat ~{d}/text
+    cat ~{d}/~{text2loc}
+  }
+  runtime {
+    docker: "ubuntu:latest"
+  }
+  output {
+    Array[String] contents = read_lines(stdout())
+  }
+}

centaur/src/main/resources/standardTestCases/wdl_biscayne/default_default/default_default.wdl

@@ -1,4 +1,4 @@
-version development
+version development-1.1
 
 workflow default_default {
   call default_default_task

centaur/src/main/resources/standardTestCases/wdl_biscayne/read_functions_windows_line_endings/read_functions_windows_line_endings.wdl

@@ -1,4 +1,4 @@
-version development
+version development-1.1
 
 struct JsonObj {
     String field1

centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type_local_inputs.json

@@ -0,0 +1,3 @@
+{
+  "directory_type.text2loc": "bar/text2"
+}

core/src/main/resources/reference.conf

@@ -470,13 +470,21 @@ languages {
         }
       }
       "biscayne" {
-        # WDL biscayne is our in-progress name for what will (probably) become WDL 1.1
+        # WDL biscayne is our in-progress name for what will become WDL 1.1
         language-factory = "languages.wdl.biscayne.WdlBiscayneLanguageFactory"
         config {
           strict-validation: true
           enabled: true
         }
       }
+      "cascades" {
+        # WDL cascades is our in-progress name for what will (probably) become WDL 2.0
+        language-factory = "languages.wdl.cascades.WdlCascadesLanguageFactory"
+        config {
+          strict-validation: true
+          enabled: true
+        }
+      }
     }
   }
 }

engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala

@@ -22,7 +22,7 @@ class ServiceInfoSpec extends AsyncFlatSpec with ScalatestRouteTest with Matcher
 
   behavior of "ServiceInfo"
 
-  val expectedResponse = WesStatusInfoResponse(Map("WDL" -> Set("draft-2", "1.0", "biscayne")),
+  val expectedResponse = WesStatusInfoResponse(Map("WDL" -> Set("draft-2", "1.0", "biscayne", "cascades")),
     List("1.0"),
     Set("ftp", "s3", "drs", "gcs", "http"),
     Map("Cromwell" -> CromwellApiService.cromwellVersion),

languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala

@@ -94,7 +94,7 @@ object LanguageFactoryUtil {
     }
 
     val firstCodeLine = fileWithoutInitialWhitespace.headOption.map(_.dropWhile(_.isWhitespace))
-    firstCodeLine.exists { line => startsWithOptions.exists(line.startsWith) }
+    firstCodeLine.exists { line => startsWithOptions.contains(line) }
   }
 
   def chooseFactory(workflowSource: WorkflowSource, wsfc: WorkflowSourceFilesCollection): ErrorOr[LanguageFactory] = {

languageFactories/wdl-biscayne/src/main/scala/languages/wdl/biscayne/WdlBiscayneLanguageFactory.scala

@@ -83,5 +83,5 @@ class WdlBiscayneLanguageFactory(override val config: Config) extends LanguageFa
     } yield validated
   }
 
-  override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version development"), List("#"))(content)
+  override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version development-1.1"), List("#"))(content)
 }

languageFactories/wdl-cascades/src/main/scala/languages/wdl/cascades/WdlCascadesLanguageFactory.scala

@@ -0,0 +1,87 @@
+package languages.wdl.cascades
+
+import java.util.concurrent.Callable
+
+import cats.syntax.either._
+import cats.instances.either._
+import cats.data.EitherT.fromEither
+import cats.effect.IO
+import com.typesafe.config.Config
+import common.Checked
+import common.transforms.CheckedAtoB
+import common.validation.ErrorOr.ErrorOr
+import common.validation.IOChecked.IOChecked
+import cromwell.core._
+import cromwell.languages.util.ImportResolver._
+import cromwell.languages.util.{LanguageFactoryUtil, ParserCache}
+import cromwell.languages.util.ParserCache.ParserCacheInputs
+import cromwell.languages.{LanguageFactory, ValidatedWomNamespace}
+import wdl.transforms.base.wdlom2wom.WomBundleToWomExecutable._
+import wdl.transforms.base.wdlom2wom._
+import wdl.transforms.cascades.ast2wdlom._
+import wdl.transforms.cascades.parsing._
+import wdl.transforms.cascades.wdlom2wom._
+import wom.ResolvedImportRecord
+import wom.core.{WorkflowJson, WorkflowOptionsJson, WorkflowSource}
+import wom.executable.WomBundle
+import wom.expression.IoFunctionSet
+import wom.transforms.WomExecutableMaker.ops._
+
+class WdlCascadesLanguageFactory(override val config: Config) extends LanguageFactory with ParserCache[WomBundle] {
+
+  override val languageName: String = "WDL"
+  override val languageVersionName: String = "Cascades"
+
+  override def validateNamespace(source: WorkflowSourceFilesCollection,
+                                 workflowSource: WorkflowSource,
+                                 workflowOptions: WorkflowOptions,
+                                 importLocalFilesystem: Boolean,
+                                 workflowIdForLogging: WorkflowId,
+                                 ioFunctions: IoFunctionSet,
+                                 importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = {
+
+    val factories: List[LanguageFactory] = List(this)
+
+    val checked: Checked[ValidatedWomNamespace] = for {
+      _ <- enabledCheck
+      bundle <- getWomBundle(workflowSource, workflowSourceOrigin = None, source.workflowOptions.asPrettyJson, importResolvers, factories)
+      executable <- createExecutable(bundle, source.inputsJson, ioFunctions)
+    } yield executable
+
+    fromEither[IO](checked)
+
+  }
+
+  override def getWomBundle(workflowSource: WorkflowSource,
+                            workflowSourceOrigin: Option[ResolvedImportRecord],
+                            workflowOptionsJson: WorkflowOptionsJson,
+                            importResolvers: List[ImportResolver],
+                            languageFactories: List[LanguageFactory],
+                            convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] = {
+
+    val converter: CheckedAtoB[FileStringParserInput, WomBundle] = stringToAst andThen wrapAst andThen astToFileElement.map(FileElementToWomBundleInputs(_, workflowOptionsJson, convertNestedScatterToSubworkflow, importResolvers, languageFactories, workflowDefinitionElementToWomWorkflowDefinition, taskDefinitionElementToWomTaskDefinition)) andThen fileElementToWomBundle
+
+    lazy val validationCallable = new Callable[ErrorOr[WomBundle]] {
+      def call: ErrorOr[WomBundle] = converter
+        .run(FileStringParserInput(workflowSource, workflowSourceOrigin.map(_.importPath).getOrElse("input.wdl")))
+        .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)).toValidated
+    }
+
+    lazy val parserCacheInputs = ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers)
+
+    for {
+      _ <- enabledCheck
+      womBundle <- retrieveOrCalculate(parserCacheInputs, validationCallable).toEither
+    } yield womBundle
+  }
+
+  override def createExecutable(womBundle: WomBundle, inputsJson: WorkflowJson, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = {
+    for {
+      _ <- enabledCheck
+      executable <- womBundle.toWomExecutable(Option(inputsJson), ioFunctions, strictValidation)
+      validated <- LanguageFactoryUtil.validateWomNamespace(executable, ioFunctions)
+    } yield validated
+  }
+
+  override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version development"), List("#"))(content)
+}

project/Merging.scala

@@ -53,7 +53,7 @@ object Merging {
           val oldStrategy = (assembly / assemblyMergeStrategy).value
           oldStrategy(x)
       }
-    case "asm-license.txt" | "module-info.class" | "overview.html" | "cobertura.properties" =>
+    case "asm-license.txt" | "module-info.class" | "overview.html" | "cobertura.properties" | "grammar.hgr" | "CHANGELOG.txt" =>
       MergeStrategy.discard
     // inspired by https://github.com/ergoplatform/explorer-backend/blob/7364ecfdeabeb691f0f25525e577d6c48240c672/build.sbt#L14-L15
     case other if other.contains("scala/annotation/nowarn.class")  => MergeStrategy.discard

services/src/test/resources/describe/wdl_1_0/relative_url_imports/description.json

@@ -19,7 +19,7 @@
   ],
   "submittedDescriptorType" : {
     "descriptorType" : "WDL",
-    "descriptorTypeVersion" : "Biscayne"
+    "descriptorTypeVersion" : "Cascades"
   },
   "importedDescriptorTypes" : [
   ],