Fixes #69, Fixes #63

ldvm · Oct 21, 2015 · 5a7b915 · 5a7b915
1 parent cffa0c3
commit 5a7b915
Show file tree

Hide file tree

Showing 17 changed files with 114 additions and 77 deletions.
diff --git a/src/app/controllers/VisualizationController.scala b/src/app/controllers/VisualizationController.scala
@@ -28,18 +28,19 @@ class VisualizationController(implicit inj: Injector) extends Controller with In
 
         DB.withSession { implicit s =>
           val files = body.files.filter(_.key == "ttlfile")
-          val maybeDataSourceId = dataSourceService.createDataSourceFromFiles(files)
-
-          maybeDataSourceId.map { i =>
-            val combine = body
-              .dataParts
-              .get("combine")
-              .flatMap(_.headOption.map(_ == "true"))
-              .getOrElse(false)
-
-            Redirect(routes.VisualizationController.discover(maybeDataSourceId.map(_.id), combine))
-          }.getOrElse {
-            Redirect(routes.ApplicationController.index()).flashing("error" -> "No data in files.")
+          val dataSourceIds = dataSourceService.createDataSourceFromFiles(files).toSeq
+
+          dataSourceIds.isEmpty match {
+            case true => Redirect(routes.ApplicationController.index()).flashing("error" -> "No data in files.")
+            case false => {
+              val combine = body
+                .dataParts
+                .get("combine")
+                .flatMap(_.headOption.map(_ == "true"))
+                .getOrElse(false)
+
+              Redirect(routes.VisualizationController.discover(dataSourceIds.map(_.id).toList, combine))
+            }
           }
         }
 
@@ -53,15 +54,15 @@ class VisualizationController(implicit inj: Injector) extends Controller with In
         request.body.get("ttlurl").map { urls =>
 
           val sanitizedList = urls.flatMap(_.split("\n")).map(_.trim).filter(_.nonEmpty)
-          val maybeDataSourceId = dataSourceService.createDataSourceFromRemoteTtl(sanitizedList)
+          val dataSourceIds = dataSourceService.createDataSourceFromRemoteTtl(sanitizedList).toSeq
 
           val combine = request
             .body
             .get("combine")
             .flatMap(_.headOption.map(_ == "true"))
             .getOrElse(false)
 
-          Redirect(routes.VisualizationController.discover(maybeDataSourceId.map(_.id), combine))
+          Redirect(routes.VisualizationController.discover(dataSourceIds.map(_.id).toList, combine))
 
         }.getOrElse {
           Redirect(routes.ApplicationController.index()).flashing(
@@ -119,12 +120,12 @@ class VisualizationController(implicit inj: Injector) extends Controller with In
     pipelineService.findEvaluationById(PipelineEvaluationId(id)).map(func).getOrElse(NotFound)
   }
 
-  def discover(dataSourceTemplateId: Option[Long], combine: Boolean = false) = DBAction { rws =>
+  def discover(dataSourceTemplateIds: List[Long], combine: Boolean = false) = DBAction { rws =>
 
     val n = if (combine) {1} else {0}
 
     val url: String = "/pipelines#/discover?" +
-      "dataSourceTemplateId=" + dataSourceTemplateId.orNull +
+      dataSourceTemplateIds.map(i => "dataSourceTemplateIds=" + i).mkString("&") +
       "&combine=" + n.toString
 
     TemporaryRedirect(url)

diff --git a/src/app/controllers/api/ComponentTemplateApiController.scala b/src/app/controllers/api/ComponentTemplateApiController.scala
@@ -4,9 +4,10 @@ import controllers.api.JsonImplicits._
 import model.entity.{ComponentTemplate, ComponentTemplateId}
 import play.api.Play.current
 import play.api.db.slick.{DBAction, _}
-import play.api.libs.json._
 import play.api.mvc.Result
 import scaldi.{Injectable, Injector}
+import play.api.libs.json._
+import play.api.libs.functional.syntax._
 
 class ComponentTemplateApiController(implicit inj: Injector) extends ApiController with Injectable {
 
@@ -67,19 +68,27 @@ class ComponentTemplateApiController(implicit inj: Injector) extends ApiControll
   }
 
   def addDatasource = DBAction { implicit rws =>
-    rws.request.body.asJson.flatMap { json =>
-      val endpointUrl = (json \ "endpointUrl").as[String]
-      val graphUris = (json \ "graphUris").as[Seq[String]]
-
-      val maybeId = dataSourceService.createDataSourceFromUris(endpointUrl, graphUris)
-      maybeId.map { id =>
-        Ok(JsObject(Seq(
-          "id" -> JsNumber(id.id)
-        )))
+
+    case class Endpoint(url: String, graphUris: Option[Seq[String]])
+    implicit val endpointReads = (
+        (__ \ 'url).read[String] and
+        (__ \ 'graphUris).readNullable[Seq[String]]
+      )(Endpoint)
+
+    rws.request.body.asJson.map { json =>
+      val endpoints = json.as[Seq[Endpoint]]
+
+      val ids = endpoints.flatMap { e =>
+        dataSourceService.createDataSourceFromUris(e.url, e.graphUris)
       }
-    }.getOrElse{
-      BadRequest
-    }
+
+      ids.size match {
+        case 0 => BadRequest
+        case _ => Ok(JsArray(
+          ids.map(id => JsObject(Seq("id" -> JsNumber(id.id))))
+        ))
+      }
+    }.getOrElse(BadRequest)
   }
 
   private def withComponentTemplate(id: Long)(func: ComponentTemplate => Result)(implicit session: Session): Result = {

diff --git a/src/app/controllers/api/PipelineApiController.scala b/src/app/controllers/api/PipelineApiController.scala
@@ -57,8 +57,8 @@ class PipelineApiController(implicit inj: Injector) extends Controller with Inje
     Ok(result)
   }
 
-  def discover(dataSourceTemplateId: Option[Long], combine: Boolean = false) =  withWebSocket { logger => implicit session =>
-    pipelineService.discover(logger, dataSourceTemplateId, combine)
+  def discover(dataSourceTemplateIds: List[Long], combine: Boolean = false) =  withWebSocket { logger => implicit session =>
+    pipelineService.discover(logger, dataSourceTemplateIds, combine)
   }
 
   def evaluate(id: Long) = withWebSocket { logger => implicit session =>

diff --git a/src/app/model/service/ComponentTemplateService.scala b/src/app/model/service/ComponentTemplateService.scala
@@ -9,7 +9,7 @@ trait ComponentTemplateService extends CrudService[ComponentTemplateId, Componen
 
   def getAllByType(implicit session: Session): Map[ComponentType, Seq[SpecificComponentTemplate]]
 
-  def getAllForDiscovery(dataSourceTemplateId: Option[Long], combine: Boolean)(implicit session: Session): (Map[ComponentType, Seq[SpecificComponentTemplate]], Option[DataSourceTemplate])
+  def getAllForDiscovery(dataSourceTemplateIds: List[Long], combine: Boolean)(implicit session: Session): (Map[ComponentType, Seq[SpecificComponentTemplate]], Seq[DataSourceTemplate])
 
   def save(component: model.dto.ComponentTemplate)(implicit session: Session): ComponentTemplateId
 

diff --git a/src/app/model/service/DataSourceService.scala b/src/app/model/service/DataSourceService.scala
@@ -14,6 +14,6 @@ trait DataSourceService extends CrudService[DataSourceTemplateId, DataSourceTemp
 
   def createDataSourceFromFiles(files: Seq[MultipartFormData.FilePart[Files.TemporaryFile]], maybeUrn: Option[UUID] = None)(implicit session: Session): Option[DataSourceTemplateId]
 
-  def createDataSourceFromUris(endpointUrl: String, graphUris: Seq[String])(implicit session: Session): Option[DataSourceTemplateId]
+  def createDataSourceFromUris(endpointUrl: String, graphUris: Option[Seq[String]])(implicit session: Session): Option[DataSourceTemplateId]
 
 }
diff --git a/src/app/model/service/PipelineService.scala b/src/app/model/service/PipelineService.scala
@@ -26,7 +26,7 @@ trait PipelineService extends CrudService[PipelineId, Pipeline, PipelineTable, P
     (ordering: PipelineTable => T = { e: PipelineTable => (e.modifiedUtc.desc, e.createdUtc.desc) })
     (implicit session: Session): Seq[Pipeline]
 
-  def discover(reporterProps: Props, dataSourceTemplateId: Option[Long], combine: Boolean = false)(implicit session: Session): PipelineDiscoveryId
+  def discover(reporterProps: Props, dataSourceTemplateIds: List[Long], combine: Boolean = false)(implicit session: Session): PipelineDiscoveryId
 
   def evaluate(pipelineId: PipelineId)(reporterProps: Props)(implicit session: Session): Option[PipelineEvaluationId]
 

diff --git a/src/app/model/service/impl/ComponentTemplateServiceImpl.scala b/src/app/model/service/impl/ComponentTemplateServiceImpl.scala
@@ -69,17 +69,17 @@ class ComponentTemplateServiceImpl(implicit inj: Injector) extends ComponentTemp
     analyzers ++ visualizers ++ transformers ++ dataSources
   }
 
-  def getAllForDiscovery(dataSourceTemplateId: Option[Long], combine: Boolean)
-    (implicit session: Session): (Map[ComponentType, Seq[SpecificComponentTemplate]], Option[DataSourceTemplate]) = {
+  def getAllForDiscovery(dataSourceTemplateIds: List[Long], combine: Boolean)
+    (implicit session: Session): (Map[ComponentType, Seq[SpecificComponentTemplate]], Seq[DataSourceTemplate]) = {
 
-    val maybeDsTemplate = dataSourceTemplateId.flatMap { i =>
+    val dsTemplates = dataSourceTemplateIds.map { i =>
       dataSourceTemplateRepository.findById(DataSourceTemplateId(i))
-    }
+    }.filter(_.isDefined).map(_.get)
 
     val datasources = if (combine) {
-      maybeDsTemplate.toSeq ++ dataSourceTemplateRepository.findPermanent
-    } else if (maybeDsTemplate.isDefined) {
-      maybeDsTemplate.toSeq
+      dsTemplates ++ dataSourceTemplateRepository.findPermanent
+    } else if (dsTemplates.nonEmpty) {
+      dsTemplates
     } else {
       dataSourceTemplateRepository.findPermanent
     }
@@ -90,7 +90,7 @@ class ComponentTemplateServiceImpl(implicit inj: Injector) extends ComponentTemp
       (ComponentType.Transformer, transformerTemplateRepository.findAllWithMandatoryDescriptors),
       (ComponentType.Visualizer, visualizerTemplateRepository.findAllWithMandatoryDescriptors)
     ).toMap,
-      maybeDsTemplate)
+      dsTemplates)
   }
 
   def save(componentTemplate: model.dto.ComponentTemplate)(implicit session: Session): ComponentTemplateId = {

diff --git a/src/app/model/service/impl/DataSourceServiceImpl.scala b/src/app/model/service/impl/DataSourceServiceImpl.scala
@@ -79,7 +79,7 @@ class DataSourceServiceImpl(implicit inj: Injector) extends DataSourceService wi
     withRandomGraphUri(None)(action)
   }
 
-  private def config(endpointUrl: String, graphUris: Seq[String]): Model = {
+  private def config(endpointUrl: String, maybeGraphUris: Option[Seq[String]]): Model = {
     val model = ModelFactory.createDefaultModel()
 
     val configResource = model.createResource()
@@ -89,7 +89,7 @@ class DataSourceServiceImpl(implicit inj: Injector) extends DataSourceService wi
     val endpointResource = model.createResource(endpointUrl)
     serviceResource.addProperty(SD.endpoint, endpointResource)
 
-    if (graphUris.nonEmpty) {
+    maybeGraphUris.foreach { graphUris =>
       val dataSetResource = model.createResource()
       serviceResource.addProperty(SD.defaultDataset, dataSetResource)
 
@@ -100,12 +100,13 @@ class DataSourceServiceImpl(implicit inj: Injector) extends DataSourceService wi
         dataSetResource.addProperty(SD.namedGraph, namedGraphResource)
       }
     }
+
     configResource.addProperty(DSPARQL.service, serviceResource)
 
     model
   }
 
-  def createDataSourceFromUris(endpointUrl: String, graphUris: Seq[String])(implicit session: Session): Option[DataSourceTemplateId] = {
+  def createDataSourceFromUris(endpointUrl: String, graphUris: Option[Seq[String]])(implicit session: Session): Option[DataSourceTemplateId] = {
 
     val resourceUri = endpointUrl
     val dataPortTemplate = model.dto.DataPortTemplate(resourceUri + "/output", None, None)
@@ -115,7 +116,7 @@ class DataSourceServiceImpl(implicit inj: Injector) extends DataSourceService wi
       resourceUri,
       Some(endpointUrl),
       None,
-      Some(config(endpointUrl, graphUris.filter(_.trim.nonEmpty))),
+      Some(config(endpointUrl, graphUris.map(_.filter(_.trim.nonEmpty)))),
       Seq(),
       Some(outputTemplate),
       Seq(),
@@ -137,7 +138,7 @@ class DataSourceServiceImpl(implicit inj: Injector) extends DataSourceService wi
       graph.datasourceUri,
       Some(name),
       maybeDescription,
-      Some(config(internalEndpoint, Seq(graph.graphUri))),
+      Some(config(internalEndpoint, Some(Seq(graph.graphUri)))),
       Seq(),
       Some(outputTemplate),
       Seq(),

diff --git a/src/app/model/service/impl/pipeline/PipelineDiscoveryAlgorithm.scala b/src/app/model/service/impl/pipeline/PipelineDiscoveryAlgorithm.scala
@@ -25,7 +25,7 @@ object PipelineDiscoveryAlgorithm {
 class PipelineDiscoveryAlgorithm(
   allComponentsByType: Map[ComponentType, Seq[SpecificComponentTemplate]],
   reporterProps: Props,
-  maybeGeneralDs: Option[DataSourceTemplate] = None,
+  maybeGeneralDs: Seq[DataSourceTemplate] = Seq(),
   maxIterations: Int = PipelineDiscoveryAlgorithm.MaxIterations
   )
   (implicit val inj: Injector, implicit val session: Session) extends SessionScoped with Injectable {
@@ -67,7 +67,7 @@ class PipelineDiscoveryAlgorithm(
           val pipelinesToSave = if (maybeGeneralDs.isEmpty) {
             createdCompletedPipelines
           } else {
-            createdCompletedPipelines.filter(_.componentInstances.exists(_.componentTemplateId == maybeGeneralDs.get.componentTemplateId))
+            createdCompletedPipelines.filter(_.componentInstances.exists(i => maybeGeneralDs.exists(_.componentTemplateId == i.componentTemplateId)))
           }
           pipelineService.saveDiscoveryResults(discoveryId, pipelinesToSave, reporter)
 

diff --git a/src/app/model/service/impl/pipeline/PipelineServiceImpl.scala b/src/app/model/service/impl/pipeline/PipelineServiceImpl.scala
@@ -233,8 +233,8 @@ class PipelineServiceImpl(implicit inj: Injector) extends PipelineService with I
     repository.findPaginatedFilteredOrdered(paginationInfo)(pipelineDiscoveryId, visualizerId)(ordering)
   }
 
-  def discover(reporterProps: Props, dataSourceTemplateId: Option[Long], combine: Boolean = false)(implicit session: Session): PipelineDiscoveryId = {
-    val allComponentsByType = componentService.getAllForDiscovery(dataSourceTemplateId, combine)
+  def discover(reporterProps: Props, dataSourceTemplateIds: List[Long], combine: Boolean = false)(implicit session: Session): PipelineDiscoveryId = {
+    val allComponentsByType = componentService.getAllForDiscovery(dataSourceTemplateIds, combine)
     new PipelineDiscoveryAlgorithm(allComponentsByType._1, reporterProps, allComponentsByType._2)
       .discoverPipelines(
         allComponentsByType._1(ComponentType.DataSource).collect { case d: DataSourceTemplate => d }

diff --git a/src/app/views/ngLayout.scala.html b/src/app/views/ngLayout.scala.html
@@ -41,7 +41,8 @@
                             "auto-size":  ['@routes.Assets.versioned("vendors/auto-size/jquery.autosize").url.replace(".js", "")'],
                             "sweet-alert":  ['@routes.Assets.versioned("vendors/sweet-alert/sweet-alert").url.replace(".js", "")'],
                             "pipeline-visualizer":  ['@routes.Assets.versioned("javascripts/lib/pipelineVisualizer").url.replace(".js", "")'],
-                            "markerclusterer":  ['@routes.Assets.versioned("javascripts/lib/gmaps-clusterer").url.replace(".js", "")']
+                            "markerclusterer":  ['@routes.Assets.versioned("javascripts/lib/gmaps-clusterer").url.replace(".js", "")'],
+                            "underscore.string": ['@routes.WebJarAssets.at(WebJarAssets.locate("underscore.string.min.js")).url.replace(".js", "")']
                         },
                         "shim":{
                             "nicescroll": ['jquery'],

diff --git a/src/build.sbt b/src/build.sbt
@@ -42,6 +42,7 @@ libraryDependencies ++= Seq(
   "org.webjars" % "angular-ui-bootstrap" % "0.12.0",
   "org.webjars" % "angular-loading-bar" % "0.5.1",
   "org.webjars" % "underscorejs" % "1.6.0-3",
+  "org.webjars" % "underscore.string" % "2.3.3",
   "org.webjars" % "highcharts" % "4.0.3",
   "org.webjars" % "highcharts-ng" % "0.0.6",
   "org.webjars" % "ng-table" % "0.3.3",

diff --git a/src/conf/routes b/src/conf/routes
@@ -32,7 +32,7 @@ GET         /assets/javascripts/angular/:ngAppName/controllers.js
 POST        /ttlupload                                                                    @controllers.VisualizationController.fromFiles()
 POST        /ttldl                                                                        @controllers.VisualizationController.fromUris()
 GET         /ttlds                                                                        @controllers.VisualizationController.dataSource()
-GET         /discover/                                                                    @controllers.VisualizationController.discover(dataSourceTemplateId : Option[Long], combine: Boolean ?= false)
+GET         /discover/                                                                    @controllers.VisualizationController.discover(dataSourceTemplateIds : List[Long], combine: Boolean ?= false)
 
 
 # API v1
@@ -53,7 +53,7 @@ GET         /api/v1/component/:id/descriptors
 # LDVM pipelines
 GET         /api/v1/pipelines/evaluate/:pipelineId                                        @controllers.api.PipelineApiController.evaluate(pipelineId: Long)
 GET         /api/v1/pipelines                                                             @controllers.api.PipelineApiController.list(skip: Int ?= 0, take: Int ?= 50, discoveryId: Option[Long] ?= None, visualizerId: Option[Long] ?= None)
-GET         /api/v1/pipelines/discover                                                    @controllers.api.PipelineApiController.discover(dataSourceTemplateId: Option[Long], combine: Boolean ?= false)
+GET         /api/v1/pipelines/discover                                                    @controllers.api.PipelineApiController.discover(dataSourceTemplateIds: List[Long], combine: Boolean ?= false)
 GET         /api/v1/pipelines/evaluations/:id                                             @controllers.api.PipelineApiController.evaluations(id: Long, skip: Int ?= 0, pageSize: Int ?= 10)
 GET         /api/v1/pipelines/visualization/:id                                           @controllers.api.PipelineApiController.visualizationById(id: Long)
 GET         /api/v1/pipelines/:id                                                         @controllers.api.PipelineApiController.findById(id: Long)

diff --git a/src/public/javascripts/angular/ldvm/api.js b/src/public/javascripts/angular/ldvm/api.js
@@ -25,7 +25,7 @@ define(['angular'], function (ng) {
         }])
         .factory('ComponentsApi', ['$resource', function ($resource) {
             return $resource(null, null, {
-                createDatasource: {url: '/api/v1/datasources/add', method: 'POST', isArray: false}
+                createDatasource: {url: '/api/v1/datasources/add', method: 'POST', isArray: true}
             });
         }])
         .factory('EvaluationApi', ['$resource', function ($resource) {

diff --git a/src/public/javascripts/angular/ldvm/controllers/discoveryController.js b/src/public/javascripts/angular/ldvm/controllers/discoveryController.js
@@ -18,9 +18,16 @@ define(['angular', './controllers'], function (ng) {
                 $scope.data = [];
 
                 var l = window.location;
+
                 var uri = "ws://" + l.host + "/api/v1/pipelines/discover";
-                if ("dataSourceTemplateId" in $routeParams && $routeParams.dataSourceTemplateId) {
-                    uri += "?dataSourceTemplateId=" + $routeParams.dataSourceTemplateId;
+                if ("dataSourceTemplateId" in $routeParams && $routeParams.dataSourceTemplateIds) {
+
+                    var params = $routeParams.dataSourceTemplateIds.map(function(p){
+                        return "dataSourceTemplateIds=" + p;
+                    });
+
+                    uri += "?" + params.join("&");
+
                     if ($routeParams.combine && $routeParams.combine > 0) {
                         uri += "&combine=true";
                     }