Csp migration

content-api/content-service/conf/application.conf

@@ -752,3 +752,14 @@ actor.timeoutMillisec = 30000
 
 #Index file validation
 isIndexHtmlValidationRequired=true
+
+cloud_storage_type=""
+cloud_storage_key=""
+cloud_storage_secret=""
+cloud_storage_container=""
+cloud_storage_endpoint=""
+
+cloudstorage.metadata.replace_absolute_path=false
+cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net"
+cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net"]
+cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"]

ontology-engine/graph-core_2.11/src/main/scala/org/sunbird/graph/GraphService.scala

@@ -1,23 +1,32 @@
 package org.sunbird.graph
 
-import java.util
-
-import org.sunbird.common.dto.{Property, Request, Response}
+import org.sunbird.common.Platform
+import org.sunbird.common.dto.{Property, Request, Response, ResponseHandler}
 import org.sunbird.graph.dac.model.{Node, SearchCriteria}
 import org.sunbird.graph.external.ExternalPropsManager
-import org.sunbird.graph.external.store.ExternalStore
 import org.sunbird.graph.service.operation.{GraphAsyncOperations, Neo4JBoltSearchOperations, NodeAsyncOperations, SearchAsyncOperations}
+import org.sunbird.graph.util.CSPMetaUtil
 
+import java.lang
 import scala.concurrent.{ExecutionContext, Future}
 
 class GraphService {
     implicit  val ec: ExecutionContext = ExecutionContext.global
+    val isrRelativePathEnabled: lang.Boolean = Platform.getBoolean("cloudstorage.metadata.replace_absolute_path", false)
 
     def addNode(graphId: String, node: Node): Future[Node] = {
+        if(isrRelativePathEnabled) {
+            val metadata = CSPMetaUtil.updateRelativePath(node.getMetadata)
+            node.setMetadata(metadata)
+        }
         NodeAsyncOperations.addNode(graphId, node)
     }
 
     def upsertNode(graphId: String, node: Node, request: Request): Future[Node] = {
+        if(isrRelativePathEnabled) {
+            val metadata = CSPMetaUtil.updateRelativePath(node.getMetadata)
+            node.setMetadata(metadata)
+        }
         NodeAsyncOperations.upsertNode(graphId, node, request)
     }
 
@@ -26,33 +35,46 @@ class GraphService {
     }
 
     def getNodeByUniqueId(graphId: String, nodeId: String, getTags: Boolean, request: Request): Future[Node] = {
-        SearchAsyncOperations.getNodeByUniqueId(graphId, nodeId, getTags, request)
+        SearchAsyncOperations.getNodeByUniqueId(graphId, nodeId, getTags, request).map(node => if(isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(node) else node)
     }
 
     def deleteNode(graphId: String, nodeId: String, request: Request): Future[java.lang.Boolean] = {
         NodeAsyncOperations.deleteNode(graphId, nodeId, request)
     }
 
     def getNodeProperty(graphId: String, identifier: String, property: String): Future[Property] = {
-        SearchAsyncOperations.getNodeProperty(graphId, identifier, property)
+        SearchAsyncOperations.getNodeProperty(graphId, identifier, property).map(property => if(isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(property) else property)
     }
-    def updateNodes(graphId: String, identifiers:util.List[String], metadata:util.Map[String,AnyRef]):Future[util.Map[String, Node]] = {
-        NodeAsyncOperations.updateNodes(graphId, identifiers, metadata)
+    def updateNodes(graphId: String, identifiers:java.util.List[String], metadata:java.util.Map[String,AnyRef]):Future[java.util.Map[String, Node]] = {
+        val updatedMetadata = if(isrRelativePathEnabled) CSPMetaUtil.updateRelativePath(metadata) else metadata
+        NodeAsyncOperations.updateNodes(graphId, identifiers, updatedMetadata)
     }
 
-    def getNodeByUniqueIds(graphId:String, searchCriteria: SearchCriteria): Future[util.List[Node]] = {
-        SearchAsyncOperations.getNodeByUniqueIds(graphId, searchCriteria)
+    def getNodeByUniqueIds(graphId:String, searchCriteria: SearchCriteria): Future[java.util.List[Node]] = {
+        SearchAsyncOperations.getNodeByUniqueIds(graphId, searchCriteria).map(nodes => if(isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(nodes) else nodes)
     }
 
     def readExternalProps(request: Request, fields: List[String]): Future[Response] = {
-        ExternalPropsManager.fetchProps(request, fields)
+        ExternalPropsManager.fetchProps(request, fields).map(res =>
+            if(isrRelativePathEnabled) {
+                val updatedResult = CSPMetaUtil.updateAbsolutePath(res.getResult)
+                val response = ResponseHandler.OK()
+                response.putAll(updatedResult)
+                response
+            } else res)
     }
 
     def saveExternalProps(request: Request): Future[Response] = {
+        val externalProps: java.util.Map[String, AnyRef] = request.getRequest
+        val updatedExternalProps = if(isrRelativePathEnabled) CSPMetaUtil.updateExternalRelativePath(externalProps) else externalProps
+        request.setRequest(updatedExternalProps)
         ExternalPropsManager.saveProps(request)
     }
 
     def updateExternalProps(request: Request): Future[Response] = {
+        val externalProps: java.util.Map[String, AnyRef] = request.getRequest
+        val updatedExternalProps = if (isrRelativePathEnabled) CSPMetaUtil.updateExternalRelativePath(externalProps) else externalProps
+        request.setRequest(updatedExternalProps)
         ExternalPropsManager.update(request)
     }
 
@@ -63,11 +85,11 @@ class GraphService {
         Neo4JBoltSearchOperations.checkCyclicLoop(graphId, endNodeId, relationType, startNodeId)
     }
 
-    def removeRelation(graphId: String, relationMap: util.List[util.Map[String, AnyRef]]) = {
+    def removeRelation(graphId: String, relationMap: java.util.List[java.util.Map[String, AnyRef]]) = {
         GraphAsyncOperations.removeRelation(graphId, relationMap)
     }
 
-    def createRelation(graphId: String, relationMap: util.List[util.Map[String, AnyRef]]) = {
+    def createRelation(graphId: String, relationMap: java.util.List[java.util.Map[String, AnyRef]]) = {
         GraphAsyncOperations.createRelation(graphId, relationMap)
     }
 }

ontology-engine/graph-core_2.11/src/main/scala/org/sunbird/graph/util/CSPMetaUtil.scala

@@ -0,0 +1,127 @@
+package org.sunbird.graph.util
+
+import org.apache.commons.collections4.MapUtils
+import org.apache.commons.lang3.StringUtils
+import org.slf4j.LoggerFactory
+import org.sunbird.common.dto.Property
+import org.sunbird.common.{JsonUtils, Platform}
+import org.sunbird.graph.dac.model.Node
+
+import scala.collection.JavaConverters._
+
+object CSPMetaUtil {
+  private[this] val logger = LoggerFactory.getLogger(classOf[CSPMetaUtil])
+
+  def updateAbsolutePath(data: java.util.Map[String, AnyRef]): java.util.Map[String, AnyRef] = {
+    val cspMeta = Platform.getStringList("cloudstorage.metadata.list", new java.util.ArrayList[String]()).asScala.toList
+    val absolutePath = Platform.getString("cloudstorage.read_base_path", "") + java.io.File.separator + Platform.getString("cloud_storage_container", "")
+    if (MapUtils.isNotEmpty(data)) {
+      val updatedMeta: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]
+      data.asScala.map(x =>
+        if (cspMeta.contains(x._1))
+          updatedMeta.put(x._1, x._2.asInstanceOf[String].replace("CLOUD_STORAGE_BASE_PATH", absolutePath))
+        else updatedMeta.put(x._1, x._2)
+      ).asJava
+      updatedMeta
+    } else data
+  }
+
+  def updateAbsolutePath(node: Node): Node = {
+    val metadata = updateAbsolutePath(node.getMetadata)
+    node.setMetadata(metadata)
+    node
+  }
+
+  def updateAbsolutePath(nodes: java.util.List[Node]): java.util.List[Node] = {
+    nodes.asScala.toList.map(node => {
+      updateAbsolutePath(node)
+    }).asJava
+  }
+
+  def updateAbsolutePath(property: Property): Property = {
+    val cspMeta = Platform.getStringList("cloudstorage.metadata.list", new java.util.ArrayList[String]())
+    val absolutePath = Platform.getString("cloudstorage.read_base_path", "") + java.io.File.separator + Platform.getString("cloud_storage_container", "")
+    if(cspMeta.contains(property.getPropertyName)) {
+      val value = property.getPropertyValue
+      value match {
+        case str: String =>
+          property.setPropertyValue(str.replace("CLOUD_STORAGE_BASE_PATH", absolutePath))
+        case _ =>
+      }
+    }
+    property
+  }
+
+  def updateRelativePath(data: java.util.Map[String, AnyRef]): java.util.Map[String, AnyRef] = {
+    logger.info("CSPMetaUtil ::: updateRelativePath util.Map[String, AnyRef] ::: data before url replace :: " + data)
+    val cspMeta: java.util.List[String] = Platform.getStringList("cloudstorage.metadata.list", new java.util.ArrayList[String]())
+    val validCSPSource: List[String] = Platform.getStringList("cloudstorage.write_base_path", new java.util.ArrayList[String]()).asScala.toList
+    val basePaths: Array[String] = validCSPSource.map(source => source + java.io.File.separator + Platform.getString("cloud_storage_container", "")).toArray
+    val repArray = getReplacementData(basePaths, "CLOUD_STORAGE_BASE_PATH")
+    val result = if (MapUtils.isNotEmpty(data)) {
+      val updatedMeta: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]
+      data.asScala.map(x =>
+        if (cspMeta.contains(x._1))
+          updatedMeta.put(x._1, getBasePath(x._1, x._2, basePaths, repArray))
+        else updatedMeta.put(x._1, x._2)
+      ).asJava
+      updatedMeta
+    } else data
+    logger.info("CSPMetaUtil ::: updateRelativePath util.Map[String, AnyRef] ::: data after url replace :: " + result)
+    result
+  }
+
+  def updateExternalRelativePath(data: java.util.Map[String, AnyRef]): java.util.Map[String, AnyRef] = {
+    logger.info("CSPMetaUtil ::: updateExternalRelativePath util.Map[String, AnyRef] ::: data before url replace :: " + data)
+
+    val values = data.get("values")
+    val updatedValues = values match {
+      case x: List[AnyRef] => {
+        val validCSPSource: List[String] = Platform.getStringList("cloudstorage.write_base_path", new java.util.ArrayList[String]()).asScala.toList
+        val basePaths: Array[String] = validCSPSource.map(source => source + java.io.File.separator + Platform.getString("cloud_storage_container", "")).toArray
+        val repArray = getReplacementData(basePaths, "CLOUD_STORAGE_BASE_PATH")
+
+        x.map(value => StringUtils.replaceEach(value.asInstanceOf[String], basePaths, repArray))
+      }
+      case _ => values
+    }
+
+    data.put("values", updatedValues)
+    logger.info("CSPMetaUtil ::: updateExternalRelativePath util.Map[String, AnyRef] ::: data after url replace :: " + data)
+    data
+  }
+
+  private def getBasePath(key: String, value: AnyRef, oldPath: Array[String], newPath: Array[String]): AnyRef = {
+    logger.info(s"CSPMetaUtil ::: getBasePath ::: Updating Path for Key : $key & Value : $value")
+    val res = if (null != value) {
+      value match {
+        case x: String => if (StringUtils.isNotBlank(x)) StringUtils.replaceEach(x, oldPath, newPath) else x
+        case y: Map[String, AnyRef] => {
+          val dStr = JsonUtils.serialize(y)
+          val result = StringUtils.replaceEach(dStr, oldPath, newPath)
+          val output: Map[String, AnyRef] = JsonUtils.deserialize(result, classOf[Map[String, AnyRef]])
+          output
+        }
+        case z: java.util.Map[String, AnyRef] => {
+          val dStr = JsonUtils.serialize(z)
+          val result = StringUtils.replaceEach(dStr, oldPath, newPath)
+          val output: java.util.Map[String, AnyRef] = JsonUtils.deserialize(result, classOf[java.util.Map[String, AnyRef]])
+          output
+        }
+      }
+    } else value
+    logger.info(s"CSPMetaUtil ::: getBasePath ::: Updated Path for Key : $key & Updated Value is : $res")
+    res
+  }
+
+  private def getReplacementData(oldPath: Array[String], repStr: String): Array[String] = {
+    val repArray = new Array[String](oldPath.length)
+    for (i <- oldPath.indices) {
+      repArray(i) = repStr
+    }
+    repArray
+  }
+
+}
+
+class CSPMetaUtil {}

ontology-engine/graph-core_2.11/src/test/resources/application.conf

@@ -486,3 +486,8 @@ languageCode {
 
 platform.language.codes=["as","bn","en","gu","hi","hoc","jun","ka","mai","mr","unx","or","san","sat","ta","te","urd"]
 objectcategorydefinition.keyspace=category_store
+
+cloudstorage.metadata.replace_absolute_path=true
+cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net"
+cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"]
+cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"]

ontology-engine/graph-engine_2.11/src/test/scala/org/sunbird/graph/BaseSpec.scala

@@ -1,7 +1,6 @@
 package org.sunbird.graph
 
 import java.io.File
-import java.util
 
 import com.datastax.driver.core.Session
 import org.apache.commons.io.FileUtils
@@ -134,7 +133,7 @@ class BaseSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll {
         node.setIdentifier("board")
         node.setNodeType("DATA_NODE")
         node.setObjectType("Category")
-        node.setMetadata(new util.HashMap[String, AnyRef]() {
+        node.setMetadata(new java.util.HashMap[String, AnyRef]() {
             {
                 put("code", "board")
                 put("orgIdFieldName", "boardIds")

taxonomy-api/taxonomy-service/conf/application.conf

@@ -316,5 +316,10 @@ languageCode {
 }
 
 platform.language.codes=["as","bn","en","gu","hi","hoc","jun","ka","mai","mr","unx","or","san","sat","ta","te","urd"]
-objectcategorydefinition.keyspace=category_store
+objectcategorydefinition.keyspace=dev_category_store
 
+cloudstorage.metadata.replace_absolute_path=true
+cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net"
+cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net"]
+cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"]
+cloud_storage_container="sunbird-content-dev"