Branch spark 69 (apache#170)

* Fixing the wrong type casting of TimeStamp to OTimeStamp when read from spark dataFrame. * SPARK-69: Problem with license when we try to read from json and write to maprdb
arjunshroff · Sep 19, 2019 · 00f7ea3 · 00f7ea3
1 parent accf448
commit 00f7ea3
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 2 deletions.
diff --git a/external/maprdb/src/main/scala/com/mapr/db/spark/utils/MapRDBUtils.scala b/external/maprdb/src/main/scala/com/mapr/db/spark/utils/MapRDBUtils.scala
@@ -27,6 +27,7 @@ private[spark] object MapRDBUtils {
       tabDesc.setAutoSplit(true)
       tabDesc.setPath(tableName)
       tabDesc.setBulkLoad(bulkMode)
+      tabDesc.setInsertionOrder(false)
       if (keys.isEmpty)
         DBClient().createTable(tabDesc)
       else

diff --git a/external/maprdb/src/test/scala/com/mapr/db/testCases/PredicateTests.scala b/external/maprdb/src/test/scala/com/mapr/db/testCases/PredicateTests.scala
@@ -10,6 +10,8 @@ import org.apache.spark.rdd.RDD
 import org.ojai.exceptions.TypeException
 import org.ojai.store.QueryCondition
 import com.mapr.db.MapRDB
+import com.mapr.db.spark.dbclient.DBClient
+
 
 object PredicateTests {
   val tableName = "/tmp/user_profiles_predicates"
@@ -29,7 +31,11 @@ object PredicateTests {
     if (MapRDB.tableExists(tableName))
       MapRDB.deleteTable(tableName)
     println("table successfully create :" + tableName)
-    MapRDB.createTable(tableName)
+    val tabDesc = DBClient().newTableDescriptor()
+    tabDesc.setAutoSplit(true)
+    tabDesc.setPath(tableName)
+    tabDesc.setInsertionOrder(false)
+    DBClient().createTable(tabDesc)
   }
 
   def runTests(sparkSession: SparkContext): Unit = {

diff --git a/external/maprdb/src/test/scala/com/mapr/db/testCases/SparkSqlAcessTests.scala b/external/maprdb/src/test/scala/com/mapr/db/testCases/SparkSqlAcessTests.scala
@@ -12,6 +12,7 @@ import org.apache.spark.sql.SparkSession
 import org.ojai.types.{ODate, OTime, OTimestamp}
 import com.mapr.org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator
 import com.mapr.db.MapRDB
+import com.mapr.db.spark.dbclient.DBClient
 
 object SparkSqlAccessTests {
   lazy val conf = new SparkConf()
@@ -31,7 +32,12 @@ object SparkSqlAccessTests {
   def tableInitialization(tableName: String): Unit = {
     if (MapRDB.tableExists(tableName))
       MapRDB.deleteTable(tableName)
-    val table = MapRDB.createTable(tableName)
+    val tabDesc = DBClient().newTableDescriptor()
+    tabDesc.setAutoSplit(true)
+    tabDesc.setPath(tableName)
+    tabDesc.setInsertionOrder(false)
+    DBClient().createTable(tabDesc)
+    val table = DBClient().getTable(tableName)
     table.insertOrReplace(getNullRecord())
     table.insertOrReplace(getBooleanRecord())
     table.insertOrReplace(getStringRecord())