Merge pull request #21 from yhuai/pr3431yin

Updates
meisam · Jan 9, 2015 · f336a16 · f336a16
2 parents 1eeb769 + baf79b5
commit f336a16
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 27 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -44,11 +44,11 @@ private[sql] case class JSONRelation(
   private def baseRDD = sqlContext.sparkContext.textFile(fileName)
 
   override val schema = userSpecifiedSchema.getOrElse(
-    JsonRDD.inferSchema(
-      baseRDD,
-      samplingRatio,
-      sqlContext.columnNameOfCorruptRecord)
-    )
+    JsonRDD.nullTypeToStringType(
+      JsonRDD.inferSchema(
+        baseRDD,
+        samplingRatio,
+        sqlContext.columnNameOfCorruptRecord)))
 
   override def buildScan() =
     JsonRDD.jsonStringToRow(baseRDD, schema, sqlContext.columnNameOfCorruptRecord)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
@@ -72,15 +72,15 @@ case class AllDataTypesScan(
         i.toDouble,
         BigDecimal(i),
         BigDecimal(i),
-        new Date(10000 + i),
+        new Date((i + 1) * 8640000),
         new Timestamp(20000 + i),
         s"varchar_$i",
         Seq(i, i + 1),
         Seq(Map(s"str_$i" -> Row(i.toLong))),
         Map(i -> i.toString),
         Map(Map(s"str_$i" -> i.toFloat) -> Row(i.toLong)),
         Row(i, i.toString),
-        Row(Seq(s"str_$i", s"str_${i + 1}"), Row(Seq(new Date(30000 + i)))))
+        Row(Seq(s"str_$i", s"str_${i + 1}"), Row(Seq(new Date((i + 2) * 8640000)))))
     }
   }
 }
@@ -101,15 +101,15 @@ class TableScanSuite extends DataSourceTest {
       i.toDouble,
       BigDecimal(i),
       BigDecimal(i),
-      new Date(10000 + i),
+      new Date((i + 1) * 8640000),
       new Timestamp(20000 + i),
       s"varchar_$i",
       Seq(i, i + 1),
       Seq(Map(s"str_$i" -> Row(i.toLong))),
       Map(i -> i.toString),
       Map(Map(s"str_$i" -> i.toFloat) -> Row(i.toLong)),
       Row(i, i.toString),
-      Row(Seq(s"str_$i", s"str_${i + 1}"), Row(Seq(new Date(30000 + i)))))
+      Row(Seq(s"str_$i", s"str_${i + 1}"), Row(Seq(new Date((i + 2) * 8640000)))))
   }.toSeq
 
   before {
@@ -126,13 +126,13 @@ class TableScanSuite extends DataSourceTest {
     sql(
       """
         |CREATE TEMPORARY TABLE tableWithSchema (
-        |stringField stRIng,
+        |`string$%Field` stRIng,
         |binaryField binary,
-        |booleanField boolean,
-        |byteField tinyint,
+        |`booleanField` boolean,
+        |ByteField tinyint,
         |shortField smaLlint,
-        |intField iNt,
-        |longField Bigint,
+        |int_Field iNt,
+        |`longField_:,<>=+/~^` Bigint,
         |floatField flOat,
         |doubleField doubLE,
         |decimalField1 decimal,
@@ -145,7 +145,7 @@ class TableScanSuite extends DataSourceTest {
         |mapFieldSimple MAP<iNt, StRing>,
         |mapFieldComplex Map<Map<stRING, fLOAT>, Struct<key:bigInt>>,
         |structFieldSimple StRuct<key:INt, Value:STrINg>,
-        |structFieldComplex StRuct<key:Array<String>, Value:struct<value:Array<date>>>
+        |structFieldComplex StRuct<key:Array<String>, Value:struct<`value_(2)`:Array<date>>>
         |)
         |USING org.apache.spark.sql.sources.AllDataTypesScanSource
         |OPTIONS (
@@ -177,13 +177,13 @@ class TableScanSuite extends DataSourceTest {
 
   test("Schema and all fields") {
     val expectedSchema = StructType(
-      StructField("stringField", StringType, true) ::
+      StructField("string$%Field", StringType, true) ::
       StructField("binaryField", BinaryType, true) ::
       StructField("booleanField", BooleanType, true) ::
-      StructField("byteField", ByteType, true) ::
+      StructField("ByteField", ByteType, true) ::
       StructField("shortField", ShortType, true) ::
-      StructField("intField", IntegerType, true) ::
-      StructField("longField", LongType, true) ::
+      StructField("int_Field", IntegerType, true) ::
+      StructField("longField_:,<>=+/~^", LongType, true) ::
       StructField("floatField", FloatType, true) ::
       StructField("doubleField", DoubleType, true) ::
       StructField("decimalField1", DecimalType.Unlimited, true) ::
@@ -209,21 +209,22 @@ class TableScanSuite extends DataSourceTest {
           StructField("key", ArrayType(StringType), true) ::
           StructField("Value",
             StructType(
-              StructField("value", ArrayType(DateType), true) :: Nil), true) ::  Nil), true) :: Nil
+              StructField("value_(2)", ArrayType(DateType), true) :: Nil), true) ::  Nil), true) ::
+      Nil
     )
 
     assert(expectedSchema == table("tableWithSchema").schema)
 
     checkAnswer(
       sql(
         """SELECT
-          | stringField,
+          | `string$%Field`,
           | cast(binaryField as string),
           | booleanField,
           | byteField,
           | shortField,
-          | intField,
-          | longField,
+          | int_Field,
+          | `longField_:,<>=+/~^`,
           | floatField,
           | doubleField,
           | decimalField1,
@@ -246,21 +247,25 @@ class TableScanSuite extends DataSourceTest {
     10)
 
   sqlTest(
-    "SELECT stringField FROM tableWithSchema",
+    "SELECT `string$%Field` FROM tableWithSchema",
     (1 to 10).map(i => Row(s"str_$i")).toSeq)
 
   sqlTest(
-    "SELECT intField FROM tableWithSchema WHERE intField < 5",
+    "SELECT int_Field FROM tableWithSchema WHERE int_Field < 5",
     (1 to 4).map(Row(_)).toSeq)
 
   sqlTest(
-    "SELECT longField * 2 FROM tableWithSchema",
+    "SELECT `longField_:,<>=+/~^` * 2 FROM tableWithSchema",
     (1 to 10).map(i => Row(i * 2.toLong)).toSeq)
 
   sqlTest(
-    "SELECT structFieldSimple.key, arrayFieldSimple[1] FROM tableWithSchema a where intField=1",
+    "SELECT structFieldSimple.key, arrayFieldSimple[1] FROM tableWithSchema a where int_Field=1",
     Seq(Seq(1, 2)))
 
+  sqlTest(
+    "SELECT structFieldComplex.Value.`value_(2)` FROM tableWithSchema",
+    (1 to 10).map(i => Row(Seq(new Date((i + 2) * 8640000)))).toSeq)
+
   test("Caching")  {
     // Cached Query Execution
     cacheTable("oneToTen")