diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala index 2be7c64612cd2..94cce9c596fcd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala @@ -446,7 +446,7 @@ private[parquet] class CatalystSchemaConverter( field.name, Types .buildGroup(REPEATED) - .addField(convertField(StructField("element", elementType, nullable))) + .addField(convertField(StructField("array", elementType, nullable))) .named(CatalystConverter.ARRAY_CONTAINS_NULL_BAG_SCHEMA_NAME)) // Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level @@ -459,7 +459,7 @@ private[parquet] class CatalystSchemaConverter( ConversionPatterns.listType( repetition, field.name, - convertField(StructField("element", elementType, nullable), REPEATED)) + convertField(StructField("array", elementType, nullable), REPEATED)) // Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by // MAP_KEY_VALUE. This is covered by `convertGroupField(field: GroupType): DataType`. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala index 35d3c33f99a06..54be9685c868c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala @@ -174,7 +174,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { """ |message root { | optional group _1 (LIST) { - | repeated int32 element; + | repeated int32 array; | } |} """.stripMargin) @@ -198,7 +198,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { |message root { | optional group _1 (LIST) { | repeated group bag { - | optional int32 element; + | optional int32 array; | } | } |} @@ -267,7 +267,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { | optional binary _1 (UTF8); | optional group _2 (LIST) { | repeated group bag { - | optional group element { + | optional group array { | required int32 _1; | required double _2; | } @@ -467,7 +467,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { """message root { | optional group f1 (LIST) { | repeated group list { - | optional int32 element; + | optional int32 array; | } | } |} @@ -478,11 +478,12 @@ class ParquetSchemaSuite extends ParquetSchemaTest { StructType(Seq( StructField( "f1", - ArrayType(IntegerType, containsNull = true), + ArrayType( + StructType(Seq(StructField("num", IntegerType))), containsNull = false), nullable = true))), """message root { | optional group f1 (LIST) { - | repeated group element { + | repeated group array { | optional int32 num; | } | } @@ -496,7 +497,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { """message root { | optional group f1 (LIST) { | repeated group list { - | required int32 element; + | required int32 array; | } | } |} @@ -505,10 +506,13 @@ class ParquetSchemaSuite extends ParquetSchemaTest { testParquetToCatalyst( "Backwards-compatibility: LIST with non-nullable element type - 2", StructType(Seq( - StructField("f1", ArrayType(IntegerType, containsNull = false), nullable = true))), + StructField("f1", + ArrayType(StructType(Seq(StructField("num", IntegerType, nullable = false))), + containsNull = false), + nullable = true))), """message root { | optional group f1 (LIST) { - | repeated group element { + | repeated group array { | required int32 num; | } | } @@ -521,7 +525,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { StructField("f1", ArrayType(IntegerType, containsNull = false), nullable = true))), """message root { | optional group f1 (LIST) { - | repeated int32 element; + | repeated int32 array; | } |} """.stripMargin) @@ -539,7 +543,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { nullable = true))), """message root { | optional group f1 (LIST) { - | repeated group element { + | repeated group array { | required binary str (UTF8); | required int32 num; | } @@ -616,7 +620,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { """message root { | optional group f1 (LIST) { | repeated group bag { - | optional int32 element; + | optional int32 array; | } | } |} @@ -648,7 +652,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { nullable = true))), """message root { | optional group f1 (LIST) { - | repeated int32 element; + | repeated int32 array; | } |} """.stripMargin)