From 2d8e754e699076c8a5915e7faf971e4bd2a5c1fd Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 17 Aug 2018 23:29:49 +0200 Subject: [PATCH] Revert test for invalid encoding --- .../datasources/json/JsonSuite.scala | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 6094db5059019..1900efb82b881 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -2224,30 +2224,19 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { } test("SPARK-23723: specified encoding is not matched to actual encoding") { - def doCount(bypassParser: Boolean, multiLine: Boolean): Long = { - var result: Long = -1 - withSQLConf(SQLConf.BYPASS_PARSER_FOR_EMPTY_SCHEMA.key -> bypassParser.toString) { - val fileName = "test-data/utf16LE.json" - val schema = new StructType().add("firstName", StringType).add("lastName", StringType) - result = spark.read.schema(schema) - .option("mode", "FAILFAST") - .option("multiline", multiLine) - .options(Map("encoding" -> "UTF-16BE", "lineSep" -> "\n")) - .json(testFile(fileName)) - .count() - } - result - } - - Seq((true, true), (false, true), (false, false)).foreach { case (bypassParser, multiLine) => - val exception = intercept[SparkException] { - doCount(bypassParser, multiLine) - } - val errMsg = exception.getMessage - assert(errMsg.contains("Malformed records are detected in record parsing")) + val fileName = "test-data/utf16LE.json" + val schema = new StructType().add("firstName", StringType).add("lastName", StringType) + val exception = intercept[SparkException] { + spark.read.schema(schema) + .option("mode", "FAILFAST") + .option("multiline", "true") + .options(Map("encoding" -> "UTF-16BE")) + .json(testFile(fileName)) + .count() } + val errMsg = exception.getMessage - assert(doCount(bypassParser = true, multiLine = false) == 5) + assert(errMsg.contains("Malformed records are detected in record parsing")) } def checkEncoding(expectedEncoding: String, pathToJsonFiles: String,