Commen why we override the limit, extend test coverage a bit

olaky · Jan 30, 2024 · 5d6ccf2 · 5d6ccf2
1 parent d8ed2c2
commit 5d6ccf2
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/JsonUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/JsonUtils.scala
@@ -30,6 +30,8 @@ object JsonUtils {
     _mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
     _mapper.registerModule(DefaultScalaModule)
 
+    // We do not want to limit the length of JSON strings in the Delta log or table data. Also note
+    // that not having a limit was the default behavior before Jackson 2.15.
     val streamReadConstraints = StreamReadConstraints
       .builder()
       .maxStringLength(Int.MaxValue)

diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala
@@ -36,7 +36,9 @@ class JsonUtilsSuite
       DataSize() -> """{}""",
       DataSize(bytesCompressed = Some(816L)) -> """{"bytesCompressed":816}""",
       DataSize(rows = Some(111L)) -> """{"rows":111}""",
+      DataSize(rows = Some(0)) -> """{"rows":0}""",
       DataSize(logicalRows = Some(111L)) -> """{"logicalRows":111}""",
+      DataSize(logicalRows = Some(-1L)) -> """{"logicalRows":-1}""",
       DataSize(bytesCompressed = Some(816L), rows = Some(111L), logicalRows = Some(111L)) ->
         """{"bytesCompressed":816,"rows":111,"logicalRows":111}"""
     )