Fix data corruption in json decoder f64-to-i64 cast (#652) (#665)

* Add failing test for JSON writer i64 bug * Add special handling for i64/u64 to json decoder array builder * Fix linter error - linter wants .flatten on a new line Co-authored-by: Christian Williams <[email protected]>
apache · Aug 6, 2021 · 51f2b2b · 51f2b2b
1 parent 64c78b8
commit 51f2b2b
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/arrow/src/json/reader.rs b/arrow/src/json/reader.rs
@@ -926,8 +926,16 @@ impl Decoder {
             rows.iter()
                 .map(|row| {
                     row.get(&col_name)
-                        .and_then(|value| value.as_f64())
-                        .and_then(num::cast::cast)
+                        .and_then(|value| {
+                            if value.is_i64() {
+                                value.as_i64().map(num::cast::cast)
+                            } else if value.is_u64() {
+                                value.as_u64().map(num::cast::cast)
+                            } else {
+                                value.as_f64().map(num::cast::cast)
+                            }
+                        })
+                        .flatten()
                 })
                 .collect::<PrimitiveArray<T>>(),
         ))
@@ -1831,6 +1839,7 @@ mod tests {
             .unwrap();
         assert_eq!(1, aa.value(0));
         assert_eq!(-10, aa.value(1));
+        assert_eq!(1627668684594000000, aa.value(2));
         let bb = batch
             .column(b.0)
             .as_any()

diff --git a/arrow/test/data/arrays.json b/arrow/test/data/arrays.json
@@ -1,3 +1,3 @@
 {"a":1, "b":[2.0, 1.3, -6.1], "c":[false, true], "d":"4"}
 {"a":-10, "b":[2.0, 1.3, -6.1], "c":[true, true], "d":"4"}
-{"a":2, "b":[2.0, null, -6.1], "c":[false, null], "d":"text"}
+{"a":1627668684594000000, "b":[2.0, null, -6.1], "c":[false, null], "d":"text"}