From 51f2b2bf4a106530e0d6ffc0108e8215c5d413e3 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 6 Aug 2021 17:00:10 -0400 Subject: [PATCH] Fix data corruption in json decoder f64-to-i64 cast (#652) (#665) * Add failing test for JSON writer i64 bug * Add special handling for i64/u64 to json decoder array builder * Fix linter error - linter wants .flatten on a new line Co-authored-by: Christian Williams --- arrow/src/json/reader.rs | 13 +++++++++++-- arrow/test/data/arrays.json | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/arrow/src/json/reader.rs b/arrow/src/json/reader.rs index 290ad4f23977..0b220eb25ab6 100644 --- a/arrow/src/json/reader.rs +++ b/arrow/src/json/reader.rs @@ -926,8 +926,16 @@ impl Decoder { rows.iter() .map(|row| { row.get(&col_name) - .and_then(|value| value.as_f64()) - .and_then(num::cast::cast) + .and_then(|value| { + if value.is_i64() { + value.as_i64().map(num::cast::cast) + } else if value.is_u64() { + value.as_u64().map(num::cast::cast) + } else { + value.as_f64().map(num::cast::cast) + } + }) + .flatten() }) .collect::>(), )) @@ -1831,6 +1839,7 @@ mod tests { .unwrap(); assert_eq!(1, aa.value(0)); assert_eq!(-10, aa.value(1)); + assert_eq!(1627668684594000000, aa.value(2)); let bb = batch .column(b.0) .as_any() diff --git a/arrow/test/data/arrays.json b/arrow/test/data/arrays.json index 5dbdd19ffc00..6de2b0324e07 100644 --- a/arrow/test/data/arrays.json +++ b/arrow/test/data/arrays.json @@ -1,3 +1,3 @@ {"a":1, "b":[2.0, 1.3, -6.1], "c":[false, true], "d":"4"} {"a":-10, "b":[2.0, 1.3, -6.1], "c":[true, true], "d":"4"} -{"a":2, "b":[2.0, null, -6.1], "c":[false, null], "d":"text"} +{"a":1627668684594000000, "b":[2.0, null, -6.1], "c":[false, null], "d":"text"}