fix: Incorrect parquet statistics written for UInt64 values > Int64::…

…MAX (#16766)
pola-rs · Jun 6, 2024 · b329894 · b329894
1 parent 5a0c803
commit b329894
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 10 deletions.
diff --git a/crates/polars-parquet/src/arrow/write/primitive/basic.rs b/crates/polars-parquet/src/arrow/write/primitive/basic.rs
@@ -1,6 +1,7 @@
 use arrow::array::{Array, PrimitiveArray};
 use arrow::types::NativeType;
 use polars_error::{polars_bail, PolarsResult};
+use polars_utils::total_ord::TotalOrd;
 
 use super::super::{utils, WriteOptions};
 use crate::arrow::read::schema::is_nullable;
@@ -176,23 +177,17 @@ where
             .then(|| {
                 array
                     .non_null_values_iter()
-                    .map(|x| {
-                        let x: P = x.as_();
-                        x
-                    })
-                    .max_by(|x, y| x.ord(y))
+                    .max_by(TotalOrd::tot_cmp)
+                    .map(T::as_)
             })
             .flatten(),
         min_value: options
             .min_value
             .then(|| {
                 array
                     .non_null_values_iter()
-                    .map(|x| {
-                        let x: P = x.as_();
-                        x
-                    })
-                    .min_by(|x, y| x.ord(y))
+                    .min_by(TotalOrd::tot_cmp)
+                    .map(T::as_)
             })
             .flatten(),
     }

diff --git a/py-polars/tests/unit/io/test_parquet.py b/py-polars/tests/unit/io/test_parquet.py
@@ -970,3 +970,15 @@ def test_hybrid_rle() -> None:
         assert "RLE_DICTIONARY" in column["encodings"]
     f.seek(0)
     assert_frame_equal(pl.read_parquet(f), df)
+
+
+def test_parquet_statistics_uint64_16683() -> None:
+    u64_max = (1 << 64) - 1
+    df = pl.Series("a", [u64_max, 0], dtype=pl.UInt64).to_frame()
+    file = io.BytesIO()
+    df.write_parquet(file, statistics=True)
+    file.seek(0)
+    statistics = pq.read_metadata(file).row_group(0).column(0).statistics
+
+    assert statistics.min == 0
+    assert statistics.max == u64_max