feat: add file statistics

python/python/lance/file.py

@@ -10,6 +10,7 @@
     LanceBufferDescriptor,
     LanceColumnMetadata,
     LanceFileMetadata,
+    LanceFileStatistics,
     LancePageMetadata,
 )
 from .lance import (
@@ -146,6 +147,9 @@ def metadata(self) -> LanceFileMetadata:
         """
         return self._reader.metadata()
 
+    def file_statistics(self) -> LanceFileStatistics:
+        return self._reader.file_statistics()
+
     def read_global_buffer(self, index: int) -> bytes:
         """
         Read a global buffer from the file at a given index
@@ -289,4 +293,5 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None:
     "LanceColumnMetadata",
     "LancePageMetadata",
     "LanceBufferDescriptor",
+    "LanceFileStatistics",
 ]

python/python/lance/lance/__init__.pyi

@@ -82,6 +82,13 @@ class LanceFileMetadata:
     global_buffers: List[LanceBufferDescriptor]
     columns: List[LanceColumnMetadata]
 
+class LanceFileStatistics:
+    columns: List[LanceColumnStatistics]
+
+class LanceColumnStatistics:
+    num_pages: int
+    size_bytes: int
+
 class _Session:
     def size_bytes(self) -> int: ...
 

python/python/tests/test_file.py

@@ -3,6 +3,7 @@
 
 import os
 
+import numpy as np
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pytest
@@ -214,6 +215,34 @@ def test_metadata(tmp_path):
     assert len(page.encoding) > 0
 
 
+def test_file_stat(tmp_path):
+    path = tmp_path / "foo.lance"
+    schema = pa.schema(
+        [pa.field("a", pa.int64()), pa.field("b", pa.list_(pa.float64(), 8))]
+    )
+
+    num_rows = 1_000_000
+
+    data1 = pa.array(range(num_rows))
+
+    # Create a fixed-size list of float64 with dimension 8
+    fixed_size_list = [np.random.rand(8).tolist() for _ in range(num_rows)]
+    data2 = pa.array(fixed_size_list, type=pa.list_(pa.float64(), 8))
+
+    with LanceFileWriter(str(path), schema) as writer:
+        writer.write_batch(pa.table({"a": data1, "b": data2}))
+    reader = LanceFileReader(str(path))
+    file_stat = reader.file_statistics()
+
+    assert len(file_stat.columns) == 2
+
+    assert file_stat.columns[0].num_pages == 1
+    assert file_stat.columns[0].size_bytes == 8_000_000
+
+    assert file_stat.columns[1].num_pages == 2
+    assert file_stat.columns[1].size_bytes == 64_000_000
+
+
 def test_round_trip_parquet(tmp_path):
     pq_path = tmp_path / "foo.parquet"
     table = pa.table({"int": [1, 2], "list_str": [["x", "yz", "abc"], ["foo", "bar"]]})

python/src/file.rs

@@ -23,7 +23,7 @@ use lance_core::cache::FileMetadataCache;
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
 use lance_file::{
     v2::{
-        reader::{BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions},
+        reader::{BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics},
         writer::{FileWriter, FileWriterOptions},
     },
     version::LanceFileVersion,
@@ -113,6 +113,33 @@ impl LanceColumnMetadata {
     }
 }
 
+#[pyclass(get_all)]
+#[derive(Clone, Debug, Serialize)]
+pub struct LanceFileStatistics {
+    columns: Vec<LanceColumnStatistics>,
+}
+
+#[pyclass(get_all)]
+#[derive(Clone, Debug, Serialize)]
+pub struct LanceColumnStatistics {
+    num_pages: usize,
+    size_bytes: usize,
+}
+
+impl LanceFileStatistics {
+    fn new(inner: &FileStatistics) -> Self {
+        let columns = inner
+            .columns
+            .iter()
+            .map(|column_stat| LanceColumnStatistics {
+                num_pages: column_stat.num_pages,
+                size_bytes: column_stat.size_bytes,
+            })
+            .collect();
+        Self { columns }
+    }
+}
+
 #[pyclass(get_all)]
 #[derive(Clone, Debug, Serialize)]
 pub struct LanceFileMetadata {
@@ -445,6 +472,11 @@ impl LanceFileReader {
         LanceFileMetadata::new(inner_meta, py)
     }
 
+    pub fn file_statistics(&self) -> LanceFileStatistics {
+        let inner_stat = self.inner.file_statistics();
+        LanceFileStatistics::new(inner_stat)
+    }
+
     pub fn read_global_buffer(&mut self, index: u32) -> PyResult<Vec<u8>> {
         let buffer_bytes = RT
             .runtime

python/src/lib.rs

@@ -37,7 +37,7 @@ use dataset::optimize::{
 use dataset::MergeInsertBuilder;
 use env_logger::Env;
 use file::{
-    LanceBufferDescriptor, LanceColumnMetadata, LanceFileMetadata, LanceFileReader,
+    LanceBufferDescriptor, LanceColumnMetadata, LanceFileMetadata, LanceFileReader, LanceFileStatistics,
     LanceFileWriter, LancePageMetadata,
 };
 use futures::StreamExt;
@@ -120,6 +120,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<LanceFileReader>()?;
     m.add_class::<LanceFileWriter>()?;
     m.add_class::<LanceFileMetadata>()?;
+    m.add_class::<LanceFileStatistics>()?;
     m.add_class::<LanceColumnMetadata>()?;
     m.add_class::<LancePageMetadata>()?;
     m.add_class::<LanceBufferDescriptor>()?;

rust/lance-file/src/v2/reader.rs

@@ -57,6 +57,18 @@ pub struct BufferDescriptor {
     pub size: u64,
 }
 
+#[derive(Debug)]
+pub struct FileStatistics {
+    pub columns: Vec<ColumnStatistics>,
+}
+
+// Currently we report the `num_pages` and `size_bytes` of each column in Lance file.
+#[derive(Debug)]
+pub struct ColumnStatistics {
+    pub num_pages: usize,
+    pub size_bytes: usize,
+}
+
 // TODO: Caching
 #[derive(Debug)]
 pub struct CachedFileMetadata {
@@ -79,6 +91,9 @@ pub struct CachedFileMetadata {
     pub num_footer_bytes: u64,
     pub major_version: u16,
     pub minor_version: u16,
+
+    // The file statistics.
+    pub file_statistics: Arc<FileStatistics>,
 }
 
 impl DeepSizeOf for CachedFileMetadata {
@@ -313,6 +328,10 @@ impl FileReader {
         &self.metadata
     }
 
+    pub fn file_statistics(&self) -> &Arc<FileStatistics> {
+        &self.metadata().file_statistics
+    }
+
     pub async fn read_global_buffer(&self, index: u32) -> Result<Bytes> {
         let buffer_desc = self.metadata.file_buffers.get(index as usize).ok_or_else(||Error::invalid_input(format!("request for global buffer at index {} but there were only {} global buffers in the file", index, self.metadata.file_buffers.len()), location!()))?;
         self.scheduler
@@ -545,6 +564,26 @@ impl FileReader {
 
         let column_infos = Self::meta_to_col_infos(column_metadatas.as_slice(), file_version);
 
+        let column_stats = column_metadatas
+            .iter()
+            .map(|col_metadata| {
+                let num_pages = col_metadata.pages.len();
+                let size_bytes = col_metadata
+                    .pages
+                    .iter()
+                    .map(|page| page.buffer_sizes.iter().sum::<u64>())
+                    .sum::<u64>();
+                ColumnStatistics {
+                    num_pages,
+                    size_bytes: size_bytes as usize,
+                }
+            })
+            .collect();
+
+        let file_stat = FileStatistics {
+            columns: column_stats,
+        };
+
         Ok(CachedFileMetadata {
             file_schema: Arc::new(schema),
             column_metadatas,
@@ -557,6 +596,7 @@ impl FileReader {
             file_buffers: gbo_table,
             major_version: footer.major_version,
             minor_version: footer.minor_version,
+            file_statistics: Arc::new(file_stat),
         })
     }