Skip to content

Commit

Permalink
Merge branch 'main' into fix-rest-catalog-request-limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@Xuanwo
Xuanwo authored Dec 14, 2024
2 parents 5bb1622 + 0ba444f commit 102483f
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 24 deletions.
4 changes: 2 additions & 2 deletions crates/iceberg/src/expr/visitors/expression_evaluator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -361,7 +361,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down
12 changes: 6 additions & 6 deletions crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1991,7 +1991,7 @@ mod test {
nan_value_counts: Default::default(),
lower_bounds: Default::default(),
upper_bounds: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2012,7 +2012,7 @@ mod test {
nan_value_counts: Default::default(),
lower_bounds: Default::default(),
upper_bounds: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2069,7 +2069,7 @@ mod test {
]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2095,7 +2095,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("dC"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2122,7 +2122,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("3str3"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand All @@ -2149,7 +2149,7 @@ mod test {
upper_bounds: HashMap::from([(3, Datum::string("イロハニホヘト"))]),

column_sizes: Default::default(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![],
equality_ids: vec![],
sort_order_id: None,
Expand Down
1 change: 1 addition & 0 deletions crates/iceberg/src/io/object_cache.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ mod tests {
.file_size_in_bytes(100)
.record_count(1)
.partition(Struct::from_iter([Some(Literal::long(100))]))
.key_metadata(None)
.build()
.unwrap(),
)
Expand Down
1 change: 1 addition & 0 deletions crates/iceberg/src/scan.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1073,6 +1073,7 @@ mod tests {
.file_size_in_bytes(100)
.record_count(1)
.partition(Struct::from_iter([Some(Literal::long(100))]))
.key_metadata(None)
.build()
.unwrap(),
)
Expand Down
31 changes: 16 additions & 15 deletions crates/iceberg/src/spec/manifest.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,7 +1074,7 @@ pub struct DataFile {
///
/// Implementation-specific key metadata for encryption
#[builder(default)]
pub(crate) key_metadata: Vec<u8>,
pub(crate) key_metadata: Option<Vec<u8>>,
/// field id: 132
/// element field id: 133
///
Expand Down Expand Up @@ -1164,8 +1164,8 @@ impl DataFile {
&self.upper_bounds
}
/// Get the Implementation-specific key metadata for the data file.
pub fn key_metadata(&self) -> &[u8] {
&self.key_metadata
pub fn key_metadata(&self) -> Option<&[u8]> {
self.key_metadata.as_deref()
}
/// Get the split offsets of the data file.
/// For example, all row group offsets in a Parquet file.
Expand Down Expand Up @@ -1378,12 +1378,13 @@ mod _serde {
nan_value_counts: Some(to_i64_entry(value.nan_value_counts)?),
lower_bounds: Some(to_bytes_entry(value.lower_bounds)?),
upper_bounds: Some(to_bytes_entry(value.upper_bounds)?),
key_metadata: Some(serde_bytes::ByteBuf::from(value.key_metadata)),
key_metadata: value.key_metadata.map(serde_bytes::ByteBuf::from),
split_offsets: Some(value.split_offsets),
equality_ids: Some(value.equality_ids),
sort_order_id: value.sort_order_id,
})
}

pub fn try_into(
self,
partition_type: &StructType,
Expand Down Expand Up @@ -1441,7 +1442,7 @@ mod _serde {
.map(|v| parse_bytes_entry(v, schema))
.transpose()?
.unwrap_or_default(),
key_metadata: self.key_metadata.map(|v| v.to_vec()).unwrap_or_default(),
key_metadata: self.key_metadata.map(|v| v.to_vec()),
split_offsets: self.split_offsets.unwrap_or_default(),
equality_ids: self.equality_ids.unwrap_or_default(),
sort_order_id: self.sort_order_id,
Expand Down Expand Up @@ -1657,7 +1658,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -1813,7 +1814,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -1880,7 +1881,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: Some(0),
Expand Down Expand Up @@ -1960,7 +1961,7 @@ mod tests {
(2, Datum::string("a")),
(3, Datum::string("x"))
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: Some(0),
Expand Down Expand Up @@ -2035,7 +2036,7 @@ mod tests {
(2, Datum::int(2)),
(3, Datum::string("x"))
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2105,7 +2106,7 @@ mod tests {
(1, Datum::long(1)),
(2, Datum::int(2)),
]),
key_metadata: vec![],
key_metadata: None,
split_offsets: vec![4],
equality_ids: vec![],
sort_order_id: None,
Expand Down Expand Up @@ -2183,7 +2184,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2214,7 +2215,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2246,7 +2247,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down Expand Up @@ -2278,7 +2279,7 @@ mod tests {
nan_value_counts: HashMap::new(),
lower_bounds: HashMap::new(),
upper_bounds: HashMap::new(),
key_metadata: Vec::new(),
key_metadata: None,
split_offsets: vec![4],
equality_ids: Vec::new(),
sort_order_id: None,
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/writer/file_writer/parquet_writer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ impl ParquetWriter {
// # TODO(#417)
// - nan_value_counts
// - distinct_counts
.key_metadata(metadata.footer_signing_key_metadata.unwrap_or_default())
.key_metadata(metadata.footer_signing_key_metadata)
.split_offsets(
metadata
.row_groups
Expand Down

0 comments on commit 102483f

Please sign in to comment.