From 7658e77f997b1ca3ceeee93af8ecec2c989c690a Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Fri, 15 Mar 2024 19:21:32 +0100 Subject: [PATCH 1/4] change serde metadata v2 --- crates/iceberg/src/spec/table_metadata.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 9893e9eea..dce5a068f 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -727,14 +727,10 @@ pub(super) mod _serde { .collect(), default_spec_id: v.default_spec_id, last_partition_id: v.last_partition_id, - properties: if v.properties.is_empty() { - None - } else { - Some(v.properties) - }, + properties: Some(v.properties), current_snapshot_id: v.current_snapshot_id.or(Some(-1)), snapshots: if v.snapshots.is_empty() { - None + Some(vec![]) } else { Some( v.snapshots From 63f0b222675a29a121d815a89e808b693637672d Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Fri, 15 Mar 2024 19:43:39 +0100 Subject: [PATCH 2/4] change default partition_specs and sort_orders --- crates/iceberg/src/spec/table_metadata.rs | 47 ++++++++++++++++------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index dce5a068f..62040b062 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -29,6 +29,7 @@ use super::{ snapshot::{Snapshot, SnapshotReference, SnapshotRetention}, PartitionSpecRef, SchemaId, SchemaRef, SnapshotRef, SortOrderRef, }; +use super::{PartitionSpec, SortOrder}; use _serde::TableMetadataEnum; @@ -297,19 +298,37 @@ impl TableMetadataBuilder { properties, } = table_creation; - if partition_spec.is_some() { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Can't create table with partition spec now", - )); - } + let partition_specs = match partition_spec { + Some(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Can't create table with partition spec now", + )) + } + None => HashMap::from([( + 0, + Arc::new(PartitionSpec { + spec_id: 0, + fields: vec![], + }), + )]), + }; - if sort_order.is_some() { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Can't create table with sort order now", - )); - } + let sort_orders = match sort_order { + Some(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Can't create table with sort order now", + )) + } + None => HashMap::from([( + 0, + Arc::new(SortOrder { + order_id: 0, + fields: vec![], + }), + )]), + }; let table_metadata = TableMetadata { format_version: FormatVersion::V2, @@ -325,14 +344,14 @@ impl TableMetadataBuilder { last_column_id: schema.highest_field_id(), current_schema_id: schema.schema_id(), schemas: HashMap::from([(schema.schema_id(), Arc::new(schema))]), - partition_specs: Default::default(), + partition_specs, default_spec_id: 0, last_partition_id: 0, properties, current_snapshot_id: None, snapshots: Default::default(), snapshot_log: vec![], - sort_orders: Default::default(), + sort_orders, metadata_log: vec![], default_sort_order_id: 0, refs: Default::default(), From 1a11ead74618a67ed8dc4d5d4f1e7c6c808c38f0 Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Fri, 15 Mar 2024 19:49:01 +0100 Subject: [PATCH 3/4] change test --- crates/iceberg/src/spec/table_metadata.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 62040b062..bc89cd13b 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -1690,9 +1690,27 @@ mod tests { .len(), 0 ); - assert_eq!(table_metadata.partition_specs.len(), 0); assert_eq!(table_metadata.properties.len(), 0); - assert_eq!(table_metadata.sort_orders.len(), 0); + assert_eq!( + table_metadata.partition_specs, + HashMap::from([( + 0, + Arc::new(PartitionSpec { + spec_id: 0, + fields: vec![] + }) + )]) + ); + assert_eq!( + table_metadata.sort_orders, + HashMap::from([( + 0, + Arc::new(SortOrder { + order_id: 0, + fields: vec![] + }) + )]) + ); } #[test] From 1f60381f9daad84e01d7329fadca3e75890354ef Mon Sep 17 00:00:00 2001 From: marvinlanhenke Date: Tue, 19 Mar 2024 07:28:56 +0100 Subject: [PATCH 4/4] use DEFAULTS --- crates/iceberg/src/spec/table_metadata.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index bc89cd13b..4892c2623 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -306,9 +306,9 @@ impl TableMetadataBuilder { )) } None => HashMap::from([( - 0, + DEFAULT_SPEC_ID, Arc::new(PartitionSpec { - spec_id: 0, + spec_id: DEFAULT_SPEC_ID, fields: vec![], }), )]), @@ -322,9 +322,9 @@ impl TableMetadataBuilder { )) } None => HashMap::from([( - 0, + DEFAULT_SORT_ORDER_ID, Arc::new(SortOrder { - order_id: 0, + order_id: DEFAULT_SORT_ORDER_ID, fields: vec![], }), )]), @@ -345,7 +345,7 @@ impl TableMetadataBuilder { current_schema_id: schema.schema_id(), schemas: HashMap::from([(schema.schema_id(), Arc::new(schema))]), partition_specs, - default_spec_id: 0, + default_spec_id: DEFAULT_SPEC_ID, last_partition_id: 0, properties, current_snapshot_id: None, @@ -353,7 +353,7 @@ impl TableMetadataBuilder { snapshot_log: vec![], sort_orders, metadata_log: vec![], - default_sort_order_id: 0, + default_sort_order_id: DEFAULT_SORT_ORDER_ID, refs: Default::default(), };