From fbd771ba73e51a547298b40d30b1d78809452e8f Mon Sep 17 00:00:00 2001 From: ZENOTME Date: Mon, 25 Dec 2023 11:52:51 +0800 Subject: [PATCH 1/7] serialize custom attr in record field --- lang/rust/avro/src/schema.rs | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 787e8596e82..830adc4badc 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -2043,6 +2043,10 @@ impl Serialize for RecordField { map.serialize_entry("aliases", aliases)?; } + for attr in &self.custom_attributes { + map.serialize_entry(attr.0, attr.1)?; + } + map.end() } } @@ -6416,6 +6420,43 @@ mod tests { } #[test] + fn avro_3920_serialize_record_with_custom_attributes() -> TestResult { + let expected = { + let mut lookup = BTreeMap::new(); + lookup.insert("value".to_owned(), 0); + Schema::Record(RecordSchema { + name: Name { + name: "LongList".to_owned(), + namespace: None, + }, + aliases: Some(vec![Alias::new("LinkedLongs").unwrap()]), + doc: None, + fields: vec![RecordField { + name: "value".to_string(), + doc: None, + default: None, + aliases: None, + schema: Schema::Long, + order: RecordFieldOrder::Ascending, + position: 0, + custom_attributes: BTreeMap::from([("field-id".to_string(), 1.into())]), + }], + lookup, + attributes: BTreeMap::from([("custom-attribute".to_string(), "value".into())]), + }) + }; + + let value = serde_json::to_value(&expected)?; + let serialized = serde_json::to_string(&value)?; + assert_eq!( + r#"{"aliases":["LinkedLongs"],"custom-attribute":"value","fields":[{"field-id":1,"name":"value","type":"long"}],"name":"LongList","type":"record"}"#, + &serialized + ); + assert_eq!(expected, Schema::parse_str(&serialized)?); + + Ok(()) + } + fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { let schema = Schema::Decimal(DecimalSchema { precision: 36, From 99e3ad124f1004a4342b55d6040a5f3379dbabe4 Mon Sep 17 00:00:00 2001 From: ZENOTME Date: Mon, 25 Dec 2023 11:53:24 +0800 Subject: [PATCH 2/7] fix clippy and typos --- lang/rust/avro/src/schema.rs | 20 ++++++++++---------- lang/rust/avro_derive/README.md | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 830adc4badc..44ae39565b7 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -4545,26 +4545,26 @@ mod tests { assert_eq!( schema.custom_attributes(), - Some(&expected_custom_attibutes()) + Some(&expected_custom_attributes()) ); } Ok(()) } - fn expected_custom_attibutes() -> BTreeMap { - let mut expected_attibutes: BTreeMap = Default::default(); - expected_attibutes.insert("string_key".to_string(), Value::String("value".to_string())); - expected_attibutes.insert("number_key".to_string(), json!(1.23)); - expected_attibutes.insert("null_key".to_string(), Value::Null); - expected_attibutes.insert( + fn expected_custom_attributes() -> BTreeMap { + let mut expected_attributes: BTreeMap = Default::default(); + expected_attributes.insert("string_key".to_string(), Value::String("value".to_string())); + expected_attributes.insert("number_key".to_string(), json!(1.23)); + expected_attributes.insert("null_key".to_string(), Value::Null); + expected_attributes.insert( "array_key".to_string(), Value::Array(vec![json!(1), json!(2), json!(3)]), ); let mut object_value: HashMap = HashMap::new(); object_value.insert("key".to_string(), Value::String("value".to_string())); - expected_attibutes.insert("object_key".to_string(), json!(object_value)); - expected_attibutes + expected_attributes.insert("object_key".to_string(), json!(object_value)); + expected_attributes } #[test] @@ -4594,7 +4594,7 @@ mod tests { assert_eq!(fields.len(), 1); let field = &fields[0]; assert_eq!(&field.name, "field_one"); - assert_eq!(field.custom_attributes, expected_custom_attibutes()); + assert_eq!(field.custom_attributes, expected_custom_attributes()); } _ => panic!("Expected Schema::Record"), } diff --git a/lang/rust/avro_derive/README.md b/lang/rust/avro_derive/README.md index 6faa215f51d..0098d38970a 100644 --- a/lang/rust/avro_derive/README.md +++ b/lang/rust/avro_derive/README.md @@ -66,4 +66,4 @@ let encoded = writer.into_inner(); ``` ### Compatibility Notes -This module is designed to work in concert with the Serde implemenation. If your use case dictates needing to manually convert to a `Value` type in order to encode then the derived schema may not be correct. +This module is designed to work in concert with the Serde implementation. If your use case dictates needing to manually convert to a `Value` type in order to encode then the derived schema may not be correct. From c341d7375da2e9a4f32fba64b15e5215ac5666ad Mon Sep 17 00:00:00 2001 From: ZENOTME Date: Tue, 26 Dec 2023 10:07:09 +0800 Subject: [PATCH 3/7] fix used attribute in schema --- lang/rust/avro/src/schema.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 44ae39565b7..72660dbe37b 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -709,10 +709,10 @@ impl RecordField { doc: field.doc(), default, aliases, - schema, order, position, - custom_attributes: RecordField::get_field_custom_attributes(field), + custom_attributes: RecordField::get_field_custom_attributes(field, &schema), + schema, }) } @@ -765,11 +765,16 @@ impl RecordField { Ok(()) } - fn get_field_custom_attributes(field: &Map) -> BTreeMap { + fn get_field_custom_attributes( + field: &Map, + schema: &Schema, + ) -> BTreeMap { let mut custom_attributes: BTreeMap = BTreeMap::new(); for (key, value) in field { match key.as_str() { "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" => continue, + key if key == "symbols" && matches!(schema, Schema::Enum(_)) => continue, + key if key == "size" && matches!(schema, Schema::Fixed(_)) => continue, _ => custom_attributes.insert(key.clone(), value.clone()), }; } @@ -3327,16 +3332,16 @@ mod tests { let schema = Schema::parse_str(raw_schema)?; assert_eq!( - "abf662f831715ff78f88545a05a9262af75d6406b54e1a8a174ff1d2b75affc4", + "b18ddbf029afaa876b54cb4de997506ef67e202bee13d8b54cb5698288c25318", format!("{}", schema.fingerprint::()) ); assert_eq!( - "6e21c350f71b1a34e9efe90970f1bc69", + "125fc6c592fde3a38cce17faf64f2d98", format!("{}", schema.fingerprint::()) ); assert_eq!( - "28cf0a67d9937bb3", + "224cc43e4e57d7a7", format!("{}", schema.fingerprint::()) ); From 9fad3aabbfa76b94d01c21d17403c37091182d72 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 4 Jan 2024 10:12:41 +0200 Subject: [PATCH 4/7] AVRO-3920: [Rust] Do not parse "logicalType" as a custom RecordField attribute Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/schema.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 72660dbe37b..2d96893be84 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -772,7 +772,8 @@ impl RecordField { let mut custom_attributes: BTreeMap = BTreeMap::new(); for (key, value) in field { match key.as_str() { - "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" => continue, + "type" | "name" | "doc" | "default" | "order" | "position" | "aliases" + | "logicalType" => continue, key if key == "symbols" && matches!(schema, Schema::Enum(_)) => continue, key if key == "size" && matches!(schema, Schema::Fixed(_)) => continue, _ => custom_attributes.insert(key.clone(), value.clone()), @@ -3332,16 +3333,16 @@ mod tests { let schema = Schema::parse_str(raw_schema)?; assert_eq!( - "b18ddbf029afaa876b54cb4de997506ef67e202bee13d8b54cb5698288c25318", + "abf662f831715ff78f88545a05a9262af75d6406b54e1a8a174ff1d2b75affc4", format!("{}", schema.fingerprint::()) ); assert_eq!( - "125fc6c592fde3a38cce17faf64f2d98", + "6e21c350f71b1a34e9efe90970f1bc69", format!("{}", schema.fingerprint::()) ); assert_eq!( - "224cc43e4e57d7a7", + "28cf0a67d9937bb3", format!("{}", schema.fingerprint::()) ); From b25fefccc55637e8f7b324feb5852ac0c597932d Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Thu, 4 Jan 2024 19:02:09 +0800 Subject: [PATCH 5/7] AVRO-3925: Fix decimal serialization format (#2673) --- lang/rust/avro/src/schema.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 2d96893be84..1864edc7821 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -49,6 +49,7 @@ fn schema_name_r() -> &'static Regex { Regex::new( r"^((?P([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?)\.)?(?P[A-Za-z_][A-Za-z0-9_]*)$", ).unwrap() + }) } From e1b2f7685bdab52a2f00b42e27e9facbd182ce75 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 4 Jan 2024 12:59:51 +0200 Subject: [PATCH 6/7] AVRO-3925: [Rust] Improve formatting Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/schema.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 1864edc7821..90cc50a80fe 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -49,7 +49,6 @@ fn schema_name_r() -> &'static Regex { Regex::new( r"^((?P([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*)?)\.)?(?P[A-Za-z_][A-Za-z0-9_]*)$", ).unwrap() - }) } @@ -6464,6 +6463,7 @@ mod tests { Ok(()) } + #[test] fn test_avro_3925_serialize_decimal_inner_fixed() -> TestResult { let schema = Schema::Decimal(DecimalSchema { precision: 36, @@ -6480,15 +6480,16 @@ mod tests { let serialized_json = serde_json::to_string_pretty(&schema)?; let expected_json = r#"{ - "type": "fixed", - "name": "decimal_36_10", - "size": 16, - "logicalType": "decimal", - "scale": 10, - "precision": 36 -}"#; + "type": "fixed", + "name": "decimal_36_10", + "size": 16, + "logicalType": "decimal", + "scale": 10, + "precision": 36 + }"#; assert_eq!(serialized_json, expected_json); + Ok(()) } @@ -6503,13 +6504,14 @@ mod tests { let serialized_json = serde_json::to_string_pretty(&schema)?; let expected_json = r#"{ - "type": "bytes", - "logicalType": "decimal", - "scale": 10, - "precision": 36 -}"#; + "type": "bytes", + "logicalType": "decimal", + "scale": 10, + "precision": 36 + }"#; assert_eq!(serialized_json, expected_json); + Ok(()) } From 2895b35d0c0351583f1ba80879296390c130d1a5 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 4 Jan 2024 13:25:14 +0200 Subject: [PATCH 7/7] Fix broken tests Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/schema.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 90cc50a80fe..f4c063df60d 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -6480,13 +6480,13 @@ mod tests { let serialized_json = serde_json::to_string_pretty(&schema)?; let expected_json = r#"{ - "type": "fixed", - "name": "decimal_36_10", - "size": 16, - "logicalType": "decimal", - "scale": 10, - "precision": 36 - }"#; + "type": "fixed", + "name": "decimal_36_10", + "size": 16, + "logicalType": "decimal", + "scale": 10, + "precision": 36 +}"#; assert_eq!(serialized_json, expected_json); @@ -6504,11 +6504,11 @@ mod tests { let serialized_json = serde_json::to_string_pretty(&schema)?; let expected_json = r#"{ - "type": "bytes", - "logicalType": "decimal", - "scale": 10, - "precision": 36 - }"#; + "type": "bytes", + "logicalType": "decimal", + "scale": 10, + "precision": 36 +}"#; assert_eq!(serialized_json, expected_json);