Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Read/write nested dictionary under map in ipc stream reader/writer #1583

Merged
merged 4 commits into from
Apr 20, 2022
Merged

Read/write nested dictionary under map in ipc stream reader/writer #1583

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
14bc547
IPC read/write nested dict in map
viirya Apr 18, 2022
e79b0f5
For review comments
viirya Apr 19, 2022
122ae13
Merge remote-tracking branch 'upstream/master' into nested_dict_in_map
viirya Apr 20, 2022
1f444f8
update comment
viirya Apr 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions arrow/src/ipc/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1483,23 +1483,33 @@ mod tests {
}

#[test]
fn test_roundtrip_stream_nested_dict_dict_in_map() {
let values = StringArray::from_iter_values(["a", "b", "c"]);
let keys = Int8Array::from_iter_values([0, 0, 1, 2, 0, 1]);
let dict_array = DictionaryArray::<Int8Type>::try_new(&keys, &values).unwrap();

let keys_array = Int32Array::from_iter_values([0, 0, 1, 2, 0, 1]);
let keys_field = Field::new("keys", DataType::Int32, false);
fn test_roundtrip_stream_nested_dict_of_map_of_dict() {
let values = StringArray::from(vec![Some("a"), None, Some("b"), Some("c")]);
tustvold marked this conversation as resolved.
Show resolved Hide resolved
let value_dict_keys = Int8Array::from_iter_values([0, 1, 1, 2, 3, 1]);
let value_dict_array =
DictionaryArray::<Int8Type>::try_new(&value_dict_keys, &values).unwrap();

let key_dict_keys = Int8Array::from_iter_values([0, 0, 2, 1, 1, 3]);
let key_dict_array =
DictionaryArray::<Int8Type>::try_new(&key_dict_keys, &values).unwrap();

let keys_field = Field::new_dict(
"keys",
DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
true,
1,
false,
);
let values_field = Field::new_dict(
"values",
DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
false,
true,
1,
false,
);
let entry_struct = StructArray::from(vec![
(keys_field, make_array(keys_array.data().clone())),
(values_field, make_array(dict_array.data().clone())),
(keys_field, make_array(key_dict_array.data().clone())),
(values_field, make_array(value_dict_array.data().clone())),
]);
let map_data_type = DataType::Map(
Box::new(Field::new(
Expand All @@ -1519,8 +1529,9 @@ mod tests {
.unwrap();
let map_array = MapArray::from(map_data);

let dict_keys = Int8Array::from_iter_values([0, 1, 1, 2, 2, 1]);
let dict_dict_array =
DictionaryArray::<Int8Type>::try_new(&keys, &map_array).unwrap();
DictionaryArray::<Int8Type>::try_new(&dict_keys, &map_array).unwrap();

let schema = Arc::new(Schema::new(vec![Field::new(
"f1",
Expand Down