From 3285a7ceba181197a16c60d4256a1ec3f2c8f634 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 15 Oct 2024 12:00:01 +0800 Subject: [PATCH 1/2] chore: Fix build after merge Signed-off-by: Xuanwo --- crates/iceberg/Cargo.toml | 2 +- .../base_writer/equality_delete_writer.rs | 48 +++++++++---------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 1307cc6f3..906df2114 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -41,7 +41,7 @@ async-std = ["dep:async-std"] tokio = ["dep:tokio"] [dependencies] -anyhow = { workspace = true } +anyhow = { workspace = true } apache-avro = { workspace = true } array-init = { workspace = true } arrow-arith = { workspace = true } diff --git a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs index ba198821a..7ea0d9ad2 100644 --- a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs @@ -22,8 +22,8 @@ use arrow_schema::{DataType, FieldRef, Fields, Schema, SchemaRef}; use itertools::Itertools; use crate::spec::{DataFile, Struct}; -use crate::writer::file_writer::FileWriter; -use crate::writer::{file_writer::FileWriterBuilder, IcebergWriter, IcebergWriterBuilder}; +use crate::writer::file_writer::{FileWriter, FileWriterBuilder}; +use crate::writer::{IcebergWriter, IcebergWriterBuilder}; use crate::{Error, ErrorKind, Result}; /// Builder for `EqualityDeleteWriter`. @@ -243,31 +243,27 @@ impl FieldProjector { #[cfg(test)] mod test { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::types::Int64Type; + use arrow_array::{ArrayRef, Int64Array, RecordBatch, StructArray}; use arrow_select::concat::concat_batches; use itertools::Itertools; - use std::{collections::HashMap, sync::Arc}; - - use arrow_array::{types::Int64Type, ArrayRef, Int64Array, RecordBatch, StructArray}; - use parquet::{ - arrow::{arrow_reader::ParquetRecordBatchReaderBuilder, PARQUET_FIELD_ID_META_KEY}, - file::properties::WriterProperties, - }; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; use tempfile::TempDir; - use crate::{ - io::{FileIO, FileIOBuilder}, - spec::{DataFile, DataFileFormat}, - writer::{ - base_writer::equality_delete_writer::{ - EqualityDeleteFileWriterBuilder, EqualityDeleteWriterConfig, FieldProjector, - }, - file_writer::{ - location_generator::{test::MockLocationGenerator, DefaultFileNameGenerator}, - ParquetWriterBuilder, - }, - IcebergWriter, IcebergWriterBuilder, - }, + use crate::io::{FileIO, FileIOBuilder}; + use crate::spec::{DataFile, DataFileFormat, Schema}; + use crate::writer::base_writer::equality_delete_writer::{ + EqualityDeleteFileWriterBuilder, EqualityDeleteWriterConfig, FieldProjector, }; + use crate::writer::file_writer::location_generator::test::MockLocationGenerator; + use crate::writer::file_writer::location_generator::DefaultFileNameGenerator; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; async fn check_parquet_data_file_with_equality_delete_write( file_io: &FileIO, @@ -511,14 +507,14 @@ mod test { let equality_ids = vec![1, 3]; let (projector, fields) = FieldProjector::new(schema.fields(), &equality_ids, PARQUET_FIELD_ID_META_KEY)?; - let delete_schema = arrow_schema::Schema::new(fields); - let delete_schema_ref = Arc::new(delete_schema.clone()); + let arrow_delete_schema = arrow_schema::Schema::new(fields); + let delete_schema: Schema = (&arrow_delete_schema).try_into()?; // prepare writer let to_write = RecordBatch::try_new(Arc::new(schema.clone()), columns).unwrap(); let pb = ParquetWriterBuilder::new( WriterProperties::builder().build(), - delete_schema_ref.clone(), + Arc::new(delete_schema.clone()), file_io.clone(), location_gen, file_name_gen, @@ -528,7 +524,7 @@ mod test { .build(EqualityDeleteWriterConfig::new( equality_ids, projector, - delete_schema.clone(), + arrow_delete_schema.clone(), None, )) .await?; From f764a2c9d6992dbb89263a59a39d8b913cebace6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 15 Oct 2024 12:06:49 +0800 Subject: [PATCH 2/2] Fix ci Signed-off-by: Xuanwo --- bindings/python/pyproject.toml | 6 +----- crates/iceberg/Cargo.toml | 2 +- deny.toml | 8 ++------ 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index f1f0a100f..f7dff6f0c 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -40,11 +40,7 @@ module-name = "pyiceberg_core.pyiceberg_core_rust" ignore = ["F403", "F405"] [tool.hatch.envs.dev] -dependencies = [ - "maturin>=1.0,<2.0", - "pytest>=8.3.2", - "pyarrow>=17.0.0", -] +dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "pyarrow>=17.0.0"] [tool.hatch.envs.dev.scripts] develop = "maturin develop" diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 906df2114..1307cc6f3 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -41,7 +41,7 @@ async-std = ["dep:async-std"] tokio = ["dep:tokio"] [dependencies] -anyhow = { workspace = true } +anyhow = { workspace = true } apache-avro = { workspace = true } array-init = { workspace = true } arrow-arith = { workspace = true } diff --git a/deny.toml b/deny.toml index 9c62e0d68..5ed9b48b9 100644 --- a/deny.toml +++ b/deny.toml @@ -24,13 +24,9 @@ allow = [ "ISC", "CC0-1.0", "Unicode-DFS-2016", - "Zlib" -] -exceptions = [ - { allow = [ - "OpenSSL", - ], name = "ring" } + "Zlib", ] +exceptions = [{ allow = ["OpenSSL"], name = "ring" }] [[licenses.clarify]] name = "ring"