Glue Catalog: Basic Setup + Test Infra (1/3) (apache#294)

* extend dependency DIRS

* create dependencies for glue

* basic setup

* rename test

* add utils/get_sdk_config

* add tests

* add list_namespace

* fix: clippy

* fix: unused

* fix: workspace

* fix: name

* use creds in test-setup

* fix: empty dependencies.rust.tsv

* fix: rename endpoint_url

* remove deps.tsv

* add hms deps.tsv

* fix deps.tsv

* fix: deps.tsv

Cargo.toml

@@ -42,6 +42,8 @@ arrow-array = { version = "51" }
 arrow-schema = { version = "51" }
 async-stream = "0.3.5"
 async-trait = "0.1"
+aws-config = "1.1.8"
+aws-sdk-glue = "1.21.0"
 bimap = "0.6"
 bitvec = "1.0.1"
 bytes = "1.5"

crates/catalog/glue/Cargo.toml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "iceberg-catalog-glue"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+rust-version = { workspace = true }
+
+categories = ["database"]
+description = "Apache Iceberg Glue Catalog Support"
+repository = { workspace = true }
+license = { workspace = true }
+keywords = ["iceberg", "glue", "catalog"]
+
+[dependencies]
+anyhow = { workspace = true }
+async-trait = { workspace = true }
+aws-config = { workspace = true }
+aws-sdk-glue = { workspace = true }
+iceberg = { workspace = true }
+log = { workspace = true }
+typed-builder = { workspace = true }
+
+[dev-dependencies]
+iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
+port_scanner = { workspace = true }
+tokio = { workspace = true }

crates/catalog/glue/README.md

@@ -0,0 +1,27 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+# Apache Iceberg Glue Catalog Official Native Rust Implementation
+
+[![crates.io](https://img.shields.io/crates/v/iceberg.svg)](https://crates.io/crates/iceberg-catalog-glue)
+[![docs.rs](https://img.shields.io/docsrs/iceberg.svg)](https://docs.rs/iceberg/latest/iceberg-catalog-glue/)
+
+This crate contains the official Native Rust implementation of Apache Iceberg Glue Catalog.
+
+See the [API documentation](https://docs.rs/iceberg-catalog-glue/latest) for examples and the full API.

crates/catalog/glue/src/catalog.rs

@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Iceberg Glue Catalog implementation.
+
+use async_trait::async_trait;
+use iceberg::table::Table;
+use iceberg::{Catalog, Namespace, NamespaceIdent, Result, TableCommit, TableCreation, TableIdent};
+use std::{collections::HashMap, fmt::Debug};
+
+use typed_builder::TypedBuilder;
+
+use crate::error::from_aws_error;
+use crate::utils::create_sdk_config;
+
+#[derive(Debug, TypedBuilder)]
+/// Glue Catalog configuration
+pub struct GlueCatalogConfig {
+    #[builder(default, setter(strip_option))]
+    uri: Option<String>,
+    #[builder(default)]
+    props: HashMap<String, String>,
+}
+
+struct GlueClient(aws_sdk_glue::Client);
+
+/// Glue Catalog
+pub struct GlueCatalog {
+    config: GlueCatalogConfig,
+    client: GlueClient,
+}
+
+impl Debug for GlueCatalog {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GlueCatalog")
+            .field("config", &self.config)
+            .finish_non_exhaustive()
+    }
+}
+
+impl GlueCatalog {
+    /// Create a new glue catalog
+    pub async fn new(config: GlueCatalogConfig) -> Self {
+        let sdk_config = create_sdk_config(&config.props, config.uri.as_ref()).await;
+
+        let client = aws_sdk_glue::Client::new(&sdk_config);
+
+        GlueCatalog {
+            config,
+            client: GlueClient(client),
+        }
+    }
+}
+
+#[async_trait]
+impl Catalog for GlueCatalog {
+    async fn list_namespaces(
+        &self,
+        parent: Option<&NamespaceIdent>,
+    ) -> Result<Vec<NamespaceIdent>> {
+        if parent.is_some() {
+            return Ok(vec![]);
+        }
+
+        let mut database_list: Vec<NamespaceIdent> = Vec::new();
+        let mut next_token: Option<String> = None;
+
+        loop {
+            let resp = match &next_token {
+                Some(token) => self.client.0.get_databases().next_token(token),
+                None => self.client.0.get_databases(),
+            };
+            let resp = resp.send().await.map_err(from_aws_error)?;
+
+            let dbs: Vec<NamespaceIdent> = resp
+                .database_list()
+                .iter()
+                .map(|db| NamespaceIdent::new(db.name().to_string()))
+                .collect();
+            database_list.extend(dbs);
+
+            next_token = resp.next_token().map(ToOwned::to_owned);
+            if next_token.is_none() {
+                break;
+            }
+        }
+
+        Ok(database_list)
+    }
+
+    async fn create_namespace(
+        &self,
+        _namespace: &NamespaceIdent,
+        _properties: HashMap<String, String>,
+    ) -> Result<Namespace> {
+        todo!()
+    }
+
+    async fn get_namespace(&self, _namespace: &NamespaceIdent) -> Result<Namespace> {
+        todo!()
+    }
+
+    async fn namespace_exists(&self, _namespace: &NamespaceIdent) -> Result<bool> {
+        todo!()
+    }
+
+    async fn update_namespace(
+        &self,
+        _namespace: &NamespaceIdent,
+        _properties: HashMap<String, String>,
+    ) -> Result<()> {
+        todo!()
+    }
+
+    async fn drop_namespace(&self, _namespace: &NamespaceIdent) -> Result<()> {
+        todo!()
+    }
+
+    async fn list_tables(&self, _namespace: &NamespaceIdent) -> Result<Vec<TableIdent>> {
+        todo!()
+    }
+
+    async fn create_table(
+        &self,
+        _namespace: &NamespaceIdent,
+        _creation: TableCreation,
+    ) -> Result<Table> {
+        todo!()
+    }
+
+    async fn load_table(&self, _table: &TableIdent) -> Result<Table> {
+        todo!()
+    }
+
+    async fn drop_table(&self, _table: &TableIdent) -> Result<()> {
+        todo!()
+    }
+
+    async fn table_exists(&self, _table: &TableIdent) -> Result<bool> {
+        todo!()
+    }
+
+    async fn rename_table(&self, _src: &TableIdent, _dest: &TableIdent) -> Result<()> {
+        todo!()
+    }
+
+    async fn update_table(&self, _commit: TableCommit) -> Result<Table> {
+        todo!()
+    }
+}

crates/catalog/glue/src/error.rs

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Iceberg Glue Catalog implementation.
+
+use anyhow::anyhow;
+use std::fmt::Debug;
+
+use iceberg::{Error, ErrorKind};
+
+/// Format AWS SDK error into iceberg error
+pub fn from_aws_error<T>(error: aws_sdk_glue::error::SdkError<T>) -> Error
+where
+    T: Debug,
+{
+    Error::new(
+        ErrorKind::Unexpected,
+        "Operation failed for hitting aws skd error".to_string(),
+    )
+    .with_source(anyhow!("aws sdk error: {:?}", error))
+}

crates/catalog/glue/src/lib.rs

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Iceberg Glue Catalog implementation.
+
+#![deny(missing_docs)]
+
+mod catalog;
+mod error;
+mod utils;
+pub use catalog::*;
+pub use utils::{
+    AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN,
+};