Merge branch 'main' into feat-add-static-table

.asf.yaml

@@ -46,7 +46,7 @@ github:
   features:
     wiki: false
     issues: true
-    projects: false
+    projects: true
   collaborators:
     - Xuanwo
     - liurenjie1024
@@ -55,7 +55,7 @@ github:
   ghp_path: /
 
 notifications:
-    commits:      [email protected]
-    issues:       [email protected]
-    pullrequests: [email protected]
-    jira_options: link label link label
+  commits: [email protected]
+  issues: [email protected]
+  pullrequests: [email protected]
+  jira_options: link label link label

.cargo/audit.toml

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[advisories]
+ignore = [
+  # rsa
+  # Marvin Attack: potential key recovery through timing sidechannels
+  # Issues: https://github.com/apache/iceberg-rust/issues/221
+  "RUSTSEC-2023-0071",
+]

.github/workflows/audit.yml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Security audit
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths:
+      - "**/Cargo.toml"
+      - "**/Cargo.lock"
+
+  pull_request:
+    paths:
+      - "**/Cargo.toml"
+      - "**/Cargo.lock"
+
+jobs:
+  security_audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install cargo-audit
+        run: cargo install cargo-audit
+      - name: Run audit check
+        run: cargo audit

.github/workflows/publish.yml

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Publish
+
+on:
+  push:
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/publish.yml"
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      # Publish package one by one instead of flooding the registry
+      max-parallel: 1
+      matrix:
+        # Order here is sensitive, as it will be used to determine the order of publishing
+        package:
+          - "crates/iceberg"
+          - "crates/catalog/hms"
+          - "crates/catalog/rest"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Dryrun ${{ matrix.package }}
+        working-directory: ${{ matrix.package }}
+        run: cargo publish --all-features --dry-run
+
+      - name: Publish ${{ matrix.package }}
+        working-directory: ${{ matrix.package }}
+        # Only publish if it's a tag and the tag is not a pre-release
+        if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }}
+        run: cargo publish --all-features
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

Cargo.toml

@@ -17,7 +17,12 @@
 
 [workspace]
 resolver = "2"
-members = ["crates/catalog/*", "crates/examples", "crates/iceberg", "crates/test_utils"]
+members = [
+  "crates/catalog/*",
+  "crates/examples",
+  "crates/iceberg",
+  "crates/test_utils",
+]
 
 [workspace.package]
 version = "0.2.0"
@@ -31,19 +36,23 @@ rust-version = "1.75.0"
 [workspace.dependencies]
 anyhow = "1.0.72"
 apache-avro = "0.16"
+array-init = "2"
 arrow-arith = { version = ">=46" }
 arrow-array = { version = ">=46" }
 arrow-schema = { version = ">=46" }
+async-stream = "0.3.5"
 async-trait = "0.1"
 bimap = "0.6"
 bitvec = "1.0.1"
+bytes = "1.5"
 chrono = "0.4"
-derive_builder = "0.13.0"
+derive_builder = "0.20.0"
 either = "1"
 env_logger = "0.11.0"
+fnv = "1"
 futures = "0.3"
-iceberg = { path = "./crates/iceberg" }
-iceberg-catalog-rest = { path = "./crates/catalog/rest" }
+iceberg = { version = "0.2.0", path = "./crates/iceberg" }
+iceberg-catalog-rest = { version = "0.2.0", path = "./crates/catalog/rest" }
 itertools = "0.12"
 lazy_static = "1"
 log = "^0.4"
@@ -52,6 +61,8 @@ murmur3 = "0.5.2"
 once_cell = "1"
 opendal = "0.45"
 ordered-float = "4.0.0"
+parquet = "50"
+pilota = "0.10.0"
 pretty_assertions = "1.4.0"
 port_scanner = "0.1.5"
 reqwest = { version = "^0.11", features = ["json"] }

Makefile

@@ -42,9 +42,11 @@ check-toml:
 
 check: check-fmt check-clippy cargo-sort check-toml
 
-unit-test:
+doc-test:
+	cargo test --no-fail-fast --doc --all-features --workspace
+
+unit-test: doc-test
 	cargo test --no-fail-fast --lib --all-features --workspace
 
-test:
-	cargo test --no-fail-fast --all-targets --all-features --workspace
-	cargo test --no-fail-fast --doc --all-features --workspace
+test: doc-test
+	cargo test --no-fail-fast --all-targets --all-features --workspace

README.md

@@ -42,7 +42,7 @@ Native Rust implementation of [Apache Iceberg](https://iceberg.apache.org/).
 | GCS         | Not Started |
 | HDFS        | Not Started |
 
-Our `FileIO` is powered by [Apache OpenDAL](https://github.com/apache/incubator-opendal), so it would be quite easy to
+Our `FileIO` is powered by [Apache OpenDAL](https://github.com/apache/opendal), so it would be quite easy to
 expand to other service.
 
 ### Table API

crates/catalog/hms/Cargo.toml

@@ -33,5 +33,12 @@ anyhow = { workspace = true }
 async-trait = { workspace = true }
 hive_metastore = { workspace = true }
 iceberg = { workspace = true }
+log = { workspace = true }
+pilota = { workspace = true }
 typed-builder = { workspace = true }
 volo-thrift = { workspace = true }
+
+[dev-dependencies]
+iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
+port_scanner = { workspace = true }
+tokio = { workspace = true }

crates/catalog/hms/README.md

@@ -0,0 +1,27 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+# Apache Iceberg HiveMetaStore Catalog Official Native Rust Implementation
+
+[![crates.io](https://img.shields.io/crates/v/iceberg.svg)](https://crates.io/crates/iceberg-catalog-hms)
+[![docs.rs](https://img.shields.io/docsrs/iceberg.svg)](https://docs.rs/iceberg/latest/iceberg-catalog-hms/)
+
+This crate contains the official Native Rust implementation of Apache Iceberg HiveMetaStore Catalog.
+
+See the [API documentation](https://docs.rs/iceberg-catalog-hms/latest) for examples and the full API.