diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebdfde01..4bfe7f2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,6 @@ jobs: - name: Cache dependencies uses: Swatinem/rust-cache@v2 - name: Rust unit tests with coverage report - # https://github.com/xd009642/tarpaulin/issues/1092#issuecomment-1407739176 run: cargo tarpaulin --engine llvm --no-dead-code --no-fail-fast --all-features --workspace -o xml --output-dir ./cov-reports --skip-clean - name: Upload coverage report uses: actions/upload-artifact@v4 @@ -91,21 +90,17 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: python/pyproject.toml - name: Setup Python venv - working-directory: ./python run: | make setup-venv source venv/bin/activate make develop - name: Python unit tests with coverage report run: | - pushd python source venv/bin/activate - coverage run --include 'hudi/*' -m pytest -v - # move to parent so the reported file paths will match the actual repo paths - popd - coverage xml --data-file=python/.coverage -o ./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml + coverage run --include 'python/hudi/*' -m pytest -v + coverage xml --data-file=.coverage -o ./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml - name: Upload coverage report uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index a8bb1d69..b34ce4f9 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -36,24 +36,22 @@ jobs: uses: apache/skywalking-eyes/header@v0.6.0 - name: Check rust code style - run: cd python && make check-rust + run: make check-rust - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.9' cache: pip - cache-dependency-path: pyproject.toml + cache-dependency-path: python/pyproject.toml - name: Install python linter dependencies - working-directory: ./python run: | make setup-venv source venv/bin/activate pip install ruff==0.5.2 mypy==1.10.1 - name: Check python code style - working-directory: ./python run: | source venv/bin/activate make check-python diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 696872d5..f79d66df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,8 +49,7 @@ view, instead, they will be linked to the corresponding issues. ## Commonly used dev commands -For most of the time, use dev commands specified in [`python/Makefile`](./python/Makefile), it applies to both Python -and Rust modules. You don't need to `cd` to the root directory and run `cargo` commands. +For most of the time, use dev commands specified in the [`Makefile`](Makefile). To setup python virtual env, run @@ -59,13 +58,10 @@ make setup-venv ``` > [!NOTE] -> This will run `python` command to setup the virtual environment. You can either change that to `python3.X`, -> or simply alias `python` to your local `python3.X` installation, for example: -> ```shell -> echo "alias python=/Library/Frameworks/Python.framework/Versions/3.12/bin/python3" >> ~/.zshrc` -> ``` +> This will run `python3` command to set up the virtual environment in `venv/`. +> Activate the virtual environment by running `source venv/bin/activate` for example. -Once activate virtual env, build the project for development by +Once a virtual environment is activated, build the project for development by ```shell make develop @@ -96,24 +92,18 @@ For Python, # For all tests make test-python # or -pytest -s +pytest -s python/tests # For a specific test case -pytest tests/test_table_read.py -s -k "test_read_table_has_correct_schema" +pytest python/tests/test_table_read.py -s -k "test_read_table_has_correct_schema" ``` ## Before creating a pull request -Run test commands to make sure the code is working as expected: +Run the below command and fix issues if any: ```shell -make test -``` - -Run check commands and follow the suggestions to fix the code: - -```shell -make check +make format check test ``` ## Create a pull request diff --git a/python/Makefile b/Makefile similarity index 80% rename from python/Makefile rename to Makefile index 948631ea..5f826b54 100644 --- a/python/Makefile +++ b/Makefile @@ -15,16 +15,19 @@ # specific language governing permissions and limitations # under the License. +SHELL := /bin/bash + .DEFAULT_GOAL := help VENV := venv -MATURIN_VERSION := $(shell grep 'requires =' pyproject.toml | cut -d= -f2- | tr -d '[ "]') +PYTHON_DIR = python +MATURIN_VERSION := $(shell grep 'requires =' $(PYTHON_DIR)/pyproject.toml | cut -d= -f2- | tr -d '[ "]') PACKAGE_VERSION := $(shell grep version Cargo.toml | head -n 1 | awk '{print $$3}' | tr -d '"' ) .PHONY: setup-venv setup-venv: ## Setup the virtualenv $(info --- Setup virtualenv ---) - python -m venv $(VENV) + python3 -m venv $(VENV) .PHONY: setup setup: ## Setup the requirements @@ -34,12 +37,12 @@ setup: ## Setup the requirements .PHONY: build build: setup ## Build Python binding of hudi-rs $(info --- Build Python binding ---) - maturin build $(MATURIN_EXTRA_ARGS) + maturin build $(MATURIN_EXTRA_ARGS) -m $(PYTHON_DIR)/Cargo.toml .PHONY: develop develop: setup ## Install Python binding of hudi-rs $(info --- Develop with Python binding ---) - maturin develop --extras=devel,pandas $(MATURIN_EXTRA_ARGS) + maturin develop --extras=devel,pandas $(MATURIN_EXTRA_ARGS) -m $(PYTHON_DIR)/Cargo.toml .PHONY: format format: format-rust format-python ## Format Rust and Python code @@ -52,7 +55,7 @@ format-rust: ## Format Rust code .PHONY: format-python format-python: ## Format Python code $(info --- Format Python code ---) - ruff format . + ruff format $(PYTHON_DIR) .PHONY: check check: check-rust check-python ## Run check on Rust and Python @@ -60,18 +63,18 @@ check: check-rust check-python ## Run check on Rust and Python .PHONY: check-rust check-rust: ## Run check on Rust $(info --- Check Rust clippy ---) - cargo clippy --all-targets --all-features --workspace -- -D warnings + cargo clippy --all-targets --all-features --workspace --no-deps -- -D warnings $(info --- Check Rust format ---) cargo fmt --all -- --check .PHONY: check-python check-python: ## Run check on Python $(info --- Check Python format ---) - ruff format --check --diff . + ruff format --check --diff $(PYTHON_DIR) $(info --- Check Python linting ---) - ruff check . + ruff check $(PYTHON_DIR) $(info --- Check Python typing ---) - mypy . + pushd $(PYTHON_DIR); mypy .; popd .PHONY: test test: test-rust test-python ## Run tests on Rust and Python @@ -84,4 +87,4 @@ test-rust: ## Run tests on Rust .PHONY: test-python test-python: ## Run tests on Python $(info --- Run Python tests ---) - pytest -s + pytest -s $(PYTHON_DIR) diff --git a/demo/run_demo.sh b/demo/run_demo.sh index cec677b9..8a9b275b 100755 --- a/demo/run_demo.sh +++ b/demo/run_demo.sh @@ -36,7 +36,7 @@ fi # install dependencies and run the demo apps docker compose exec -T runner /bin/bash -c " - cd /opt/hudi-rs/python && \ + cd /opt/hudi-rs && \ make setup develop && \ cd /opt/hudi-rs/demo/sql-datafusion && ./run.sh &&\ cd /opt/hudi-rs/demo/table-api-python && ./run.sh && \ diff --git a/python/tests/conftest.py b/python/tests/conftest.py index b1fd5668..41522739 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -16,8 +16,8 @@ # under the License. import os +import pathlib import zipfile -from pathlib import Path import pytest @@ -34,7 +34,7 @@ def _extract_testing_table(zip_file_path, target_path) -> str: ] ) def get_sample_table(request, tmp_path) -> str: - fixture_path = "tests/table" + fixture_path = pathlib.Path(__file__).parent.joinpath("table") table_name = request.param - zip_file_path = Path(fixture_path).joinpath(f"{table_name}.zip") + zip_file_path = pathlib.Path(fixture_path).joinpath(f"{table_name}.zip") return _extract_testing_table(zip_file_path, tmp_path)