diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index adbdae6e1de58..2d1b3e3557470 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,145 +21,332 @@ on: push: paths: - '.github/workflows/rust.yml' - - 'ci/docker/*rust*' - - 'ci/scripts/rust_*.sh' - - 'ci/scripts/util_*.sh' - 'rust/**' - 'format/Flight.proto' pull_request: paths: - '.github/workflows/rust.yml' - - 'ci/docker/*rust*' - - 'ci/scripts/rust_*.sh' - - 'ci/scripts/util_*.sh' - 'rust/**' - 'format/Flight.proto' -env: - DOCKER_BUILDKIT: 0 - COMPOSE_DOCKER_CLI_BUILD: 1 - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - jobs: - debian: - name: AMD64 Debian 10 Rust ${{ matrix.rust }} + # build the library, a compilation step used by multiple steps below + linux-build-lib: + name: AMD64 Debian 10 Rust ${{ matrix.rust }} build libraries runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} strategy: - fail-fast: false matrix: + arch: [amd64] rust: [nightly-2020-11-24] - env: - RUST: ${{ matrix.rust }} + container: + image: ${{ matrix.arch }}/rust steps: - - name: Checkout Arrow - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - name: Cache Cargo + uses: actions/cache@v2 with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v1 - with: - path: .docker - key: debian-10-rust-${{ hashFiles('rust/**/**.rs', 'rust/**/Cargo.toml') }} - restore-keys: debian-10-rust- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: /github/home/.cargo + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + # these represent compiled steps of both dependencies and arrow + # and thus are specific for a particular OS, arch and rust version. + path: /github/home/target + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}- + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Build run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run debian-rust - - name: Fix Cache Permissions - run: sudo chmod -R o+r .docker - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - shell: bash - run: archery docker push debian-rust + export CARGO_HOME="/github/home/.cargo" + export CARGO_TARGET_DIR="/github/home/target" + cd rust + cargo build - windows: - name: AMD64 Windows 2019 Rust ${{ matrix.rust }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + # test the crate + linux-test: + name: AMD64 Debian 10 Rust ${{ matrix.rust }} test workspace + needs: [linux-build-lib] + runs-on: ubuntu-latest strategy: - fail-fast: false matrix: + arch: [amd64] rust: [nightly-2020-11-24] + container: + image: ${{ matrix.arch }}/rust + env: + ARROW_TEST_DATA: /__w/arrow/arrow/testing/data + PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data steps: - - name: Install Rust - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v2 with: - toolchain: ${{ matrix.rust }} - override: true - components: rustfmt - - name: Install Flatbuffers - shell: bash - run: choco install flatc - - name: Checkout Arrow - uses: actions/checkout@v2 + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Cache Build Artifacts - uses: actions/cache@v1 + path: /github/home/.cargo + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 with: - path: rust/target - key: windows-rust-v1-${{ hashFiles('rust/**/**.rs', 'rust/**/Cargo.toml') }} - restore-keys: windows-rust-v1- - - name: Build - shell: bash - run: ci/scripts/rust_build.sh $(pwd) $(pwd)/build - - name: Test + path: /github/home/target + # this key equals the ones on `linux-build-lib` for re-use + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Run tests + run: | + export CARGO_HOME="/github/home/.cargo" + export CARGO_TARGET_DIR="/github/home/target" + cd rust + cargo test + # test datafusion examples + cd datafusion + cargo run --example csv_sql + cargo run --example parquet_sql + + # test the --features "simd" of the arrow crate + linux-test-simd: + name: AMD64 Debian 10 Rust ${{ matrix.rust }} test arrow simd + needs: [linux-build-lib] + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + rust: [nightly-2020-11-24] + container: + image: ${{ matrix.arch }}/rust + env: + ARROW_TEST_DATA: /__w/arrow/arrow/testing/data + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /github/home/.cargo + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /github/home/target + # this key equals the ones on `linux-build-lib` for re-use + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Run tests + run: | + export CARGO_HOME="/github/home/.cargo" + export CARGO_TARGET_DIR="/github/home/target" + cd rust/arrow + cargo test --features "simd" + + # test the arrow crate with stable rust + linux-test-stable: + name: AMD64 Debian 10 Rust ${{ matrix.rust }} test arrow + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + rust: [stable] + container: + image: ${{ matrix.arch }}/rust + env: + ARROW_TEST_DATA: /__w/arrow/arrow/testing/data + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /github/home/.cargo + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /github/home/target + # this key equals the ones on `linux-build-lib` for re-use + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Run tests + run: | + export CARGO_HOME="/github/home/.cargo" + export CARGO_TARGET_DIR="/github/home/target" + cd rust/arrow + cargo test + cargo run --example builders + cargo run --example dynamic_types + cargo run --example read_csv + cargo run --example read_csv_infer_schema + + windows-and-macos: + name: ${{ matrix.os }} Rust ${{ matrix.rust }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-latest] + rust: [nightly-2020-11-24] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + # this particular action (and not cache@v2) is necessary to avoid build errors on mac due to caching + # (see its README) + - uses: Swatinem/rust-cache@v1 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Run tests shell: bash - run: ci/scripts/rust_test.sh $(pwd) $(pwd)/build + run: | + export ARROW_TEST_DATA=$(pwd)/testing/data + export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data + cd rust + cargo test - macos: - name: AMD64 MacOS 10.15 Rust ${{ matrix.rust }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + clippy: + name: Clippy + needs: [linux-build-lib] + runs-on: ubuntu-latest strategy: - fail-fast: false matrix: + arch: [amd64] rust: [nightly-2020-11-24] + container: + image: ${{ matrix.arch }}/rust steps: - - name: Install Rust - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v2 + with: + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /github/home/.cargo + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /github/home/target + # this key equals the ones on `linux-build-lib` for re-use + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} + - uses: actions-rs/toolchain@v1 with: toolchain: ${{ matrix.rust }} - override: true + default: true components: rustfmt, clippy - - name: Install Flatbuffers - shell: bash - run: brew install flatbuffers - - name: Checkout Arrow - uses: actions/checkout@v2 + - name: Run clippy + run: | + export CARGO_HOME="/github/home/.cargo" + export CARGO_TARGET_DIR="/github/home/target" + cd rust + cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names + + coverage: + name: Coverage + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + rust: [nightly-2020-11-24] + steps: + - uses: actions/checkout@v2 with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - # Note we don't cache the build artifacts for MacOSX as it was causing intermittent failures - # see https://issues.apache.org/jira/browse/ARROW-9628 - - name: Run Clippy - shell: bash - run: ci/scripts/rust_lint.sh $(pwd) - - name: Build - shell: bash - run: ci/scripts/rust_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/rust_test.sh $(pwd) $(pwd)/build + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /home/runner/.cargo + # this key is not equal because the user is different than on a container (runner vs github) + key: cargo-coverage-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /home/runner/target + # this key is not equal because coverage uses different compilation flags. + key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}- + - name: Run coverage + run: | + export CARGO_HOME="/home/runner/.cargo" + export CARGO_TARGET_DIR="/home/runner/target" + + export ARROW_TEST_DATA=$(pwd)/testing/data + export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data + + # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0 + # see https://github.com/xd009642/tarpaulin/issues/618 + cargo install --version 0.16.0 cargo-tarpaulin + cd rust + cargo tarpaulin --out Xml + - name: Report coverage + continue-on-error: true + run: bash <(curl -s https://codecov.io/bash) + + # test FFI against the C-Data interface exposed by pyarrow + pyarrow-integration-test: + name: Pyarrow C data interface integration test + runs-on: ubuntu-latest + strategy: + matrix: + rust: [nightly-2020-11-24] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + default: true + components: rustfmt + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /home/runner/.cargo + key: cargo-maturin-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /home/runner/target + # this key is not equal because maturin uses different compilation flags. + key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}- + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Run tests + run: | + export CARGO_HOME="/home/runner/.cargo" + export CARGO_TARGET_DIR="/home/runner/target" + + cd rust/arrow-pyarrow-integration-testing + + python -m venv venv + source venv/bin/activate + + pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0 + maturin develop + python -m unittest discover tests diff --git a/.github/workflows/rust_cron.yml b/.github/workflows/rust_cron.yml deleted file mode 100644 index 11af0aba36f7f..0000000000000 --- a/.github/workflows/rust_cron.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Rust Cron - -on: - push: - paths: - - '.github/workflows/rust_cron.yml' - pull_request: - paths: - - '.github/workflows/rust_cron.yml' - schedule: - - cron: 0 */12 * * * - -jobs: - coverage: - name: AMD64 Debian 10 Rust ${{ matrix.rust }} Coverage - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - strategy: - fail-fast: false - matrix: - rust: [nightly-2020-11-24] - env: - RUST: ${{ matrix.rust }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Run coverage - shell: bash - run: | - echo ${RUST} > rust/rust-toolchain && - ci/scripts/rust_coverage.sh `pwd` `pwd`/build $RUST - - name: Report coverage - continue-on-error: true - shell: bash - run: bash <(curl -s https://codecov.io/bash) - - pyarrow-integration: - name: AMD64 Debian 10 Rust ${{ matrix.rust }} Pyarrow integration - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - strategy: - fail-fast: false - matrix: - rust: [nightly-2020-11-19] - env: - RUST: ${{ matrix.rust }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Run test - shell: bash - run: | - echo ${RUST} > rust/rust-toolchain && - ci/scripts/rust_pyarrow_integration.sh `pwd` `pwd`/build $RUST diff --git a/ci/docker/debian-10-rust.dockerfile b/ci/docker/debian-10-rust.dockerfile deleted file mode 100644 index 99507cf796ac2..0000000000000 --- a/ci/docker/debian-10-rust.dockerfile +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -FROM ${arch}/rust - -# install pre-requisites for building flatbuffers -RUN apt-get update -y && \ - apt-get install -y build-essential cmake && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# install flatbuffers -ARG flatbuffers=1.11.0 -RUN wget -q -O - https://github.com/google/flatbuffers/archive/v${flatbuffers}.tar.gz | tar -xzf - && \ - cd flatbuffers-${flatbuffers} && \ - cmake -G "Unix Makefiles" && \ - make install -j4 && \ - cd / && \ - rm -rf flatbuffers-${flatbuffers} - -ARG rust=nightly-2020-11-24 - -# freeze the version for deterministic builds -RUN rustup default ${rust} && \ - rustup component add rustfmt --toolchain ${rust}-x86_64-unknown-linux-gnu && \ - rustup toolchain add stable-x86_64-unknown-linux-gnu - -# Compile a dummy program, so that the dependencies are compiled and cached on a layer -# see https://stackoverflow.com/a/58474618/931303 for details -# We do not compile any of the workspace or we defeat the purpose of caching - we only -# compile their external dependencies. - -ENV CARGO_HOME="/rust/cargo" \ - CARGO_TARGET_DIR="/rust/target" \ - RUSTFLAGS="-D warnings" - -# The artifact of the steps below is a "${CARGO_TARGET_DIR}" containing -# compiled dependencies. Create the directories and place an empty lib.rs -# files. -COPY rust /arrow/rust -RUN mkdir -p \ - /arrow/rust/arrow-pyarrow-integration-testing/src \ - /arrow/rust/arrow-flight/src \ - /arrow/rust/arrow/src \ - /arrow/rust/benchmarks/src \ - /arrow/rust/datafusion/src \ - /arrow/rust/integration-testing/src \ - /arrow/rust/parquet/src \ - /arrow/rust/parquet_derive/src \ - /arrow/rust/parquet_derive_test/src && \ - touch \ - /arrow/rust/arrow-pyarrow-integration-testing/src/lib.rs \ - /arrow/rust/arrow-flight/src/lib.rs \ - /arrow/rust/arrow/src/lib.rs \ - /arrow/rust/benchmarks/src/lib.rs \ - /arrow/rust/datafusion/src/lib.rs \ - /arrow/rust/integration-testing/src/lib.rs \ - /arrow/rust/parquet/src/lib.rs \ - /arrow/rust/parquet_derive/src/lib.rs \ - /arrow/rust/parquet_derive_test/src/lib.rs - -# Compile dependencies for the whole workspace -RUN cd /arrow/rust && cargo build --workspace --lib --all-features diff --git a/ci/scripts/rust_coverage.sh b/ci/scripts/rust_coverage.sh deleted file mode 100755 index a8c1c362594eb..0000000000000 --- a/ci/scripts/rust_coverage.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/rust -build_dir=${2}/rust -rust=${3} - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data -export CARGO_TARGET_DIR=${build_dir} - -pushd ${source_dir} - -rustup default ${rust} -rustup component add rustfmt --toolchain ${rust}-x86_64-unknown-linux-gnu -# 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0 -# see https://github.com/xd009642/tarpaulin/issues/618 -cargo install --version 0.16.0 cargo-tarpaulin - -cargo tarpaulin --out Xml - -popd diff --git a/ci/scripts/rust_lint.sh b/ci/scripts/rust_lint.sh deleted file mode 100755 index 95ba5e3879bf9..0000000000000 --- a/ci/scripts/rust_lint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Entrypoint to run rust linter on CI (automated tests) -# -# Usage: rust_lint.sh - -set -e - -source_dir=${1}/rust - -pushd ${source_dir}/arrow - cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names -popd diff --git a/ci/scripts/rust_pyarrow_integration.sh b/ci/scripts/rust_pyarrow_integration.sh deleted file mode 100755 index e4208114144f0..0000000000000 --- a/ci/scripts/rust_pyarrow_integration.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/rust -build_dir=${2}/rust -rust=${3} - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data -export CARGO_TARGET_DIR=${build_dir} - -pushd ${source_dir}/arrow-pyarrow-integration-testing - -#rustup default ${rust} -#rustup component add rustfmt --toolchain ${rust}-x86_64-unknown-linux-gnu -python3 -m venv venv -venv/bin/pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0 - -source venv/bin/activate -maturin develop -python -m unittest discover tests - -popd diff --git a/ci/scripts/rust_test.sh b/ci/scripts/rust_test.sh deleted file mode 100755 index 881cdfb36cb5c..0000000000000 --- a/ci/scripts/rust_test.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/rust - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data - -pushd ${source_dir} - -# run unit tests excluding arrow, which is run separately -cargo test --workspace --exclude arrow - -# test datafusion examples -pushd datafusion -cargo run --example csv_sql -cargo run --example parquet_sql -popd - -# ARROW-10757 clean artefacts to reclaim space -cargo clean - -# run unit tests with SIMD on -pushd arrow -cargo test --features "simd" -# run unit tests with stable Rust -cargo +stable test - -# test arrow examples -cargo run --example builders -cargo run --example dynamic_types -cargo run --example read_csv -cargo run --example read_csv_infer_schema -popd - -popd diff --git a/docker-compose.yml b/docker-compose.yml index 195fc0104d235..bf5c06cfabdd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,7 +102,6 @@ x-hierarchy: - debian-java: - debian-java-jni - debian-js - - debian-rust - fedora-cpp: - fedora-python - ubuntu-cpp: @@ -903,35 +902,6 @@ services: /bin/bash -c " /arrow/ci/scripts/r_sanitize.sh /arrow" - ################################ Rust ####################################### - - debian-rust: - # Usage: - # docker-compose build debian-rust - # docker-compose run debian-rust - image: ${REPO}:${ARCH}-debian-10-rust-${RUST} - build: - context: . - dockerfile: ci/docker/debian-10-rust.dockerfile - cache_from: - - ${REPO}:${ARCH}-debian-10-rust-${RUST} - args: - arch: ${ARCH} - rust: ${RUST} - shm_size: *shm-size - environment: - CARGO_HOME: /rust/cargo - CARGO_TARGET_DIR: /build/rust - volumes: &rust-volumes - - .:/arrow:delegated - - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${ARCH}-debian-${DEBIAN}-rust:/build:delegated - command: &rust-command > - /bin/bash -c " - echo ${RUST} > /arrow/rust/rust-toolchain && - if [ ! -d "/build/rust" ]; then mv /rust/target /build/rust; fi && - /arrow/ci/scripts/rust_build.sh /arrow /build && - /arrow/ci/scripts/rust_test.sh /arrow /build" - ################################# Go ######################################## debian-go: diff --git a/rust/README.md b/rust/README.md index 9265d2f23562f..9aa725f168505 100644 --- a/rust/README.md +++ b/rust/README.md @@ -52,10 +52,33 @@ You can find more details about each crate on their respective READMEs. ## Developer's guide to Arrow Rust -Before running tests and examples, it is necessary to set up the local development environment. +### How to compile + +This is a standard cargo project with workspaces. To build it, you need to have `rust` and `cargo`: + +```bash +cd /rust && cargo build +``` + +You can also use rust's official docker image: + +```bash +docker run --rm -v $(pwd)/rust:/rust -it rust /bin/bash -c "cd /rust && cargo build" +``` + +The command above assumes that are in the root directory of the project, not in the same +directory as this README.md. + +You can also compile specific workspaces, + +```bash +cd /rust/arrow && cargo build +``` ### Git Submodules +Before running tests and examples, it is necessary to set up the local development environment. + The tests rely on test data that is contained in git submodules. To pull down this data run the following: @@ -115,19 +138,3 @@ Search for `allow(clippy::` in the codebase to identify lints that are ignored/a * If you are introducing a line that returns a lint warning or error, you may disable the lint on that line. * If you have several lints on a function or module, you may disable the lint on the function or module. * If a lint is pervasive across multiple modules, you may disable it at the crate level. - -## CI and Dockerized builds - -There are currently multiple CI systems that build the project and they all use the same docker image. It is possible to run the same build locally. - -From the root of the Arrow project, run the following command to build the Docker image that the CI system uses to build the project. - -```bash -docker-compose build debian-rust -``` - -Run the following command to build the project in the same way that the CI system will build the project. Note that this currently does cause some files to be written to your local workspace. - -```bash -docker-compose run --rm debian-rust bash -```