Fix rust compile issues and add GH action to run build validations an…

…d tests (microsoft#18346) ### Description This PR gets the onnxruntime Rust bindings to a foundation where they can be extended and validated as the onnxruntime progresses. Specifically, the PR does the following. - fixes some of the existing compilation issues due to missing some enums output tensor data types. - introduces a `just vendor` task that will vendor the source code from the onnxruntime to enable a common base directory within the crate directory rather than using a relative parent path. This enables `crate package` to be able to archive the onnxruntime native code, which will enable consumers of the onnxruntime-sys crate to be able to compile on their target. - introduces a GH action to lint the Rust code (rustfmt, clippy), build the library, validate through tests, and validate crate can package correctly. TODOs: - [x] This PR is based on microsoft#18200 and will need to be rebased once that PR is merged. ### Motivation and Context  This is the first step to getting new onnxruntime Rust crates published through this project, which will unblock community Rust projects which would like to take a dependency on onnxruntime Rust. Follow up work to enable publication of onnxruntime Rust crates: - change name of the crates to be published (onnxruntime-rs and onnxruntime-sys are already taken and we'll need new names) - update authors / license to reflect contributions from previous maintainer(s) and new maintainers - introduce a crate publish GH action or ADO pipeline --------- Signed-off-by: David Justice <[email protected]>
AmusementClub · Nov 9, 2023 · 2c22b49 · 2c22b49
1 parent 8d50313
commit 2c22b49
Show file tree

Hide file tree

Showing 8 changed files with 244 additions and 58 deletions.
diff --git a/.github/actions/rust-toolchain-setup/action.yml b/.github/actions/rust-toolchain-setup/action.yml
@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-action.json
+
+name: 'Rust toolchain setup'
+description: 'Common setup steps for GitHub workflows for Rust projects'
+
+runs:
+  using: composite
+  steps:
+    - uses: dtolnay/[email protected]
+      with:
+        components: clippy, rustfmt
+    - uses: extractions/setup-just@v1
+      with:
+        just-version: '1.15.0' # optional semver specification, otherwise latest
+
+    ###
+    ### Linux setup
+    ###
+    - name: rustup
+      # We need to use the nightly rust tool change to enable registry-auth / to connect to ADO feeds.
+      if: ${{ (runner.os == 'Linux') }}
+      run: |
+        rustup set profile minimal
+        rustup install
+      shell: bash
+    # - name: Cargo login
+    #   if: ${{ (runner.os == 'Linux') }}
+    #   run: just cargo-login-ci
+    #   shell: bash
+
+      ###
+      ### Windows setup
+      ###
+    - name: rustup
+      # We need to use the nightly rust tool change to enable registry-auth / to connect to ADO feeds.
+      if: ${{ (runner.os == 'Windows') }}
+      run: |
+        rustup set profile minimal
+        rustup install
+      shell: pwsh
+    # - name: Cargo login
+    #   if: ${{ (runner.os == 'Windows') }}
+    #   run: just cargo-login-ci-windows
+    #   shell: pwsh
diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml
@@ -0,0 +1,132 @@
+name: Rust
+
+on: [pull_request]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_LOG: onnxruntime=debug,onnxruntime-sys=debug
+  RUST_BACKTRACE: 1
+  MANIFEST_PATH: ${{ github.workspace }}/rust/Cargo.toml
+
+jobs:
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rust-toolchain-setup
+      - name: vendor onnxruntime source
+        run: just vendor
+      - name: fmt
+        run: cargo fmt --all -- --check
+
+  download:
+    name: Download prebuilt ONNX Runtime archive from build.rs
+    runs-on: ubuntu-latest
+    env:
+      ORT_RUST_STRATEGY=download
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rust-toolchain-setup
+      - run: rustup target install x86_64-unknown-linux-gnu
+      - run: rustup target install x86_64-apple-darwin
+      - run: rustup target install i686-pc-windows-msvc
+      - run: rustup target install x86_64-pc-windows-msvc
+      # ******************************************************************
+      - name: Download prebuilt archive (CPU, x86_64-unknown-linux-gnu)
+        run: cargo build --target x86_64-unknown-linux-gnu  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (CPU, x86_64-unknown-linux-gnu)
+        run: ls -lh target/x86_64-unknown-linux-gnu/debug/build/onnxruntime-sys-*/out/onnxruntime-linux-x64-1.*.tgz
+      # ******************************************************************
+      - name: Download prebuilt archive (CPU, x86_64-apple-darwin)
+        run: cargo build --target x86_64-apple-darwin  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (CPU, x86_64-apple-darwin)
+        run: ls -lh target/x86_64-apple-darwin/debug/build/onnxruntime-sys-*/out/onnxruntime-osx-x64-1.*.tgz
+      # ******************************************************************
+      - name: Download prebuilt archive (CPU, i686-pc-windows-msvc)
+        run: cargo build --target i686-pc-windows-msvc  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (CPU, i686-pc-windows-msvc)
+        run: ls -lh target/i686-pc-windows-msvc/debug/build/onnxruntime-sys-*/out/onnxruntime-win-x86-1.*.zip
+      # ******************************************************************
+      - name: Download prebuilt archive (CPU, x86_64-pc-windows-msvc)
+        run: cargo build --target x86_64-pc-windows-msvc  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (CPU, x86_64-pc-windows-msvc)
+        run: ls -lh target/x86_64-pc-windows-msvc/debug/build/onnxruntime-sys-*/out/onnxruntime-win-x64-1.*.zip
+      # ******************************************************************
+      - name: Download prebuilt archive (GPU, x86_64-unknown-linux-gnu)
+        env:
+          ORT_USE_CUDA: "yes"
+        run: cargo build --target x86_64-unknown-linux-gnu  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (GPU, x86_64-unknown-linux-gnu)
+        run: ls -lh target/x86_64-unknown-linux-gnu/debug/build/onnxruntime-sys-*/out/onnxruntime-linux-x64-gpu-1.*.tgz
+      # ******************************************************************
+      - name: Download prebuilt archive (GPU, x86_64-pc-windows-msvc)
+        env:
+          ORT_USE_CUDA: "yes"
+        run: cargo build --target x86_64-pc-windows-msvc  --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Verify prebuilt archive downloaded (GPU, x86_64-pc-windows-msvc)
+        run: ls -lh target/x86_64-pc-windows-msvc/debug/build/onnxruntime-sys-*/out/onnxruntime-win-gpu-x64-1.*.zip
+
+  test:
+    name: Test Suite
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          [
+            x86_64-unknown-linux-gnu,
+            x86_64-apple-darwin,
+            x86_64-pc-windows-msvc,
+            i686-pc-windows-msvc,
+          ]
+        include:
+          - target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-apple-darwin
+            os: macos-latest
+          - target: x86_64-pc-windows-msvc
+            os: windows-latest
+          - target: i686-pc-windows-msvc
+            os: windows-latest
+    env:
+      CARGO_BUILD_TARGET: ${{ matrix.target }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rust-toolchain-setup
+      - name: vendor onnxruntime source
+        run: just vendor
+      - run: rustup target install ${{ matrix.target }}
+      - name: Install additional packages (macOS)
+        if: contains(matrix.target, 'x86_64-apple-darwin')
+        run: brew install libomp
+      - name: Build (cargo build)
+        run: cargo build --all --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Build tests (cargo test)
+        run: cargo test --no-run --manifest-path ${{ env.MANIFEST_PATH }}
+      - name: Build onnxruntime with 'model-fetching' feature
+        run: cargo build --manifest-path ${{ env.MANIFEST_PATH }} --features model-fetching
+      - name: Test onnxruntime-sys
+        run: cargo build --package onnxruntime-sys -- --test-threads=1 --nocapture
+      - name: Test onnxruntime
+        run: cargo test --manifest-path ${{ env.MANIFEST_PATH }} --features model-fetching -- --test-threads=1 --nocapture
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rust-toolchain-setup
+      - name: vendor onnxruntime source
+        run: just vendor
+      - run: clippy --all-features --manifest-path ${{ env.MANIFEST_PATH }} -- -D warnings
+
+  package-sys:
+    name: Package onnxruntime-sys
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rust-toolchain-setup
+      - name: vendor onnxruntime source
+        run: just vendor
+      - run: cargo package --allow-dirty --package onnxruntime-sys
diff --git a/rust/justfile b/rust/justfile
@@ -0,0 +1,13 @@
+
+
+vendor:
+    mkdir -p ./onnxruntime-sys/vendor/onnxruntime-src
+    cp -rf ../onnxruntime ./onnxruntime-sys/vendor/onnxruntime-src
+    cp -rf ../cmake ./onnxruntime-sys/vendor/onnxruntime-src
+    rm -rf ./onnxruntime-sys/vendor/onnxruntime-src/cmake/external/onnx
+    cp -rf ../include ./onnxruntime-sys/vendor/onnxruntime-src
+    mkdir -p ./onnxruntime-sys/vendor/onnxruntime-src/tools
+    cp -rf ../tools/ci_build ./onnxruntime-sys/vendor/onnxruntime-src/tools
+    cp -rf ../samples ./onnxruntime-sys/vendor/onnxruntime-src
+    cp -f ../requirements.txt.in ./onnxruntime-sys/vendor/onnxruntime-src
+    cp -f ../VERSION_NUMBER ./onnxruntime-sys/vendor/onnxruntime-src
diff --git a/rust/onnxruntime-sys/.gitignore b/rust/onnxruntime-sys/.gitignore
@@ -0,0 +1 @@
+vendor
diff --git a/rust/onnxruntime-sys/Cargo.toml b/rust/onnxruntime-sys/Cargo.toml
@@ -3,25 +3,24 @@ authors = ["Nicolas Bigaouette <[email protected]>"]
 edition = "2018"
 name = "onnxruntime-sys"
 version = "0.0.14"
-
 links = "onnxruntime"
-
 description = "Unsafe wrapper around Microsoft's ONNX Runtime"
 documentation = "https://docs.rs/onnxruntime-sys"
 homepage = "https://github.com/microsoft/onnxruntime"
 license = "MIT OR Apache-2.0"
 readme = "../README.md"
 repository = "https://github.com/microsoft/onnxruntime"
-
 categories = ["science"]
 keywords = ["neuralnetworks", "onnx", "bindings"]
+include = ["src", "example", "vendor", "build.rs"]
 
 [dependencies]
 libloading = "0.7"
 
 [build-dependencies]
 bindgen = "0.63"
 cmake = "0.1"
+anyhow = "1.0"
 
 # Used on unix
 flate2 = "1.0"