Skip to content

Commit

Permalink
refactored the CI
Browse files Browse the repository at this point in the history
- standarized all CI to use Ubuntu 22
- refactored the justfile
- moved the rust integration test to e2e
- moved the oci validation go version to e2e
- change the file filter action to `tj-actions/changed-files`
- refactored the dependencies installation to `just prepare` recipe

Signed-off-by: yihuaf <[email protected]>
  • Loading branch information
yihuaf committed Jun 6, 2023
1 parent 5cbf857 commit 44b2543
Show file tree
Hide file tree
Showing 5 changed files with 159 additions and 97 deletions.
4 changes: 0 additions & 4 deletions .github/scripts/dependency.sh

This file was deleted.

53 changes: 51 additions & 2 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ jobs:
- uses: actions/checkout@v3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- name: Install just
uses: taiki-e/install-action@just
- name: Install requirements
run: just prepare
- name: Build youki
run: just youki-release
- name: Upload youki binary
Expand Down Expand Up @@ -77,3 +77,52 @@ jobs:
# # - name: Debug
# # if: ${{ always() }}
# # uses: mxschmitt/action-tmate@v3

oci-validation-go:
runs-on: ubuntu-22.04
needs: [youki-build]
timeout-minutes: 15
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Install just
uses: taiki-e/install-action@just
- uses: actions/setup-go@v3
with:
go-version: '1.19.9'
cache: true
cache-dependency-path: tests/oci-runtime-tests/src/github.com/opencontainers/runtime-tools/go.sum
- name: Download youki binary
uses: actions/download-artifact@v3
with:
name: youki
- name: Add the permission to run
run: chmod +x ./youki
- name: Run integration tests
run: just oci-tests

oci-validation-rust:
runs-on: ubuntu-22.04
needs: [youki-build]
timeout-minutes: 15
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Install just
uses: taiki-e/install-action@just
- name: Download youki binary
uses: actions/download-artifact@v3
with:
name: youki
- name: Add the permission to run
run: chmod +x ./youki
- name: Build
run: just runtimetest rust-oci-tests-bin
- name: Validate tests on youki
run: just rust-oci-tests
30 changes: 17 additions & 13 deletions .github/workflows/integration_tests_validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,39 +9,43 @@ on:

jobs:
changes:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
timeout-minutes: 15
outputs:
dirs: ${{ steps.filter.outputs.changes }}
any_modified: ${{ steps.filter.outputs.any_modified }}
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
- uses: tj-actions/changed-files@v36
id: filter
with:
filters: |
./integration_test:
- './tests/rust-integration-tests/**'
files: |
integration_test/tests/rust-integration-tests/**
files_ignore: |
**.md
- name: List all changed files
run: |
for file in ${{ steps.filter.outputs.all_modified_files }}; do
echo "$file was changed"
done
validate:
needs: [changes]
if: ${{ !contains(needs.changes.outputs.dirs, '[]') }}
runs-on: ubuntu-20.04
if: needs.changes.outputs.any_modified == 'true'
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- name: Install just
uses: taiki-e/install-action@just
- name: Install requirements
run: just prepare
- name: Install runc 1.1.0
run: |
wget -q https://github.com/opencontainers/runc/releases/download/v1.1.0/runc.amd64
sudo mv runc.amd64 /usr/bin/runc
sudo chmod 755 /usr/bin/runc
- name: Build
run: just youki-release runtimetest rust-oci-tests-bin
run: just runtimetest rust-oci-tests-bin
- name: Validate tests on runc
run: just validate-rust-oci-runc
- name: Validate tests on youki
run: just rust-oci-tests
100 changes: 36 additions & 64 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,70 +9,64 @@ on:

jobs:
changes:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
timeout-minutes: 15
outputs:
dirs: ${{ steps.filter.outputs.changes }}
any_modified: ${{ steps.filter.outputs.any_modified }}
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
- uses: tj-actions/changed-files@v36
id: filter
with:
filters: |
crates/youki:
- 'crates/youki/**'
crates/libcontainer:
- 'crates/libcontainer/**'
crates/libcgroups:
- 'crates/libcgroups/**'
tests/rust-integration-tests/runtimetest:
- './tests/rust-integration-tests/runtimetest/**'
tests/rust-integration-tests/integration_test:
- './tests/rust-integration-tests/integration_test/**'
tests/rust-integration-tests/test_framework:
- './tests/rust-integration-tests/test_framework/**'
files_ignore: |
docs
LICENSE
**.md
- name: List all changed files
run: |
for file in ${{ steps.filter.outputs.all_modified_files }}; do
echo "$file was changed"
done
check:
needs: [changes]
if: ${{ !contains(needs.changes.outputs.dirs, '[]') }}
runs-on: ubuntu-20.04
if: needs.changes.outputs.any_modified == 'true'
runs-on: ubuntu-22.04
timeout-minutes: 15
strategy:
matrix:
dirs: ${{ fromJSON(needs.changes.outputs.dirs) }}
steps:
- uses: actions/checkout@v3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- run: rustup component add rustfmt clippy
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt, clippy
- uses: taiki-e/install-action@just
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev
- name: Check formatting
run: cargo fmt --all -- --check
working-directory: ${{matrix.dirs}}
- name: Check clippy lints
working-directory: ${{matrix.dirs}}
run: cargo clippy --all-targets --all-features -- -D warnings
run: just prepare
- name: Check formatting and lints
run: just lint

tests:
runs-on: ubuntu-20.04
needs: [changes]
if: needs.changes.outputs.any_modified == 'true'
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- uses: actions/checkout@v3
- name: Install rus
run: rustup show
- run: sudo apt-get -y update
- run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1
- uses: taiki-e/install-action@just
- name: Install requirements
run: just prepare
- name: Run tests
run: |
export LD_LIBRARY_PATH=$HOME/.wasmedge/lib
cd ./crates && cargo test --all --all-features --no-fail-fast
coverage:
runs-on: ubuntu-20.04
needs: [changes]
if: needs.changes.outputs.any_modified == 'true'
runs-on: ubuntu-22.04
timeout-minutes: 15
name: Run test coverage
name: run test coverage
steps:
- uses: actions/checkout@v3
- name: Setup Rust toolchain and cache
Expand All @@ -83,37 +77,15 @@ jobs:
uses: taiki-e/install-action@v1
with:
tool: [email protected]
- uses: taiki-e/install-action@just
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
run: just prepare
- name: Run Test Coverage for youki
run: |
cargo llvm-cov clean --workspace
cargo llvm-cov --no-report
cargo llvm-cov --no-run --lcov --ignore-filename-regex "libcgroups/src/systemd/dbus/systemd_api.rs" --output-path ./coverage.lcov
- name: Upload Youki Code Coverage Results
uses: codecov/codecov-action@v2
uses: codecov/codecov-action@v3
with:
file: ./coverage.lcov

integration_tests:
runs-on: ubuntu-20.04
timeout-minutes: 15
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- name: Install just
uses: taiki-e/install-action@just
- uses: actions/setup-go@v3
with:
go-version: "1.17.6"
cache: true
cache-dependency-path: tests/oci-runtime-tests/src/github.com/opencontainers/runtime-tools/go.sum
- name: Build
run: just youki-release
- name: Run integration tests
run: just oci-tests
69 changes: 55 additions & 14 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,30 @@ alias youki := youki-dev

ROOT := `git rev-parse --show-toplevel`


# build

# build all binaries
build-all: youki-release rust-oci-tests-bin runtimetest

# build youki in dev mode
youki-dev:
./scripts/build.sh -o {{ROOT}} -c youki
./scripts/build.sh -o {{ ROOT }} -c youki

# build youki in release mode
youki-release:
./scripts/build.sh -o {{ROOT}} -r -c youki
./scripts/build.sh -o {{ ROOT }} -r -c youki

# build runtimetest binary
runtimetest:
./scripts/build.sh -o {{ROOT}} -r -c runtimetest
./scripts/build.sh -o {{ ROOT }} -r -c runtimetest

# build rust oci tests binary
rust-oci-tests-bin:
./scripts/build.sh -o {{ROOT}} -r -c integration-test

./scripts/build.sh -o {{ ROOT }} -r -c integration-test


# Tests


# run oci tests
test-oci: oci-tests rust-oci-tests

Expand All @@ -46,13 +43,12 @@ featuretest:
./scripts/features_test.sh

# run oci integration tests
oci-tests: youki-release
./scripts/oci_integration_tests.sh {{ROOT}}

oci-tests:
./scripts/oci_integration_tests.sh {{ ROOT }}

# run rust oci integration tests
rust-oci-tests: youki-release runtimetest rust-oci-tests-bin
./scripts/rust_integration_tests.sh {{ROOT}}/youki
./scripts/rust_integration_tests.sh {{ ROOT }}/youki

# validate rust oci integration tests on runc
validate-rust-oci-runc: runtimetest rust-oci-tests-bin
Expand All @@ -63,7 +59,7 @@ containerd-test: youki-dev
VAGRANT_VAGRANTFILE=Vagrantfile.containerd2youki vagrant up
VAGRANT_VAGRANTFILE=Vagrantfile.containerd2youki vagrant provision --provision-with test

#misc
# misc

# run bpftrace hack
hack-bpftrace:
Expand All @@ -72,8 +68,53 @@ hack-bpftrace:
# run linting on project
lint:
cargo fmt --all -- --check
cargo clippy --all-targets --all-features -- -D warnings
cargo clippy --all --all-targets --all-features -- -D warnings

# run format on project
format:
cargo fmt --all

# cleans up generated artifacts
clean:
./scripts/clean.sh {{ROOT}}
./scripts/clean.sh {{ ROOT }}

prepare:
#!/usr/bin/env bash
set -euo pipefail
# Check if system is Ubuntu
if [[ -f /etc/lsb-release ]]; then
source /etc/lsb-release
if [[ $DISTRIB_ID == "Ubuntu" ]]; then
echo "System is Ubuntu"
sudo apt-get -y update
sudo apt-get install -y \
pkg-config \
libsystemd-dev \
libdbus-glib-1-dev \
build-essential \
libelf-dev \
libseccomp-dev \
libclang-dev \
libssl-dev
exit 0
fi
fi

# Check if system is Fedora
if [[ -f /etc/fedora-release ]]; then
echo "System is Fedora"
sudo dnf -y update
sudo dnf install -y \
pkg-config \
systemd-devel \
dbus-devel \
elfutils-libelf-devel \
libseccomp-devel \
clang-devel \
openssl-devel
exit 0
fi

echo "Unknown system for `prepare` automation. You are on your own. Good luck!"
exit 1

0 comments on commit 44b2543

Please sign in to comment.