Benchmark Asterinas #119
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Asterinas | |
| on: | |
| # In case of manual trigger, use workflow_dispatch | |
| workflow_dispatch: | |
| schedule: | |
| # Schedule to run on every day at 20:00 UTC (04:00 Beijing Time) | |
| - cron: '0 20 * * *' | |
| jobs: | |
| Benchmarks: | |
| runs-on: self-hosted | |
| strategy: | |
| matrix: | |
| benchmark: | |
| - sysbench/cpu_lat | |
| - sysbench/thread_lat | |
| # Memory-related benchmarks | |
| - lmbench/mem_read_bw | |
| - lmbench/mem_write_bw | |
| - lmbench/mem_copy_bw | |
| - lmbench/mem_pagefault_lat | |
| - lmbench/mem_mmap_bw | |
| - lmbench/mem_mmap_lat | |
| # Process-related benchmarks | |
| - lmbench/process_getppid_lat | |
| - lmbench/process_ctx_lat | |
| - lmbench/process_fork_lat | |
| - lmbench/process_exec_lat | |
| - lmbench/process_shell_lat | |
| # Signal-related benchmarks | |
| - lmbench/signal_catch_lat | |
| - lmbench/signal_install_lat | |
| - lmbench/signal_prot_lat | |
| # IPC-related benchmarks | |
| - lmbench/pipe_lat | |
| - lmbench/pipe_bw | |
| - lmbench/fifo_lat | |
| - lmbench/semaphore_lat | |
| - lmbench/unix_lat | |
| - lmbench/unix_bw | |
| - lmbench/unix_connect_lat | |
| # Syscall-related benchmarks | |
| - lmbench/vfs_fstat_lat | |
| - lmbench/vfs_open_lat | |
| - lmbench/vfs_stat_lat | |
| - lmbench/vfs_write_lat | |
| - lmbench/vfs_read_lat | |
| - lmbench/vfs_select_lat | |
| - lmbench/vfs_fcntl_lat | |
| - lmbench/vfs_read_pagecache_bw | |
| # File-related benchmarks | |
| - lmbench/ramfs_create_delete_files_0k_ops | |
| - lmbench/ramfs_create_delete_files_10k_ops | |
| - lmbench/ext2_create_delete_files_0k_ops | |
| - lmbench/ext2_create_delete_files_10k_ops | |
| - lmbench/ext2_copy_files_bw | |
| # Network-related benchmark | |
| - lmbench/tcp_loopback_bw_4k | |
| - lmbench/tcp_loopback_bw_64k | |
| - lmbench/tcp_loopback_lat | |
| - lmbench/tcp_loopback_connect_lat | |
| - lmbench/tcp_loopback_select_lat | |
| - lmbench/tcp_loopback_http_bw | |
| - lmbench/udp_loopback_lat | |
| - iperf3/tcp_virtio_bw | |
| - nginx/http_req10k_conc1_bw | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| container: | |
| image: asterinas/asterinas:0.9.1 | |
| options: --device=/dev/kvm | |
| env: | |
| # Need to set up proxy since the self-hosted CI server is located in China, | |
| # which has poor network connection to the official Rust crate repositories. | |
| RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static | |
| RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up the environment | |
| run: | | |
| chmod +x test/benchmark/bench_linux_and_aster.sh | |
| # Set up git due to the network issue on the self-hosted runner | |
| git config --global --add safe.directory /__w/asterinas/asterinas | |
| git config --global http.sslVerify false | |
| git config --global http.version HTTP/1.1 | |
| - name: Run benchmark | |
| uses: nick-invision/retry@v2 # Retry the benchmark command in case of failure | |
| with: | |
| timeout_minutes: 20 | |
| max_attempts: 3 | |
| command: | | |
| make install_osdk | |
| BENCHMARK_TYPE=$(jq -r '.benchmark_type' test/benchmark/${{ matrix.benchmark }}/config.json) | |
| bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }} $BENCHMARK_TYPE | |
| - name: Set up benchmark configuration | |
| run: | | |
| ALERT_THRESHOLD=$(jq -r '.alert_threshold' test/benchmark/${{ matrix.benchmark }}/config.json) | |
| ALERT_TOOL=$(jq -r '.alert_tool' test/benchmark/${{ matrix.benchmark }}/config.json) | |
| TITLE=$(jq -r '.title' test/benchmark/${{ matrix.benchmark }}/config.json) | |
| DESCRIPTION=$(jq -r '.description' test/benchmark/${{ matrix.benchmark }}/config.json) | |
| if [ "$ALERT_THRESHOLD" = "null" ]; then | |
| ALERT_THRESHOLD="130%" | |
| fi | |
| if [ "$ALERT_TOOL" = "null" ]; then | |
| ALERT_TOOL="customSmallerIsBetter" | |
| fi | |
| if [ "$TITLE" = "null" ]; then | |
| TITLE="${{ matrix.benchmark }}" | |
| fi | |
| echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV | |
| echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV | |
| echo "TITLE=$TITLE" >> $GITHUB_ENV | |
| echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV | |
| BENCHMARK_NAME=$(basename "${{ matrix.benchmark }}") | |
| echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV | |
| BENCHMARK_STORE_DIR=$(dirname "${{ matrix.benchmark }}") | |
| echo "BENCHMARK_STORE_DIR=$BENCHMARK_STORE_DIR" >> $GITHUB_ENV | |
| BENCHMARK_SUMMARY=test/benchmark/${BENCHMARK_STORE_DIR}/summary.json | |
| echo "BENCHMARK_SUMMARY=$BENCHMARK_SUMMARY" >> $GITHUB_ENV | |
| - name: Store benchmark results | |
| uses: asterinas/github-action-benchmark@v4 | |
| with: | |
| name: ${{ env.BENCHMARK_NAME }} | |
| tool: ${{ env.ALERT_TOOL }} | |
| output-file-path: result_${{ env.BENCHMARK_NAME }}.json | |
| benchmark-data-dir-path: ${{ env.BENCHMARK_STORE_DIR }} | |
| github-token: ${{ secrets.BENCHMARK_SECRET }} | |
| gh-repository: 'github.com/asterinas/benchmark' | |
| auto-push: true | |
| alert-threshold: ${{ env.ALERT_THRESHOLD }} | |
| comment-on-alert: true | |
| fail-on-alert: true | |
| chart-title: ${{ env.TITLE }} | |
| chart-description: ${{ env.DESCRIPTION }} | |
| summary-json-path: ${{ env.BENCHMARK_SUMMARY }} |