From eddb526e06708e64557089dff59ad60e759ab443 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 14 Nov 2024 23:14:01 -0600 Subject: [PATCH 1/2] ci: make scripts more uniform Perform the following style changes: - Only use uppercase variable names if they are exported - Only use `${...}` syntax when joining with other strings, `$...` otherwise - Use `CARGO_TERM_VERBOSE` rather than always passing `-v` - Indent is always four spaces - Fix shellcheck errors in all shell scripts - Rewrap some long or complex commands - Replace redundant `| \` with just `|` --- .github/workflows/full_ci.yml | 1 + ci/android-install-ndk.sh | 10 +- ci/android-install-sdk.sh | 8 +- ci/build.sh | 167 +++++++------- .../wasm32-unknown-emscripten/node-wrapper.sh | 10 +- ci/emscripten.sh | 6 +- ci/install-musl.sh | 20 +- ci/install-rust.sh | 67 +++--- ci/run-docker.sh | 74 +++---- ci/run.sh | 204 ++++++++++-------- ci/style.sh | 3 +- ci/test-runner-linux | 14 +- ci/wasi.sh | 4 +- 13 files changed, 313 insertions(+), 275 deletions(-) diff --git a/.github/workflows/full_ci.yml b/.github/workflows/full_ci.yml index 826d11d68d583..a5c41bb2b3807 100644 --- a/.github/workflows/full_ci.yml +++ b/.github/workflows/full_ci.yml @@ -6,6 +6,7 @@ on: types: [opened, synchronize, reopened] env: + CARGO_TERM_VERBOSE: true LIBC_CI: 1 jobs: diff --git a/ci/android-install-ndk.sh b/ci/android-install-ndk.sh index b57370d81e6f7..eb666e2db5da1 100644 --- a/ci/android-install-ndk.sh +++ b/ci/android-install-ndk.sh @@ -2,10 +2,10 @@ set -ex -NDK=android-ndk-r27 -wget --tries=20 -q https://dl.google.com/android/repository/${NDK}-linux.zip -unzip -q ${NDK}-linux.zip +ndk=android-ndk-r27 +wget --tries=20 -q "https://dl.google.com/android/repository/${ndk}-linux.zip" +unzip -q "${ndk}-linux.zip" -mv ./${NDK}/toolchains/llvm/prebuilt/linux-x86_64 /android +mv "./${ndk}/toolchains/llvm/prebuilt/linux-x86_64" /android -rm -rf ./${NDK}-linux.zip ./${NDK} +rm -rf "./${ndk}-linux.zip" "./${ndk}" diff --git a/ci/android-install-sdk.sh b/ci/android-install-sdk.sh index 5e7037df044ef..b7f2b8e1443fc 100644 --- a/ci/android-install-sdk.sh +++ b/ci/android-install-sdk.sh @@ -9,10 +9,10 @@ set -ex # located in https://github.com/appunite/docker by just wrapping it in a script # which apparently magically accepts the licenses. -SDK=6609375 +sdk=6609375 mkdir -p sdk/cmdline-tools -wget -q --tries=20 https://dl.google.com/android/repository/commandlinetools-linux-${SDK}_latest.zip -unzip -q -d sdk/cmdline-tools commandlinetools-linux-${SDK}_latest.zip +wget -q --tries=20 "https://dl.google.com/android/repository/commandlinetools-linux-${sdk}_latest.zip" +unzip -q -d sdk/cmdline-tools "commandlinetools-linux-${sdk}_latest.zip" case "$1" in arm | armv7) @@ -69,4 +69,4 @@ echo "no" | --name "${1}" \ --package "${image}" | grep -v = || true -rm -rf commandlinetools-linux-${SDK}_latest.zip emulator-linux_x64-9058569.zip +rm -rf "commandlinetools-linux-${sdk}_latest.zip" emulator-linux_x64-9058569.zip diff --git a/ci/build.sh b/ci/build.sh index d8e9dadbc14f2..aad5fe849e3a1 100644 --- a/ci/build.sh +++ b/ci/build.sh @@ -10,29 +10,27 @@ set -ex : "${TOOLCHAIN?The TOOLCHAIN environment variable must be set.}" : "${OS?The OS environment variable must be set.}" -RUST=${TOOLCHAIN} -VERBOSE=-v +rust="$TOOLCHAIN" -echo "Testing Rust ${RUST} on ${OS}" +echo "Testing Rust $rust on $OS" -if [ "${TOOLCHAIN}" = "nightly" ] ; then +if [ "$TOOLCHAIN" = "nightly" ] ; then rustup component add rust-src fi test_target() { - BUILD_CMD="${1}" - TARGET="${2}" - NO_STD="${3}" + build_cmd="${1}" + target="${2}" + no_std="${3}" # If there is a std component, fetch it: - if [ "${NO_STD}" != "1" ]; then + if [ "${no_std}" != "1" ]; then # FIXME: rustup often fails to download some artifacts due to network # issues, so we retry this N times. N=5 n=0 - until [ $n -ge $N ] - do - if rustup target add "${TARGET}" --toolchain "${RUST}" ; then + until [ $n -ge $N ]; do + if rustup target add "$target" --toolchain "$rust" ; then break fi n=$((n+1)) @@ -41,56 +39,75 @@ test_target() { fi # Test that libc builds without any default features (no std) - if [ "${NO_STD}" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" "$VERBOSE" --no-default-features --target "${TARGET}" + if [ "$no_std" != "1" ]; then + cargo "+$rust" "$build_cmd" --no-default-features --target "$target" else # FIXME: With `build-std` feature, `compiler_builtins` emits a lof of lint warnings. - RUSTFLAGS="-A improper_ctypes_definitions" cargo "+${RUST}" "${BUILD_CMD}" \ - -Z build-std=core,alloc "$VERBOSE" --no-default-features --target "${TARGET}" + RUSTFLAGS="-A improper_ctypes_definitions" \ + cargo "+$rust" "$build_cmd" \ + -Z build-std=core,alloc \ + --no-default-features \ + --target "$target" fi + # Test that libc builds with default features (e.g. std) # if the target supports std - if [ "$NO_STD" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" "$VERBOSE" --target "${TARGET}" + if [ "$no_std" != "1" ]; then + cargo "+$rust" "$build_cmd" --target "$target" else - RUSTFLAGS="-A improper_ctypes_definitions" cargo "+${RUST}" "${BUILD_CMD}" \ - -Z build-std=core,alloc "$VERBOSE" --target "${TARGET}" + RUSTFLAGS="-A improper_ctypes_definitions" \ + cargo "+$rust" "${build_cmd}" \ + -Z build-std=core,alloc \ + --target "$target" fi # Test that libc builds with the `extra_traits` feature - if [ "${NO_STD}" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" "$VERBOSE" --no-default-features --target "${TARGET}" \ - --features extra_traits + if [ "$no_std" != "1" ]; then + cargo "+$rust" "$build_cmd" \ + --no-default-features \ + --features extra_traits \ + --target "$target" else - RUSTFLAGS="-A improper_ctypes_definitions" cargo "+${RUST}" "${BUILD_CMD}" \ - -Z build-std=core,alloc "$VERBOSE" --no-default-features \ - --target "${TARGET}" --features extra_traits + RUSTFLAGS="-A improper_ctypes_definitions" \ + cargo "+$rust" "$build_cmd" \ + -Z build-std=core,alloc \ + --no-default-features \ + --features extra_traits \ + --target "$target" fi # Test the 'const-extern-fn' feature on nightly - if [ "${RUST}" = "nightly" ]; then - if [ "${NO_STD}" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" "$VERBOSE" --no-default-features --target "${TARGET}" \ - --features const-extern-fn + if [ "${rust}" = "nightly" ]; then + if [ "${no_std}" != "1" ]; then + cargo "+$rust" "$build_cmd" \ + --no-default-features \ + --features const-extern-fn \ + --target "$target" else - RUSTFLAGS="-A improper_ctypes_definitions" cargo "+${RUST}" "${BUILD_CMD}" \ - -Z build-std=core,alloc "$VERBOSE" --no-default-features \ - --target "${TARGET}" --features const-extern-fn + RUSTFLAGS="-A improper_ctypes_definitions" \ + cargo "+$rust" "$build_cmd" \ + -Z build-std=core,alloc \ + --no-default-features \ + --features const-extern-fn \ + --target "$target" fi fi # Also test that it builds with `extra_traits` and default features: - if [ "$NO_STD" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" "$VERBOSE" --target "${TARGET}" \ + if [ "$no_std" != "1" ]; then + cargo "+$rust" "$build_cmd" \ + --target "$target" \ --features extra_traits else - RUSTFLAGS="-A improper_ctypes_definitions" cargo "+${RUST}" "${BUILD_CMD}" \ - -Z build-std=core,alloc "$VERBOSE" --target "${TARGET}" \ + RUSTFLAGS="-A improper_ctypes_definitions" \ + cargo "+$rust" "$build_cmd" \ + -Z build-std=core,alloc \ + --target "$target" \ --features extra_traits fi } -RUST_LINUX_TARGETS="\ +rust_linux_targets="\ aarch64-linux-android \ aarch64-unknown-linux-gnu \ arm-linux-androideabi \ @@ -113,7 +130,7 @@ x86_64-unknown-linux-musl \ x86_64-unknown-netbsd \ " -RUST_GT_1_13_LINUX_TARGETS="\ +rust_gt_1_13_linux_targets="\ arm-unknown-linux-musleabi \ arm-unknown-linux-musleabihf \ armv7-unknown-linux-musleabihf \ @@ -121,16 +138,16 @@ sparc64-unknown-linux-gnu \ wasm32-unknown-emscripten \ x86_64-linux-android \ " -RUST_GT_1_19_LINUX_TARGETS="\ +rust_gt_1_19_linux_targets="\ aarch64-unknown-linux-musl \ sparcv9-sun-solaris \ wasm32-unknown-unknown \ " -RUST_GT_1_24_LINUX_TARGETS="\ +rust_gt_1_24_linux_targets="\ i586-unknown-linux-musl \ " -RUST_NIGHTLY_LINUX_TARGETS="\ +rust_nightly_linux_targets="\ aarch64-unknown-fuchsia \ armv5te-unknown-linux-gnueabi \ armv5te-unknown-linux-musleabi \ @@ -145,65 +162,63 @@ x86_64-unknown-linux-gnux32 \ x86_64-unknown-redox \ " -RUST_APPLE_TARGETS="\ +rust_apple_targets="\ aarch64-apple-ios \ " -RUST_NIGHTLY_APPLE_TARGETS="\ +rust_nightly_apple_targets="\ aarch64-apple-darwin \ " # Must start with `x86_64-pc-windows-msvc` first. -RUST_NIGHTLY_WINDOWS_TARGETS="\ +rust_nightly_windows_targets="\ x86_64-pc-windows-msvc \ x86_64-pc-windows-gnu \ i686-pc-windows-msvc \ " # The targets are listed here alphabetically -TARGETS="" +targets="" case "${OS}" in linux*) - TARGETS="${RUST_LINUX_TARGETS}" + targets="$rust_linux_targets" - if [ "${RUST}" != "1.13.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_13_LINUX_TARGETS}" - if [ "${RUST}" != "1.19.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_19_LINUX_TARGETS}" - if [ "${RUST}" != "1.24.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_24_LINUX_TARGETS}" + if [ "$rust" != "1.13.0" ]; then + targets="$targets $rust_gt_1_13_linux_targets" + if [ "$rust" != "1.19.0" ]; then + targets="$targets $rust_gt_1_19_linux_targets" + if [ "${rust}" != "1.24.0" ]; then + targets="$targets $rust_gt_1_24_linux_targets" fi fi fi - if [ "${RUST}" = "nightly" ]; then - TARGETS="${TARGETS} ${RUST_NIGHTLY_LINUX_TARGETS}" + if [ "$rust" = "nightly" ]; then + targets="$targets $rust_nightly_linux_targets" fi ;; macos*) - TARGETS="${RUST_APPLE_TARGETS}" + targets="$rust_apple_targets" - if [ "${RUST}" = "nightly" ]; then - TARGETS="${TARGETS} ${RUST_NIGHTLY_APPLE_TARGETS}" + if [ "$rust" = "nightly" ]; then + targets="$targets $rust_nightly_apple_targets" fi ;; windows*) - TARGETS=${RUST_NIGHTLY_WINDOWS_TARGETS} - - ;; - *) + targets=${rust_nightly_windows_targets} ;; + *) ;; esac -for TARGET in $TARGETS; do - if echo "$TARGET"|grep -q "$FILTER"; then +for target in $targets; do + if echo "$target" | grep -q "$FILTER"; then if [ "${OS}" = "windows" ]; then - TARGET="$TARGET" sh ./ci/install-rust.sh - test_target build "$TARGET" + TARGET="$target" sh ./ci/install-rust.sh + test_target build "$target" else - test_target build "$TARGET" + test_target build "$target" fi fi done @@ -211,7 +226,7 @@ done # Targets which are not available via rustup and must be built with -Zbuild-std # FIXME(hexagon): hexagon-unknown-linux-musl should be tested but currently has # duplicate symbol errors from `compiler_builtins`. -RUST_LINUX_NO_CORE_TARGETS="\ +rust_linux_no_core_targets="\ aarch64-pc-windows-msvc \ aarch64-unknown-freebsd \ aarch64-unknown-hermit \ @@ -275,24 +290,24 @@ x86_64-unknown-openbsd \ x86_64-wrs-vxworks \ " -if [ "${RUST}" = "nightly" ] && [ "${OS}" = "linux" ]; then - for TARGET in $RUST_LINUX_NO_CORE_TARGETS; do - if echo "$TARGET"|grep -q "$FILTER"; then - test_target build "$TARGET" 1 +if [ "${rust}" = "nightly" ] && [ "${OS}" = "linux" ]; then + for target in $rust_linux_no_core_targets; do + if echo "$target" | grep -q "$FILTER"; then + test_target build "$target" 1 fi done fi -RUST_APPLE_NO_CORE_TARGETS="\ +rust_apple_no_core_targets="\ armv7s-apple-ios \ i686-apple-darwin \ i386-apple-ios \ " -if [ "${RUST}" = "nightly" ] && [ "${OS}" = "macos" ]; then - for TARGET in $RUST_APPLE_NO_CORE_TARGETS; do - if echo "$TARGET" | grep -q "$FILTER"; then - test_target build "$TARGET" 1 +if [ "${rust}" = "nightly" ] && [ "${OS}" = "macos" ]; then + for target in $rust_apple_no_core_targets; do + if echo "$target" | grep -q "$FILTER"; then + test_target build "$target" 1 fi done fi diff --git a/ci/docker/wasm32-unknown-emscripten/node-wrapper.sh b/ci/docker/wasm32-unknown-emscripten/node-wrapper.sh index b1936f041070a..369439269a683 100755 --- a/ci/docker/wasm32-unknown-emscripten/node-wrapper.sh +++ b/ci/docker/wasm32-unknown-emscripten/node-wrapper.sh @@ -2,10 +2,10 @@ set -e -me=$1 +me="$1" shift -dir=$(dirname $me) -file=$(basename $me) +dir=$(dirname "$me") +file=$(basename "$me") -cd $dir -exec node $file "$@" +cd "$dir" +exec node "$file" "$@" diff --git a/ci/emscripten.sh b/ci/emscripten.sh index b99d2cfbe5397..b17a40c1a287d 100644 --- a/ci/emscripten.sh +++ b/ci/emscripten.sh @@ -4,12 +4,12 @@ set -ex # Note: keep in sync with: # https://github.com/rust-lang/rust/blob/master/src/ci/docker/scripts/emscripten.sh -EMSDK_VERSION=3.1.68 +emsdk_version=3.1.68 git clone https://github.com/emscripten-core/emsdk.git /emsdk-portable cd /emsdk-portable -./emsdk install "${EMSDK_VERSION}" -./emsdk activate "${EMSDK_VERSION}" +./emsdk install "$emsdk_version" +./emsdk activate "$emsdk_version" # Compile and cache libc # shellcheck disable=SC1091 diff --git a/ci/install-musl.sh b/ci/install-musl.sh index 7ea50916c4caf..fae46ee928c9f 100644 --- a/ci/install-musl.sh +++ b/ci/install-musl.sh @@ -1,17 +1,17 @@ -#!/usr/bin/env sh +#!/bin/sh # # Install musl and musl-sanitized linux kernel headers # to musl-{$1} directory set -ex -MUSL_VERSION=1.1.24 -MUSL="musl-${MUSL_VERSION}" +musl_version=1.1.24 +musl="musl-${musl_version}" # Download, configure, build, and install musl: -curl --retry 5 https://www.musl-libc.org/releases/${MUSL}.tar.gz | tar xzf - +curl --retry 5 https://www.musl-libc.org/releases/${musl}.tar.gz | tar xzf - -cd $MUSL +cd "$musl" case ${1} in aarch64) musl_arch=aarch64 @@ -62,15 +62,15 @@ esac # shellcheck disable=SC2103 cd .. -rm -rf $MUSL +rm -rf "$musl" # Download, configure, build, and install musl-sanitized kernel headers: -KERNEL_HEADER_VER="4.19.88" +kernel_header_ver="4.19.88" curl --retry 5 -L \ - "https://github.com/sabotage-linux/kernel-headers/archive/v${KERNEL_HEADER_VER}.tar.gz" | \ + "https://github.com/sabotage-linux/kernel-headers/archive/v${kernel_header_ver}.tar.gz" | tar xzf - ( - cd kernel-headers-${KERNEL_HEADER_VER} + cd "kernel-headers-${kernel_header_ver}" make ARCH="${kernel_arch}" prefix="/musl-${musl_arch}" install -j4 ) -rm -rf kernel-headers-${KERNEL_HEADER_VER} +rm -rf kernel-headers-${kernel_header_ver} diff --git a/ci/install-rust.sh b/ci/install-rust.sh index d7a035af21689..a0b841807b106 100644 --- a/ci/install-rust.sh +++ b/ci/install-rust.sh @@ -10,45 +10,46 @@ if [ -n "$TOOLCHAIN" ]; then else toolchain=nightly fi + if [ "$OS" = "windows" ]; then - : "${TARGET?The TARGET environment variable must be set.}" - rustup set profile minimal - rustup update --force "$toolchain-$TARGET" - rustup default "$toolchain-$TARGET" + : "${TARGET?The TARGET environment variable must be set.}" + rustup set profile minimal + rustup update --force "$toolchain-$TARGET" + rustup default "$toolchain-$TARGET" else - rustup set profile minimal - rustup update --force "$toolchain" - rustup default "$toolchain" + rustup set profile minimal + rustup update --force "$toolchain" + rustup default "$toolchain" fi if [ -n "$TARGET" ]; then - echo "Install target" - rustup target add "$TARGET" + echo "Install target" + rustup target add "$TARGET" fi if [ -n "$INSTALL_RUST_SRC" ]; then - echo "Install rust-src" - rustup component add rust-src + echo "Install rust-src" + rustup component add rust-src fi if [ "$OS" = "windows" ]; then - if [ "$ARCH_BITS" = "i686" ]; then - echo "Install MinGW32" - choco install mingw --x86 --force - fi + if [ "$ARCH_BITS" = "i686" ]; then + echo "Install MinGW32" + choco install mingw --x86 --force + fi - echo "Find GCC libraries" - gcc -print-search-dirs - /usr/bin/find "C:\ProgramData\Chocolatey" -name "crt2*" - /usr/bin/find "C:\ProgramData\Chocolatey" -name "dllcrt2*" - /usr/bin/find "C:\ProgramData\Chocolatey" -name "libmsvcrt*" + echo "Find GCC libraries" + gcc -print-search-dirs + /usr/bin/find "C:\ProgramData\Chocolatey" -name "crt2*" + /usr/bin/find "C:\ProgramData\Chocolatey" -name "dllcrt2*" + /usr/bin/find "C:\ProgramData\Chocolatey" -name "libmsvcrt*" - if [ -n "$ARCH_BITS" ]; then - echo "Fix MinGW" - for i in crt2.o dllcrt2.o libmingwex.a libmsvcrt.a ; do - cp -f "/C/ProgramData/Chocolatey/lib/mingw/tools/install/mingw$ARCH_BITS/$ARCH-w64-mingw32/lib/$i" "$(rustc --print sysroot)/lib/rustlib/$TARGET/lib" - done - fi + if [ -n "$ARCH_BITS" ]; then + echo "Fix MinGW" + for i in crt2.o dllcrt2.o libmingwex.a libmsvcrt.a ; do + cp -f "/C/ProgramData/Chocolatey/lib/mingw/tools/install/mingw$ARCH_BITS/$ARCH-w64-mingw32/lib/$i" "$(rustc --print sysroot)/lib/rustlib/$TARGET/lib" + done + fi fi echo "Query rust and cargo versions" @@ -63,11 +64,11 @@ rustup show echo "Generate lockfile" N=5 n=0 -until [ $n -ge $N ] -do - if cargo generate-lockfile; then - break - fi - n=$((n+1)) - sleep 1 +until [ $n -ge $N ]; do + if cargo generate-lockfile; then + break + fi + + n=$((n+1)) + sleep 1 done diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 042303846f3fe..a812ee7ab183a 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/sh # Disable SC2086 as it confuses the docker command. # shellcheck disable=SC2086 @@ -10,18 +10,18 @@ set -ex # Default to assuming the CARGO_HOME is one directory up (to account for a `bin` # subdir) from where the `cargo` binary in `$PATH` lives. -DEFAULT_CARGO_HOME="$(dirname "$(dirname "$(command -v cargo)")")" +default_cargo_home="$(dirname "$(dirname "$(command -v cargo)")")" # If the CARGO_HOME env var is already set, use that. If it isn't set use the # default. -CARGO_HOME="${CARGO_HOME:-$DEFAULT_CARGO_HOME}" +export CARGO_HOME="${CARGO_HOME:-$default_cargo_home}" echo "${HOME}" pwd # Avoid "no space left on device" failure. if [ "${1}" = "aarch64-linux-android" ] ; then - docker system prune -af - docker system df + docker system prune -af + docker system df fi run() { @@ -37,21 +37,21 @@ run() { fi docker run \ - --rm \ - --user "$(id -u)":"$(id -g)" \ - --env LIBC_CI \ - --env LIBC_CI_ZBUILD_STD \ - --env CARGO_HOME=/cargo \ - --env CARGO_TARGET_DIR=/checkout/target \ - --volume "$CARGO_HOME":/cargo \ - --volume "$(rustc --print sysroot)":/rust:ro \ - --volume "$(pwd)":/checkout:ro \ - --volume "$(pwd)"/target:/checkout/target \ - $kvm \ - --init \ - --workdir /checkout \ - "libc-${1}" \ - sh -c "HOME=/tmp PATH=\$PATH:/rust/bin exec ci/run.sh ${1}" + --rm \ + --user "$(id -u)":"$(id -g)" \ + --env LIBC_CI \ + --env LIBC_CI_ZBUILD_STD \ + --env CARGO_HOME=/cargo \ + --env CARGO_TARGET_DIR=/checkout/target \ + --volume "$CARGO_HOME":/cargo \ + --volume "$(rustc --print sysroot)":/rust:ro \ + --volume "$(pwd)":/checkout:ro \ + --volume "$(pwd)"/target:/checkout/target \ + $kvm \ + --init \ + --workdir /checkout \ + "libc-${1}" \ + sh -c "HOME=/tmp PATH=\$PATH:/rust/bin exec ci/run.sh ${1}" } build_switch() { @@ -69,23 +69,23 @@ build_switch() { cp "$(command -v rustup)" "$(rustc --print sysroot)/bin" docker run \ - --rm \ - --user "$(id -u)":"$(id -g)" \ - --env LIBC_CI \ - --env CARGO_HOME=/cargo \ - --env CARGO_TARGET_DIR=/checkout/target \ - --volume "$CARGO_HOME":/cargo \ - --volume "$(rustc --print sysroot)":/rust:ro \ - --volume "$(pwd)":/checkout:ro \ - --volume "$(pwd)"/target:/checkout/target \ - --volume ~/.rustup:/.rustup:Z \ - $kvm \ - --init \ - --workdir /checkout \ - libc-switch \ - sh -c "HOME=/tmp RUSTUP_HOME=/tmp PATH=\$PATH:/rust/bin rustup default nightly \ - && rustup component add rust-src --target ci/switch.json \ - && cargo build -Z build-std=core,alloc --target ci/switch.json" + --rm \ + --user "$(id -u)":"$(id -g)" \ + --env LIBC_CI \ + --env CARGO_HOME=/cargo \ + --env CARGO_TARGET_DIR=/checkout/target \ + --volume "$CARGO_HOME":/cargo \ + --volume "$(rustc --print sysroot)":/rust:ro \ + --volume "$(pwd)":/checkout:ro \ + --volume "$(pwd)"/target:/checkout/target \ + --volume ~/.rustup:/.rustup:Z \ + $kvm \ + --init \ + --workdir /checkout \ + libc-switch \ + sh -c "HOME=/tmp RUSTUP_HOME=/tmp PATH=\$PATH:/rust/bin rustup default nightly \ + && rustup component add rust-src --target ci/switch.json \ + && cargo build -Z build-std=core,alloc --target ci/switch.json" } if [ -z "${1}" ]; then diff --git a/ci/run.sh b/ci/run.sh index 4de8087699e24..e01dd3c14a574 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -5,9 +5,9 @@ set -ex -MIRRORS_URL="https://ci-mirrors.rust-lang.org/libc" +mirrors_url="https://ci-mirrors.rust-lang.org/libc" -TARGET="${1}" +target="$1" # If we're going to run tests inside of a qemu image, then we don't need any of # the scripts below. Instead, download the image, prepare a filesystem which has @@ -16,104 +16,126 @@ TARGET="${1}" # It's assume that all images, when run with two disks, will run the `run.sh` # script from the second which we place inside. if [ "$QEMU" != "" ]; then - tmpdir=/tmp/qemu-img-creation - mkdir -p "${tmpdir}" + tmpdir=/tmp/qemu-img-creation + mkdir -p "${tmpdir}" - if [ -z "${QEMU#*.gz}" ]; then - # image is .gz : download and uncompress it - qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" | \ - gunzip -d > "${tmpdir}/${qemufile}" + if [ -z "${QEMU#*.gz}" ]; then + # image is .gz : download and uncompress it + qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')" + if [ ! -f "${tmpdir}/${qemufile}" ]; then + curl --retry 5 "${mirrors_url}/${QEMU}" | + gunzip -d > "${tmpdir}/${qemufile}" + fi + elif [ -z "${QEMU#*.xz}" ]; then + # image is .xz : download and uncompress it + qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')" + if [ ! -f "${tmpdir}/${qemufile}" ]; then + curl --retry 5 "${mirrors_url}/${QEMU}" | + unxz > "${tmpdir}/${qemufile}" + fi + else + # plain qcow2 image: just download it + qemufile="$(echo "${QEMU}" | sed 's/\//__/g')" + if [ ! -f "${tmpdir}/${qemufile}" ]; then + curl --retry 5 "${mirrors_url}/${QEMU}" \ + > "${tmpdir}/${qemufile}" + fi fi - elif [ -z "${QEMU#*.xz}" ]; then - # image is .xz : download and uncompress it - qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" | \ - unxz > "${tmpdir}/${qemufile}" - fi - else - # plain qcow2 image: just download it - qemufile="$(echo "${QEMU}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" \ - > "${tmpdir}/${qemufile}" - fi - fi - # Create a mount a fresh new filesystem image that we'll later pass to QEMU. - # This will have a `run.sh` script will which use the artifacts inside to run - # on the host. - rm -f "${tmpdir}/libc-test.img" - mkdir "${tmpdir}/mount" + # Create a mount a fresh new filesystem image that we'll later pass to QEMU. + # This will have a `run.sh` script will which use the artifacts inside to run + # on the host. + rm -f "${tmpdir}/libc-test.img" + mkdir "${tmpdir}/mount" - # Do the standard rigamarole of cross-compiling an executable and then the - # script to run just executes the binary. - cargo build \ - --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" \ - --test main ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} - rm "${CARGO_TARGET_DIR}/${TARGET}"/debug/main-*.d - cp "${CARGO_TARGET_DIR}/${TARGET}"/debug/main-* "${tmpdir}"/mount/libc-test - # shellcheck disable=SC2016 - echo 'exec $1/libc-test' > "${tmpdir}/mount/run.sh" + # Do the standard rigamarole of cross-compiling an executable and then the + # script to run just executes the binary. + cargo build \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + --test main ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + rm "${CARGO_TARGET_DIR}/${target}"/debug/main-*.d + cp "${CARGO_TARGET_DIR}/${target}"/debug/main-* "${tmpdir}"/mount/libc-test + # shellcheck disable=SC2016 + echo 'exec $1/libc-test' > "${tmpdir}/mount/run.sh" - du -sh "${tmpdir}/mount" - genext2fs \ - --root "${tmpdir}/mount" \ - --size-in-blocks 100000 \ - "${tmpdir}/libc-test.img" + du -sh "${tmpdir}/mount" + genext2fs \ + --root "${tmpdir}/mount" \ + --size-in-blocks 100000 \ + "${tmpdir}/libc-test.img" - # Pass -snapshot to prevent tampering with the disk images, this helps when - # running this script in development. The two drives are then passed next, - # first is the OS and second is the one we just made. Next the network is - # configured to work (I'm not entirely sure how), and then finally we turn off - # graphics and redirect the serial console output to out.log. - qemu-system-x86_64 \ - -m 1024 \ - -snapshot \ - -drive if=virtio,file="${tmpdir}/${qemufile}" \ - -drive if=virtio,file="${tmpdir}/libc-test.img" \ - -net nic,model=virtio \ - -net user \ - -nographic \ - -vga none 2>&1 | tee "${CARGO_TARGET_DIR}/out.log" - exec grep -E "^(PASSED)|(test result: ok)" "${CARGO_TARGET_DIR}/out.log" + # Pass -snapshot to prevent tampering with the disk images, this helps when + # running this script in development. The two drives are then passed next, + # first is the OS and second is the one we just made. Next the network is + # configured to work (I'm not entirely sure how), and then finally we turn off + # graphics and redirect the serial console output to out.log. + qemu-system-x86_64 \ + -m 1024 \ + -snapshot \ + -drive if=virtio,file="${tmpdir}/${qemufile}" \ + -drive if=virtio,file="${tmpdir}/libc-test.img" \ + -net nic,model=virtio \ + -net user \ + -nographic \ + -vga none 2>&1 | tee "${CARGO_TARGET_DIR}/out.log" + exec grep -E "^(PASSED)|(test result: ok)" "${CARGO_TARGET_DIR}/out.log" fi -if [ "$TARGET" = "s390x-unknown-linux-gnu" ]; then - # FIXME: s390x-unknown-linux-gnu often fails to test due to timeout, - # so we retry this N times. - N=5 - n=0 - passed=0 - until [ $n -ge $N ] - do - if [ "$passed" = "0" ]; then - if cargo test --no-default-features --manifest-path libc-test/Cargo.toml --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} ; then - passed=$((passed+1)) - continue - fi - elif [ "$passed" = "1" ]; then - if cargo test --manifest-path libc-test/Cargo.toml --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} ; then - passed=$((passed+1)) - continue - fi - elif [ "$passed" = "2" ]; then - if cargo test --features extra_traits --manifest-path libc-test/Cargo.toml --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"}; then - break - fi - fi - n=$((n+1)) - sleep 1 - done +if [ "$target" = "s390x-unknown-linux-gnu" ]; then + # FIXME: s390x-unknown-linux-gnu often fails to test due to timeout, + # so we retry this N times. + N=5 + n=0 + passed=0 + until [ $n -ge $N ]; do + if [ "$passed" = "0" ]; then + if cargo test \ + --no-default-features \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + then + passed=$((passed+1)) + continue + fi + elif [ "$passed" = "1" ]; then + if cargo test \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + then + passed=$((passed+1)) + continue + fi + elif [ "$passed" = "2" ]; then + if cargo test \ + --features extra_traits \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + then + break + fi + fi + n=$((n+1)) + sleep 1 + done else - cargo test --no-default-features --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + cargo test \ + --no-default-features \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} - cargo test --manifest-path libc-test/Cargo.toml --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + cargo test \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} - RUST_BACKTRACE=1 cargo test --features extra_traits --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} + RUST_BACKTRACE=1 cargo test \ + --features extra_traits \ + --manifest-path libc-test/Cargo.toml \ + --target "$target" \ + ${LIBC_CI_ZBUILD_STD+"-Zbuild-std"} fi diff --git a/ci/style.sh b/ci/style.sh index c8d49e163de96..59f8452552b73 100755 --- a/ci/style.sh +++ b/ci/style.sh @@ -11,8 +11,7 @@ rustfmt -V cargo fmt --all -- --check if shellcheck --version ; then - # GHA's shellcheck is too old (0.4.6) and cannot handle SC2153 correctly. - shellcheck -e SC2103 -e SC2153 ci/*.sh + shellcheck ci/*.sh else echo "shellcheck not found" exit 1 diff --git a/ci/test-runner-linux b/ci/test-runner-linux index 3ce551944d888..9ab029b0a7df2 100755 --- a/ci/test-runner-linux +++ b/ci/test-runner-linux @@ -2,8 +2,8 @@ set -e -arch=$1 -prog=$2 +arch="$1" +prog="$2" cd /qemu/init echo "#!/bin/sh\n/prog --color=never" > run_prog.sh @@ -13,11 +13,11 @@ find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz cd .. timeout 30s qemu-system-$arch \ - -m 1024 \ - -nographic \ - -kernel kernel \ - -initrd initrd.gz \ - -append init=/run_prog.sh > output || true + -m 1024 \ + -nographic \ + -kernel kernel \ + -initrd initrd.gz \ + -append init=/run_prog.sh > output || true # remove kernel messages tr -d '\r' < output | grep -Ev '^\[' diff --git a/ci/wasi.sh b/ci/wasi.sh index a06c1f45af11a..1b72d356fac06 100644 --- a/ci/wasi.sh +++ b/ci/wasi.sh @@ -18,8 +18,8 @@ apt-get install -y --no-install-recommends \ wasmtime=24.0.0 wasi_sdk=24 -curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime/wasmtime-v$wasmtime-x86_64-linux.tar.xz | \ - tar xJf - +curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime/wasmtime-v$wasmtime-x86_64-linux.tar.xz | + tar xJf - mv wasmtime-v$wasmtime-x86_64-linux wasmtime # The pre-built `*.deb` files for wasi-sdk install to `/opt/wasi-sdk` From 4230b417cc04965938055e9580010ba653cb12a9 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 14 Nov 2024 23:18:29 -0600 Subject: [PATCH 2/2] ci: run shellcheck on all scripts --- ci/style.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/style.sh b/ci/style.sh index 59f8452552b73..131632ff21dd4 100755 --- a/ci/style.sh +++ b/ci/style.sh @@ -11,7 +11,7 @@ rustfmt -V cargo fmt --all -- --check if shellcheck --version ; then - shellcheck ci/*.sh + find . -name '*.sh' -exec shellcheck {} ';' else echo "shellcheck not found" exit 1