adding ci files (xfstests)

.github/scripts/build_ubuntu_defconfig.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2024 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euox pipefail
+d=$(dirname "${BASH_SOURCE[0]}")
+. $d/series/utils.sh
+
+logs=$(get_logs_dir)
+f=${logs}/build_ubuntu_defconfig.log
+
+date -Iseconds | tee -a ${f}
+echo "Build an ubuntu kernel" | tee -a ${f}
+echo "Top 16 commits" | tee -a ${f}
+git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f}
+
+kernel_base_sha=$(git log -1 --pretty=%H $(git log -1 --reverse --pretty=%H .github)^)
+echo "build_name $(git describe --tags ${kernel_base_sha})" | tee -a ${f}
+build_name=$(git describe --tags ${kernel_base_sha})
+
+# Build the kernel that will run LTP
+export CI_TRIPLE="riscv64-linux-gnu"
+cp $d/series/kconfigs/ubuntu_defconfig arch/riscv/configs/
+$d/series/kernel_builder.sh rv64 testsuites plain gcc | tee -a ${f}
+
+kernel_dir="/build/$(gen_kernel_name rv64 testsuites plain gcc)"
+echo $build_name > $kernel_dir/kernel_version
+#tar cJvf --exclude $(basename $kernel_path) modules.tar.xz /build/$(gen_kernel_name rv64 testsuites plain gcc)/

.github/scripts/defconfig.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2024 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+d=$(dirname "${BASH_SOURCE[0]}")
+. $d/series/utils.sh
+
+logs=$(get_logs_dir)
+f=${logs}/defconfig.log
+
+date -Iseconds | tee -a ${f}
+echo "Top 16 commits" | tee -a ${f}
+git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f}
+
+${d}/series/build_only_defconfig.sh | tee -a ${f}
+${d}/series/test_only_defconfig.sh | tee -a ${f}

.github/scripts/kselftest.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2023 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+d=$(dirname "${BASH_SOURCE[0]}")
+. $d/series/utils.sh
+
+logs=$(get_logs_dir)
+f=${logs}/kselftest.log
+
+date -Iseconds | tee -a ${f}
+echo "Build, boot, and run kselftests on various kernels" | tee -a ${f}
+echo "Top 16 commits" | tee -a ${f}
+git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f}
+
+kernel_base_sha=$(git log -1 --pretty=%H $(git log -1 --reverse --pretty=%H .github)^)
+echo "build_name $(git describe --tags ${kernel_base_sha})" | tee -a ${f}
+
+${d}/series/build_only_kselftest.sh | tee -a ${f}
+${d}/series/test_only_kselftest.sh | tee -a ${f}

.github/scripts/libhugetlbfs.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2024 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euox pipefail
+d=$(dirname "${BASH_SOURCE[0]}")
+. $d/series/utils.sh
+
+logs=$(get_logs_dir)
+f=${logs}/libhugetlbfs.log
+
+KERNEL_PATH=$(find "$1" -name '*vmlinuz*')
+mv $KERNEL_PATH $KERNEL_PATH.gz
+gunzip  $KERNEL_PATH.gz
+
+build_name=$(cat "$1/kernel_version")
+
+# The Docker image comes with a prebuilt python environment with all tuxrun
+# dependencies
+source /build/.env/bin/activate
+
+mkdir -p /build/squad_json/
+
+/build/tuxrun/run --runtime null --device qemu-riscv64 --kernel $KERNEL_PATH --tests libhugetlbfs --results /build/squad_json/libhugetlbfs.json --log-file-text /build/squad_json/libhugetlbfs.log --timeouts libhugetlbfs=480  --overlay /build/libhugetlbfs.tar.xz || true
+
+# Convert JSON to squad datamodel
+python3 /build/my-linux/.github/scripts/series/tuxrun_to_squad_json.py --result-path /build/squad_json/libhugetlbfs.json --testsuite libhugetlbfs
+python3 /build/my-linux/.github/scripts/series/generate_metadata.py --logs-path /build/squad_json/ --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME}
+
+curl --header "Authorization: token $SQUAD_TOKEN" --form tests=@/build/squad_json/libhugetlbfs.squad.json --form log=@/build/squad_json/libhugetlbfs.log --form metadata=@/build/squad_json/metadata.json https://mazarinen.tail1c623.ts.net/api/submit/riscv-linux/linux-all/${build_name}/qemu

.github/scripts/ltp.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2023 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euox pipefail
+d=$(dirname "${BASH_SOURCE[0]}")
+. $d/series/utils.sh
+
+logs=$(get_logs_dir)
+f=${logs}/ltp.log
+
+KERNEL_PATH=$(find "$1" -name '*vmlinuz*')
+mv $KERNEL_PATH $KERNEL_PATH.gz
+gunzip  $KERNEL_PATH.gz
+
+build_name=$(cat "$1/kernel_version")
+
+# The Docker image comes with a prebuilt python environment with all tuxrun
+# dependencies
+source /build/.env/bin/activate
+
+# TODO ltp-controllers is too slow for now because of cgroup_fj_stress.sh
+# but I haven't found an easy to skip this one from tuxrun
+ltp_tests=( "ltp-commands"  "ltp-syscalls" "ltp-mm" "ltp-hugetlb" "ltp-crypto" "ltp-cve" "ltp-containers" "ltp-fs" "ltp-sched" )
+
+mkdir -p /build/squad_json/
+parallel_log=$(mktemp -p ${ci_root})
+
+for ltp_test in ${ltp_tests[@]}; do
+    echo "/build/tuxrun/run --runtime null --device qemu-riscv64 --kernel $KERNEL_PATH --tests $ltp_test --results /build/squad_json/$ltp_test.json --log-file-text /build/squad_json/$ltp_test.log --timeouts $ltp_test=480 || true"
+done | parallel -j $(($(nproc)/4)) --colsep ' ' --joblog ${parallel_log}
+
+cat ${parallel_log}
+rm ${parallel_log}
+
+for ltp_test in ${ltp_tests[@]}; do
+    # Convert JSON to squad datamodel
+    python3 /build/my-linux/.github/scripts/series/tuxrun_to_squad_json.py --result-path /build/squad_json/$ltp_test.json --testsuite $ltp_test
+    python3 /build/my-linux/.github/scripts/series/generate_metadata.py --logs-path /build/squad_json/ --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME}
+
+    curl --header "Authorization: token $SQUAD_TOKEN" --form tests=@/build/squad_json/$ltp_test.squad.json --form log=@/build/squad_json/$ltp_test.log --form metadata=@/build/squad_json/metadata.json https://mazarinen.tail1c623.ts.net/api/submit/riscv-linux/linux-all/${build_name}/qemu
+done

.github/scripts/patches.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2023 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+d=$(dirname "${BASH_SOURCE[0]}")
+
+parallel_log=$(mktemp -p /build)
+basesha=$(git log -1 --pretty=%H .github/scripts/patches.sh)
+patches=( $(git rev-list --reverse ${basesha}..HEAD) )
+
+patch_tot=${#patches[@]}
+num_commits=$((patch_tot + 3))
+
+date -Iseconds
+echo "Run PW tests"
+echo "Top ${num_commits} commits"
+git log -${num_commits} --abbrev=12 --pretty="commit %h (\"%s\")"
+
+tm=$(mktemp -p /build)
+rc=0
+cnt=1
+
+# Linear
+for i in "${patches[@]}"; do
+    \time --quiet -o $tm -f "took %es" \
+	  bash ${d}/patches/patch_tester.sh ${i} ${cnt} ${patch_tot} || rc=1
+    echo "::notice::Patch ${cnt}/${patch_tot} $(cat $tm)"
+    cnt=$(( cnt + 1 ))
+done
+rm $tm
+exit 0
+
+# Parallel... (slower?)
+parallel -j 4 --joblog ${parallel_log} --colsep=, bash ${d}/patches/patch_tester.sh \
+	 {1} {2} {3} :::: <(
+    for i in "${patches[@]}"; do
+        echo ${i},${cnt},${patch_tot}
+        cnt=$(( cnt + 1 ))
+    done) || rc=1
+cat ${parallel_log}
+

.github/scripts/patches/patch_tester.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2023 Rivos Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+d=$(dirname "${BASH_SOURCE[0]}")
+
+sha1=$1
+patch_num=$2
+patch_tot=$3
+
+tm=$(mktemp -p /build)
+
+worktree=$(mktemp -d -p /build)
+git worktree add $worktree ${sha1} &>/dev/null
+cd $worktree
+
+rc=0
+tests=( $(ls ${d}/tests/*.sh) )
+tcnt=1
+for j in "${tests[@]}"; do
+    git reset --hard ${sha1} &>/dev/null
+    msg="Patch ${patch_num}/${patch_tot}: Test ${tcnt}/${#tests[@]}: ${j}"
+    echo "::group::${msg}"
+    testrc=0
+    \time --quiet -o $tm -f "took %es" \
+          bash ${j} || testrc=$?
+    echo "::endgroup::"
+    if (( $testrc == 250 )); then
+        rc=1
+        echo "::warning::WARN ${msg} $(cat $tm)"
+    elif (( $testrc )); then
+        rc=1
+        echo "::error::FAIL ${msg} $(cat $tm)"
+    else
+        echo "::notice::OK ${msg} $(cat $tm)"
+    fi
+    tcnt=$(( tcnt + 1 ))
+done
+
+git worktree remove $worktree &>/dev/null || true
+rm $tm
+exit $rc

.github/scripts/patches/tests/build_rv32_defconfig.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2022 by Rivos Inc.
+
+tmpdir=$(mktemp -d -p /build)
+tmpfile=$(mktemp -p /build)
+rc=0
+
+tuxmake --wrapper ccache --target-arch riscv --directory . \
+        --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \
+        --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \
+        -o $tmpdir --toolchain llvm -z none -k rv32_defconfig \
+        CROSS_COMPILE=riscv64-linux- \
+        >$tmpfile 2>&1 || rc=1
+
+if [ $rc -ne 0 ]; then
+    echo "Full log:"
+    cat $tmpfile
+    echo "warnings/errors:"
+    grep "\(warning\|error\):" $tmpfile
+fi
+
+rm -rf $tmpdir $tmpfile
+
+exit $rc

.github/scripts/patches/tests/build_rv64_clang_allmodconfig.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2019 Netronome Systems, Inc.
+
+# Modified tests/patch/build_defconfig_warn.sh for RISC-V builds
+
+tmpfile_e=$(mktemp -p /build)
+tmpfile_o=$(mktemp -p /build)
+tmpfile_n=$(mktemp -p /build)
+tmpdir_b=$(mktemp -d -p /build)
+tmpdir_o=$(mktemp -d -p /build)
+
+rc=0
+
+build() {
+    tuxmake --wrapper ccache --target-arch riscv -e PATH=$PATH --directory . \
+        --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \
+        --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \
+        -o $tmpdir_o -b $tmpdir_b --toolchain llvm -z none --kconfig allmodconfig \
+        -K CONFIG_WERROR=n -K CONFIG_RANDSTRUCT_NONE=y -K CONFIG_SAMPLES=n \
+        -K CONFIG_DRM_WERROR=n \
+        W=1 CROSS_COMPILE=riscv64-linux- \
+        config default \
+        >$1 2>&1
+}
+
+echo "Redirect to $tmpfile_o and $tmpfile_n"
+echo "Tree base:"
+HEAD=$(git rev-parse HEAD)
+git log -1 --pretty='%h ("%s")' HEAD~
+
+echo "Building the whole tree with the patch"
+time build $tmpfile_e || rc=1
+if [ $rc -eq 1 ]; then
+        echo "error:"
+        grep "\(error\):" $tmpfile_e
+        rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e
+        exit $rc
+fi
+
+git checkout -q HEAD~
+echo "Building the tree before the patch"
+time build $tmpfile_o
+incumbent=$(grep -c "\(warning\|error\):" $tmpfile_o)
+
+git checkout -q $HEAD
+echo "Building the tree with the patch"
+time build $tmpfile_n || rc=1
+if [ $rc -eq 1 ]; then
+        echo "error/warning:"
+        grep "\(warning\|error\):" $tmpfile_n
+        rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b
+        exit $rc
+fi
+
+current=$(grep -c "\(warning\|error\):" $tmpfile_n)
+if [ $current -gt $incumbent ]; then
+        echo "New errors added:"
+
+        tmpfile_errors_before=$(mktemp -p /build)
+        tmpfile_errors_now=$(mktemp -p /build)
+        grep "\(warning\|error\):" $tmpfile_o | sort | uniq -c > $tmpfile_errors_before
+        grep "\(warning\|error\):" $tmpfile_n | sort | uniq -c > $tmpfile_errors_now
+
+        diff -U 0 $tmpfile_errors_before $tmpfile_errors_now
+
+        rm $tmpfile_errors_before $tmpfile_errors_now
+
+        echo "Per-file breakdown"
+        tmpfile_fo=$(mktemp -p /build)
+        tmpfile_fn=$(mktemp -p /build)
+
+        echo "error/warning file pre:"
+        grep "\(warning\|error\):" $tmpfile_o | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \
+          > $tmpfile_fo
+        echo "error/warning file post:"
+        grep "\(warning\|error\):" $tmpfile_n | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \
+          > $tmpfile_fn
+
+        diff -U 0 $tmpfile_fo $tmpfile_fn
+        rm $tmpfile_fo $tmpfile_fn
+        echo "pre: $incumbent post: $current"
+        rc=1
+fi
+
+rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e
+exit $rc