From 5423f338ce7f9e3eaa735529db92466f38244c7b Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 16 Mar 2022 13:34:14 -0700 Subject: [PATCH] rebase --- Jenkinsfile | 112 +++++++++++++++++- docker/build.sh | 13 ++ ...git_change_docs.sh => git_change_files.sh} | 9 +- 3 files changed, 129 insertions(+), 5 deletions(-) rename tests/scripts/{git_change_docs.sh => git_change_files.sh} (81%) diff --git a/Jenkinsfile b/Jenkinsfile index 44f7b591cfaf4..b9a72e1905767 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,8 +83,10 @@ tvm_multilib_tsim = 'build/libvta_tsim.so, ' + // command to start a docker container docker_run = 'docker/bash.sh' +docker_build = 'docker/build.sh' // timeout in minutes max_time = 240 +rebuild_docker_images = false def per_exec_ws(folder) { return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder @@ -200,11 +202,21 @@ stage('Sanity Check') { init_git() is_docs_only_build = sh ( returnStatus: true, - script: './tests/scripts/git_change_docs.sh', + script: './tests/scripts/git_change_files.sh docs/', label: 'Check for docs only changes', ) skip_ci = should_skip_ci(env.CHANGE_ID) skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_files.sh docker/', + label: 'Check for docs only changes', + ) + if (rebuild_docker_images) { + // Exit before linting so we can use the newly created Docker images + // to run the lint + return + } sh ( script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", label: 'Run lint', @@ -214,6 +226,104 @@ stage('Sanity Check') { } } +def build_image(image_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + hash = "${env.BRANCH_NAME}-${hash}" + sh( + script: "${docker_build} ${image_name} --spec ${image_name}:${hash}", + label: 'Building docker image' + ) + sh( + script: "docker rmi ${image_name}:${hash}", + label: 'Removing docker image' + ) + sh "echo NYI: Uploading docker image to registry..." +} + +if (rebuild_docker_images) { + stage('Docker Image Build') { + // TODO in a follow up PR: Upload to ECR, find tag and use in + // subsequent builds + parallel 'ci-lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_lint') + } + } + }, 'ci-cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_cpu') + } + } + }, 'ci-gpu': { + node('GPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_gpu') + } + } + }, 'ci-qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_qemu') + } + } + }, 'ci-i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_i386') + } + } + }, 'ci-arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_arm') + } + } + }, 'ci-wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_wasm') + } + } + }, 'ci-hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('ci_hexagon') + } + } + } + } + // // TODO: Once we are able to use the built images, enable this step + // // If the docker images changed, we need to run the image build before the lint + // // can run since it requires a base docker image. Most of the time the images + // // aren't build though so it's faster to use the same node that checks for + // // docker changes to run the lint in the usual case. + // stage('Sanity Check (re-run)') { + // timeout(time: max_time, unit: 'MINUTES') { + // node('CPU') { + // ws(per_exec_ws('tvm/sanity')) { + // init_git() + // sh ( + // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + // label: 'Run lint', + // ) + // } + // } + // } + // } +} // Run make. First try to do an incremental make from a previous workspace in hope to // accelerate the compilation. If something is wrong, clean the workspace and then diff --git a/docker/build.sh b/docker/build.sh index 39fe7a0242461..ed67b638c79b3 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -24,6 +24,7 @@ # [--dockerfile ] [-it] # [--net=host] [--cache-from ] # [--name CONTAINER_NAME] [--context-path ] +# [--spec DOCKER_IMAGE_SPEC] # [] # # CONTAINER_TYPE: Type of the docker container used the run the build, @@ -36,6 +37,9 @@ # this optional value is not supplied (via the --dockerfile # flag), will use Dockerfile.CONTAINER_TYPE in default # +# DOCKER_IMAGE_SPEC: Override the default logic to determine the image name and +# tag +# # IMAGE_NAME: An image to be as a source for cached layers when building the # Docker image requested. # @@ -73,6 +77,11 @@ if [[ "$1" == "-it" ]]; then shift 1 fi +if [[ "$1" == "--spec" ]]; then + OVERRIDE_IMAGE_SPEC="$2" + shift 2 +fi + if [[ "$1" == "--net=host" ]]; then CI_DOCKER_EXTRA_PARAMS+=('--net=host') CI_DOCKER_BUILD_EXTRA_PARAMS+=("--network=host") @@ -162,6 +171,10 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') # Compose the full image spec with "name:tag" e.g. "tvm.ci_cpu:v0.03" DOCKER_IMG_SPEC="${DOCKER_IMG_NAME}:${DOCKER_IMAGE_TAG}" +if [[ -n ${OVERRIDE_IMAGE_SPEC+x} ]]; then + DOCKER_IMG_SPEC="$OVERRIDE_IMAGE_SPEC" +fi + # Print arguments. echo "WORKSPACE: ${WORKSPACE}" echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}" diff --git a/tests/scripts/git_change_docs.sh b/tests/scripts/git_change_files.sh similarity index 81% rename from tests/scripts/git_change_docs.sh rename to tests/scripts/git_change_files.sh index 35b1bb7ec34ce..e7e244686344d 100755 --- a/tests/scripts/git_change_docs.sh +++ b/tests/scripts/git_change_files.sh @@ -20,19 +20,20 @@ set -eux FOUND_ONE_FILE=0 -SAW_NON_DOC_CHANGES=0 +SAW_NON_RELEVANT_CHANGES=0 +FOLDER_WITH_CHANGES="$1" changed_files=$(git diff --no-commit-id --name-only -r origin/main) for file in $changed_files; do FOUND_ONE_FILE=1 - if ! grep -q "docs/" <<< "$file"; then - SAW_NON_DOC_CHANGES=1 + if ! grep -q "$FOLDER_WITH_CHANGES" <<< "$file"; then + SAW_NON_RELEVANT_CHANGES=1 break fi done -if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_DOC_CHANGES} -eq 1 ]; then +if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_RELEVANT_CHANGES} -eq 1 ]; then exit 0 else exit 1