[VL] Optimize GHA workflow with nightly built docker image (part-2) (#…

…4348)
apache · Jan 15, 2024 · 8fb2c35 · 8fb2c35
1 parent 38da97d
commit 8fb2c35
Show file tree

Hide file tree

Showing 14 changed files with 265 additions and 163 deletions.
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
diff --git a/tools/gluten-te/centos/build.sh b/tools/gluten-te/centos/build.sh
@@ -51,11 +51,6 @@ CACHE_GLUTEN_BRANCH=${CACHE_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
 # Backend type
 BUILD_BACKEND_TYPE=${BUILD_BACKEND_TYPE:-$DEFAULT_BUILD_BACKEND_TYPE}
 
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
-
-DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
-
 ##
 
 BUILD_DOCKER_BUILD_ARGS=

diff --git a/tools/gluten-te/centos/buildenv.sh b/tools/gluten-te/centos/buildenv.sh
@@ -18,13 +18,9 @@ set -ex
 
 BASEDIR=$(dirname $0)
 
-source "$BASEDIR/defaults.conf"
+source "$BASEDIR/shared.sh"
 
-# Enable buildkit
-export DOCKER_BUILDKIT=1
-export BUILDKIT_PROGRESS=plain
-
-# Docker registry used to pull pre-built images to speed-up builds
+# Docker registry used to pull layer cache to speed-up builds
 DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}
 
 # Docker registry to push pre-built images
@@ -40,16 +36,6 @@ USE_ALI_MAVEN_MIRROR=${USE_ALI_MAVEN_MIRROR:-$DEFAULT_USE_ALI_MAVEN_MIRROR}
 # Set timezone name
 TIMEZONE=${TIMEZONE:-$DEFAULT_TIMEZONE}
 
-# Set operating system
-OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
-
-# Set os version
-OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
-
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
-
-DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
 
 if [ "$USE_ALI_MAVEN_MIRROR" == "ON" ]
 then

diff --git a/tools/gluten-te/centos/cbash-build.sh b/tools/gluten-te/centos/cbash-build.sh
@@ -18,7 +18,10 @@ set -ex
 
 BASEDIR=$(dirname $0)
 
-source "$BASEDIR/build.sh"
+source "$BASEDIR/shared.sh"
+
+# Docker registry used to pull layer cache to speed-up builds
+DOCKER_PULL_REGISTRY=${DOCKER_PULL_REGISTRY:-$DEFAULT_DOCKER_PULL_REGISTRY}
 
 # Non-interactive during docker run
 NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
@@ -32,6 +35,14 @@ EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
 # Whether to mount Maven cache
 MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
 
+if [ -n "$DOCKER_PULL_REGISTRY" ]
+then
+  docker pull $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
+  docker tag $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
+else
+  $BASEDIR/build.sh
+fi
+
 CBASH_DOCKER_RUN_ARGS=
 if [ "$NON_INTERACTIVE" != "ON" ]
 then
@@ -46,6 +57,7 @@ CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt seccomp=unconfined"
+CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -e NUM_THREADS=$(nproc)"
 if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
 then
   CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.m2/repository:/root/.m2/repository"
@@ -56,4 +68,4 @@ CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
 CBASH_BASH_ARGS="$*"
 BASH_ARGS="$CBASH_BASH_ARGS"
 
-docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE bash -c "cd /opt/gluten && $BASH_ARGS"
+docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE bash -c "cd /opt/gluten && $BASH_ARGS"
diff --git a/tools/gluten-te/centos/defaults.conf b/tools/gluten-te/centos/defaults.conf
@@ -46,7 +46,10 @@ DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
 # Build will result in this image
 DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build
 
-# Docker registry used to pull pre-built images to speed-up builds
+# Docker registry to push pre-built images
+DEFAULT_DOCKER_PULL_REGISTRY=
+
+# Docker registry used to pull layer cache to speed-up builds
 DEFAULT_DOCKER_CACHE_REGISTRY=
 
 # Docker registry to push pre-built images

diff --git a/tools/gluten-te/centos/gha/gha-checkout/checkout.sh b/tools/gluten-te/centos/gha/gha-checkout/checkout.sh
@@ -26,16 +26,24 @@ then
   exit 1
 fi
 
+if [ -z "$GITHUB_JOB" ]
+then
+  echo "Unable to parse GITHUB_JOB."
+  exit 1
+fi
+
+if [ -z "$GITHUB_SHA" ]
+then
+  echo "Unable to parse GITHUB_SHA."
+  exit 1
+fi
+
 export EXTRA_DOCKER_OPTIONS="$EXTRA_DOCKER_OPTIONS --name gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID --detach -v $BASEDIR/scripts:/opt/scripts"
 export NON_INTERACTIVE=ON
 
 $BASEDIR/../../cbash-build.sh 'sleep 14400'
 
 # The target branches
 TARGET_GLUTEN_REPO=${TARGET_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-FALLBACK_GLUTEN_BRANCH=${FALLBACK_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
-FALLBACK_GLUTEN_COMMIT="$(git ls-remote $TARGET_GLUTEN_REPO $FALLBACK_GLUTEN_BRANCH | awk '{print $1;}')"
-
-TARGET_GLUTEN_COMMIT="${GITHUB_SHA:-$FALLBACK_GLUTEN_COMMIT}"
 
-$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $TARGET_GLUTEN_COMMIT"
+$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $GITHUB_SHA"
diff --git a/tools/gluten-te/centos/shared.sh b/tools/gluten-te/centos/shared.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+SHARED_BASEDIR=$(dirname $0)
+
+source "$SHARED_BASEDIR/defaults.conf"
+
+# Enable buildkit
+export DOCKER_BUILDKIT=1
+export BUILDKIT_PROGRESS=plain
+
+# Set operating system
+OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
+
+# Set os version
+OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
+
+# Buildenv will result in this image
+DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
+
+# Build will result in this image
+DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
+
+DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
+
+DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
diff --git a/tools/gluten-te/ubuntu/build.sh b/tools/gluten-te/ubuntu/build.sh
@@ -50,11 +50,6 @@ CACHE_GLUTEN_BRANCH=${CACHE_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
 # Backend type
 BUILD_BACKEND_TYPE=${BUILD_BACKEND_TYPE:-$DEFAULT_BUILD_BACKEND_TYPE}
 
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
-
-DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
-
 ##
 
 BUILD_DOCKER_BUILD_ARGS=

diff --git a/tools/gluten-te/ubuntu/buildenv.sh b/tools/gluten-te/ubuntu/buildenv.sh
@@ -18,13 +18,9 @@ set -ex
 
 BASEDIR=$(dirname $0)
 
-source "$BASEDIR/defaults.conf"
+source "$BASEDIR/shared.sh"
 
-# Enable buildkit
-export DOCKER_BUILDKIT=1
-export BUILDKIT_PROGRESS=plain
-
-# Docker registry used to pull pre-built images to speed-up builds
+# Docker registry used to pull layer cache to speed-up builds
 DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}
 
 # Docker registry to push pre-built images
@@ -40,15 +36,6 @@ USE_ALI_MAVEN_MIRROR=${USE_ALI_MAVEN_MIRROR:-$DEFAULT_USE_ALI_MAVEN_MIRROR}
 # Set timezone name
 TIMEZONE=${TIMEZONE:-$DEFAULT_TIMEZONE}
 
-# Set operating system
-OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
-OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
-
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
-
-DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
-
 if [ "$USE_ALI_MAVEN_MIRROR" == "ON" ]
 then
   MAVEN_MIRROR_URL='https://maven.aliyun.com/repository/public'

diff --git a/tools/gluten-te/ubuntu/cbash-build.sh b/tools/gluten-te/ubuntu/cbash-build.sh
@@ -18,7 +18,10 @@ set -ex
 
 BASEDIR=$(dirname $0)
 
-source "$BASEDIR/build.sh"
+source "$BASEDIR/shared.sh"
+
+# Docker registry used to pull layer cache to speed-up builds
+DOCKER_PULL_REGISTRY=${DOCKER_PULL_REGISTRY:-$DEFAULT_DOCKER_PULL_REGISTRY}
 
 # Non-interactive during docker run
 NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
@@ -32,6 +35,14 @@ EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
 # Whether to mount Maven cache
 MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
 
+if [ -n "$DOCKER_PULL_REGISTRY" ]
+then
+  docker pull $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
+  docker tag $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
+else
+  $BASEDIR/build.sh
+fi
+
 CBASH_DOCKER_RUN_ARGS=
 if [ "$NON_INTERACTIVE" != "ON" ]
 then
@@ -46,7 +57,7 @@ CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
 CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt seccomp=unconfined"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -e NUM_THREADS=30"
+CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -e NUM_THREADS=$(nproc)"
 if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
 then
   CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.m2/repository:/root/.m2/repository"

diff --git a/tools/gluten-te/ubuntu/defaults.conf b/tools/gluten-te/ubuntu/defaults.conf
@@ -47,7 +47,10 @@ DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
 # Build will result in this image
 DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build
 
-# Docker registry used to pull pre-built images to speed-up builds
+# Docker registry to push pre-built images
+DEFAULT_DOCKER_PULL_REGISTRY=
+
+# Docker registry used to pull layer cache to speed-up builds
 DEFAULT_DOCKER_CACHE_REGISTRY=
 
 # Docker registry to push pre-built images

diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/checkout.sh b/tools/gluten-te/ubuntu/gha/gha-checkout/checkout.sh
@@ -26,16 +26,24 @@ then
   exit 1
 fi
 
+if [ -z "$GITHUB_JOB" ]
+then
+  echo "Unable to parse GITHUB_JOB."
+  exit 1
+fi
+
+if [ -z "$GITHUB_SHA" ]
+then
+  echo "Unable to parse GITHUB_SHA."
+  exit 1
+fi
+
 export EXTRA_DOCKER_OPTIONS="$EXTRA_DOCKER_OPTIONS --name gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID --detach -v $BASEDIR/scripts:/opt/scripts"
 export NON_INTERACTIVE=ON
 
 $BASEDIR/../../cbash-build.sh 'sleep 14400'
 
 # The target branches
 TARGET_GLUTEN_REPO=${TARGET_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-FALLBACK_GLUTEN_BRANCH=${FALLBACK_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
-FALLBACK_GLUTEN_COMMIT="$(git ls-remote $TARGET_GLUTEN_REPO $FALLBACK_GLUTEN_BRANCH | awk '{print $1;}')"
-
-TARGET_GLUTEN_COMMIT="${GITHUB_SHA:-$FALLBACK_GLUTEN_COMMIT}"
 
-$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $TARGET_GLUTEN_COMMIT"
+$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $GITHUB_SHA"
diff --git a/tools/gluten-te/ubuntu/shared.sh b/tools/gluten-te/ubuntu/shared.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+SHARED_BASEDIR=$(dirname $0)
+
+source "$SHARED_BASEDIR/defaults.conf"
+
+# Enable buildkit
+export DOCKER_BUILDKIT=1
+export BUILDKIT_PROGRESS=plain
+
+# Set operating system
+OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
+
+# Set os version
+OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
+
+# Buildenv will result in this image
+DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
+
+# Build will result in this image
+DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
+
+DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
+
+DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
diff --git a/tools/gluten-te/ubuntu/tpc.sh b/tools/gluten-te/ubuntu/tpc.sh
@@ -75,7 +75,7 @@ then
   exit 1
 fi
 
-DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE="$DOCKER_SELECTED_TARGET_IMAGE_TPC-$OS_IMAGE"
+DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE="$DOCKER_SELECTED_TARGET_IMAGE_TPC-$OS_IMAGE:$OS_IMAGE_TAG"
 
 TPC_DOCKER_BUILD_ARGS=
 TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
@@ -96,6 +96,7 @@ JDWP_SERVER_PORT=${JDWP_SERVER_PORT:-$DEFAULT_JDWP_SERVER_PORT}
 
 if [ -n "$DOCKER_CACHE_REGISTRY" ]
 then
+  docker pull $DOCKER_CACHE_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE || echo "Required image not found in cache registry."
   TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
 fi