publish multiple hadoop versions

Adikteev · Nov 8, 2016 · a20419a · a20419a
1 parent 9c3d007
commit a20419a
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 25 deletions.
diff --git a/bin/dist.sh b/bin/dist.sh
@@ -1,17 +1,15 @@
 #!/bin/bash
 #
-# Builds spark, and creates a properties file spark_dist_uri.properties
-# containing the URL of the distribution.
-#
+# 1. Builds a spark distribution
+# 2. Uploads it to S3
+# 3. Creates a properties file spark_dist_uri.properties containing the URL of the distribution.
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SPARK_DIR="${DIR}/../../spark"
 
 check_env() {
     # Check env early, before starting the cluster:
-    if [ -z "$S3_BUCKET" \
-            -o -z "${S3_PREFIX+x}" \
-            -o -z "$DIST_NAME" ]; then
+    if [ -z "$S3_BUCKET" -o -z "${S3_PREFIX+x}" ]; then
         echo "Missing required env. See check in ${BIN_DIR}/dist.sh."
         env
         exit 1
@@ -20,7 +18,7 @@ check_env() {
 
 function write_properties() {
     SPARK_FILENAME=$(basename $(ls ${SPARK_DIR}/spark*.tgz))
-    SPARK_DIST_URI="http://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/${SPARK_FILENAME}"
+    SPARK_DIST_URI="http://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/spark/${GIT_COMMIT}/${SPARK_FILENAME}"
     echo "SPARK_DIST_URI=${SPARK_DIST_URI}" > spark_dist_uri.properties
 }
 

diff --git a/bin/jenkins-dist-publish.sh b/bin/jenkins-dist-publish.sh
@@ -6,26 +6,43 @@
 set -e -x -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SPARK_DIR="${DIR}/../../spark"
 SPARK_BUILD_DIR=${DIR}/..
 
-function run() {
+function run_tests() {
     # test
     source bin/jenkins.sh
     install_cli
     docker_login
-    DIST_NAME="spark-${GIT_COMMIT}" make dist && export $(cat spark_dist_uri.properties)
+    make dist && export $(cat spark_dist_uri.properties)
     make universe && export $(cat "${WORKSPACE}/stub-universe.properties")
     make test
 
-    # publish
-    rename_dist
-    AWS_ACCESS_KEY_ID=${PROD_AWS_ACCESS_KEY_ID} \
-                     AWS_SECRET_ACCESS_KEY=${PROD_AWS_SECRET_ACCESS_KEY} \
-                     S3_BUCKET=${PROD_S3_BUCKET} \
-                     S3_PREFIX=${PROD_S3_PREFIX} \
-                     upload_to_s3
+}
+
+function publish() {
+    HADOOP_VERSIONS=( "2.4" "2.6" "2.7" )
+    for HADOOP_VERSION in "${HADOOP_VERSIONS[@]}"
+    do
+        pushd "${SPARK_DIR}"
+        ./build/mvn help:all-profiles | grep "hadoop-${HADOOP_VERSION}"
+        PROFILE_EXISTS=$?
+        popd
+
+        if [ ${PROFILE_EXISTS} -eq 0 ]; then
+            HADOOP_VERSION=${HADOOP_VERSION} make dist
+            rename_dist
+            AWS_ACCESS_KEY_ID=${PROD_AWS_ACCESS_KEY_ID} \
+                             AWS_SECRET_ACCESS_KEY=${PROD_AWS_SECRET_ACCESS_KEY} \
+                             S3_BUCKET=${PROD_S3_BUCKET} \
+                             S3_PREFIX=${PROD_S3_PREFIX} \
+                             upload_to_s3
+        fi
+        popd
+    done
 }
 
 pushd "${SPARK_BUILD_DIR}"
-run
+run_tests
+publish
 popd
diff --git a/bin/jenkins-dist-test.sh b/bin/jenkins-dist-test.sh
@@ -9,7 +9,7 @@ function run() {
     source bin/jenkins.sh
     install_cli
     docker_login
-    DIST_NAME="spark-${GIT_COMMIT}" make dist && export $(cat spark_dist_uri.properties)
+    make dist && export $(cat spark_dist_uri.properties)
     make universe && export $(cat "${WORKSPACE}/stub-universe.properties")
     make test
 }

diff --git a/bin/jenkins.sh b/bin/jenkins.sh
@@ -8,21 +8,24 @@ SPARK_DIR="${DIR}/../../spark"
 SPARK_BUILD_DIR="${DIR}/.."
 
 function make_distribution {
-    # Env var: DIST_NAME
+    # Env: HADOOP_VERSION
+    HADOOP_VERSION=${HADOOP_VERSION:-2.6}
 
     pushd "${SPARK_DIR}"
 
     if [[ -n "${SPARK_DIST_URI}" ]]; then
         wget "${SPARK_DIST_URI}"
     else
         if [ -f make-distribution.sh ]; then
-            ./make-distribution.sh -Phadoop-2.4 -DskipTests
+            # location of make-distribution.sh before Spark 2.0
+            ./make-distribution.sh --tgz "-Phadoop-${HADOOP_VERSION}" -Phive -Phive-thriftserver -DskipTests
         else
-            ./dev/make-distribution.sh -Pmesos -Phadoop-2.6 -Phive -Phive-thriftserver -Psparkr -DskipTests
+            # location of make-distribution.sh after Spark 2.0
+            ./dev/make-distribution.sh --tgz -Pmesos "-Phadoop-${HADOOP_VERSION}" -Phive -Phive-thriftserver -Psparkr -DskipTests
         fi
 
-        mv dist ${DIST_NAME}
-        tar czf ${DIST_NAME}.tgz ${DIST_NAME}
+        # mv dist ${DIST_NAME}
+        # tar czf ${DIST_NAME}.tgz ${DIST_NAME}
     fi
 
     popd
@@ -37,8 +40,8 @@ function rename_dist {
     # rename to spark-<tag>
     tar xvf spark-*.tgz
     rm spark-*.tgz
-    mv spark-* spark-${VERSION}
-    tar czf spark-${VERSION}.tgz spark-${VERSION}
+    mv spark-* "spark-${VERSION}-bin-${HADOOP_VERSION}"
+    tar czf "spark-${VERSION}-bin-${HADOOP_VERSION}.tgz" "spark-${VERSION}-bin-${HADOOP_VERSION}"
 
     popd
 }