TEZ-4300: Download protoc automatically compile/development time

build-tools/.gitignore

@@ -0,0 +1,2 @@
+protobuf
+

build-tools/install-protobuf.sh

@@ -16,7 +16,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar -xzvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install
+set -x
+PROTOBUF_VERSION=${1:-2.5.0}
+PROTOBUF_MAJOR_VERSION=`echo $PROTOBUF_VERSION | cut -d. -f1`
+if [ -n "$ZSH_VERSION" ]; then
+   SCRIPT_DIR="${0:a:h}"
+elif [ -n "$BASH_VERSION" ]; then
+   SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+function install_protobuf(){
+    # before protobuf 3, there is no pre-compiled executables are host on github, let's try to build and make it
+    if (( PROTOBUF_MAJOR_VERSION < 3 )); then
+        wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz"
+        tar -xzvf "protobuf-$PROTOBUF_VERSION.tar.gz"
+        rm "protobuf-$PROTOBUF_VERSION.tar.gz"
+        cd "protobuf-$PROTOBUF_VERSION" && ./configure --prefix=/usr && make && sudo make install
+    # since protobuf 3, there are precompiled protoc on github, let's quickly download and use it
+    else
+        case "$(uname -s)" in
+            Darwin)
+                FILE_NAME="protoc-$PROTOBUF_VERSION-osx-x86_64"
+                ;;
+            Linux)
+                echo 'Linux'
+                FILE_NAME="protoc-$PROTOBUF_VERSION-linux-x86_64"
+                ;;
+            *)
+                echo "Unsupported OS returned by uname -s, you'll have to install protobuf 3.x manually"
+                exit 1
+                ;;
+        esac
+        rm -f $FILE_NAME.zip #cleanup unfinished file if any
+        wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/$FILE_NAME.zip"
+        mkdir $SCRIPT_DIR/protobuf
+        unzip -o "$FILE_NAME.zip" -d $SCRIPT_DIR/protobuf
+        rm "$FILE_NAME.zip"
+    fi
+}
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+    PROTOBUF_INSTALLED_VERSION=`$SCRIPT_DIR/protobuf/bin/protoc --version`
+else
+    PROTOBUF_INSTALLED_VERSION=`protoc --version`
+fi
+
+if [ $? -eq 0 ]; then
+    PROTOBUF_INSTALLED_VERSION=`echo $PROTOBUF_INSTALLED_VERSION | tr -s ' ' | cut -d ' ' -f 2`
+    if [ $PROTOBUF_INSTALLED_VERSION == $PROTOBUF_VERSION ]; then
+        echo "Current protobuf version is equal to the requested ($PROTOBUF_INSTALLED_VERSION), exiting..."
+    else
+        echo "Current protobuf version ($PROTOBUF_INSTALLED_VERSION) is not equal to the requested ($PROTOBUF_VERSION), installing $PROTOBUF_VERSION"
+        install_protobuf
+    fi
+
+    echo $PROTOBUF_INSTALLED_VERSION
+else
+    echo "protoc --version command had non-zero return value, need to install probuf"
+    install_protobuf
+fi

build-tools/protocw

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+### This is a protoc wrapper for tez, which can dinamically call protoc from a downloaded protobuf.
+
+if [ -n "$ZSH_VERSION" ]; then
+   SCRIPT_DIR="${0:a:h}"
+elif [ -n "$BASH_VERSION" ]; then
+   SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+    $SCRIPT_DIR/protobuf/bin/protoc "$@"
+else
+    protoc "$@"
+fi

pom.xml

@@ -64,7 +64,7 @@
     <jersey.version>1.19</jersey.version>
     <slf4j.version>1.7.30</slf4j.version>
     <protobuf.version>2.5.0</protobuf.version>
-    <protoc.path>${env.PROTOC_PATH}</protoc.path>
+    <protoc.path>${maven.multiModuleProjectDirectory}/build-tools/protocw</protoc.path>
     <scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>
     <frontend-maven-plugin.version>1.4</frontend-maven-plugin.version>
     <findbugs-maven-plugin.version>3.0.5</findbugs-maven-plugin.version>

tez-api/pom.xml

@@ -145,6 +145,24 @@
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>
       </plugin>
+      <plugin>
+        <artifactId>exec-maven-plugin</artifactId>
+        <groupId>org.codehaus.mojo</groupId>
+        <version>1.6.0</version>
+        <executions>
+          <execution>
+            <id>Install protobuf</id>
+            <phase>initialize</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>${basedir}/../build-tools/install-protobuf.sh</executable>
+              <arguments>${protobuf.version}</arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-maven-plugins</artifactId>