TEZ-4300: Download protoc automatically compile/development time

BUILDING.txt

@@ -117,8 +117,10 @@ It's important to note that maven will still include tez-ui project, but all of
 ----------------------------------------------------------------------------------
 Protocol Buffer compiler:
 
-The version of Protocol Buffer compiler, protoc, must be 2.5.0 and match the
-version of the protobuf JAR.
+The version of Protocol Buffer compiler, protoc, can be defined on-the-fly as:
+ $ mvn clean install -DskipTests -pl ./tez-api -Dprotobuf.version=3.7.1
+
+The default version is defined in the root pom.xml.
 
 If you have multiple versions of protoc in your system, you can set in your 
 build shell the PROTOC_PATH environment variable to point to the one you 

build-tools/.gitignore

@@ -0,0 +1,2 @@
+protobuf
+

build-tools/install-protobuf.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -16,7 +16,73 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar -xzvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install
+# This script attempts to install an arbitrary version of protobuf if needed.
+# The desired version should be the first parameter: $1.
+# Typical usage: the script is automatically called from tez-api (by maven) during the build process.
+
+# This script runs from build-tools folder. The user can remove
+# the dynamically installed protobuf anytime like:
+# rm -rf ./build-tools/protobuf/ #from root folder
+
+set -x
+PROTOBUF_VERSION=${1:-2.5.0}
+PROTOBUF_MAJOR_VERSION=$(echo "$PROTOBUF_VERSION" | cut -d. -f1)
+if [ -n "$ZSH_VERSION" ]; then
+   SCRIPT_DIR="${0:a:h}"
+else
+   SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+function install_protobuf {
+    # before protobuf 3, there is no pre-compiled executables are host on github, let's try to build and make it
+    if (( PROTOBUF_MAJOR_VERSION < 3 )); then
+        wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz"
+        tar -xzvf "protobuf-$PROTOBUF_VERSION.tar.gz"
+        rm "protobuf-$PROTOBUF_VERSION.tar.gz"
+        cd "protobuf-$PROTOBUF_VERSION" && ./configure --prefix=/usr && make && sudo make install
+    # since protobuf 3, there are precompiled protoc executables on github, let's quickly download and use it
+    else
+        ARCH=`uname -m`
+        case "$(uname -s)" in
+            Darwin)
+                FILE_NAME="protoc-$PROTOBUF_VERSION-osx-$ARCH"
+                ;;
+            Linux)
+                if test $ARCH = "aarch64"; then
+                    ARCH="aarch_64"
+                fi
+                FILE_NAME="protoc-$PROTOBUF_VERSION-linux-$ARCH"
+                ;;
+            *)
+                echo "Unsupported OS returned by uname -s, you'll have to install protobuf 3.x manually"
+                exit 1
+                ;;
+        esac
+        rm -f "$FILE_NAME.zip" #cleanup unfinished file if any
+        wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/$FILE_NAME.zip"
+        mkdir "$SCRIPT_DIR/protobuf"
+        unzip -o "$FILE_NAME.zip" -d "$SCRIPT_DIR/protobuf"
+        rm "$FILE_NAME.zip"
+    fi
+}
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+    PROTOBUF_INSTALLED_VERSION=$("$SCRIPT_DIR/protobuf/bin/protoc" --version)
+else
+    PROTOBUF_INSTALLED_VERSION=$(protoc --version)
+fi
+
+PROTOC_EXIT_CODE=$?
+
+if [ $PROTOC_EXIT_CODE -eq 0 ]; then
+    PROTOBUF_INSTALLED_VERSION=$(echo "$PROTOBUF_INSTALLED_VERSION" | tr -s ' ' | cut -d ' ' -f 2)
+    if [ "$PROTOBUF_INSTALLED_VERSION" == "$PROTOBUF_VERSION" ]; then
+        echo "Current protobuf version is equal to the requested ($PROTOBUF_INSTALLED_VERSION), exiting..."
+    else
+        echo "Current protobuf version ($PROTOBUF_INSTALLED_VERSION) is not equal to the requested ($PROTOBUF_VERSION), installing $PROTOBUF_VERSION"
+        install_protobuf
+    fi
+else
+    echo "protoc --version command had non-zero return value, need to install probuf"
+    install_protobuf
+fi

build-tools/protocw

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### This is a protoc wrapper for tez, which can dinamically call protoc from a downloaded protobuf.
+
+if [ -n "$ZSH_VERSION" ]; then
+   SCRIPT_DIR="${0:a:h}"
+else
+   SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+    "$SCRIPT_DIR/protobuf/bin/protoc" "$@"
+else
+    protoc "$@"
+fi
+exit $?

pom.xml

@@ -1310,6 +1310,28 @@
         </dependency>
       </dependencies>
     </profile>
+    <!-- This is because <protoc.path>${env.PROTOC_PATH}</protoc.path> above
+         doesn't let us define a default value in the absence of env.PROTOC_PATH.
+         By defining this profile, the following order is considered:
+
+         0. protoc.path == env.PROTOC_PATH by pom.xml, if protoc.path is not defined
+         1. -Dprotoc.path: if defined, it wins
+         2. env.PROTOC_PATH: if protoc.path is not defined, but env.PROTOC_PATH is defined, env.PROTOC_PATH wins
+            (because protoc.path ==> env.PROTOC_PATH)
+         3. if neither -Dprotoc.path, nor PROTOC_PATH is defined, protocw script will run
+            (which can run protoc from the PATH, or an automatically installed version from build-tools/protobuf)
+    -->
+    <profile>
+        <id>protoc-path-env-variable-not-defined</id>
+        <activation>
+            <property>
+                <name>!env.PROTOC_PATH</name>
+            </property>
+        </activation>
+        <properties>
+            <protoc.path>${basedir}/../build-tools/protocw</protoc.path>
+        </properties>
+    </profile>
   </profiles>
 
   <reporting>

tez-api/pom.xml

@@ -145,6 +145,28 @@
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>
       </plugin>
+      <!-- This plugin takes care of on-the-fly installation of the needed protobuf version.
+           The needed version is always what's defined as protobuf.version in the pom,
+           so if user wants to change protobuf version quickly in development time,
+           supposed to change only protobuf.version and then rebuild tez-api. -->
+      <plugin>
+        <artifactId>exec-maven-plugin</artifactId>
+        <groupId>org.codehaus.mojo</groupId>
+        <version>1.6.0</version>
+        <executions>
+          <execution>
+            <id>Install protobuf</id>
+            <phase>initialize</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>${basedir}/../build-tools/install-protobuf.sh</executable>
+              <arguments>${protobuf.version}</arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-maven-plugins</artifactId>

tez-plugins/tez-protobuf-history-plugin/pom.xml

@@ -85,8 +85,21 @@
           </execution>
         </executions>
       </plugin>
-
     </plugins>
   </build>
 
+  <profiles>
+    <!-- For further details, please refer to the profile definition in root pom.xml -->
+    <profile>
+        <id>protoc-path-env-variable-not-defined</id>
+        <activation>
+            <property>
+                <name>!env.PROTOC_PATH</name>
+            </property>
+        </activation>
+        <properties>
+            <protoc.path>${basedir}/../../build-tools/protocw</protoc.path>
+        </properties>
+    </profile>
+  </profiles>
 </project>