feat: hermetic build scripts to use a single output/generation folder

library_generation/README.md

@@ -7,37 +7,38 @@ The script, `generate_library.sh`, allows you to generate a GAPIC client library
 Use Linux environment and install java runtime environment (8 or above).
 
 ## Prerequisite
-Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to the current
-working directory (refers as `$cwd`, a directory contains `generate_library.sh`).
+Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to an `output`
+directory located in the working directory
+(likely `library_generation/output` if planning to call from the same folder).
 The directory structure should be the same as import statements in protos.
 
-For example, we want to generate from `folder1/folder2/protoA`, so `proto_path` 
-should be set to `folder1/folder2` (a relative path from `$cwd`). 
+For example, you want to generate from `folder1/folder2/protoA`, so `proto_path` 
+should be set to `folder1/folder2` (a relative path from `output`). 
 protoA imports protoB as `folder3/folder4/protoB`, then there should 
-be `folder3/folder4` (containing protoB) in `$cwd`.
+be `folder3/folder4` (containing protoB) in `output`.
 
 In order to generate a GAPIC library, you need to pull `google/` from [googleapis](https://github.com/googleapis/googleapis)
-and put it into `$cwd` since protos in `google/` are likely referenced by 
+and put it into `output` since protos in `google/` are likely referenced by 
 protos from which the library are generated.
 
 ## Parameters to run `generate_library.sh`
 
 You need to run the script with the following parameters.
 
 ### proto_path
-A directory in `$cwd` and copy proto files into it. 
-The absolute path of `proto_path` is `$cwd/$proto_path`. 
+A directory in `$cwd/output` and copy proto files into it. 
+The absolute path of `proto_path` is `$cwd/output/$proto_path`. 
 
 Use `-p` or `--proto_path` to specify the value.
 
 ### destination_path 
-A directory within `$cwd`. 
+A directory within `$cwd/output`. 
 This is the path in which the generated library will reside. 
-The absolute path of `destination_path` is `$cwd/$destination_path`. 
+The absolute path of `destination_path` is `$cwd/output/$destination_path`. 
 
 Use `-d` or `--destination_path` to specify the value.
 
-Note that you do not need to create `$detination_path` beforehand.
+Note that you do not need to create `$destination_path` beforehand.
 
 The directory structure of the generated library is
 ```

library_generation/generate_library.sh

@@ -54,6 +54,7 @@ shift # past argument or value
 done
 
 script_dir=$(dirname "$(readlink -f "$0")")
+output_folder="$(pwd)/output"
 # source utility functions
 source "${script_dir}"/utilities.sh
 
@@ -81,15 +82,17 @@ if [ -z "${os_architecture}" ]; then
   os_architecture="linux-x86_64"
 fi
 
-mkdir -p "${destination_path}"
+
+mkdir -p "${output_folder}/${destination_path}"
 ##################### Section 0 #####################
 # prepare tooling
 #####################################################
 # the order of services entries in gapic_metadata.json is relevant to the
 # order of proto file, sort the proto files with respect to their name to
 # get a fixed order.
-proto_files=$(find "${proto_path}" -type f  -name "*.proto" | sort)
 folder_name=$(extract_folder_name "${destination_path}")
+pushd "${output_folder}"
+proto_files=$(find "${proto_path}" -type f  -name "*.proto" | sort)
 # download gapic-generator-java, protobuf and grpc plugin.
 download_tools "${gapic_generator_version}" "${protobuf_version}" "${grpc_version}" "${os_architecture}"
 ##################### Section 1 #####################
@@ -153,9 +156,11 @@ for proto_src in ${proto_files}; do
   mkdir -p "${destination_path}/proto-${folder_name}/src/main/proto"
   rsync -R "${proto_src}" "${destination_path}/proto-${folder_name}/src/main/proto"
 done
+popd # output_folder
 ##################### Section 4 #####################
 # rm tar files
 #####################################################
-cd "${destination_path}"
+pushd cd "${output_folder}/${destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
+popd
 set +x

library_generation/test/generate_library_integration_test.sh

@@ -45,11 +45,14 @@ done
 script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}/../utilities.sh"
 library_generation_dir="${script_dir}"/..
-cd "${library_generation_dir}"
+output_folder="$(pwd)/output"
+mkdir -p "${output_folder}"
+pushd "${output_folder}"
 # checkout the master branch of googleapis/google (proto files) and WORKSPACE
 echo "Checking out googlapis repository..."
 sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning"
-cd googleapis
+pushd googleapis
+cp -r google "${output_folder}"
 # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
 gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
 echo "The version of gapic-generator-java is ${gapic_generator_version}."
@@ -82,6 +85,8 @@ os_architecture="linux-x86_64"
 if [[ "$os_type" == *"macos"* ]]; then
   os_architecture="osx-x86_64"
 fi
+popd
+popd
 echo "OS Architecture is ${os_architecture}."
 # generate GAPIC client library
 echo "Generating library from ${proto_path}, to ${destination_path}..."
@@ -99,19 +104,19 @@ echo "Generating library from ${proto_path}, to ${destination_path}..."
 echo "Generate library finished."
 echo "Checking out googleapis-gen repository..."
 
+pushd "${output_folder}"
 sparse_clone "${googleapis_gen_url}" "${proto_path}"
 
 echo "Compare generation result..."
 RESULT=0
-diff -r "googleapis-gen/${proto_path}/${destination_path}" "${destination_path}" -x "*gradle*" || RESULT=$?
+diff -r "googleapis-gen/${proto_path}/${destination_path}" "${output_folder}/${destination_path}" -x "*gradle*" || RESULT=$?
 
 if [ ${RESULT} == 0 ] ; then
   echo "SUCCESS: Comparison finished, no difference is found."
 else
   echo "FAILURE: Differences found."
 fi
 
-cd ..
-rm -rf googleapis
+popd
 
 exit ${RESULT}

library_generation/test/generate_library_unit_tests.sh

@@ -181,15 +181,14 @@ generate_library_failed_with_invalid_generator_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
     -d ../"${destination}" \
     --gapic_generator_version 1.99.0 \
     --protobuf_version 23.2 \
     --grpc_version 1.55.1 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"
@@ -199,15 +198,14 @@ generate_library_failed_with_invalid_protobuf_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
     -d ../"${destination}" \
     --gapic_generator_version 2.24.0 \
     --protobuf_version 22.99 \
     --grpc_version 1.55.1 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"
@@ -217,14 +215,13 @@ generate_library_failed_with_invalid_grpc_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
-    -d ../"${destination}" \
+    -d ../output/"${destination}" \
     --gapic_generator_version 2.24.0 \
     --grpc_version 0.99.0 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"

library_generation/utilities.sh

@@ -137,13 +137,15 @@ get_protobuf_version() {
 }
 
 download_tools() {
+  pushd "${output_folder}"
   local gapic_generator_version=$1
   local protobuf_version=$2
   local grpc_version=$3
   local os_architecture=$4
   download_generator "${gapic_generator_version}"
   download_protobuf "${protobuf_version}" "${os_architecture}"
   download_grpc_plugin "${grpc_version}" "${os_architecture}"
+  popd
 }
 
 download_generator() {
@@ -177,7 +179,7 @@ download_protobuf() {
     rm "protobuf-${protobuf_version}.zip"
   fi
 
-  protoc_path=protobuf-${protobuf_version}/bin
+  protoc_path="${output_folder}/protobuf-${protobuf_version}/bin"
 }
 
 download_grpc_plugin() {
@@ -257,13 +259,13 @@ sparse_clone() {
   clone_dir=$(basename "${repo_url%.*}")
   rm -rf "${clone_dir}"
   git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}"
-  cd "${clone_dir}"
+  pushd "${clone_dir}"
   if [ -n "${commitish}" ]; then
     git checkout "${commitish}"
   fi
   git sparse-checkout set --no-cone ${paths}
   git checkout
-  cd ..
+  popd
 }
 
 # takes a versions.txt file and returns its version

showcase/scripts/generate_showcase.sh

@@ -11,6 +11,7 @@ source "${lib_gen_scripts_dir}/utilities.sh"
 readonly perform_cleanup=$1
 
 cd "${SCRIPT_DIR}"
+mkdir -p "${SCRIPT_DIR}/output"
 
 # clone gapic-showcase
 if [ ! -d schema ]; then
@@ -23,7 +24,7 @@ if [ ! -d schema ]; then
   showcase_version=$(grep -e '<gapic-showcase.version>' "${SCRIPT_DIR}/../gapic-showcase/pom.xml" | cut -d'>' -f2 | cut -d'<' -f1)
     sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}"
   cd gapic-showcase
-  mv schema ..
+  mv schema ../output
   cd ..
   rm -rdf gapic-showcase
 fi
@@ -32,7 +33,7 @@ if [ ! -d google ];then
     rm -rdf googleapis
   fi
   sparse_clone https://github.com/googleapis/googleapis.git "WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/longrunning google/iam/v1 google/cloud/location google/type"
-  mv googleapis/google .
+  mv googleapis/google output
   rm -rdf googleapis
 fi
 
@@ -41,8 +42,8 @@ ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-
 rest_numeric_enums="false"
 transport="grpc+rest"
 include_samples="false"
-rm -rdf showcase-output
-mkdir showcase-output
+rm -rdf output/showcase-output
+mkdir output/showcase-output
 set +e
 bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \
   --proto_path "schema/google/showcase/v1beta1" \
@@ -54,7 +55,7 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \
 
 exit_code=$?
 if [ "${exit_code}" -ne 0 ]; then
-  rm -rdf showcase-output
+  rm -rdf output
   exit "${exit_code}"
 fi
 set +x

showcase/scripts/showcase_utilities.sh

@@ -3,5 +3,5 @@
 function cleanup {
   script_dir=$1
   cd "${script_dir}"
-  rm -rdf gapic-generator-java* google schema protobuf-* protoc-gen-grpc-java* showcase-output out
+  rm -rdf output gapic-generator-java*
 }

showcase/scripts/verify.sh

@@ -12,16 +12,26 @@ readonly REPLACE_SOURCE='false'
 readonly PERFORM_CLEANUP='false'
 bash "${SCRIPT_DIR}/generate_components.sh" "${REPLACE_SOURCE}" "${PERFORM_CLEANUP}"
 
-# compare with proto library
 PROTO_PROJECT_DIR='proto-gapic-showcase-v1beta1'
-diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/showcase-output/proto-showcase-output/src"
-
-# compare with grpc library
 GRPC_PROJECT_DIR='grpc-gapic-showcase-v1beta1'
-diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/showcase-output/grpc-showcase-output/src/main/java/com"
-
-# compare with gapic library
 GAPIC_PROJECT_DIR='gapic-showcase'
-diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/showcase-output/gapic-showcase-output/src/main/java"
+
+{
+  # compare with proto library
+  (diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/output/showcase-output/proto-showcase-output/src") &&
+
+  # compare with grpc library
+  (diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/output/showcase-output/grpc-showcase-output/src/main/java/com") &&
+
+  # compare with gapic library
+  (diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/output/showcase-output/gapic-showcase-output/src/main/java") 
+} || {
+  failure="true"
+}
 
 cleanup $SCRIPT_DIR
+
+if [ "${failure}" == "true" ]; then
+  exit 1
+fi
+