feat: generate_library.sh with postprocessing

.github/workflows/verify_library_generation.yaml

@@ -14,6 +14,7 @@ jobs:
       matrix:
         java: [ 8 ]
         os: [ ubuntu-22.04, macos-12 ]
+        post_processing: [ 'true', 'false' ]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -22,11 +23,35 @@ jobs:
         java-version: ${{ matrix.java }}
         distribution: temurin
         cache: maven
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - name: install docker (ubuntu)
+      if: matrix.os == 'ubuntu-22.04'
+      run: |
+        set -x
+        # install docker
+        sudo apt install containerd -y
+        sudo apt install -y docker.io docker-compose
+
+        # launch docker
+        sudo systemctl start docker
+    - name: install docker (macos)
+      if: matrix.os == 'macos-12'
+      run: |
+        brew update --preinstall
+        brew install docker docker-compose qemu 
+        brew upgrade qemu
+        colima start
+        docker run --user $(id -u):$(id -g) --rm hello-world
     - name: Run integration tests
       run: |
         set -x
+        git config --global user.email "[email protected]"
+        git config --global user.name "Github Workflow"
         library_generation/test/generate_library_integration_test.sh \
-        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git
+        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \
+        --enable_postprocessing "${{ matrix.post_processing }}"
   unit_tests:
     strategy:
       matrix:

library_generation/README.md

@@ -21,6 +21,12 @@ In order to generate a GAPIC library, you need to pull `google/` from [googleapi
 and put it into `output` since protos in `google/` are likely referenced by 
 protos from which the library are generated.
 
+In order to generate a post-processed GAPIC library, you need to pull the
+original repository (e.g. google-cloud-java/java-asset) and pass it as
+`repository_path`. This repository will be the source of truth for pre-existing
+pom.xml files, owlbot.py and .OwlBot.yaml files. See the option belows for
+custom postprocessed generations (e.g. custom `versions.txt` file).
+
 ## Parameters to run `generate_library.sh`
 
 You need to run the script with the following parameters.
@@ -150,8 +156,38 @@ Use `--include_samples` to specify the value.
 Choose the protoc binary type from https://github.com/protocolbuffers/protobuf/releases.
 Default is "linux-x86_64".
 
-## An example to generate a client library
+### enable_postprocessing (optional)
+Whether to enable the post-processing steps (usage of owlbot) in the generation
+of this library
+Default is "true".
+
+### repository_path (optional)
+Relative path from `output_folder` to the location of the original,
+post-processed source code of the library being generated. It is necessary when
+`enable_postprocessing` is `"true"`
+
+### versions_file (optional)
+It must point to a versions.txt file containing the versions the post-processed
+poms will have. It is required when `enable_postprocessing` is `"true"`
+
+
+## An example to generate a non post-processed client library
+```bash
+library_generation/generate_library.sh \
+-p google/cloud/confidentialcomputing/v1 \
+-d google-cloud-confidentialcomputing-v1-java \
+--gapic_generator_version 2.24.0 \
+--protobuf_version 23.2 \
+--grpc_version 1.55.1 \
+--gapic_additional_protos "google/cloud/common_resources.proto google/cloud/location/locations.proto" \
+--transport grpc+rest \
+--rest_numeric_enums true \
+--enable_postprocessing false \
+--include_samples true
 ```
+
+## An example to generate a library with postprocessing
+```bash
 library_generation/generate_library.sh \
 -p google/cloud/confidentialcomputing/v1 \
 -d google-cloud-confidentialcomputing-v1-java \
@@ -161,5 +197,8 @@ library_generation/generate_library.sh \
 --gapic_additional_protos "google/cloud/common_resources.proto google/cloud/location/locations.proto" \
 --transport grpc+rest \
 --rest_numeric_enums true \
+--enable_postprocessing true \
+--repository_path "java-logging" \
+--versions_file "path/to/versions.txt" \
 --include_samples true
 ```

library_generation/generate_library.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
-set -x
 
 # parse input parameters
 while [[ $# -gt 0 ]]; do
@@ -61,10 +60,22 @@ case $key in
     include_samples="$2"
     shift
     ;;
+  --enable_postprocessing)
+    enable_postprocessing="$2"
+    shift
+    ;;
+  --repository_path)
+    repository_path="$2"
+    shift
+    ;;
   --os_architecture)
     os_architecture="$2"
     shift
     ;;
+  --versions_file)
+    versions_file="$2"
+    shift
+    ;;
   *)
     echo "Invalid option: [$1]"
     exit 1
@@ -74,6 +85,7 @@ shift # past argument or value
 done
 
 script_dir=$(dirname "$(readlink -f "$0")")
+# source utility functions
 source "${script_dir}"/utilities.sh
 output_folder="$(get_output_folder)"
 
@@ -117,17 +129,20 @@ if [ -z "${include_samples}" ]; then
   include_samples="true"
 fi
 
+if [ -z "$enable_postprocessing" ]; then
+  enable_postprocessing="true"
+fi
+
 if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
 
-
 mkdir -p "${output_folder}/${destination_path}"
 ##################### Section 0 #####################
 # prepare tooling
 #####################################################
 # the order of services entries in gapic_metadata.json is relevant to the
-# order of proto file, sort the proto files with respect to their name to
+# order of proto file, sort the proto files with respect to their bytes to
 # get a fixed order.
 folder_name=$(extract_folder_name "${destination_path}")
 pushd "${output_folder}"
@@ -137,7 +152,7 @@ case "${proto_path}" in
     find_depth="-maxdepth 1"
     ;;
 esac
-proto_files=$(find "${proto_path}" ${find_depth} -type f  -name "*.proto" | sort)
+proto_files=$(find "${proto_path}" ${find_depth} -type f  -name "*.proto" | LC_COLLATE=C sort)
 # include or exclude certain protos in grpc plugin and gapic generator java.
 case "${proto_path}" in
   "google/cloud")
@@ -280,5 +295,30 @@ popd # output_folder
 #####################################################
 pushd "${output_folder}/${destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
-popd
-set +x
+popd # destination path
+##################### Section 5 #####################
+# post-processing
+#####################################################
+source "${script_dir}/post_processing/postprocessing_utilities.sh"
+if [ "${enable_postprocessing}" != "true" ];
+then
+  echo "post processing is disabled"
+  exit 0
+fi
+if [ -z "${versions_file}" ];then
+  echo "no versions.txt argument provided. Please provide one in order to enable post-processing"
+  exit 1
+fi
+workspace="${output_folder}/workspace"
+if [ -d "${workspace}" ]; then
+  rm -rdf "${workspace}"
+fi
+
+mkdir -p "${workspace}"
+
+run_owlbot_postprocessor "${workspace}" \
+  "${script_dir}" \
+  "${output_folder}/${destination_path}" \
+  "${repository_path}" \
+  "${proto_path}" \
+  "${versions_file}"

library_generation/post_processing/postprocessing_utilities.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# Main functions to interact with owlbot post-processor and postprocessing
+# scripts
+
+# returns the metadata json path if given, or defaults to the one found in
+# $repository_path
+# Arguments
+# 1 - repository_path: path from output_folder to the location of the library
+# containing .repo-metadata. It assumes the existence of google-cloud-java in
+# the output folder
+# 2 - output_folder: root for the generated libraries, used in conjunction with
+get_repo_metadata_json() {
+  local repository_path=$1
+  local output_folder=$2
+  >&2 echo 'Attempting to obtain .repo-metadata.json from repository_path'
+  local default_metadata_json_path="${output_folder}/${repository_path}/.repo-metadata.json"
+  if [ -f "${default_metadata_json_path}" ]; then
+    echo "${default_metadata_json_path}"
+  else
+    >&2 echo 'failed to obtain json from repository_path'
+    exit 1
+  fi
+}
+
+# returns the owlbot image sha contained in google-cloud-java. This is default
+# behavior that may be overriden by a custom value in the future.
+# Arguments
+# 1 - output_folder: root for the generated libraries, used in conjunction with
+# 2 - repository_root: usually "google-cloud-java". The .OwlBot.yaml
+# file is looked into its .github folder
+get_owlbot_sha() {
+  local output_folder=$1
+  local repository_root=$2
+  if [ ! -d "${output_folder}/${repository_root}" ];
+  then
+    >&2 echo 'No repository to infer owlbot_sha was provided. This is necessary for post-processing' >&2
+    exit 1
+  fi
+  >&2 echo "Attempting to obtain owlbot_sha from monorepo folder"
+  owlbot_sha=$(grep 'sha256' "${output_folder}/${repository_root}/.github/.OwlBot.lock.yaml" | cut -d: -f3)
+  echo "${owlbot_sha}"
+}
+
+# Runs the owlbot post-processor docker image.
+# Arguments
+# 1 - workspace: the location of the grpc,proto and gapic libraries to be
+# processed
+# owlbot
+# 4 - scripts_root: location of the generation scripts
+# 5 - destination_path: used to transfer the raw grpc, proto and gapic libraries
+# 6 - repository_path: path from output_folder to the location of the source of
+# truth/pre-existing poms. This can either be a folder in google-cloud-java or
+# the root of a HW library
+# 7 - proto_path: googleapis path of the library. This is used to prepare the
+# folder structure to run `owlbot-cli copy-code`
+function run_owlbot_postprocessor {
+  workspace=$1
+  scripts_root=$2
+  destination_path=$3
+  repository_path=$4
+  proto_path=$5
+  versions_file=$6
+
+  repository_root=$(echo "${repository_path}" | cut -d/ -f1)
+  repo_metadata_json_path=$(get_repo_metadata_json "${repository_path}" "${output_folder}")
+  owlbot_sha=$(get_owlbot_sha "${output_folder}" "${repository_root}")
+
+  # read or infer owlbot sha
+
+  cp "${repo_metadata_json_path}" "${workspace}"/.repo-metadata.json
+
+  # call owl-bot-copy
+  owlbot_staging_folder="${workspace}/owl-bot-staging"
+  mkdir -p "${owlbot_staging_folder}"
+  owlbot_postprocessor_image="gcr.io/cloud-devrel-public-resources/owlbot-java@sha256:${owlbot_sha}"
+
+
+
+  # copy existing pom, owlbot and version files if the source of truth repo is present
+  if [[ -d "${output_folder}/${repository_path}" ]]; then
+    rsync -avm \
+      --include='*/' \
+      --include='*.xml' \
+      --include='package-info.java' \
+      --include='owlbot.py' \
+      --include='.OwlBot.yaml' \
+      --exclude='*' \
+      "${output_folder}/${repository_path}/" \
+      "${workspace}"
+  fi
+
+  echo 'Running owl-bot-copy'
+  pre_processed_libs_folder="${destination_path}/pre-processed"
+  mkdir -p "${pre_processed_libs_folder}/${proto_path}/$(basename "${destination_path}")"
+  find "${destination_path}" -mindepth 1 -maxdepth 1 -type d -not -name 'pre-processed' \
+    -exec cp -pr {} "${pre_processed_libs_folder}/${proto_path}/$(basename "${destination_path}")" \;
+  pushd "${pre_processed_libs_folder}"
+  # create an empty repository so owl-bot-copy can process this as a repo
+  # (cannot process non-git-repositories)
+  git init
+  git commit --allow-empty -m 'empty commit'
+  popd # pre_processed_libs_folder
+
+  docker run --rm \
+    --user $(id -u):$(id -g) \
+    -v "${workspace}:/repo" \
+    -v "${pre_processed_libs_folder}:/pre-processed-libraries" \
+    -w /repo \
+    --env HOME=/tmp \
+    gcr.io/cloud-devrel-public-resources/owlbot-cli:latest \
+    copy-code \
+    --source-repo-commit-hash=none \
+    --source-repo=/pre-processed-libraries \
+    --config-file=.OwlBot.yaml
+
+
+  echo 'running owl-bot post-processor'
+  versions_file_arg=""
+  if [ -f "${versions_file}" ];then
+    versions_file_arg="-v ${versions_file}:/versions.txt"
+  fi
+  # run the postprocessor
+  docker run --rm \
+    -v "${workspace}:/workspace" \
+    ${versions_file_arg} \
+    --user $(id -u):$(id -g) \
+    "${owlbot_postprocessor_image}"
+}