From 4918c1974ac7943e4c239e8bf25bf2e61ab353f5 Mon Sep 17 00:00:00 2001
From: Shiyan Xu <2701446+xushiyan@users.noreply.github.com>
Date: Thu, 11 Jul 2024 20:34:40 -0500
Subject: [PATCH] chore: add scripts to streamline source release (#64)

- add script to create src release
- add script to verify src release
- enable release branch CI
---
 .github/workflows/ci.yml      |   1 +
 release/create_src_release.sh | 116 ++++++++++++++++++++++++++++++
 release/verify_src_release.sh | 131 ++++++++++++++++++++++++++++++++++
 3 files changed, 248 insertions(+)
 create mode 100755 release/create_src_release.sh
 create mode 100755 release/verify_src_release.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 18d9f8ea..16913b17 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,6 +21,7 @@ on:
   push:
     branches:
       - main
+      - release/**
   pull_request:
     branches:
       - main
diff --git a/release/create_src_release.sh b/release/create_src_release.sh
new file mode 100755
index 00000000..982ec3fd
--- /dev/null
+++ b/release/create_src_release.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -o errexit
+set -o nounset
+
+# check git branch
+curr_branch=$(git rev-parse --abbrev-ref HEAD)
+if [[ ! $curr_branch == release/* ]]; then
+  echo "Branch does not start with 'release/'"
+  exit 1
+else
+  echo "❗️  On branch $curr_branch"
+fi
+
+# check release version
+hudi_version=${HUDI_VERSION}
+if [ -z "${hudi_version}" ]; then
+  echo "HUDI_VERSION is not set; Please specify the target hudi version to release, e.g., 0.1.0-rc.1"
+  exit 1
+else
+  echo "❗️  Releasing Hudi version $hudi_version"
+fi
+
+# check release version against git tag
+curr_tag=$(git describe --exact-match --tags)
+if [[ ! "$curr_tag" == "release-$hudi_version" ]]; then
+  echo "Tag '$curr_tag' does not match with release version release-$hudi_version"
+  exit 1
+else
+  echo "❗️  On tag $curr_tag"
+fi
+
+# make sure a desired key is specified
+gpg_user_id=${GPG_USER_ID}
+if [ -z "${gpg_user_id}" ]; then
+  echo "GPG_USER_ID is not set; run 'gpg --list-secret-keys --keyid-format=long' to choose a key."
+  exit 1
+else
+  echo "❗️  Signing using key '$gpg_user_id'"
+fi
+
+work_dir="$TMPDIR$(date +'%Y-%m-%d-%H-%M-%S')"
+hudi_src_rel_dir="$work_dir/hudi-rs-$hudi_version"
+echo ">>> Archiving branch $curr_branch to $hudi_src_rel_dir"
+mkdir -p "$hudi_src_rel_dir"
+git archive --format=tgz --output="$hudi_src_rel_dir/hudi-rs-$hudi_version.src.tgz" "$curr_branch"
+echo "Done archiving."
+
+pushd "$hudi_src_rel_dir"
+echo ">>> Generating signature"
+for i in *.tgz; do
+  echo "$i"
+  gpg --local-user "$gpg_user_id" --armor --output "$i.asc" --detach-sig "$i"
+done
+echo ">>> Checking signature"
+for i in *.tgz; do
+  echo "$i"
+  gpg --local-user "$gpg_user_id" --verify "$i.asc" "$i"
+done
+echo ">>> Generating sha512sum"
+if [ "$(uname)" == "Darwin" ]; then
+  SHASUM="shasum -a 512"
+else
+  SHASUM="sha512sum"
+fi
+for i in *.tgz; do
+  echo "$i"
+  $SHASUM "$i" >"$i.sha512"
+done
+echo ">>> Checking sha512sum"
+for i in *.tgz; do
+  echo "$i"
+  $SHASUM --check "$i.sha512"
+done
+
+echo "✅  SUCCESS! Created source release at $hudi_src_rel_dir"
+for i in $(ls -a "$hudi_src_rel_dir"); do echo "|___$i"; done
+popd
+
+svn_dev_url="https://dist.apache.org/repos/dist/dev/hudi"
+svn_dir="$work_dir/svn_dev"
+echo ">>> Checking out svn dev to $svn_dir"
+svn co -q $svn_dev_url --depth=immediates "$svn_dir"
+echo ">>> Checking if the same version dir exists in svn"
+hudi_src_rel_svn_dir="$svn_dir/$(basename $hudi_src_rel_dir)"
+if [ -d $hudi_src_rel_svn_dir ]; then
+  echo "❌  Version $(basename $hudi_src_rel_svn_dir) already exists!"
+  exit 1
+fi
+echo ">>> Copying source release files to $svn_dir"
+cp -r "$hudi_src_rel_dir" "$svn_dir/"
+echo "✅  SUCCESS! Placed source release at $hudi_src_rel_svn_dir"
+for i in $(ls -a "$hudi_src_rel_svn_dir"); do echo "|___$i"; done
+
+echo "❗️  [ACTION REQUIRED] Manually inspect and commit the source release to SVN."
+echo "1️⃣  cd $svn_dir"
+echo "2️⃣  svn add $(basename $hudi_src_rel_svn_dir)"
+echo "3️⃣  svn commit -m 'add $(basename $hudi_src_rel_svn_dir)'"
diff --git a/release/verify_src_release.sh b/release/verify_src_release.sh
new file mode 100755
index 00000000..2356fdd4
--- /dev/null
+++ b/release/verify_src_release.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -o errexit
+set -o nounset
+
+if [[ $# -ne 2 ]]; then
+  echo "Usage: $0 <dev|release> <hudi_version>"
+  exit 1
+fi
+
+repo=$1
+hudi_version=$2
+
+dev_pattern="^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$"
+release_pattern="^[0-9]+\.[0-9]+\.[0-9]+$"
+
+if [[ "$repo" == "dev" ]]; then
+  if [[ ! "$hudi_version" =~ $dev_pattern ]]; then
+    echo "ERROR: For 'dev' repo, version must be in format X.Y.Z-[alpha|beta|rc].N (e.g., 0.1.0-rc.1)"
+    exit 1
+  fi
+elif [[ "$repo" == "release" ]]; then
+  if [[ ! "$hudi_version" =~ $release_pattern ]]; then
+    echo "ERROR: For 'release' repo, version must be in format X.Y.Z (e.g., 0.1.0)"
+    exit 1
+  fi
+else
+  echo "ERROR: Invalid repository type. Use 'dev' or 'release'."
+  exit 1
+fi
+
+work_dir="$TMPDIR$(date +'%Y-%m-%d-%H-%M-%S')/svn_dir"
+hudi_artifact="hudi-rs-$hudi_version"
+svn_url="https://dist.apache.org/repos/dist/$repo/hudi/$hudi_artifact"
+echo "Checking out src release from $svn_url to $work_dir"
+svn co -q "$svn_url" "$work_dir"
+
+cd "$work_dir"
+src="$hudi_artifact.src.tgz"
+pub_key="$src.asc"
+checksum="$src.sha512"
+echo ">>> Verifying artifacts exist..."
+artifacts=($src $pub_key $checksum)
+for artifact in "${artifacts[@]}"; do
+  if [ ! -f "$artifact" ]; then
+    echo "ERROR: Artifact $artifact does not exist."
+    exit 1
+  fi
+done
+echo "<<< OK"
+
+echo ">>> Verifying checksum..."
+if [ "$(uname)" == "Darwin" ]; then
+  SHASUM="shasum -a 512"
+else
+  SHASUM="sha512sum"
+fi
+$SHASUM "$src" >"$work_dir/src.sha512"
+diff -u "$checksum" "$work_dir/src.sha512"
+echo "<<< OK"
+
+echo ">>> Verifying signature..."
+curl -s "https://dist.apache.org/repos/dist/$repo/hudi/KEYS" >"$work_dir/KEYS"
+gpg -q --import "$work_dir/KEYS"
+gpg --verify "$pub_key" "$src"
+echo "<<< OK"
+
+echo "Un-tarring the source release artifact"
+mkdir "$hudi_artifact"
+tar -xzf "$src" -C "$hudi_artifact"
+cd "$hudi_artifact"
+
+echo ">>> Verifying no DISCLAIMER..."
+if [ -f "./DISCLAIMER" ]; then
+  echo "ERROR: DISCLAIMER file should not be present."
+  exit 1
+fi
+echo "<<< OK"
+
+echo ">>> Verifying LICENSE file present..."
+if [ ! -f "./LICENSE" ]; then
+  echo "ERROR: LICENSE file is missing."
+  exit 1
+fi
+echo "<<< OK"
+
+echo ">>> Verifying NOTICE file present..."
+if [ ! -f "./NOTICE" ]; then
+  echo "ERROR: NOTICE file is missing."
+  exit 1
+fi
+echo "<<< OK"
+
+echo ">>> Verifying licenses..."
+docker run -it --rm -v $(pwd):/github/workspace apache/skywalking-eyes header check
+echo "<<< OK"
+
+echo ">>> Verifying no binary files..."
+find_binary_files() {
+  find . -type f \
+    -not -path "*/tests/data/*" \
+    -not -path "*/tests/table/*" \
+    -not -name "*.json" -not -name "*.xml" \
+    -exec file -I '{}' \; |
+    grep -viE 'directory|text/'
+}
+numBinaryFiles=$(find_binary_files | wc -l)
+if ((numBinaryFiles > 0)); then
+  echo "ERROR: There were non-text files in the source release. Please check:"
+  find_binary_files
+  exit 1
+fi
+echo "<<< OK"