From 0f4548c1fae56a3ad3d8adb3a7b2753390db26aa Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Fri, 29 Jul 2022 15:41:57 -0600 Subject: [PATCH 1/3] add support for creating changelogs for rpm releases The function rpm_release will be used to generate the following files: * cl-md - the new entry to use for CHANGELOG.md * cl-spec - the spec file %changelog section update * git-commit-msg - the git commit message The initial CHANGELOG.md is generated using spec_cl_to_cl_md. Then use rpm_release for subsequent releases. --- bz-manage.sh | 201 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 191 insertions(+), 10 deletions(-) diff --git a/bz-manage.sh b/bz-manage.sh index d8ae6b46..fe312845 100755 --- a/bz-manage.sh +++ b/bz-manage.sh @@ -22,6 +22,7 @@ STATUS=${STATUS:-POST} BZ_URL=${URL:-https://bugzilla.redhat.com/buglist.cgi} LIMIT=${LIMIT:-100} BASE_URL="${BZ_URL}?limit=${LIMIT}&component=${COMPONENT}&${ITR_FIELD}=${ITR}" +show_url="https://bugzilla.redhat.com/show_bug.cgi?id=" setitmdtm() { local field val bzlist baseurl @@ -179,13 +180,193 @@ clone_check() { return $rc } -case "${1:-}" in -setitm) setitmdtm itm ;; -setdtm) setitmdtm dtm ;; -reset_dev_wb) reset_dev_wb ;; -get_commit_msg) get_commit_msg ;; -get_cl) get_cl ;; -new) shift; new_bz "$@" ;; -clone_check) clone_check ;; -*) echo Error - see documentation; exit 1 ;; -esac +fmt_summary() { + local summary roles + summary="$1" + roles="$2" + if [ -n "$roles" ]; then + if [[ "$summary" =~ ^${roles}[\ -]+(.+)$ ]]; then + echo "$roles - ${BASH_REMATCH[1]}" + else + echo "$roles - $summary" + fi + else + echo "$summary" + fi +} + +rpm_release() { + # look up BZ for given ITR and status (default status is POST) + # input is new version - N-V-R format + # generate 3 files + # - cl-spec - the new %changelog entry for the spec file - user will + # need to edit for name, email + # - git-commit-msg - in the format required by dist-git + # - cl-md - the new entry for CHANGELOG.md + version="$1"; shift + local queryurl jq bz summary itr sum_prefix bz_prefix + queryurl="${BASE_URL}&bug_status=${STATUS}" + jq='.bugs[] | ((.id|tostring) + "|" + (.whiteboard|gsub("role:"; "")) + "|" + .cf_doc_type + "|" + .summary)' + datesec=$(date +%s) + get_bzs "${queryurl}" "$jq" -r | sort -k 2 -t \| > bzs.raw + cat > "${CL_MD:-cl-md}" < - $version" > "${CL_SPEC:-cl-spec}" + echo "Package update" > "${GIT_COMMIT_MSG:-git-commit-msg}" + new_features=false + while IFS=\| read -r bz roles doc_text summary; do + fix_summary=$(fmt_summary "$summary" "$roles") + if [ "$doc_text" = Enhancement ]; then + echo "- [${fix_summary}](${show_url}$bz)" >> "${CL_MD:-cl-md}" + echo "- Resolves:rhbz#${bz} : ${fix_summary}" >> "${CL_SPEC:-cl-spec}" + { echo "" + echo "$fix_summary" + echo "Resolves:rhbz#${bz}"; } >> "${GIT_COMMIT_MSG:-git-commit-msg}" + new_features=true + fi + done < bzs.raw + if [ "$new_features" = false ]; then + echo "- none" >> "${CL_MD:-cl-md}" + fi + echo "" >> "${CL_MD:-cl-md}" + { echo "### Bug Fixes"; echo ""; } >> "${CL_MD:-cl-md}" + fixes=false + while IFS=\| read -r bz roles doc_text summary; do + if [ "$doc_text" = "Bug Fix" ]; then + echo "- [${fix_summary}](${show_url}$bz)" >> "${CL_MD:-cl-md}" + echo "- Resolves:rhbz#${bz} : ${fix_summary}" >> "${CL_SPEC:-cl-spec}" + { echo "" + echo "$fix_summary" + echo "Resolves:rhbz#${bz}"; } >> "${GIT_COMMIT_MSG:-git-commit-msg}" + fixes=true + fi + done < bzs.raw + if [ "$fixes" = false ]; then + { echo "- none"; echo ""; } >> "${CL_MD:-cl-md}" + fi + while IFS=\| read -r bz roles doc_text summary; do + if [ "$doc_text" != Enhancement ] && [ "$doc_text" != "Bug Fix" ]; then + echo "- Resolves:rhbz#${bz} : ${fix_summary}" >> "${CL_SPEC:-cl-spec}" + { echo "" + echo "$fix_summary" + echo "Resolves:rhbz#${bz}"; } >> "${GIT_COMMIT_MSG:-git-commit-msg}" + fixes=true + fi + done < bzs.raw +} + +format_bz_for_md() { + local bz roles doc_text summary nf_file bf_file jq output fix_summary + bz="$1" + nf_file="$2" + bf_file="$3" + jq='.bugs[] | ((.whiteboard|gsub("role:"; "")) + "|" + .cf_doc_type + "|" + .summary)' + IFS=\| read -r roles doc_text summary <<< "$(bugzilla query -b "$bz" --json | jq -r "$jq")" + if [ "${doc_text:-}" = Enhancement ]; then + output="$nf_file" + elif [ "${doc_text:-}" = "Bug Fix" ]; then + output="$bf_file" + else + echo Skipping "$bz" "${doc_text:-no doc text}" "${summary:-no summary}" + return 0 + fi + if [ -n "$roles" ]; then + if [[ "$summary" =~ ^${roles}[\ -]+(.+)$ ]]; then + fix_summary="$roles - ${BASH_REMATCH[1]}" + else + fix_summary="$roles - $summary" + fi + else + fix_summary="$summary" + fi + if [ -f "$output" ]; then + if grep -F -q "$fix_summary" "$output" || grep -q "\<${bz}\>" "$output"; then + # already in there - skip + return 0 + fi + fi + echo "- [${fix_summary}](${show_url}$bz)" >> "$output" +} + +make_cl_for_version() { + local version datestr cl_file cl_nf_file cl_bf_file + version="$1" + datestr="$2" + cl_file="$3" + cl_nf_file="$4" + cl_bf_file="$5" + if [ ! -f "$cl_nf_file" ] && [ ! -f "$cl_bf_file" ]; then + return 0 # no cl for this version + fi + { echo "" + echo ["$version"] - "$datestr" + echo ---------------------------- + echo "" + echo "### New Features" + echo ""; } >> "$cl_file" + if [ -f "$cl_nf_file" ]; then + cat "$cl_nf_file" >> "$cl_file" + else + echo "- none" >> "$cl_file" + fi + { echo "" + echo "### Bug Fixes" + echo ""; } >> "$cl_file" + if [ -f "$cl_bf_file" ]; then + cat "$cl_bf_file" >> "$cl_file" + else + echo "- none" >> "$cl_file" + fi + rm -f "$cl_nf_file" "$cl_bf_file" +} + +spec_cl_to_cl_md() { + local spec print cur_ver mon day year version datestr cl_file cl_nf_file cl_bf_file + spec="$1" + print=false + version="" + cur_ver="" + cl_file=".cl.md" + cl_nf_file=".cl-nf.md" + cl_bf_file=".cl-bf.md" + { echo Changelog + echo =========; } > "$cl_file" + while read -r line ; do + if [ "$line" = %changelog ]; then + print=true + continue + fi + if [ "$print" = false ]; then + continue + fi + if [[ "$line" =~ ^\*\ +[a-zA-Z]+\ +([a-zA-Z]+)\ +([0-9]+)\ +([0-9]+)\ .*-\ +([0-9]+\.[0-9]+(\.[0-9]+)?)- ]]; then + mon="${BASH_REMATCH[1]}" + day="${BASH_REMATCH[2]}" + year="${BASH_REMATCH[3]}" + version="${BASH_REMATCH[4]}" + need_new_datestr=false + if [ -n "$cur_ver" ] && [ "$version" != "$cur_ver" ]; then + make_cl_for_version "$cur_ver" "$datestr" "$cl_file" "$cl_nf_file" "$cl_bf_file" + need_new_datestr=true + fi + if [ -z "${datestr:-}" ] || [ "$need_new_datestr" = true ]; then + datestr=$(date --date="$mon $day $year" +%Y-%m-%d) + fi + cur_ver="$version" + elif [[ "$line" =~ bz.?([0-9]{7}) ]]; then + while [[ "$line" =~ bz.?([0-9]{7}) ]]; do + bz="${BASH_REMATCH[1]}" + format_bz_for_md "$bz" "$cl_nf_file" "$cl_bf_file" + line="${line/$bz}" + done + fi + done < "$spec" + make_cl_for_version "$cur_ver" "$datestr" "$cl_file" "$cl_nf_file" "$cl_bf_file" +} + +"$@" From 9f10c6377daa69cf678235f61dbd7388891fb6ec Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Tue, 2 Aug 2022 19:56:57 -0600 Subject: [PATCH 2/3] use git commit subjects for changelog if no version If we are using the commit hash to build the collection, there will not be a version and a corresponding changelog. In that case, use the git commit titles and list them as Bug Fixes. * This will inevitably be wrong - there will be New Features and Other Changes listed erroneously - but there is no way to know * When there is an actual release with a version and a changelog, some of the changelog information will be duplicated - the new release will list some of the old changelog items again. I think this is a minor problem to have as long as releases using commit hashes are rare, and the changelog isn't really used in Galaxy anyway. --- release_collection.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/release_collection.py b/release_collection.py index 71c6047f..8da3897b 100755 --- a/release_collection.py +++ b/release_collection.py @@ -110,6 +110,10 @@ def comp_versions(cur_ref, new_ref): return 1 except ValueError as exc: logging.debug(f"Could not compare version {cur_ref} to {new_ref}: {exc}") + if cur_ref == new_ref: + return 0 + else: + return -1 # assume new is "newer" than cur def get_latest_tag_hash(args, rolename, cur_ref, org, repo): @@ -157,8 +161,23 @@ def get_latest_tag_hash(args, rolename, cur_ref, org, repo): describe_cmd = ["git", "describe", "--tags", "--long", "--abbrev=40"] describe_output = run_cmd(describe_cmd, roledir) tag, n_commits, g_hash = describe_output.stdout.strip().rsplit("-", 2) + if n_commits != "0": + # get commit messages to use for changelog + log_cmd = [ + "git", + "log", + "--oneline", + "--no-merges", + "--reverse", + "--pretty=format:- %s", + f"{cur_ref}..", + ] + log_output = run_cmd(log_cmd, roledir) + commit_msgs = log_output.stdout.replace("\\r", "") + else: + commit_msgs = "" # commit hash - skip leading "g" - return (tag, g_hash[1:], n_commits == "0") + return (tag, g_hash[1:], n_commits == "0", commit_msgs) def process_ignore_and_lint_files(args, coll_dir): @@ -201,7 +220,7 @@ def process_ignore_and_lint_files(args, coll_dir): yaml.safe_dump(ansible_lint, open(os.path.join(coll_dir, ".ansible-lint"), "w")) -def get_role_changelog(args, rolename, cur_ref, new_ref): +def get_role_changelog(args, rolename, cur_ref, new_ref, commit_msgs): """ Retrieve the matched changelogs from CHANGELOG.md and return them as a string. @@ -209,11 +228,15 @@ def get_role_changelog(args, rolename, cur_ref, new_ref): _changelog = "" if comp_versions(cur_ref, new_ref) >= 0: return _changelog + _changelog = "### {}\n".format(rolename) + if commit_msgs: + # make a fake changelog for compact_coll_changelog + _changelog = "{}\n##### Bug Fixes\n\n{}".format(_changelog, commit_msgs) + return _changelog _changelogmd = os.path.join(args.src_path, rolename, "CHANGELOG.md") if not os.path.exists(_changelogmd): - return _changelog + return "" _print = False - _changelog = "### {}\n".format(rolename) with open(_changelogmd, "r") as cl_fd: for _cl in cl_fd: cl = _cl.rstrip() @@ -233,7 +256,7 @@ def get_role_changelog(args, rolename, cur_ref, new_ref): _changelog = "{}\n#### {}".format(_changelog, cl) elif not cl.startswith("----"): _changelog = "{}\n{}".format(_changelog, cl) - logging.info(f"get_role_changelog - returning\n{_changelog}") + logging.info("get_role_changelog - returning\n%s", _changelog) return _changelog @@ -472,7 +495,7 @@ def update_collection(args, galaxy, coll_rel): for rolename in args.include: if not args.skip_git: cur_ref = coll_rel[rolename]["ref"] - tag, cm_hash, tag_is_latest = get_latest_tag_hash( + tag, cm_hash, tag_is_latest, commit_msgs = get_latest_tag_hash( args, rolename, coll_rel[rolename]["ref"], @@ -498,7 +521,7 @@ def update_collection(args, galaxy, coll_rel): "The role %s is updated. Updating the changelog.", rolename ) _changelog = get_role_changelog( - args, rolename, cur_ref, coll_rel[rolename]["ref"] + args, rolename, cur_ref, coll_rel[rolename]["ref"], commit_msgs ) coll_changelog = "{}\n{}".format(coll_changelog, _changelog) role_to_collection( @@ -556,7 +579,7 @@ def update_collection(args, galaxy, coll_rel): galaxy["version"], datetime.now().date() ) ) - clf.write(compact_coll_changelog(coll_changelog)) + clf.write(compact_coll_changelog(coll_changelog) + "\n") # Existing changelogs orig_cl_file = "lsr_role2collection/COLLECTION_CHANGELOG.md" with open(orig_cl_file, "r") as origclf: From 55005ed95c42308eb8892909b19cba003bd58bd0 Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Wed, 3 Aug 2022 11:03:53 -0600 Subject: [PATCH 3/3] refactor get_latest_tag_hash to return commit_msgs in all cases --- release_collection.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/release_collection.py b/release_collection.py index 8da3897b..eac27637 100755 --- a/release_collection.py +++ b/release_collection.py @@ -144,40 +144,40 @@ def get_latest_tag_hash(args, rolename, cur_ref, org, repo): # determine what is the main branch, check it out, and update it mmatch = re.search(r"origin/HEAD -> origin/(\w+)", branch_output.stdout) main_branch = mmatch.group(1) - _ = run_cmd(["bash", "-c", f"git checkout {main_branch}; git pull"], roledir) - if args.no_update: - # make sure cur_ref is checked out - _ = run_cmd(["git", "checkout", cur_ref], roledir) - return (None, None, None) + _ = run_cmd(["bash", "-c", f"git checkout {main_branch}; git pull --tags"], roledir) + commit_msgs = "" # see if there have been any commits since the last time we checked count_output = run_cmd( ["bash", "-c", f"git log --oneline {cur_ref}.. | wc -l"], roledir, ) + tag, commit_hash, n_commits = None, None, "0" if count_output.stdout == "0": logging.debug(f"no changes to role {rolename} since ref {cur_ref}") - return (None, None, None) - # get latest tag and commit hash - describe_cmd = ["git", "describe", "--tags", "--long", "--abbrev=40"] - describe_output = run_cmd(describe_cmd, roledir) - tag, n_commits, g_hash = describe_output.stdout.strip().rsplit("-", 2) - if n_commits != "0": - # get commit messages to use for changelog - log_cmd = [ - "git", - "log", - "--oneline", - "--no-merges", - "--reverse", - "--pretty=format:- %s", - f"{cur_ref}..", - ] - log_output = run_cmd(log_cmd, roledir) - commit_msgs = log_output.stdout.replace("\\r", "") else: - commit_msgs = "" - # commit hash - skip leading "g" - return (tag, g_hash[1:], n_commits == "0", commit_msgs) + # get latest tag and commit hash + describe_cmd = ["git", "describe", "--tags", "--long", "--abbrev=40"] + describe_output = run_cmd(describe_cmd, roledir) + tag, n_commits, g_hash = describe_output.stdout.strip().rsplit("-", 2) + # commit hash - skip leading "g" + commit_hash = g_hash[1:] + if n_commits != "0": + # get commit messages to use for changelog + log_cmd = [ + "git", + "log", + "--oneline", + "--no-merges", + "--reverse", + "--pretty=format:- %s", + f"{cur_ref}..", + ] + log_output = run_cmd(log_cmd, roledir) + commit_msgs = log_output.stdout.replace("\\r", "") + if args.no_update: + # make sure cur_ref is checked out + _ = run_cmd(["git", "checkout", cur_ref], roledir) + return (tag, commit_hash, n_commits == "0", commit_msgs) def process_ignore_and_lint_files(args, coll_dir):