From 9a4cf0e7cd22fbe2853abb0a45f76a7eb1f74f0a Mon Sep 17 00:00:00 2001 From: bigwhite Date: Thu, 31 Oct 2019 16:19:55 +0800 Subject: [PATCH 1/2] add release automation script --- release_tool/README.md | 13 +++ release_tool/prepare_release_branch.py | 153 +++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 release_tool/README.md create mode 100644 release_tool/prepare_release_branch.py diff --git a/release_tool/README.md b/release_tool/README.md new file mode 100644 index 00000000..98203bc7 --- /dev/null +++ b/release_tool/README.md @@ -0,0 +1,13 @@ +# Release tool + +Use script to bump the release verison and create the release PR to merge to develop branch. + +- If you have generated your github access token, use the following command to bump versions and send PR. + ```bash + python prepare_release_branch.py -v -n -a + ``` + +- If the access token is not given, this script only bumps the release version and push the commit to remote repo. You need to go to github web page to create your PR manually. + ``` + python prepare_release_branch.py -v -n + ``` diff --git a/release_tool/prepare_release_branch.py b/release_tool/prepare_release_branch.py new file mode 100644 index 00000000..689a2e08 --- /dev/null +++ b/release_tool/prepare_release_branch.py @@ -0,0 +1,153 @@ +import argparse +import os +import sys +import logging +import json +import re +import subprocess +import requests + +logging.getLogger().setLevel(logging.INFO) +root_repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def parse_args(): + def validate_version_str(version): + v_str = str(version).strip() + if not v_str: + raise argparse.ArgumentTypeError('verison str can not be emtpy.') + err_message = 'version string should be of format "major.minor.hotfix"' + numbers = v_str.split('.') + if len(numbers) != 3: + raise argparse.ArgumentTypeError(err_message) + for n in numbers: + valid = False + try: + v = int(n) + valid = (v>=0) + except: + valid = False + if not valid: + raise argparse.ArgumentTypeError(err_message) + return v_str + + def validate_token(token): + t = token.strip() + if not t: + raise argparse.ArgumentTypeError('token can not be empty') + return t + + parser = argparse.ArgumentParser( + 'prepare_release_branch.py', + description='eventgen release branch tool.\ncreate the release branch, set the right version and push the pull request.') + parser.add_argument('-v', '--verbose', default=False, action='store_true', help='enable the verbose logging') + parser.add_argument('-n', '--version_str', type=validate_version_str, required=True) + parser.add_argument('-a', '--token', help='your github access token.', default=None, type=validate_token) + return parser.parse_args(sys.argv[1:]) + +def setup_logging(verbose=None): + l = logging.DEBUG if verbose is True else logging.INFO + logging.getLogger().setLevel(l) + +def setup_env(): + ''' + by default, we use this hard code current working dir. + because curent working dir has impact about the child sh process. + we need to setup it before launching any process. + if there is concrete requirement about setting the current + working dir, we can change it to cmd arguemnt. + ''' + logging.debug(f'try to change current working directory to {root_repo_dir}') + os.chdir(root_repo_dir) + +def run_sh_cmd(args, exit_on_error=None): + should_exit_on_error = True if exit_on_error is None else exit_on_error + child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = child.communicate() + outs = out.decode('utf-8') + errs = err.decode('utf-8') + if child.returncode == 0: + logging.debug(f'execute sh command {args} success.') + logging.debug(f'children output:\n{outs}') + return True + logging.error(f'execute sh cmd {args} fail.\nchildren output:\n{outs}\n{errs}') + if should_exit_on_error: + assert False, 'sh command fails.' + return False + +def get_release_branch_name(version_str): + v = version_str.replace('.', '_') + return f'release/{v}' + +def replace_version(ver): + ver_json_file = os.path.join(root_repo_dir, 'splunk_eventgen', 'version.json') + with open(ver_json_file, 'w') as fp: + json.dump({'version': ver}, fp) + app_conf = os.path.join(root_repo_dir, 'splunk_eventgen', 'splunk_app', 'default', 'app.conf') + app_conf_content = [] + with open(app_conf, 'r') as fp: + app_conf_content = fp.readlines() + app_pattern = re.compile(r'version\s*=') + with open(app_conf, 'w') as fp: + for line in app_conf_content: + lstr = line.strip() + if app_pattern.search(lstr): + fp.write(f'version = {ver}\n') + else: + fp.write(f'{lstr}\n') + +def commit_version_files(ver): + ver_json_file = os.path.join('splunk_eventgen', 'version.json') + app_conf = os.path.join('splunk_eventgen', 'splunk_app', 'default', 'app.conf') + run_sh_cmd(['git', 'add', ver_json_file]) + run_sh_cmd(['git', 'add', app_conf]) + run_sh_cmd(['git', 'commit', '-m', f'update eventgen version to {ver}'], False) + logging.info('committed version files.') + +def create_pr(ver, token): + release_branch = get_release_branch_name(ver) + response = requests.post( + 'https://api.github.com/repos/splunk/eventgen/pulls', + json={'title': f'Release eventgen {ver}', 'head': release_branch, 'base': 'develop', 'body': + 'As the title'}, headers={ + 'Accept': 'application/vnd.github.full+json', + 'Content-Type': 'application/json', + 'Authorization': f'token {token}'}) + response.raise_for_status() + data = response.json() + pr_url = data['url'] + logging.info(f'Pull request is created:\n\t{pr_url}') + + +if __name__ == '__main__': + arg_values = parse_args() + if arg_values is None: + sys.exit(1) + setup_logging(arg_values.verbose) + setup_env() + + logging.info('checkout to the develop branch and pull the latest change...') + run_sh_cmd(['git', 'checkout', 'develop']) + run_sh_cmd(['git', 'pull']) + + logging.info('check out the release branch') + release_branch = get_release_branch_name(arg_values.version_str) + branch_exist = run_sh_cmd(['git','show-ref','--verify',f'refs/heads/{release_branch}'], False) + if not branch_exist: + run_sh_cmd(['git', 'checkout', '-b', release_branch]) + else: + run_sh_cmd(['git', 'checkout', release_branch]) + + replace_version(arg_values.version_str) + logging.info(f'verison is replaced with {arg_values.version_str}.') + + commit_version_files(arg_values.version_str) + + run_sh_cmd(['git', 'push', 'origin', release_branch]) + logging.info(f'release branch {release_branch} is pushed to remote repo.') + + if arg_values.token: + create_pr(arg_values.version_str, arg_values.token) + else: + pr_url = 'https://github.com/splunk/eventgen/compare' + logging.info('create pull reqeust manually by visiting this url:\n{pr_url}') From 4256b094e0be871d8fbc62e2e46b5ecbe39cd603 Mon Sep 17 00:00:00 2001 From: bigwhite Date: Thu, 31 Oct 2019 17:59:59 +0800 Subject: [PATCH 2/2] update CHANGELOG.md automatically --- docs/CHANGELOG.md | 143 ++++++++++--------------- release_tool/README.md | 6 +- release_tool/prepare_release_branch.py | 23 +++- 3 files changed, 78 insertions(+), 94 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 30f02118..39bae45d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,89 +1,54 @@ -6.5.0 -- Added metrics output mode -- Fixed regex token replacement issue -- Added test coverage information -- Increased functional test coverage -- Eventgen server complete revamp and standalone mode support -- Added contributor license -- Updated Dockerfile -- Added documentation -- Fixed bugs / stability / optimized speed - -6.4.0 -- Fixed exception log error -- Fixed CircleCI status badage error -- Fixed navigation error for app if installed with Splunk Stream -- Fixed generatorWorkers not working error -- Fixed interval error when end = 1 -- Fixed fileName in global stanza error -- Added 3rd party libs in SA-Eventgen App -- Added httpeventAllowFailureCount for httpevent -- Added 3rd party libs in license credit -- Disabled logging queue in multiprocess mode -- Changed implementation of extendIndex for better performance - -6.3.6 -- Added functional tests for jinja template and modular input feature -- Fixed default jinja template directory is not correctly resolved when sampleDir is set issue -- Fixed verbose flag not working in splunk_eventgen command line issue -- Fixed index, source, sourcetype are not correct when using splunkstream mode issue -- Fixed ssh to container issue -- Fixed perdayvolume without end setting error -- Updated documentation for better reading and remove unrelated part - -6.3.5 -- Added extendIndexes feature to support a list of indexes -- Fixed timer and token logic -- Changed end=-1 to continuously iterate without stopping -- Changed end=0 to not execute -- Added a linter for code quality -- Updated docs / docs format -- Added a suite of functional tests - -6.3.4: -- Documentation cleanup -- Jinja template bugfix in app -- Implementation of 'timeMultiple’ option -- Templates for bugs/feature requests -- Fixed Jinja test configuration stanzas -- Default behavior for 'count' edge cases - -6.3.3: -- Added performance metrics compared to Eventgen 5.x -- New config option for generation-time metrics: outputCounter -- Jinja template fixes -- Timestamp parsing fix -- Output queueing fix for outputMode splunkstream -- Count rater fixes, now supports indefinite generation - -6.3.2: -- Fixed verbosity bug -- Added documentation - -6.3.1: -- Fixed Eventgen Volume APIs -- Improved Eventgen Server Logging -- Corrected Eventgen Server and Controller conf syncing issue -- Adding verbosity options (ERROR, INFO, DEBUG) to Eventgen modinput -- Implemented future event generation support in replay mode -- Fixed Jinja template's missing default values -- Adjusted logging message levels for less verbosity -- Fixed event count off by 1 issue -- Fixed unnecessary empty data generators being created -- Updated dependency list - -6.3.0: -- Bug fixes for the customer issues -- Documentation upgrade -- Code refactoring for version unification -- Logging improvements - -6.2.1: -- Fixing SA-Eventgen Dashboard and log searching -- Improving internal logging and fixing splunkd logging issue -- Fixing timestamping in default generator -- Fixing custom plugin integration -- Fixing SA-Eventgen app settings -- Supporting Eventgen 5 backward compatibility with additional features -- Better modinput process management -- Minor Bugfixes with various customer cases +**7.0.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.0.0) + +**6.5.2**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.2) + + +**6.5.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.1) + + +**6.5.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.0) + +**6.4.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.4.0) + +**6.3.6**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.6) + +**6.3.5**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.5) + +**6.3.4**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.4) + +**6.3.3**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.3) + +**6.3.2**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.2) + +**6.3.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.1) + +**6.3.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.0) + +**6.2.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.2.1) + diff --git a/release_tool/README.md b/release_tool/README.md index 98203bc7..9cbecfe7 100644 --- a/release_tool/README.md +++ b/release_tool/README.md @@ -2,12 +2,14 @@ Use script to bump the release verison and create the release PR to merge to develop branch. -- If you have generated your github access token, use the following command to bump versions and send PR. +**Note: this script only works with python3.** + +- If you have generated your github access token, you can use the following command to bump versions and send PR automatically. ```bash python prepare_release_branch.py -v -n -a ``` -- If the access token is not given, this script only bumps the release version and push the commit to remote repo. You need to go to github web page to create your PR manually. +- If the access token is not given, this script only is only used to bump the release version and push the commit to remote repo. You need to go to github web page to create your PR manually. ``` python prepare_release_branch.py -v -n ``` diff --git a/release_tool/prepare_release_branch.py b/release_tool/prepare_release_branch.py index 689a2e08..f2d9df77 100644 --- a/release_tool/prepare_release_branch.py +++ b/release_tool/prepare_release_branch.py @@ -95,12 +95,29 @@ def replace_version(ver): fp.write(f'version = {ver}\n') else: fp.write(f'{lstr}\n') + logging.info(f'verison is replaced with {ver}.') -def commit_version_files(ver): + +def update_changelog(ver): + changelog_file = os.path.join(root_repo_dir, 'docs', 'CHANGELOG.md') + content = None + with open(changelog_file, 'r') as fp: + content = fp.readlines() + new_content = f'**{ver}**:\n\n' + f'- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/{ver})\n\n' + with open(changelog_file, 'w') as fp: + fp.write(new_content) + for l in content: + fp.write(l) + logging.info('CHANGELOG.md is updated.') + + +def commit_updated_files(ver): ver_json_file = os.path.join('splunk_eventgen', 'version.json') app_conf = os.path.join('splunk_eventgen', 'splunk_app', 'default', 'app.conf') + changelog = os.path.join('docs', 'CHANGELOG.md') run_sh_cmd(['git', 'add', ver_json_file]) run_sh_cmd(['git', 'add', app_conf]) + run_sh_cmd(['git', 'add', changelog]) run_sh_cmd(['git', 'commit', '-m', f'update eventgen version to {ver}'], False) logging.info('committed version files.') @@ -139,9 +156,9 @@ def create_pr(ver, token): run_sh_cmd(['git', 'checkout', release_branch]) replace_version(arg_values.version_str) - logging.info(f'verison is replaced with {arg_values.version_str}.') + update_changelog(arg_values.version_str) - commit_version_files(arg_values.version_str) + commit_updated_files(arg_values.version_str) run_sh_cmd(['git', 'push', 'origin', release_branch]) logging.info(f'release branch {release_branch} is pushed to remote repo.')