diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55e888809..65ccef689 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,15 @@ jobs: - name: Version Snapshot run: pip freeze - name: Test + id: Test run: make pytest + - name: Fetch packages metadata (if Test step failed) + # Note: checking for general failure() first is needed according to docs + if: ${{ failure() && steps.Test.conclusion == 'failure' }} + env: + days: 7 # check for newer package versions in past X days + run: | + python3 ./.github/workflows/scripts/fetch_packages_metadata.py $days test-ui: name: Test UI diff --git a/.github/workflows/scripts/fetch_packages_metadata.py b/.github/workflows/scripts/fetch_packages_metadata.py new file mode 100644 index 000000000..3137e0b4b --- /dev/null +++ b/.github/workflows/scripts/fetch_packages_metadata.py @@ -0,0 +1,82 @@ +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +from itertools import chain, islice +import subprocess +import sys + +from packaging import version +import requests + + +def get_info_from_pypi(pkg): + url = f"https://pypi.org/pypi/{pkg}/json" + try: + res = requests.get(url).json() + return res + except Exception as err: + print(f"request to pypi failed for <{pkg}>: {err}") + return None + + +def get_pkg_list(): + out = subprocess.check_output(["python", "-m", "pip", "freeze"]) + req_list = out.decode().split("\n") + return req_list + + +def check_release_date(pkg_list): + """returns packages that were updated in the past `max_days` days, formatted for printing""" + pkgs_output = [] + for row in pkg_list: + try: + pkg, v = row.split("==") + except Exception: + if len(row) > 1: + print(f"skipping {row}") + continue + + res = get_info_from_pypi(pkg) + if not res: + continue + + latest = res["info"]["version"] + latest_upload_time = res["releases"][latest][0]["upload_time"] + + # continue if release older than argv[1] + today = datetime.utcnow() + release_date = datetime.fromisoformat(latest_upload_time) + if today - release_date > timedelta(days=max_days): + continue + + sign = ">" + if version.parse(latest) == version.parse(v): + sign = "=" + + pkgs_output.append( + f"Newer version for <{pkg}>:\n - released {latest} {latest_upload_time[:10]}\t({sign})\n - current {v}\n" + ) + return pkgs_output + + +if __name__ == "__main__": + max_days = 7 # default + if len(sys.argv) > 1: + max_days = int(sys.argv[1]) + + print(f"Packages that were updated in the last {max_days} days:\n") + + pkg_list = get_pkg_list() + l = len(pkg_list) + + workers = 3 + parts = list(range(0, l, l // workers)) + parts[-1] = l + + with ThreadPoolExecutor(max_workers=workers) as executor: + results = executor.map( + check_release_date, + [islice(pkg_list, parts[i], parts[i + 1]) for i in range(len(parts) - 1)], + ) + + for item in chain(*results): + print(item) diff --git a/Makefile b/Makefile index 735a87bcb..3f842583a 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ lint-dependencies: @$(PYTHON_PIP) install -q -r lint_requirements.txt lint-server: lint-dependencies - $(PYTHON) -m flake8 elyra + $(PYTHON) -m flake8 elyra .github @echo $(BLACK_CMD) @$(BLACK_CMD) || (echo "Black formatting encountered issues. Use 'make black-format' to apply the suggested changes."; exit 1)