From caecce8023df06a56814a01b61392b224d0f6b1d Mon Sep 17 00:00:00 2001 From: tal66 <77445020+tal66@users.noreply.github.com> Date: Fri, 10 Jun 2022 22:05:10 +0300 Subject: [PATCH 1/4] CI: fetch packages info if server tests fail --- .github/workflows/build.yml | 8 +++ .../scripts/fetch_packages_metadata.py | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 .github/workflows/scripts/fetch_packages_metadata.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55e888809..cfee0564d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,15 @@ jobs: - name: Version Snapshot run: pip freeze - name: Test + id: Test run: make pytest + - name: Fetch packages metadata (Test step failed) + # Note: checking for general failure() first is needed according to docs + if: ${{ failure() && steps.Test.conclusion == 'failure' }} + env: + days: 7 # check for newer package versions in past X days + run: | + python3 ./.github/workflows/scripts/fetch_packages_metadata.py $days test-ui: name: Test UI diff --git a/.github/workflows/scripts/fetch_packages_metadata.py b/.github/workflows/scripts/fetch_packages_metadata.py new file mode 100644 index 000000000..72e6bf4e0 --- /dev/null +++ b/.github/workflows/scripts/fetch_packages_metadata.py @@ -0,0 +1,69 @@ +from requests import get +from datetime import datetime, timedelta +from concurrent.futures import ThreadPoolExecutor +from itertools import islice +import sys +import subprocess + + +def get_info_from_pypi(pkg): + url = f"https://pypi.org/pypi/{pkg}/json" + try: + res = get(url).json() + return res + except Exception as e: + print(f"request to pypi failed for <{pkg}>: {e}") + return None + + +def get_outdated_pkg_list(): + out = subprocess.check_output(["python", "-m", "pip", "list", "--outdated", "--format", "freeze"]) + req_list = out.decode().split("\n") + return req_list + + +def check_release_date(pkg_list): + for row in pkg_list: + try: + pkg, v = row.split("==") + except Exception: + print(f"skipping {row}") + continue + + res = get_info_from_pypi(pkg) + if not res: + continue + + latest = res["info"]["version"] + latest_upload_time = res["releases"][latest][0]["upload_time"] + + # continue if release older than argv[1] + today = datetime.utcnow() + release_date = datetime.fromisoformat(latest_upload_time) + if today - release_date > timedelta(days=max_days): + continue + + print(f"Newer version for <{pkg}>:") + print(f" - released {latest} {latest_upload_time[:10]}") + print(f" - current {v}") + print("") + + +if __name__ == "__main__": + max_days = 7 # default + if len(sys.argv) > 1: + max_days = int(sys.argv[1]) + + print(f"packages that were updated in the last {max_days} days:") + + pkg_list = get_outdated_pkg_list() + l = len(pkg_list) + workers = 3 + parts = [i for i in range(0, l, l // workers)] + parts[-1] = l + + with ThreadPoolExecutor(max_workers=workers) as executor: + executor.map( + check_release_date, + [islice(pkg_list, parts[i], parts[i + 1]) for i in range(len(parts) - 1)], + ) From 0b061de2b21f5a58784bea78b7c199b9404b7abf Mon Sep 17 00:00:00 2001 From: tal66 <77445020+tal66@users.noreply.github.com> Date: Fri, 10 Jun 2022 22:40:33 +0300 Subject: [PATCH 2/4] small print change --- .github/workflows/scripts/fetch_packages_metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/fetch_packages_metadata.py b/.github/workflows/scripts/fetch_packages_metadata.py index 72e6bf4e0..095627a2f 100644 --- a/.github/workflows/scripts/fetch_packages_metadata.py +++ b/.github/workflows/scripts/fetch_packages_metadata.py @@ -27,7 +27,8 @@ def check_release_date(pkg_list): try: pkg, v = row.split("==") except Exception: - print(f"skipping {row}") + if len(row) > 1: + print(f"skipping {row}") continue res = get_info_from_pypi(pkg) @@ -55,6 +56,7 @@ def check_release_date(pkg_list): max_days = int(sys.argv[1]) print(f"packages that were updated in the last {max_days} days:") + print() pkg_list = get_outdated_pkg_list() l = len(pkg_list) From d5f6c7cdaae3fe0273a1616e81f993d347ca834f Mon Sep 17 00:00:00 2001 From: tal66 <77445020+tal66@users.noreply.github.com> Date: Sat, 11 Jun 2022 00:11:47 +0300 Subject: [PATCH 3/4] postpone printing because of threading --- .../workflows/scripts/fetch_packages_metadata.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/fetch_packages_metadata.py b/.github/workflows/scripts/fetch_packages_metadata.py index 095627a2f..a1a986837 100644 --- a/.github/workflows/scripts/fetch_packages_metadata.py +++ b/.github/workflows/scripts/fetch_packages_metadata.py @@ -1,7 +1,7 @@ from requests import get from datetime import datetime, timedelta from concurrent.futures import ThreadPoolExecutor -from itertools import islice +from itertools import islice, chain import sys import subprocess @@ -23,6 +23,7 @@ def get_outdated_pkg_list(): def check_release_date(pkg_list): + pkgs_output = [] for row in pkg_list: try: pkg, v = row.split("==") @@ -44,10 +45,10 @@ def check_release_date(pkg_list): if today - release_date > timedelta(days=max_days): continue - print(f"Newer version for <{pkg}>:") - print(f" - released {latest} {latest_upload_time[:10]}") - print(f" - current {v}") - print("") + pkgs_output.append( + f"Newer version for <{pkg}>:\n - released {latest} {latest_upload_time[:10]}\n - current {v}\n" + ) + return pkgs_output if __name__ == "__main__": @@ -65,7 +66,10 @@ def check_release_date(pkg_list): parts[-1] = l with ThreadPoolExecutor(max_workers=workers) as executor: - executor.map( + results = executor.map( check_release_date, [islice(pkg_list, parts[i], parts[i + 1]) for i in range(len(parts) - 1)], ) + + for item in chain(*results): + print(item) From 0b406a75d123a93caf279d7f313f912eb85a8cfa Mon Sep 17 00:00:00 2001 From: tal66 <77445020+tal66@users.noreply.github.com> Date: Sat, 11 Jun 2022 10:16:37 +0300 Subject: [PATCH 4/4] changed pkg list to pip freeze, formatting --- .github/workflows/build.yml | 2 +- .../scripts/fetch_packages_metadata.py | 35 +++++++++++-------- Makefile | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfee0564d..65ccef689 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,7 +94,7 @@ jobs: - name: Test id: Test run: make pytest - - name: Fetch packages metadata (Test step failed) + - name: Fetch packages metadata (if Test step failed) # Note: checking for general failure() first is needed according to docs if: ${{ failure() && steps.Test.conclusion == 'failure' }} env: diff --git a/.github/workflows/scripts/fetch_packages_metadata.py b/.github/workflows/scripts/fetch_packages_metadata.py index a1a986837..3137e0b4b 100644 --- a/.github/workflows/scripts/fetch_packages_metadata.py +++ b/.github/workflows/scripts/fetch_packages_metadata.py @@ -1,28 +1,31 @@ -from requests import get -from datetime import datetime, timedelta from concurrent.futures import ThreadPoolExecutor -from itertools import islice, chain -import sys +from datetime import datetime, timedelta +from itertools import chain, islice import subprocess +import sys + +from packaging import version +import requests def get_info_from_pypi(pkg): url = f"https://pypi.org/pypi/{pkg}/json" try: - res = get(url).json() + res = requests.get(url).json() return res - except Exception as e: - print(f"request to pypi failed for <{pkg}>: {e}") + except Exception as err: + print(f"request to pypi failed for <{pkg}>: {err}") return None -def get_outdated_pkg_list(): - out = subprocess.check_output(["python", "-m", "pip", "list", "--outdated", "--format", "freeze"]) +def get_pkg_list(): + out = subprocess.check_output(["python", "-m", "pip", "freeze"]) req_list = out.decode().split("\n") return req_list def check_release_date(pkg_list): + """returns packages that were updated in the past `max_days` days, formatted for printing""" pkgs_output = [] for row in pkg_list: try: @@ -45,8 +48,12 @@ def check_release_date(pkg_list): if today - release_date > timedelta(days=max_days): continue + sign = ">" + if version.parse(latest) == version.parse(v): + sign = "=" + pkgs_output.append( - f"Newer version for <{pkg}>:\n - released {latest} {latest_upload_time[:10]}\n - current {v}\n" + f"Newer version for <{pkg}>:\n - released {latest} {latest_upload_time[:10]}\t({sign})\n - current {v}\n" ) return pkgs_output @@ -56,13 +63,13 @@ def check_release_date(pkg_list): if len(sys.argv) > 1: max_days = int(sys.argv[1]) - print(f"packages that were updated in the last {max_days} days:") - print() + print(f"Packages that were updated in the last {max_days} days:\n") - pkg_list = get_outdated_pkg_list() + pkg_list = get_pkg_list() l = len(pkg_list) + workers = 3 - parts = [i for i in range(0, l, l // workers)] + parts = list(range(0, l, l // workers)) parts[-1] = l with ThreadPoolExecutor(max_workers=workers) as executor: diff --git a/Makefile b/Makefile index 735a87bcb..3f842583a 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ lint-dependencies: @$(PYTHON_PIP) install -q -r lint_requirements.txt lint-server: lint-dependencies - $(PYTHON) -m flake8 elyra + $(PYTHON) -m flake8 elyra .github @echo $(BLACK_CMD) @$(BLACK_CMD) || (echo "Black formatting encountered issues. Use 'make black-format' to apply the suggested changes."; exit 1)