CI: fetch packages info if server tests fail

.github/workflows/build.yml

@@ -92,7 +92,15 @@ jobs:
       - name: Version Snapshot
         run: pip freeze
       - name: Test
+        id: Test
         run: make pytest
+      - name: Fetch packages metadata (if Test step failed)
+        # Note: checking for general failure() first is needed according to docs
+        if: ${{ failure() && steps.Test.conclusion == 'failure' }}
+        env:
+          days: 7 # check for newer package versions in past X days
+        run: |
+          python3 ./.github/workflows/scripts/fetch_packages_metadata.py $days
 
   test-ui:
     name: Test UI

.github/workflows/scripts/fetch_packages_metadata.py

@@ -0,0 +1,82 @@
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+from itertools import chain, islice
+import subprocess
+import sys
+
+from packaging import version
+import requests
+
+
+def get_info_from_pypi(pkg):
+    url = f"https://pypi.org/pypi/{pkg}/json"
+    try:
+        res = requests.get(url).json()
+        return res
+    except Exception as err:
+        print(f"request to pypi failed for <{pkg}>: {err}")
+        return None
+
+
+def get_pkg_list():
+    out = subprocess.check_output(["python", "-m", "pip", "freeze"])
+    req_list = out.decode().split("\n")
+    return req_list
+
+
+def check_release_date(pkg_list):
+    """returns packages that were updated in the past `max_days` days, formatted for printing"""
+    pkgs_output = []
+    for row in pkg_list:
+        try:
+            pkg, v = row.split("==")
+        except Exception:
+            if len(row) > 1:
+                print(f"skipping {row}")
+            continue
+
+        res = get_info_from_pypi(pkg)
+        if not res:
+            continue
+
+        latest = res["info"]["version"]
+        latest_upload_time = res["releases"][latest][0]["upload_time"]
+
+        # continue if release older than argv[1]
+        today = datetime.utcnow()
+        release_date = datetime.fromisoformat(latest_upload_time)
+        if today - release_date > timedelta(days=max_days):
+            continue
+
+        sign = ">"
+        if version.parse(latest) == version.parse(v):
+            sign = "="
+
+        pkgs_output.append(
+            f"Newer version for <{pkg}>:\n - released {latest}  {latest_upload_time[:10]}\t({sign})\n - current  {v}\n"
+        )
+    return pkgs_output
+
+
+if __name__ == "__main__":
+    max_days = 7  # default
+    if len(sys.argv) > 1:
+        max_days = int(sys.argv[1])
+
+    print(f"Packages that were updated in the last {max_days} days:\n")
+
+    pkg_list = get_pkg_list()
+    l = len(pkg_list)
+
+    workers = 3
+    parts = list(range(0, l, l // workers))
+    parts[-1] = l
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        results = executor.map(
+            check_release_date,
+            [islice(pkg_list, parts[i], parts[i + 1]) for i in range(len(parts) - 1)],
+        )
+
+    for item in chain(*results):
+        print(item)

Makefile

@@ -109,7 +109,7 @@ lint-dependencies:
 	@$(PYTHON_PIP) install -q -r lint_requirements.txt
 
 lint-server: lint-dependencies
-	$(PYTHON) -m flake8 elyra
+	$(PYTHON) -m flake8 elyra .github
 	@echo $(BLACK_CMD)
 	@$(BLACK_CMD) || (echo "Black formatting encountered issues.  Use 'make black-format' to apply the suggested changes."; exit 1)