diff --git a/.gitignore b/.gitignore index 6367b4b3..a1aed8ab 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ _build env.sh *.swp .libraries/* +.gitlibs/* .cp_org/* .blinka/* .vscode diff --git a/tools/ci_status.py b/tools/ci_status.py index be3061b9..c7d830a9 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -16,8 +16,10 @@ from typing import Optional import argparse +import time from github.Repository import Repository from github.Workflow import Workflow +from github.WorkflowRun import WorkflowRun from github.GithubException import GithubException from library_functions import StrPath from iterate_libraries import ( @@ -56,6 +58,46 @@ def run_gh_rest_check( return workflow_runs[0].conclusion +def run_gh_rest_rerun( + lib_repo: Repository, + user: Optional[str] = None, + branch: Optional[str] = None, + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, +) -> bool: + """Uses ``PyGithub`` to rerun the CI status of a repository + + :param Repository lib_repo: The repo as a github.Repository.Repository object + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` is + provided, any workflow name is acceptable; the default is ``"build.yml"`` + :param int rerun_level: The level at which rerun should occur (0 = none, + 1 = failed, 2 = all) + :return: The requested runs conclusion + :rtype: bool + """ + if not rerun_level: + return False + if rerun_level == 1: + result = ( + run_gh_rest_check(lib_repo, user, branch, workflow_filename) == "success" + ) + if rerun_level == 2 or not result: + arg_dict = {} + if user is not None: + arg_dict["actor"] = user + if branch is not None: + arg_dict["branch"] = branch + workflow: Workflow = lib_repo.get_workflow(workflow_filename) + latest_run: WorkflowRun = workflow.get_runs(**arg_dict)[0] + latest_run.rerun() + return True + return False + + def check_build_status( lib_repo: Repository, user: Optional[str] = None, @@ -105,6 +147,55 @@ def check_build_status( return None +# pylint: disable=too-many-arguments +def rerun_workflow( + lib_repo: Repository, + user: Optional[str] = None, + branch: Optional[str] = None, + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, + debug: bool = False, +): + """Uses ``PyGithub`` to rerun the CI of the Adafruit + CircuitPython Bundle repositories + + :param Repository lib_repo: The repo as a github.Repository.Repository object + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` + is provided, any workflow name is acceptable; the defail is `"build.yml"` + :param int rerun_level: The level at which rerun should occur (0 = none, + 1 = failed, 2 = all) + :param bool debug: Whether debug statements should be printed to the standard + output + :return: The result of the workflow run, or ``None`` if it could not be + determined + :rtype: bool|None + """ + if lib_repo.archived: + return False + + try: + result = run_gh_rest_rerun( + lib_repo, user, branch, workflow_filename, rerun_level + ) + if debug and result: + print("***", "Library", lib_repo.name, "workflow was rerun!", "***") + return result + except GithubException: + if debug: + print( + "???", + "Library", + lib_repo.name, + "had an issue occur", + "???", + ) + return None + + def check_build_statuses( gh_token: str, user: Optional[str] = None, @@ -112,6 +203,7 @@ def check_build_statuses( workflow_filename: Optional[str] = "build.yml", *, debug: bool = False, + local_folder: str = "", ) -> list[RemoteLibFunc_IterResult[bool]]: """Checks all the libraries in the Adafruit CircuitPython Bundle to get the latest build status with the requested information @@ -125,6 +217,7 @@ def check_build_statuses( provided, any workflow name is acceptable; the defail is `"build.yml"` :param bool debug: Whether debug statements should be printed to the standard output + :param str local_folder: A path to a local folder containing extra repositories :return: A list of tuples containing paired Repoistory objects and build statuses :rtype: list @@ -133,6 +226,49 @@ def check_build_statuses( return iter_remote_bundle_with_func( gh_token, [(check_build_status, (user, branch, workflow_filename), {"debug": debug})], + local_folder=local_folder, + ) + + +def rerun_workflows( + gh_token: str, + user: Optional[str] = None, + branch: Optional[str] = "main", + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, + *, + debug: bool = False, + local_folder: str = "", +) -> list[RemoteLibFunc_IterResult[bool]]: + """Reruns the CI of all the libraries in the Adafruit CircuitPython Bundle. + + :param str gh_token: The Github token to be used for with the Github API + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` is + provided, any workflow name is acceptable; the defail is `"build.yml"` + :param int rerun_level: The level at which reruns should occur (0 = none, + 1 = failed, 2 = all) + :param bool debug: Whether debug statements should be printed to + the standard output + :param str local_folder: A path to a local folder containing extra repositories + :return: A list of tuples containing paired Repoistory objects and build + statuses + :rtype: list + """ + + return iter_remote_bundle_with_func( + gh_token, + [ + ( + rerun_workflow, + (user, branch, workflow_filename, rerun_level), + {"debug": debug}, + ) + ], + local_folder=local_folder, ) @@ -193,12 +329,52 @@ def save_build_statuses( parser.add_argument( "--debug", action="store_true", help="Print debug text during execution" ) + parser.add_argument( + "--rerun-level", + metavar="R", + type=int, + dest="rerun_level", + default=0, + help="Level to rerun CI workflows (0 = none, 1 = failed, 2 = all)", + ) + parser.add_argument( + "--local-folder", + metavar="L", + type=str, + dest="local_folder", + default="", + help="An additional folder to check and run", + ) args = parser.parse_args() + if args.rerun_level: + if args.debug: + print("Rerunning workflows...") + rerun_workflows( + args.gh_token, + args.user, + args.branch, + args.workflow, + args.rerun_level, + debug=args.debug, + local_folder=args.local_folder, + ) + if args.debug: + print("Waiting 10 minutes to allow workflows to finish running...") + time.sleep(600) + + if args.debug: + print("Checking workflows statuses...") results = check_build_statuses( - args.gh_token, args.user, args.branch, args.workflow, debug=args.debug + args.gh_token, + args.user, + args.branch, + args.workflow, + debug=args.debug, + local_folder=args.local_folder, ) + fail_list = [ repo_name.name for repo_name, repo_results in results if not repo_results[0] ] diff --git a/tools/iterate_libraries.py b/tools/iterate_libraries.py index 6440b781..d54996e7 100644 --- a/tools/iterate_libraries.py +++ b/tools/iterate_libraries.py @@ -16,8 +16,9 @@ import os import glob +import pathlib from collections.abc import Sequence, Iterable -from typing import TypeVar +from typing import TypeVar, Any, Union, List from typing_extensions import TypeAlias import parse from github import Github @@ -65,9 +66,25 @@ _BUNDLE_BRANCHES = ("drivers", "helpers") +def perform_func( + item: Any, + func_workflow: Union[RemoteLibFunc_IterInstruction, LocalLibFunc_IterInstruction], +) -> Union[List[RemoteLibFunc_IterResult], List[LocalLibFunc_IterResult]]: + """ + Perform the given function + """ + func_results = [] + for func, args, kwargs in func_workflow: + result = func(item, *args, **kwargs) + func_results.append(result) + return func_results + + def iter_local_bundle_with_func( bundle_path: StrPath, func_workflow: Iterable[LocalLibFunc_IterInstruction], + *, + local_folder: str = "", ) -> list[LocalLibFunc_IterResult]: """Iterate through the libraries and run a given function with the provided arguments @@ -85,6 +102,9 @@ def iter_local_bundle_with_func( # Initialize list of results results = [] + # Keep track of all libraries iterated + iterated = set() + # Loop through each bundle branch for branch_name in _BUNDLE_BRANCHES: @@ -94,20 +114,30 @@ def iter_local_bundle_with_func( # Enter each library in the bundle for library_path in libraries_path_list: - func_results = [] - - for func, args, kwargs in func_workflow: - result = func(library_path, *args, **kwargs) - func_results.append(result) + iterated.add(os.path.split(library_path)[1].lower()) + func_results = perform_func(library_path, func_workflow) results.append((library_path, func_results)) + if local_folder: + additional = { + os.path.split(pathname)[1].lower() + for pathname in glob.glob(os.path.join(local_folder, "*")) + } + diff = additional.difference(iterated) + for unused in diff: + unused_func_results = perform_func(unused, func_workflow) + results.append((unused, unused_func_results)) + return results # pylint: disable=too-many-locals def iter_remote_bundle_with_func( - gh_token: str, func_workflow: RemoteLibFunc_IterInstruction + gh_token: str, + func_workflow: RemoteLibFunc_IterInstruction, + *, + local_folder: str = "", ) -> list[RemoteLibFunc_IterResult]: """Iterate through the remote bundle, accessing each library's git repo using the GitHub RESTful API (specifically using ``PyGithub``) @@ -129,6 +159,9 @@ def iter_remote_bundle_with_func( # Initialize list of results results = [] + # Keep track of all libraries iterated + iterated = set() + # Loop through each bundle branch for branch_name in _BUNDLE_BRANCHES: @@ -144,13 +177,19 @@ def iter_remote_bundle_with_func( repo_name: str = repo_name_result.named["repo_name"] repo = github_client.get_repo(f"adafruit/{repo_name}") + iterated.add(repo_name.lower()) - func_results = [] - - for func, args, kwargs in func_workflow: - result = func(repo, *args, **kwargs) - func_results.append(result) - + func_results = perform_func(repo, func_workflow) results.append((repo, func_results)) + if local_folder: + additional = { + path.name.lower() for path in pathlib.Path(local_folder).glob("*") + } + diff = additional.difference(iterated) + for unused in diff: + unused_repo = github_client.get_repo(f"adafruit/{unused}") + unused_func_results = perform_func(unused_repo, func_workflow) + results.append((unused_repo, unused_func_results)) + return results diff --git a/tools/run_black.sh b/tools/run_black.sh new file mode 100644 index 00000000..a85bf7ad --- /dev/null +++ b/tools/run_black.sh @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2023 Alec Delaney +# +# SPDX-License-Identifier: MIT + +rm -rf .gitlibs +mkdir .gitlibs +cd .libraries +for repo in *; do + cd ../.gitlibs + git clone https://github.com/adafruit/$repo.git + cd $repo + pre-commit run --all-files + git add -A + git commit -m "Run pre-commit" + git push + cd .. +done