diff --git a/adabot/__init__.py b/adabot/__init__.py index e69de29..39fe1f0 100644 --- a/adabot/__init__.py +++ b/adabot/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2023 Tim Cocks +# +# SPDX-License-Identifier: MIT + +"""AdaBot is a friendly helper bot that works across the web to make people's +lives better.""" + +REQUESTS_TIMEOUT = 30 diff --git a/adabot/arduino_libraries.py b/adabot/arduino_libraries.py index adbeb3b..d0d9339 100644 --- a/adabot/arduino_libraries.py +++ b/adabot/arduino_libraries.py @@ -11,7 +11,7 @@ import requests -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT logger = logging.getLogger(__name__) ch = logging.StreamHandler(stream=sys.stdout) @@ -86,7 +86,8 @@ def is_arduino_library(repo): + repo["name"] + "/" + repo["default_branch"] - + "/library.properties" + + "/library.properties", + timeout=REQUESTS_TIMEOUT, ) return lib_prop_file.ok @@ -116,7 +117,8 @@ def validate_library_properties(repo): + repo["name"] + "/" + repo["default_branch"] - + "/library.properties" + + "/library.properties", + timeout=REQUESTS_TIMEOUT, ) if not lib_prop_file.ok: # print("{} skipped".format(repo["name"])) @@ -193,7 +195,8 @@ def validate_actions(repo): + repo["name"] + "/" + repo["default_branch"] - + "/.github/workflows/githubci.yml" + + "/.github/workflows/githubci.yml", + timeout=REQUESTS_TIMEOUT, ) return repo_has_actions.ok @@ -309,7 +312,10 @@ def main(verbosity=1, output_file=None): # pylint: disable=missing-function-doc logger.setLevel("CRITICAL") try: - reply = requests.get("http://downloads.arduino.cc/libraries/library_index.json") + reply = requests.get( + "http://downloads.arduino.cc/libraries/library_index.json", + timeout=REQUESTS_TIMEOUT, + ) if not reply.ok: logging.error( "Could not fetch http://downloads.arduino.cc/libraries/library_index.json" diff --git a/adabot/circuitpython_libraries.py b/adabot/circuitpython_libraries.py index 7e9f2a6..59c8e75 100644 --- a/adabot/circuitpython_libraries.py +++ b/adabot/circuitpython_libraries.py @@ -17,7 +17,7 @@ import github as pygithub import requests -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT from adabot import pypi_requests as pypi from adabot.lib import circuitpython_library_validators as cirpy_lib_vals from adabot.lib import common_funcs @@ -223,7 +223,7 @@ def run_library_checks(validators, kw_args, error_depth): resp = requests.get( "https://raw.githubusercontent.com/adafruit/" "CircuitPython_Community_Bundle/main/.gitmodules", - timeout=30, + timeout=REQUESTS_TIMEOUT, ) community_bundle_submodules = resp.text community_library_count = community_bundle_submodules.count("submodule") diff --git a/adabot/circuitpython_library_download_stats.py b/adabot/circuitpython_library_download_stats.py index ec61559..dc5bb17 100644 --- a/adabot/circuitpython_library_download_stats.py +++ b/adabot/circuitpython_library_download_stats.py @@ -17,7 +17,7 @@ from google.cloud import bigquery import google.oauth2.service_account -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT from adabot.lib import common_funcs # Setup ArgumentParser @@ -60,7 +60,7 @@ def retrieve_piwheels_stats(): """Get data dump of piwheels download stats""" stats = {} - response = requests.get(PIWHEELS_PACKAGES_URL) + response = requests.get(PIWHEELS_PACKAGES_URL, timeout=REQUESTS_TIMEOUT) if response.ok: packages = response.json() stats = { diff --git a/adabot/circuitpython_library_patches.py b/adabot/circuitpython_library_patches.py index 19002cd..b4a7b3e 100644 --- a/adabot/circuitpython_library_patches.py +++ b/adabot/circuitpython_library_patches.py @@ -13,6 +13,7 @@ import sh from sh.contrib import git +from adabot import REQUESTS_TIMEOUT from adabot.lib import common_funcs @@ -100,7 +101,8 @@ def get_patches(run_local): return_list = [] if not run_local: contents = requests.get( - "https://api.github.com/repos/adafruit/adabot/contents/patches" + "https://api.github.com/repos/adafruit/adabot/contents/patches", + timeout=REQUESTS_TIMEOUT ) if contents.ok: for patch in contents.json(): diff --git a/adabot/circuitpython_library_release.py b/adabot/circuitpython_library_release.py index 8177ee4..918f078 100644 --- a/adabot/circuitpython_library_release.py +++ b/adabot/circuitpython_library_release.py @@ -48,7 +48,7 @@ def make_release(new_tag, logger, test_run=False): def get_pypi_name(): """ - return the shorthand pypi project name + return the shorthand project name used for pypi, docs, etc. """ data = toml.load("pyproject.toml") @@ -126,7 +126,7 @@ def get_release_info(): } -def get_compare_url(tag_name): +def get_compare_url(tag_name, compare_to_tag_name="main"): """ Get the URL to the GitHub compare page for the latest release compared to current main. @@ -138,7 +138,9 @@ def get_compare_url(tag_name): if not remote_url.startswith("https"): return "Sorry, Unknown Remotes" - compare_url = remote_url.replace(".git", f"/compare/{tag_name}...main") + compare_url = remote_url.replace( + ".git", f"/compare/{tag_name}...{compare_to_tag_name}" + ) return compare_url diff --git a/adabot/lib/assign_hacktober_label.py b/adabot/lib/assign_hacktober_label.py index 1824ed9..a4763c2 100644 --- a/adabot/lib/assign_hacktober_label.py +++ b/adabot/lib/assign_hacktober_label.py @@ -11,7 +11,7 @@ import datetime import requests -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT from adabot.lib import common_funcs cli_args = argparse.ArgumentParser(description="Hacktoberfest Label Assigner") @@ -73,7 +73,9 @@ def get_open_issues(repo): ) if response.links.get("next"): - response = requests.get(response.links["next"]["url"]) + response = requests.get( + response.links["next"]["url"], timeout=REQUESTS_TIMEOUT + ) else: break diff --git a/adabot/lib/circuitpython_library_validators.py b/adabot/lib/circuitpython_library_validators.py index d08f973..bcf668b 100644 --- a/adabot/lib/circuitpython_library_validators.py +++ b/adabot/lib/circuitpython_library_validators.py @@ -21,7 +21,7 @@ import parse import github as pygithub -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT from adabot.lib import common_funcs from adabot.lib import assign_hacktober_label as hacktober @@ -231,7 +231,7 @@ def rtd_yml_base(self): "%20if%20cookiecutter.sphinx_docs%20in%20%5B'y'%2C%20'yes'%5D%20%25" "%7D.readthedocs.yaml%7B%25%20endif%20%25%7D" ) - rtd_yml = requests.get(rtd_yml_dl_url) + rtd_yml = requests.get(rtd_yml_dl_url, timeout=REQUESTS_TIMEOUT) if rtd_yml.ok: try: self._rtd_yaml_base = yaml.safe_load(rtd_yml.text) @@ -255,7 +255,7 @@ def pcc_versions(self): "circuitpython/main/%7B%7B%20cookiecutter.__dirname%20%7D%7D/.pre-" "commit-config.yaml" ) - pcc_yml = requests.get(pcc_yml_dl_url) + pcc_yml = requests.get(pcc_yml_dl_url, timeout=REQUESTS_TIMEOUT) if pcc_yml.ok: try: pcc_yaml_base = yaml.safe_load(pcc_yml.text) @@ -463,7 +463,7 @@ def _filter_file_diffs(filenames): def _validate_readme(self, download_url): # We use requests because file contents are hosted by # githubusercontent.com, not the API domain. - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_README_DOWNLOAD_FAILED] @@ -509,7 +509,7 @@ def _validate_py_for_u_modules(self, download_url): """ # We use requests because file contents are hosted by # githubusercontent.com, not the API domain. - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_PYFILE_DOWNLOAD_FAILED] @@ -547,7 +547,7 @@ def _validate_actions_build_yml(self, actions_build_info): """ download_url = actions_build_info["download_url"] - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_PYFILE_DOWNLOAD_FAILED] @@ -557,7 +557,7 @@ def _validate_actions_build_yml(self, actions_build_info): def _validate_pre_commit_config_yaml(self, file_info): download_url = file_info["download_url"] - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_PYFILE_DOWNLOAD_FAILED] @@ -594,7 +594,7 @@ def _validate_pre_commit_config_yaml(self, file_info): def _validate_pyproject_toml(self, file_info): """Check pyproject.toml for pypi compatibility""" download_url = file_info["download_url"] - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_TOMLFILE_DOWNLOAD_FAILED] return [] @@ -602,7 +602,7 @@ def _validate_pyproject_toml(self, file_info): def _validate_requirements_txt(self, repo, file_info, check_blinka=True): """Check requirements.txt for pypi compatibility""" download_url = file_info["download_url"] - contents = requests.get(download_url, timeout=30) + contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT) if not contents.ok: return [ERROR_PYFILE_DOWNLOAD_FAILED] @@ -717,7 +717,9 @@ def validate_contents(self, repo): if ".readthedocs.yaml" in files: filename = ".readthedocs.yaml" file_info = content_list[files.index(filename)] - rtd_contents = requests.get(file_info["download_url"]) + rtd_contents = requests.get( + file_info["download_url"], timeout=REQUESTS_TIMEOUT + ) if rtd_contents.ok: try: rtd_yml = yaml.safe_load(rtd_contents.text) @@ -735,7 +737,9 @@ def validate_contents(self, repo): if len(self._pcc_versions) or self.pcc_versions != "": filename = ".pre-commit-config.yaml" file_info = content_list[files.index(filename)] - pcc_contents = requests.get(file_info["download_url"]) + pcc_contents = requests.get( + file_info["download_url"], timeout=REQUESTS_TIMEOUT + ) if pcc_contents.ok: try: pcc_yml = yaml.safe_load(pcc_contents.text) @@ -888,7 +892,8 @@ def validate_readthedocs(self, repo): return [] if not self.rtd_subprojects: rtd_response = requests.get( - "https://readthedocs.org/api/v2/project/74557/subprojects/", timeout=15 + "https://readthedocs.org/api/v2/project/74557/subprojects/", + timeout=REQUESTS_TIMEOUT, ) if not rtd_response.ok: return [ERROR_RTD_SUBPROJECT_FAILED] @@ -937,7 +942,7 @@ def validate_readthedocs(self, repo): url = f"https://readthedocs.org/api/v3/projects/{rtd_slug}/builds/" rtd_token = os.environ["RTD_TOKEN"] headers = {"Authorization": f"token {rtd_token}"} - response = requests.get(url, headers=headers) + response = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT) json_response = response.json() error_message = json_response.get("detail") @@ -981,7 +986,7 @@ def validate_core_driver_page(self, repo): "https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/" "main/docs/drivers.rst" ), - timeout=15, + timeout=REQUESTS_TIMEOUT, ) if not driver_page.ok: return [ERROR_DRIVERS_PAGE_DOWNLOAD_FAILED] diff --git a/adabot/lib/common_funcs.py b/adabot/lib/common_funcs.py index 554fa4c..65b9b09 100644 --- a/adabot/lib/common_funcs.py +++ b/adabot/lib/common_funcs.py @@ -12,7 +12,7 @@ import os import re import requests -from adabot import github_requests as gh_reqs +from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT from adabot import pypi_requests as pypi CORE_REPO_URL = "/repos/adafruit/circuitpython" @@ -96,7 +96,7 @@ def get_bundle_submodules(): # master branch of the bundle is the canonical source of the bundle release. result = requests.get( "https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/main/.gitmodules", - timeout=15, + timeout=REQUESTS_TIMEOUT, ) if result.status_code != 200: # output_handler("Failed to access bundle .gitmodules file from GitHub!", quiet=True) diff --git a/adabot/pypi_requests.py b/adabot/pypi_requests.py index 59c3b45..00a8076 100644 --- a/adabot/pypi_requests.py +++ b/adabot/pypi_requests.py @@ -9,6 +9,8 @@ import requests +from adabot import REQUESTS_TIMEOUT + def _fix_url(url): if url.startswith("/"): @@ -18,4 +20,4 @@ def _fix_url(url): def get(url, **kwargs): """Process a GET request from pypi.org""" - return requests.get(_fix_url(url), timeout=30, **kwargs) + return requests.get(_fix_url(url), timeout=REQUESTS_TIMEOUT, **kwargs) diff --git a/tools/docs_status.py b/tools/docs_status.py index 985c61e..97d1cbb 100644 --- a/tools/docs_status.py +++ b/tools/docs_status.py @@ -21,10 +21,12 @@ import requests from github.Repository import Repository from github.ContentFile import ContentFile + from iterate_libraries import ( iter_remote_bundle_with_func, RemoteLibFunc_IterResult, ) +from adabot import REQUESTS_TIMEOUT def check_docs_status( @@ -64,7 +66,7 @@ def check_docs_status( # GET the latest documentation build runs url = f"https://readthedocs.org/api/v3/projects/{rtd_slug}/builds/" headers = {"Authorization": f"token {rtd_token}"} - response = requests.get(url, headers=headers) + response = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT) json_response: dict[str, Any] = response.json() # Return the results of the latest run diff --git a/tools/file_compare.py b/tools/file_compare.py index d3b8b55..705c5c0 100644 --- a/tools/file_compare.py +++ b/tools/file_compare.py @@ -20,6 +20,7 @@ import requests from requests.structures import CaseInsensitiveDict +from adabot import REQUESTS_TIMEOUT from adabot.lib.common_funcs import list_repos @@ -54,9 +55,9 @@ def compare(git_file: str, token: Optional[str] = None) -> list: headers = CaseInsensitiveDict() headers["Authorization"] = f"token {token}" - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT) else: - resp = requests.get(url) + resp = requests.get(url, timeout=REQUESTS_TIMEOUT) if resp.status_code != 200: print(name) diff --git a/tools/find_text.py b/tools/find_text.py index 5b994f8..8305a5a 100644 --- a/tools/find_text.py +++ b/tools/find_text.py @@ -19,6 +19,7 @@ import requests +from adabot import REQUESTS_TIMEOUT from adabot.lib.common_funcs import list_repos argumentList = sys.argv[1:] @@ -129,7 +130,9 @@ def prettyprint(info, results): for repo in all_repos: INFO = "getting {} for: {}".format(FILE, repo["name"]) - response = requests.get(URL_TEMPLATE.format(repo["name"], FILE)) + response = requests.get( + URL_TEMPLATE.format(repo["name"], FILE), timeout=REQUESTS_TIMEOUT + ) result = [] if response.status_code == 404: RESULTS["file_not_found"].append(repo["html_url"])