Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix sync. Again. #1072

Merged
merged 7 commits into from
May 12, 2021
Merged

Fix sync. Again. #1072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9241be7
Get rich traceback to use main stderr rich console
ewels May 11, 2021
b20c0c4
Wrap whole sync PR code in try /except
ewels May 11, 2021
5bf75fc
Explicitly disable requests_cache on all sync API calls
ewels May 11, 2021
bcef01a
Move debug log of API response further up the execution
ewels May 11, 2021
45d843e
Better logging of exceptions, fix sync problem
ewels May 11, 2021
67132b8
Merge branch 'master' into fix-sync
ewels May 11, 2021
7b85d4f
Changelog, fix tests
ewels May 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

### General

* Fixed a bug in the Docker image build for tools that failed due to an extra hyphen.
* Fixed a bug in the Docker image build for tools that failed due to an extra hyphen. [[#1069](https://github.com/nf-core/tools/pull/1069)]
* Regular release sync fix - this time it was to do with JSON serialisation [[#1072](https://github.com/nf-core/tools/pull/1072)]

## [v1.14 - Brass Chicken :chicken:](https://github.com/nf-core/tools/releases/tag/1.14) - [2021-05-11]

Expand Down
8 changes: 5 additions & 3 deletions nf_core/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@


def run_nf_core():
# Set up rich stderr console
stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors())

# Set up the rich traceback
rich.traceback.install(width=200, word_wrap=True, extra_lines=1)
rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1)

# Print nf-core header to STDERR
stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors())
# Print nf-core header
stderr.print("\n[green]{},--.[grey39]/[green],-.".format(" " * 42), highlight=False)
stderr.print("[blue] ___ __ __ __ ___ [green]/,-._.--~\\", highlight=False)
stderr.print("[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False)
Expand Down
123 changes: 75 additions & 48 deletions nf_core/sync.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import re
import requests
import requests_cache
import rich
import shutil
import time

Expand Down Expand Up @@ -323,43 +324,66 @@ def make_pull_request(self):
"base": self.from_branch,
}

stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors())

while True:
r = requests.post(
url="https://api.github.com/repos/{}/pulls".format(self.gh_repo),
data=json.dumps(pr_content),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
try:
self.gh_pr_returned_data = json.loads(r.content)
returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4)
r_headers_pp = json.dumps(r.headers, indent=4)
except:
self.gh_pr_returned_data = r.content
returned_data_prettyprint = r.content
r_headers_pp = r.headers

# PR worked
if r.status_code == 201:
self.pr_url = self.gh_pr_returned_data["html_url"]
log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}\n\n{r_headers_pp}")
log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}")
break

# Returned 403 error - too many simultaneous requests
# https://github.com/nf-core/tools/issues/911
if r.status_code == 403:
log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r_headers_pp}")
wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0))))
if wait_time == 0:
log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait")
wait_time = random.randrange(10, 60)
log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..")
time.sleep(wait_time)

# Something went wrong
else:
log.debug("Submitting PR to GitHub API")
returned_data_prettyprint = ""
r_headers_pp = ""
with requests_cache.disabled():
r = requests.post(
url="https://api.github.com/repos/{}/pulls".format(self.gh_repo),
data=json.dumps(pr_content),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
try:
self.gh_pr_returned_data = json.loads(r.content)
returned_data_prettyprint = json.dumps(dict(self.gh_pr_returned_data), indent=4)
r_headers_pp = json.dumps(dict(r.headers), indent=4)
except:
self.gh_pr_returned_data = r.content
returned_data_prettyprint = r.content
r_headers_pp = r.headers
log.error("Could not parse JSON response from GitHub API!")
stderr.print_exception()

# Dump the responses to the log just in case..
log.debug(f"PR response from GitHub. Data:\n{returned_data_prettyprint}\n\nHeaders:\n{r_headers_pp}")

# PR worked
if r.status_code == 201:
self.pr_url = self.gh_pr_returned_data["html_url"]
log.debug(f"GitHub API PR worked, return code 201")
log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}")
break

# Returned 403 error - too many simultaneous requests
# https://github.com/nf-core/tools/issues/911
if r.status_code == 403:
log.debug(f"GitHub API PR failed with 403 error")
wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0))))
if wait_time == 0:
log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait")
wait_time = random.randrange(10, 60)
log.warning(
f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds.."
)
time.sleep(wait_time)

# Something went wrong
else:
raise PullRequestException(
f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}"
)
# Don't catch the PullRequestException that we raised inside
except PullRequestException:
raise
# Do catch any other exceptions that we hit
except Exception as e:
stderr.print_exception()
raise PullRequestException(
f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}"
f"Something went badly wrong - {e}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}"
)

def close_open_template_merge_prs(self):
Expand All @@ -371,10 +395,11 @@ def close_open_template_merge_prs(self):

# Look for existing pull-requests
list_prs_url = f"https://api.github.com/repos/{self.gh_repo}/pulls"
list_prs_request = requests.get(
url=list_prs_url,
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
with requests_cache.disabled():
list_prs_request = requests.get(
url=list_prs_url,
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
try:
list_prs_json = json.loads(list_prs_request.content)
list_prs_pp = json.dumps(list_prs_json, indent=4)
Expand Down Expand Up @@ -412,18 +437,20 @@ def close_open_pr(self, pr):
f"This pull-request is now outdated and has been closed in favour of {self.pr_url}\n\n"
f"Please use {self.pr_url} to merge in the new changes from the nf-core template as soon as possible."
)
comment_request = requests.post(
url=pr["comments_url"],
data=json.dumps({"body": comment_text}),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
with requests_cache.disabled():
comment_request = requests.post(
url=pr["comments_url"],
data=json.dumps({"body": comment_text}),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)

# Update the PR status to be closed
pr_request = requests.patch(
url=pr["url"],
data=json.dumps({"state": "closed"}),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
with requests_cache.disabled():
pr_request = requests.patch(
url=pr["url"],
data=json.dumps({"state": "closed"}),
auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
)
try:
pr_request_json = json.loads(pr_request.content)
pr_request_pp = json.dumps(pr_request_json, indent=4)
Expand Down