diff --git a/CHANGELOG.md b/CHANGELOG.md index cbdaa7f57c..31fcc2bb73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) - Use black also to format python files in workflows ([#1563](https://github.com/nf-core/tools/pull/1563)) - Add check for mimetype in the `input` parameter. ([#1647](https://github.com/nf-core/tools/issues/1647)) +- Check that the singularity and docker tags are parsable. Add `--fail-warned` flag to `nf-core modules lint` ([#1654](https://github.com/nf-core/tools/issues/1654)) ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e66d9cead2..f457d9e0c6 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -643,10 +643,11 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-k", "--key", type=str, metavar="", multiple=True, help="Run only these lint tests") @click.option("-a", "--all", is_flag=True, help="Run on all modules") +@click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") -def lint(ctx, tool, dir, key, all, local, passed, fix_version): +def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): """ Lint one or more modules in a directory. @@ -659,6 +660,7 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version): try: module_lint = nf_core.modules.ModuleLint( dir, + fail_warned, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], diff --git a/nf_core/download.py b/nf_core/download.py index ea1e6b33d6..89188a1fc9 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -479,7 +479,7 @@ def find_container_images(self): # Don't recognise this, throw a warning else: - log.error(f"[red]Cannot parse container string, skipping: [green]{match}") + log.error(f"[red]Cannot parse container string, skipping: [green]'{file}'") if this_container: containers_raw.append(this_container) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 8351a11d06..a4664ee05a 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -70,13 +70,14 @@ class ModuleLint(ModuleCommand): from .module_todos import module_todos from .module_version import module_version - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=None): + def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull=False, base_path=None): self.dir = dir try: self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) except LookupError as e: raise UserWarning(e) + self.fail_warned = fail_warned self.passed = [] self.warned = [] self.failed = [] @@ -245,7 +246,7 @@ def get_installed_modules(self): # Filter local modules if os.path.exists(local_modules_dir): - local_modules = sorted([x for x in local_modules if x.endswith(".nf")]) + local_modules = sorted([x for x in os.listdir(local_modules_dir) if x.endswith(".nf")]) # nf-core/modules if self.repo_type == "modules": @@ -342,7 +343,11 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): if local: self.main_nf(mod, fix_version, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] - self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] + warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] + if not self.fail_warned: + self.warned += warned + else: + self.failed += warned # Otherwise run all the lint tests else: @@ -353,7 +358,12 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] - self.warned += [LintResult(mod, *m) for m in mod.warned] + warned = [LintResult(mod, *m) for m in mod.warned] + if not self.fail_warned: + self.warned += warned + else: + self.failed += warned + self.failed += [LintResult(mod, *m) for m in mod.failed] def _print_results(self, show_passed=False): diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 531181d88e..ee069743b2 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -241,7 +241,6 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) else: self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) - for l in lines: if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] @@ -249,11 +248,23 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - singularity_tag = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l).group(1) + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l) + if match is not None: + singularity_tag = match.group(1) + self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) + else: + self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) + singularity_tag = None if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - docker_tag = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l).group(1) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l) + if match is not None: + docker_tag = match.group(1) + self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) + else: + self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) + docker_tag = None # Check that all bioconda packages have build numbers # Also check for newer versions @@ -434,6 +445,15 @@ def _container_type(line): if re.search("bioconda::", line): return "bioconda" if line.startswith("https://containers") or line.startswith("https://depot"): - return "singularity" + # Look for a http download URL. + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 + url_regex = ( + r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)" + ) + url_match = re.search(url_regex, line, re.S) + if url_match: + return "singularity" + else: + return None if line.startswith("biocontainers/") or line.startswith("quay.io/"): return "docker" diff --git a/nf_core/utils.py b/nf_core/utils.py index 822d5a6811..a0a01b999e 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -327,6 +327,7 @@ def setup_requests_cachedir(): "backend": "sqlite", } + logging.getLogger("requests_cache").setLevel(logging.WARNING) try: if not os.path.exists(cachedir): os.makedirs(cachedir)