diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 10d0d7dc02..2bd49f9f00 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -5,7 +5,13 @@ on: push: branches: - dev + paths-ignore: + - "docs/**" + - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index e52eee1331..8ce4fc60bf 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -4,7 +4,11 @@ on: push: branches: - dev + paths-ignore: + - "CHANGELOG.md" pull_request: + paths-ignore: + - "CHANGELOG.md" release: types: [published] diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c6cfc1fa9..4c406f9a41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### Download - Improved container image resolution and prioritization of http downloads over Docker URIs ([#2364](https://github.com/nf-core/tools/pull/2364)). +- Registries provided with `-l`/`--container-library` will be ignored for modules with explicit container registry specifications ([#2403](https://github.com/nf-core/tools/pull/2403)). ### Linting @@ -42,9 +43,9 @@ - `params.max_multiqc_email_size` is no longer required ([#2273](https://github.com/nf-core/tools/pull/2273)) - Remove `cleanup = true` from `test_full.config` in pipeline template ([#2279](https://github.com/nf-core/tools/pull/2279)) - Fix usage docs for specifying `params.yaml` ([#2279](https://github.com/nf-core/tools/pull/2279)) -- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk] -- Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) -- Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) +- Added stub in modules template ([#2277](https://github.com/nf-core/tools/pull/2277)) [Contributed by @nvnieuwk] +- Move registry definitions out of profile scope ([#2286])() +- Remove `aws_tower` profile ([#2287])() - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) - Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301)) @@ -85,7 +86,7 @@ _In addition, `-r` / `--revision` has been changed to a parameter that can be pr ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) -- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])(https://github.com/nf-core/tools/pull/2278/files) +- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])() - Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281)) - Warn about null/None/empty default values in `nextflow_schema.json` ([#3328](https://github.com/nf-core/tools/pull/2328)) - Fix linting when creating a pipeline skipping some parts of the template and add CI test ([#2330](https://github.com/nf-core/tools/pull/2330)) @@ -116,7 +117,7 @@ _In addition, `-r` / `--revision` has been changed to a parameter that can be pr - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` -- Remove `.cff` files from `.editorconfig` [(#2145)[https://github.com/nf-core/tools/pull/2145]] +- Remove `.cff` files from `.editorconfig` [[#2145](https://github.com/nf-core/tools/pull/2145)] - Simplify pipeline README ([#2186](https://github.com/nf-core/tools/issues/2186)) - Added support for the apptainer container engine via `-profile apptainer`. ([#2244](https://github.com/nf-core/tools/issues/2244)) [Contributed by @jfy133] - Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io` ([#2133](https://github.com/nf-core/tools/pull/2133)) @@ -1143,7 +1144,7 @@ making a pull-request. See [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md) - Move `params`section from `base.config` to `nextflow.config` - Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment - Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` -- Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com +- Added support for nf-tower in the travis tests, using public mailbox - Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG - Adjusted `.travis.yml` checks to allow for `patch` branches to be tested - Add Python 3.7 dependency to the `environment.yml` file diff --git a/nf_core/download.py b/nf_core/download.py index cf95d27a14..9ca786b5e3 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1081,7 +1081,10 @@ def get_singularity_images(self, current_revision=""): continue except ContainerError.ImageNotFound as e: # Try other registries - continue + if e.error_log.absoluteURI: + break # there no point in trying other registries if absolute URI was specified. + else: + continue except ContainerError.InvalidTag as e: # Try other registries continue @@ -1089,7 +1092,10 @@ def get_singularity_images(self, current_revision=""): # Try other registries log.error(e.message) log.error(e.helpmessage) - continue + if e.error_log.absoluteURI: + break # there no point in trying other registries if absolute URI was specified. + else: + continue else: # The else clause executes after the loop completes normally. # This means the library loop completed without breaking, indicating failure for all libraries (registries) @@ -1235,8 +1241,18 @@ def singularity_pull_image(self, container, out_path, cache_path, library, progr Various exceptions possible from `subprocess` execution of Singularity. """ output_path = cache_path or out_path - # Pull using singularity - address = f"docker://{library}/{container.replace('docker://', '')}" + + # Sometimes, container still contain an explicit library specification, which + # resulted in attempted pulls e.g. from docker://quay.io/quay.io/qiime2/core:2022.11 + # Thus, if an explicit registry is specified, the provided -l value is ignored. + container_parts = container.split("/") + if len(container_parts) > 2: + address = container + absolute_URI = True + else: + address = f"docker://{library}/{container.replace('docker://', '')}" + absolute_URI = False + if shutil.which("singularity"): singularity_command = ["singularity", "pull", "--name", output_path, address] elif shutil.which("apptainer"): @@ -1270,6 +1286,7 @@ def singularity_pull_image(self, container, out_path, cache_path, library, progr container=container, registry=library, address=address, + absolute_URI=absolute_URI, out_path=out_path if out_path else cache_path or "", singularity_command=singularity_command, error_msg=lines, @@ -1555,10 +1572,11 @@ def bare_clone(self, destination): class ContainerError(Exception): """A class of errors related to pulling containers with Singularity/Apptainer""" - def __init__(self, container, registry, address, out_path, singularity_command, error_msg): + def __init__(self, container, registry, address, absolute_URI, out_path, singularity_command, error_msg): self.container = container self.registry = registry self.address = address + self.absolute_URI = absolute_URI self.out_path = out_path self.singularity_command = singularity_command self.error_msg = error_msg @@ -1615,10 +1633,15 @@ class ImageNotFound(FileNotFoundError): def __init__(self, error_log): self.error_log = error_log - self.message = ( - f'[bold red]"Pulling "{self.error_log.container}" from "{self.error_log.address}" failed.[/]\n' - ) - self.helpmessage = f'Saving image of "{self.error_log.container}" failed.\nPlease troubleshoot the command \n"{" ".join(self.error_log.singularity_command)}" manually.f\n' + if not self.error_log.absolute_URI: + self.message = ( + f'[bold red]"Pulling "{self.error_log.container}" from "{self.error_log.address}" failed.[/]\n' + ) + self.helpmessage = f'Saving image of "{self.error_log.container}" failed.\nPlease troubleshoot the command \n"{" ".join(self.error_log.singularity_command)}" manually.f\n' + else: + self.message = f'[bold red]"The pipeline requested the download of non-existing container image "{self.error_log.address}"[/]\n' + self.helpmessage = f'Please try to rerun \n"{" ".join(self.error_log.singularity_command)}" manually with a different registry.f\n' + super().__init__(self.message) class InvalidTag(AttributeError): @@ -1646,6 +1669,11 @@ class OtherError(RuntimeError): def __init__(self, error_log): self.error_log = error_log - self.message = f'[bold red]"{self.error_log.container}" failed for unclear reasons.[/]\n' - self.helpmessage = f'Pulling of "{self.error_log.container}" failed.\nPlease troubleshoot the command \n"{" ".join(self.error_log.singularity_command)}" manually.\n' + if not self.error_log.absolute_URI: + self.message = f'[bold red]"{self.error_log.container}" failed for unclear reasons.[/]\n' + self.helpmessage = f'Pulling of "{self.error_log.container}" failed.\nPlease troubleshoot the command \n"{" ".join(self.error_log.singularity_command)}" manually.\n' + else: + self.message = f'[bold red]"The pipeline requested the download of non-existing container image "{self.error_log.address}"[/]\n' + self.helpmessage = f'Please try to rerun \n"{" ".join(self.error_log.singularity_command)}" manually with a different registry.f\n' + super().__init__(self.message, self.helpmessage, self.error_log)