diff --git a/.github/workflows/pytest-frozen-ubuntu-20.04.yml b/.github/workflows/pytest-frozen-ubuntu-20.04.yml index b015376633..5faf8ce605 100644 --- a/.github/workflows/pytest-frozen-ubuntu-20.04.yml +++ b/.github/workflows/pytest-frozen-ubuntu-20.04.yml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - pytest: + pytest-frozen: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 415765cfa2..f5d0cca950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ ### Download - Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). +- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)). ### Linting diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d9381761f7..7faa27e2d4 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -221,7 +221,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-r", "--revision", multiple=True, - help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2.", + help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2`", ) @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( diff --git a/nf_core/download.py b/nf_core/download.py index 8832eca5d3..db98b17f22 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -108,7 +108,7 @@ def __init__( self.force = force self.tower = tower self.include_configs = None - self.container = container + self.container = container if not singularity_cache_index else "singularity" self.singularity_cache = ( singularity_cache if not singularity_cache_index else "remote" ) # if a singularity_cache_index is given, use the file and overrule choice. @@ -157,7 +157,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...['+str(len(self.revision)-2)+' more revisions]...,'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -228,28 +228,29 @@ def download_workflow_static(self): # Collect all required singularity images if self.container == "singularity": - self.find_container_images(revision_dirname) + self.find_container_images(os.path.join(self.outdir, revision_dirname)) - try: - self.get_singularity_images() - except OSError as e: - log.critical(f"[red]{e}[/]") - sys.exit(1) + try: + self.get_singularity_images(current_revision=item[0]) + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Compress into an archive if self.compress_type is not None: log.info("Compressing output into archive") self.compress_download() - def download_workflow_tower(self): + def download_workflow_tower(self, location=None): """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") self.workflow_repo = WorkflowRepo( - remote_url=f"git@github.com:{self.pipeline}.git", + remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, + location=location if location else None, # manual location is required for the tests to work in_cache=False, ) @@ -261,17 +262,17 @@ def download_workflow_tower(self): # extract the required containers if self.container == "singularity": - for commit in self.wf_sha.values(): + for revision, commit in self.wf_sha.items(): # Checkout the repo in the current revision self.workflow_repo.checkout(commit) # Collect all required singularity images self.find_container_images(self.workflow_repo.access()) - try: - self.get_singularity_images() - except OSError as e: - log.critical(f"[red]{e}[/]") - sys.exit(1) + try: + self.get_singularity_images(current_revision=revision) + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: @@ -412,30 +413,47 @@ def prompt_singularity_cachedir_creation(self): if cachedir_path: os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path - # Ask if user wants this set in their .bashrc - bashrc_path = os.path.expanduser("~/.bashrc") - if not os.path.isfile(bashrc_path): - bashrc_path = os.path.expanduser("~/.bash_profile") - if not os.path.isfile(bashrc_path): - bashrc_path = False - if bashrc_path: + """ + Optionally, create a permanent entry for the NXF_SINGULARITY_CACHEDIR in the terminal profile. + Currently support for bash and zsh. + ToDo: "sh", "bash", "dash", "ash","csh", "tcsh", "ksh", "zsh", "fish", "cmd", "powershell", "pwsh"? + """ + + if os.environ["SHELL"] == "/bin/bash": + shellprofile_path = os.path.expanduser("~/~/.bash_profile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = os.path.expanduser("~/.bashrc") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + elif os.environ["SHELL"] == "/bin/zsh": + shellprofile_path = os.path.expanduser("~/.zprofile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = os.path.expanduser("~/.zshenv") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + else: + shellprofile_path = os.path.expanduser("~/.profile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + + if shellprofile_path: stderr.print( - f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(bashrc_path)}[/] file ." + f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(shellprofile_path)}[/] file ." "This will then be automatically set every time you open a new terminal. We can add the following line to this file for you: \n" f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]' ) append_to_file = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(shellprofile_path)}[/] ?[/]" ) if append_to_file: - with open(os.path.expanduser(bashrc_path), "a") as f: + with open(os.path.expanduser(shellprofile_path), "a") as f: f.write( "\n\n#######################################\n" f"## Added by `nf-core download` v{nf_core.__version__} ##\n" + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' + "\n#######################################\n" ) - log.info(f"Successfully wrote to [blue]{bashrc_path}[/]") + log.info(f"Successfully wrote to [blue]{shellprofile_path}[/]") log.warning( "You will need reload your terminal after the download completes for this to take effect." ) @@ -620,7 +638,7 @@ def wf_use_local_configs(self, revision_dirname): with open(nfconfig_fn, "w") as nfconfig_fh: nfconfig_fh.write(nfconfig) - def find_container_images(self, revision_dirname): + def find_container_images(self, workflow_directory): """Find container image names for workflow. Starts by using `nextflow config` to pull out any process.container @@ -662,7 +680,7 @@ def find_container_images(self, revision_dirname): containers_raw = [] if not self.containers else self.containers # Use linting code to parse the pipeline nextflow config - self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, revision_dirname)) + self.nf_config = nf_core.utils.fetch_wf_config(workflow_directory) # Find any config variables that look like a container for k, v in self.nf_config.items(): @@ -670,7 +688,7 @@ def find_container_images(self, revision_dirname): containers_raw.append(v.strip('"').strip("'")) # Recursive search through any DSL2 module files for container spec lines. - for subdir, _, files in os.walk(os.path.join(self.outdir, revision_dirname, "modules")): + for subdir, _, files in os.walk(os.path.join(workflow_directory, "modules")): for file in files: if file.endswith(".nf"): file_path = os.path.join(subdir, file) @@ -745,14 +763,14 @@ def find_container_images(self, revision_dirname): # Remove duplicates and sort self.containers = sorted(list(set(containers_raw))) - def get_singularity_images(self): + def get_singularity_images(self, current_revision=""): """Loop through container names and download Singularity images""" if len(self.containers) == 0: log.info("No container names found in workflow") else: log.info( - f"Found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in workflow." + f"Processing workflow revision {current_revision}, found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in total." ) with DownloadProgress() as progress: @@ -1087,6 +1105,7 @@ def __init__( remote_url, revision, commit, + location=None, hide_progress=False, in_cache=True, ): @@ -1118,7 +1137,7 @@ def __init__( self.retries = 0 # retries for setting up the locally cached repository self.hide_progress = hide_progress - self.setup_local_repo(remote_url, in_cache=in_cache) + self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) # expose some instance attributes self.tags = self.repo.tags @@ -1155,7 +1174,7 @@ def retry_setup_local_repo(self, skip_confirm=False): else: raise LookupError("Exiting due to error with locally cached Git repository.") - def setup_local_repo(self, remote, in_cache=True): + def setup_local_repo(self, remote, location=None, in_cache=True): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -1163,13 +1182,15 @@ def setup_local_repo(self, remote, in_cache=True): Args: remote (str): git url of remote - commit (str): name of branch to checkout from (optional) - hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. + location (Path): location where the clone should be created/cached. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. Sets self.repo """ + if location: + self.local_repo_dir = os.path.join(location, self.fullname) + else: + self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) - self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) try: if not os.path.exists(self.local_repo_dir): try: diff --git a/tests/data/testdata_remote_containers.txt b/tests/data/testdata_remote_containers.txt new file mode 100644 index 0000000000..93cf46f2f6 --- /dev/null +++ b/tests/data/testdata_remote_containers.txt @@ -0,0 +1,37 @@ +./depot.galaxyproject.org-singularity-bbmap-38.93--he522d1c_0.img +./depot.galaxyproject.org-singularity-bedtools-2.30.0--hc088bd4_0.img +./depot.galaxyproject.org-singularity-bioconductor-dupradar-1.18.0--r40_1.img +./depot.galaxyproject.org-singularity-bioconductor-summarizedexperiment-1.20.0--r40_0.img +./depot.galaxyproject.org-singularity-bioconductor-tximeta-1.8.0--r40_0.img +./depot.galaxyproject.org-singularity-fastqc-0.11.9--0.img +./depot.galaxyproject.org-singularity-gffread-0.12.1--h8b12597_0.img +./depot.galaxyproject.org-singularity-hisat2-2.2.1--h1b792b2_3.img +./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-59cdd445419f14abac76b31dd0d71217994cbcc9-0.img +./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0.img +./depot.galaxyproject.org-singularity-mulled-v2-8849acf39a43cdd6c839a369a74c0adc823e2f91-ab110436faf952a33575c64dd74615a84011450b-0.img +./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-0e773bb207600fcb4d38202226eb20a33c7909b6-0.img +./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-38aed4501da19db366dc7c8d52d31d94e760cfaf-0.img +./depot.galaxyproject.org-singularity-mulled-v2-cf0123ef83b3c38c13e3b0696a3f285d3f20f15b-64aad4a4e144878400649e71f42105311be7ed87-0.img +./depot.galaxyproject.org-singularity-multiqc-1.11--pyhdfd78af_0.img +./depot.galaxyproject.org-singularity-multiqc-1.13--pyhdfd78af_0.img +./depot.galaxyproject.org-singularity-perl-5.26.2.img +./depot.galaxyproject.org-singularity-picard-2.26.10--hdfd78af_0.img +./depot.galaxyproject.org-singularity-picard-2.27.4--hdfd78af_0.img +./depot.galaxyproject.org-singularity-preseq-3.1.2--h445547b_2.img +./depot.galaxyproject.org-singularity-python-3.9--1.img +./depot.galaxyproject.org-singularity-qualimap-2.2.2d--1.img +./depot.galaxyproject.org-singularity-rseqc-3.0.1--py37h516909a_1.img +./depot.galaxyproject.org-singularity-salmon-1.5.2--h84f40af_0.img +./depot.galaxyproject.org-singularity-samtools-1.15.1--h1170115_0.img +./depot.galaxyproject.org-singularity-sortmerna-4.3.4--h9ee0642_0.img +./depot.galaxyproject.org-singularity-stringtie-2.2.1--hecb563c_2.img +./depot.galaxyproject.org-singularity-subread-2.0.1--hed695b0_0.img +./depot.galaxyproject.org-singularity-trim-galore-0.6.7--hdfd78af_0.img +./depot.galaxyproject.org-singularity-ubuntu-20.04.img +./depot.galaxyproject.org-singularity-ucsc-bedclip-377--h0b8a92a_2.img +./depot.galaxyproject.org-singularity-ucsc-bedgraphtobigwig-377--h446ed27_1.img +./depot.galaxyproject.org-singularity-umi_tools-1.1.2--py38h4a8c8d9_0.img +These entries should not be used: +On October 5, 2011, the 224-meter containership MV Rena struck a reef close to New Zealand’s coast and broke apart. That spells disaster, no? +MV Rena + diff --git a/tests/test_download.py b/tests/test_download.py index d1a770a630..41fb9c625f 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -3,16 +3,20 @@ import hashlib import os +import re import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest import nf_core.create import nf_core.utils -from nf_core.download import DownloadWorkflow +from nf_core.download import DownloadWorkflow, WorkflowRepo +from nf_core.synced_repo import SyncedRepo +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR from .utils import with_temporary_file, with_temporary_folder @@ -169,6 +173,32 @@ def test_singularity_pull_image_singularity_not_installed(self, tmp_dir, mock_ri with pytest.raises(OSError): download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) + # + # Test for '--singularity-cache remote --singularity-cache-index'. Provide a list of containers already available in a remote location. + # + @with_temporary_folder + def test_remote_container_functionality(self, tmp_dir): + os.environ["NXF_SINGULARITY_CACHEDIR"] = "foo" + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + outdir=os.path.join(tmp_dir, "new"), + revision="3.9", + compress_type="none", + singularity_cache_index=Path(__file__).resolve().parent / "data/testdata_remote_containers.txt", + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + # test if settings are changed accordingly. + assert download_obj.singularity_cache == "remote" and download_obj.container == "singularity" + assert isinstance(download_obj.containers_remote, list) and len(download_obj.containers_remote) == 0 + # read in the file + download_obj.read_remote_containers() + assert len(download_obj.containers_remote) == 33 + assert "depot.galaxyproject.org-singularity-salmon-1.5.2--h84f40af_0.img" in download_obj.containers_remote + assert "MV Rena" not in download_obj.containers_remote # decoy in test file + # # Tests for the main entry method 'download_workflow' # @@ -184,6 +214,65 @@ def test_download_workflow_with_success(self, tmp_dir, mock_download_image, mock container="singularity", revision="1.6", compress_type="none", + singularity_cache="copy", ) + download_obj.include_configs = True # suppress prompt, because stderr.is_interactive doesn't. download_obj.download_workflow() + + # + # Test Download for Tower + # + @with_temporary_folder + def test_download_workflow_for_tower(self, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + tower=True, + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + + download_obj.get_revision_hash() + + # download_obj.wf_download_url is not set for tower downloads, but the sha values are + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 2 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + + # The outdir for multiple revisions is the pipeline name and date: e.g. nf-core-rnaseq_2023-04-27_18-54 + assert bool(re.search(r"nf-core-rnaseq_\d{4}-\d{2}-\d{1,2}_\d{1,2}-\d{1,2}", download_obj.outdir, re.S)) + + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_tower(location=tmp_dir) + + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + # corroborate that the other revisions are inaccessible to the user. + assert len(download_obj.workflow_repo.tags) == len(download_obj.revision) + + # manually test container image detection for 3.7 revision + download_obj.workflow_repo.checkout(download_obj.wf_sha["3.7"]) + assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 0 + download_obj.find_container_images(download_obj.workflow_repo.access()) + assert len(download_obj.containers) == 30 # 30 containers for 3.7 + assert ( + "https://depot.galaxyproject.org/singularity/bbmap:38.93--he522d1c_0" in download_obj.containers + ) # direct definition + assert ( + "https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0" + in download_obj.containers + ) # indirect definition via $container variable.