Writing additional tests for the --tower download functionality.

nf-core · May 2, 2023 · a7f8d69 · a7f8d69
1 parent 073a9c2
commit a7f8d69
Show file tree

Hide file tree

Showing 6 changed files with 188 additions and 40 deletions.
diff --git a/.github/workflows/pytest-frozen-ubuntu-20.04.yml b/.github/workflows/pytest-frozen-ubuntu-20.04.yml
@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pytest:
+  pytest-frozen:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -37,6 +37,7 @@
 ### Download
 
 - Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)).
+- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)).
 
 ### Linting
 

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
@@ -221,7 +221,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all
     "-r",
     "--revision",
     multiple=True,
-    help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2.",
+    help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2`",
 )
 @click.option("-o", "--outdir", type=str, help="Output directory")
 @click.option(

diff --git a/nf_core/download.py b/nf_core/download.py
@@ -108,7 +108,7 @@ def __init__(
         self.force = force
         self.tower = tower
         self.include_configs = None
-        self.container = container
+        self.container = container if not singularity_cache_index else "singularity"
         self.singularity_cache = (
             singularity_cache if not singularity_cache_index else "remote"
         )  # if a singularity_cache_index is given, use the file and overrule choice.
@@ -157,7 +157,7 @@ def download_workflow(self):
             sys.exit(1)
 
         summary_log = [
-            f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'",
+            f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...['+str(len(self.revision)-2)+' more revisions]...,'+self.revision[-1]}'",
             f"Pull containers: '{self.container}'",
         ]
         if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None:
@@ -228,28 +228,29 @@ def download_workflow_static(self):
 
             # Collect all required singularity images
             if self.container == "singularity":
-                self.find_container_images(revision_dirname)
+                self.find_container_images(os.path.join(self.outdir, revision_dirname))
 
-            try:
-                self.get_singularity_images()
-            except OSError as e:
-                log.critical(f"[red]{e}[/]")
-                sys.exit(1)
+                try:
+                    self.get_singularity_images(current_revision=item[0])
+                except OSError as e:
+                    log.critical(f"[red]{e}[/]")
+                    sys.exit(1)
 
         # Compress into an archive
         if self.compress_type is not None:
             log.info("Compressing output into archive")
             self.compress_download()
 
-    def download_workflow_tower(self):
+    def download_workflow_tower(self, location=None):
         """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline"""
 
         log.info("Collecting workflow from GitHub")
 
         self.workflow_repo = WorkflowRepo(
-            remote_url=f"git@github.com:{self.pipeline}.git",
+            remote_url=f"https://github.com/{self.pipeline}.git",
             revision=self.revision if self.revision else None,
             commit=self.wf_sha.values() if bool(self.wf_sha) else None,
+            location=location if location else None,  # manual location is required for the tests to work
             in_cache=False,
         )
 
@@ -261,17 +262,17 @@ def download_workflow_tower(self):
 
         # extract the required containers
         if self.container == "singularity":
-            for commit in self.wf_sha.values():
+            for revision, commit in self.wf_sha.items():
                 # Checkout the repo in the current revision
                 self.workflow_repo.checkout(commit)
                 # Collect all required singularity images
                 self.find_container_images(self.workflow_repo.access())
 
-            try:
-                self.get_singularity_images()
-            except OSError as e:
-                log.critical(f"[red]{e}[/]")
-                sys.exit(1)
+                try:
+                    self.get_singularity_images(current_revision=revision)
+                except OSError as e:
+                    log.critical(f"[red]{e}[/]")
+                    sys.exit(1)
 
         # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set)
         if self.compress_type is not None:
@@ -412,30 +413,47 @@ def prompt_singularity_cachedir_creation(self):
                 if cachedir_path:
                     os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path
 
-                    # Ask if user wants this set in their .bashrc
-                    bashrc_path = os.path.expanduser("~/.bashrc")
-                    if not os.path.isfile(bashrc_path):
-                        bashrc_path = os.path.expanduser("~/.bash_profile")
-                        if not os.path.isfile(bashrc_path):
-                            bashrc_path = False
-                    if bashrc_path:
+                    """
+                    Optionally, create a permanent entry for the NXF_SINGULARITY_CACHEDIR in the terminal profile.
+                    Currently support for bash and zsh.
+                    ToDo: "sh", "bash", "dash", "ash","csh", "tcsh", "ksh", "zsh", "fish", "cmd", "powershell", "pwsh"?
+                    """
+
+                    if os.environ["SHELL"] == "/bin/bash":
+                        shellprofile_path = os.path.expanduser("~/~/.bash_profile")
+                        if not os.path.isfile(shellprofile_path):
+                            shellprofile_path = os.path.expanduser("~/.bashrc")
+                            if not os.path.isfile(shellprofile_path):
+                                shellprofile_path = False
+                    elif os.environ["SHELL"] == "/bin/zsh":
+                        shellprofile_path = os.path.expanduser("~/.zprofile")
+                        if not os.path.isfile(shellprofile_path):
+                            shellprofile_path = os.path.expanduser("~/.zshenv")
+                            if not os.path.isfile(shellprofile_path):
+                                shellprofile_path = False
+                    else:
+                        shellprofile_path = os.path.expanduser("~/.profile")
+                        if not os.path.isfile(shellprofile_path):
+                            shellprofile_path = False
+
+                    if shellprofile_path:
                         stderr.print(
-                            f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(bashrc_path)}[/] file ."
+                            f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(shellprofile_path)}[/] file ."
                             "This will then be automatically set every time you open a new terminal. We can add the following line to this file for you: \n"
                             f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]'
                         )
                         append_to_file = rich.prompt.Confirm.ask(
-                            f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]"
+                            f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(shellprofile_path)}[/] ?[/]"
                         )
                         if append_to_file:
-                            with open(os.path.expanduser(bashrc_path), "a") as f:
+                            with open(os.path.expanduser(shellprofile_path), "a") as f:
                                 f.write(
                                     "\n\n#######################################\n"
                                     f"## Added by `nf-core download` v{nf_core.__version__} ##\n"
                                     + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"'
                                     + "\n#######################################\n"
                                 )
-                            log.info(f"Successfully wrote to [blue]{bashrc_path}[/]")
+                            log.info(f"Successfully wrote to [blue]{shellprofile_path}[/]")
                             log.warning(
                                 "You will need reload your terminal after the download completes for this to take effect."
                             )
@@ -620,7 +638,7 @@ def wf_use_local_configs(self, revision_dirname):
         with open(nfconfig_fn, "w") as nfconfig_fh:
             nfconfig_fh.write(nfconfig)
 
-    def find_container_images(self, revision_dirname):
+    def find_container_images(self, workflow_directory):
         """Find container image names for workflow.
 
         Starts by using `nextflow config` to pull out any process.container
@@ -662,15 +680,15 @@ def find_container_images(self, revision_dirname):
         containers_raw = [] if not self.containers else self.containers
 
         # Use linting code to parse the pipeline nextflow config
-        self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, revision_dirname))
+        self.nf_config = nf_core.utils.fetch_wf_config(workflow_directory)
 
         # Find any config variables that look like a container
         for k, v in self.nf_config.items():
             if k.startswith("process.") and k.endswith(".container"):
                 containers_raw.append(v.strip('"').strip("'"))
 
         # Recursive search through any DSL2 module files for container spec lines.
-        for subdir, _, files in os.walk(os.path.join(self.outdir, revision_dirname, "modules")):
+        for subdir, _, files in os.walk(os.path.join(workflow_directory, "modules")):
             for file in files:
                 if file.endswith(".nf"):
                     file_path = os.path.join(subdir, file)
@@ -745,14 +763,14 @@ def find_container_images(self, revision_dirname):
         # Remove duplicates and sort
         self.containers = sorted(list(set(containers_raw)))
 
-    def get_singularity_images(self):
+    def get_singularity_images(self, current_revision=""):
         """Loop through container names and download Singularity images"""
 
         if len(self.containers) == 0:
             log.info("No container names found in workflow")
         else:
             log.info(
-                f"Found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in workflow."
+                f"Processing workflow revision {current_revision}, found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in total."
             )
 
             with DownloadProgress() as progress:
@@ -1087,6 +1105,7 @@ def __init__(
         remote_url,
         revision,
         commit,
+        location=None,
         hide_progress=False,
         in_cache=True,
     ):
@@ -1118,7 +1137,7 @@ def __init__(
         self.retries = 0  # retries for setting up the locally cached repository
         self.hide_progress = hide_progress
 
-        self.setup_local_repo(remote_url, in_cache=in_cache)
+        self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache)
 
         # expose some instance attributes
         self.tags = self.repo.tags
@@ -1155,21 +1174,23 @@ def retry_setup_local_repo(self, skip_confirm=False):
         else:
             raise LookupError("Exiting due to error with locally cached Git repository.")
 
-    def setup_local_repo(self, remote, in_cache=True):
+    def setup_local_repo(self, remote, location=None, in_cache=True):
         """
         Sets up the local git repository. If the repository has been cloned previously, it
         returns a git.Repo object of that clone. Otherwise it tries to clone the repository from
         the provided remote URL and returns a git.Repo of the new clone.
 
         Args:
             remote (str): git url of remote
-            commit (str): name of branch to checkout from (optional)
-            hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False.
+            location (Path): location where the clone should be created/cached.
             in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False.
         Sets self.repo
         """
+        if location:
+            self.local_repo_dir = os.path.join(location, self.fullname)
+        else:
+            self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname)
 
-        self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname)
         try:
             if not os.path.exists(self.local_repo_dir):
                 try:

diff --git a/tests/data/testdata_remote_containers.txt b/tests/data/testdata_remote_containers.txt
@@ -0,0 +1,37 @@
+./depot.galaxyproject.org-singularity-bbmap-38.93--he522d1c_0.img
+./depot.galaxyproject.org-singularity-bedtools-2.30.0--hc088bd4_0.img
+./depot.galaxyproject.org-singularity-bioconductor-dupradar-1.18.0--r40_1.img
+./depot.galaxyproject.org-singularity-bioconductor-summarizedexperiment-1.20.0--r40_0.img
+./depot.galaxyproject.org-singularity-bioconductor-tximeta-1.8.0--r40_0.img
+./depot.galaxyproject.org-singularity-fastqc-0.11.9--0.img
+./depot.galaxyproject.org-singularity-gffread-0.12.1--h8b12597_0.img
+./depot.galaxyproject.org-singularity-hisat2-2.2.1--h1b792b2_3.img
+./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-59cdd445419f14abac76b31dd0d71217994cbcc9-0.img
+./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0.img
+./depot.galaxyproject.org-singularity-mulled-v2-8849acf39a43cdd6c839a369a74c0adc823e2f91-ab110436faf952a33575c64dd74615a84011450b-0.img
+./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-0e773bb207600fcb4d38202226eb20a33c7909b6-0.img
+./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-38aed4501da19db366dc7c8d52d31d94e760cfaf-0.img
+./depot.galaxyproject.org-singularity-mulled-v2-cf0123ef83b3c38c13e3b0696a3f285d3f20f15b-64aad4a4e144878400649e71f42105311be7ed87-0.img
+./depot.galaxyproject.org-singularity-multiqc-1.11--pyhdfd78af_0.img
+./depot.galaxyproject.org-singularity-multiqc-1.13--pyhdfd78af_0.img
+./depot.galaxyproject.org-singularity-perl-5.26.2.img
+./depot.galaxyproject.org-singularity-picard-2.26.10--hdfd78af_0.img
+./depot.galaxyproject.org-singularity-picard-2.27.4--hdfd78af_0.img
+./depot.galaxyproject.org-singularity-preseq-3.1.2--h445547b_2.img
+./depot.galaxyproject.org-singularity-python-3.9--1.img
+./depot.galaxyproject.org-singularity-qualimap-2.2.2d--1.img
+./depot.galaxyproject.org-singularity-rseqc-3.0.1--py37h516909a_1.img
+./depot.galaxyproject.org-singularity-salmon-1.5.2--h84f40af_0.img
+./depot.galaxyproject.org-singularity-samtools-1.15.1--h1170115_0.img
+./depot.galaxyproject.org-singularity-sortmerna-4.3.4--h9ee0642_0.img
+./depot.galaxyproject.org-singularity-stringtie-2.2.1--hecb563c_2.img
+./depot.galaxyproject.org-singularity-subread-2.0.1--hed695b0_0.img
+./depot.galaxyproject.org-singularity-trim-galore-0.6.7--hdfd78af_0.img
+./depot.galaxyproject.org-singularity-ubuntu-20.04.img
+./depot.galaxyproject.org-singularity-ucsc-bedclip-377--h0b8a92a_2.img
+./depot.galaxyproject.org-singularity-ucsc-bedgraphtobigwig-377--h446ed27_1.img
+./depot.galaxyproject.org-singularity-umi_tools-1.1.2--py38h4a8c8d9_0.img
+These entries should not be used:
+On October 5, 2011, the 224-meter containership MV Rena struck a reef close to New Zealand’s coast and broke apart. That spells disaster, no?
+MV Rena
+