From ad6e31bf0e3ea2282190f78b8ed5ca433f7d6b14 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Oct 2022 12:03:49 +0000 Subject: [PATCH 1/2] NF-Core Sync includes template.yaml as option Fixes #1879 --- CHANGELOG.md | 1 + nf_core/__main__.py | 5 +++-- nf_core/create.py | 1 + nf_core/sync.py | 5 +++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb6955ecb..2a59ccbfd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Fix error in tagging GitPod docker images during releases +- `nf-core sync` now supports the template YAML file using `-t/--template-yaml`. ### Modules diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9df0b2ff97..461a901ec9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1054,7 +1054,8 @@ def bump_version(new_version, dir, nextflow): @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") -def sync(dir, from_branch, pull_request, github_repository, username): +@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") +def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -1071,7 +1072,7 @@ def sync(dir, from_branch, pull_request, github_repository, username): nf_core.utils.is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username) + sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) try: sync_obj.sync() except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: diff --git a/nf_core/create.py b/nf_core/create.py index 1cd4fecba0..8b028d90af 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -39,6 +39,7 @@ class PipelineCreate(object): force (bool): Overwrites a given workflow directory with the same name. Defaults to False. May the force be with you. outdir (str): Path to the local output directory. + template_yaml (str): Path to template.yml file for pipeline creation settings. """ def __init__( diff --git a/nf_core/sync.py b/nf_core/sync.py index a663f1c7a7..365d6d1371 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -52,6 +52,7 @@ class PipelineSync(object): required_config_vars (list): List of nextflow variables required to make template pipeline gh_username (str): GitHub username gh_repo (str): GitHub repository name + template_yaml (str): Path to template.yml file for pipeline creation settings. """ def __init__( @@ -61,6 +62,7 @@ def __init__( make_pr=False, gh_repo=None, gh_username=None, + template_yaml_path=None, ): """Initialise syncing object""" @@ -77,6 +79,8 @@ def __init__( self.gh_repo = gh_repo self.pr_url = "" + self.template_yaml_path = template_yaml_path + # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api self.gh_api.lazy_init() @@ -233,6 +237,7 @@ def make_template_pipeline(self): force=True, outdir=self.pipeline_dir, author=self.wf_config["manifest.author"].strip('"').strip("'"), + template_yaml_path=self.template_yaml_path, plain=True, ).init_pipeline() From 713987a123a4c5b369f95509310be4dcf18481d9 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Oct 2022 16:06:01 +0000 Subject: [PATCH 2/2] nf-core sync supports template.yaml within repository - Cache template.yml file before running nf-core sync. - If syncing fails, reset to start --- nf_core/sync.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 365d6d1371..b3211da7dd 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -80,6 +80,10 @@ def __init__( self.pr_url = "" self.template_yaml_path = template_yaml_path + # Save contents of template.yml for using outside of git. + if self.template_yaml_path is not None: + with open(self.template_yaml_path, "r") as template_yaml: + self.template_yaml_cache = template_yaml.read() # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api @@ -208,7 +212,7 @@ def delete_template_branch_files(self): # Delete everything log.info("Deleting all files in 'TEMPLATE' branch") for the_file in os.listdir(self.pipeline_dir): - if the_file == ".git": + if the_file == ".git" or the_file == self.template_yaml_path: continue file_path = os.path.join(self.pipeline_dir, the_file) log.debug(f"Deleting {file_path}") @@ -229,17 +233,29 @@ def make_template_pipeline(self): # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) - nf_core.create.PipelineCreate( - name=self.wf_config["manifest.name"].strip('"').strip("'"), - description=self.wf_config["manifest.description"].strip('"').strip("'"), - version=self.wf_config["manifest.version"].strip('"').strip("'"), - no_git=True, - force=True, - outdir=self.pipeline_dir, - author=self.wf_config["manifest.author"].strip('"').strip("'"), - template_yaml_path=self.template_yaml_path, - plain=True, - ).init_pipeline() + # Re-write the template yaml from cache which may have been updated + if self.template_yaml_path and self.template_yaml_cache: + with open(self.template_yaml_path, "w") as template_path: + template_path.write(self.template_yaml_cache) + + try: + nf_core.create.PipelineCreate( + name=self.wf_config["manifest.name"].strip('"').strip("'"), + description=self.wf_config["manifest.description"].strip('"').strip("'"), + version=self.wf_config["manifest.version"].strip('"').strip("'"), + no_git=True, + force=True, + outdir=self.pipeline_dir, + author=self.wf_config["manifest.author"].strip('"').strip("'"), + template_yaml_path=self.template_yaml_path, + plain=True, + ).init_pipeline() + except Exception as err: + # If sync fails, remove template_yaml_path before raising error. + os.remove(self.template_yaml_path) + # Reset to where you were to prevent git getting messed up. + self.repo.git.reset("--hard") + raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") def commit_template_changes(self): """If we have any changes with the new template files, make a git commit"""