Merge pull request #880 from ewels/pipeline-jinja

Pipeline jinja

.github/workflows/create-lint-wf.yml

@@ -39,6 +39,7 @@ jobs:
           nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc
 
       - name: Upload log file artifact
+        if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
           name: nf-core-log-file

CHANGELOG.md

@@ -21,6 +21,8 @@
 * The parameters `--max_memory` and `--max_time` are now validated against a regular expression [[#793](https://github.com/nf-core/tools/issues/793)]
   * Must be written in the format `123.GB` / `456.h` with any of the prefixes listed in the [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#memory)
   * Bare numbers no longer allowed, avoiding people from trying to specify GB and actually specifying bytes.
+* Finally dropped the wonderful [cookiecutter](https://github.com/cookiecutter/cookiecutter) library that was behind the first pipeline template that led to nf-core [[#880](https://github.com/nf-core/tools/pull/880)]
+  * Now rendering templates directly using [Jinja](https://jinja.palletsprojects.com/), which is what cookiecutter was doing anyway
 
 ### Modules
 

docs/api/_src/lint_tests/template_strings.rst

@@ -0,0 +1,4 @@
+template_strings
+================
+
+.. automethod:: nf_core.lint.PipelineLint.template_strings

nf_core/__main__.py

@@ -277,18 +277,18 @@ def validate_wf_name_prompt(ctx, opts, value):
 )
 @click.option("-d", "--description", prompt=True, required=True, type=str, help="A short description of your pipeline")
 @click.option("-a", "--author", prompt=True, required=True, type=str, help="Name of the main author(s)")
-@click.option("--new-version", type=str, default="1.0dev", help="The initial version number to use")
+@click.option("--version", type=str, default="1.0dev", help="The initial version number to use")
 @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository")
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists")
 @click.option("-o", "--outdir", type=str, help="Output directory for new pipeline (default: pipeline name)")
-def create(name, description, author, new_version, no_git, force, outdir):
+def create(name, description, author, version, no_git, force, outdir):
     """
     Create a new pipeline using the nf-core template.
 
     Uses the nf-core template to make a skeleton Nextflow pipeline with all required
     files, boilerplate code and bfest-practices.
     """
-    create_obj = nf_core.create.PipelineCreate(name, description, author, new_version, no_git, force, outdir)
+    create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir)
     create_obj.init_pipeline()
 
 

nf_core/create.py

@@ -2,16 +2,16 @@
 """Creates a nf-core pipeline matching the current
 organization's specification based on a template.
 """
-import click
-import cookiecutter.main, cookiecutter.exceptions
+from genericpath import exists
 import git
+import jinja2
 import logging
+import mimetypes
 import os
+import pathlib
 import requests
 import shutil
 import sys
-import tempfile
-import textwrap
 
 import nf_core
 
@@ -25,35 +25,32 @@ class PipelineCreate(object):
         name (str): Name for the pipeline.
         description (str): Description for the pipeline.
         author (str): Authors name of the pipeline.
-        new_version (str): Version flag. Semantic versioning only. Defaults to `1.0dev`.
+        version (str): Version flag. Semantic versioning only. Defaults to `1.0dev`.
         no_git (bool): Prevents the creation of a local Git repository for the pipeline. Defaults to False.
         force (bool): Overwrites a given workflow directory with the same name. Defaults to False.
             May the force be with you.
         outdir (str): Path to the local output directory.
     """
 
-    def __init__(self, name, description, author, new_version="1.0dev", no_git=False, force=False, outdir=None):
+    def __init__(self, name, description, author, version="1.0dev", no_git=False, force=False, outdir=None):
         self.short_name = name.lower().replace(r"/\s+/", "-").replace("nf-core/", "").replace("/", "-")
-        self.name = "nf-core/{}".format(self.short_name)
+        self.name = f"nf-core/{self.short_name}"
         self.name_noslash = self.name.replace("/", "-")
         self.name_docker = self.name.replace("nf-core", "nfcore")
         self.description = description
         self.author = author
-        self.new_version = new_version
+        self.version = version
         self.no_git = no_git
         self.force = force
         self.outdir = outdir
         if not self.outdir:
             self.outdir = os.path.join(os.getcwd(), self.name_noslash)
 
     def init_pipeline(self):
-        """Creates the nf-core pipeline.
-
-        Launches cookiecutter, that will ask for required pipeline information.
-        """
+        """Creates the nf-core pipeline. """
 
         # Make the new pipeline
-        self.run_cookiecutter()
+        self.render_template()
 
         # Init the git repository and make the first commit
         if not self.no_git:
@@ -66,69 +63,87 @@ def init_pipeline(self):
             + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]"
         )
 
-    def run_cookiecutter(self):
-        """Runs cookiecutter to create a new nf-core pipeline."""
-        log.info("Creating new nf-core pipeline: {}".format(self.name))
+    def render_template(self):
+        """Runs Jinja to create a new nf-core pipeline."""
+        log.info(f"Creating new nf-core pipeline: '{self.name}'")
 
         # Check if the output directory exists
         if os.path.exists(self.outdir):
             if self.force:
-                log.warning("Output directory '{}' exists - continuing as --force specified".format(self.outdir))
+                log.warning(f"Output directory '{self.outdir}' exists - continuing as --force specified")
             else:
-                log.error("Output directory '{}' exists!".format(self.outdir))
+                log.error(f"Output directory '{self.outdir}' exists!")
                 log.info("Use -f / --force to overwrite existing files")
                 sys.exit(1)
         else:
             os.makedirs(self.outdir)
 
-        # Build the template in a temporary directory
-        self.tmpdir = tempfile.mkdtemp()
-        template = os.path.join(os.path.dirname(os.path.realpath(nf_core.__file__)), "pipeline-template/")
-        cookiecutter.main.cookiecutter(
-            template,
-            extra_context={
-                "name": self.name,
-                "description": self.description,
-                "author": self.author,
-                "name_noslash": self.name_noslash,
-                "name_docker": self.name_docker,
-                "short_name": self.short_name,
-                "version": self.new_version,
-                "nf_core_version": nf_core.__version__,
-            },
-            no_input=True,
-            overwrite_if_exists=self.force,
-            output_dir=self.tmpdir,
+        # Run jinja2 for each file in the template folder
+        env = jinja2.Environment(
+            loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True
         )
+        template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template")
+        copy_ftypes = ["image", "application/java-archive"]
+        object_attrs = vars(self)
+        object_attrs["nf_core_version"] = nf_core.__version__
+
+        # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980
+        template_files = list(pathlib.Path(template_dir).glob("**/*"))
+        template_files += list(pathlib.Path(template_dir).glob("*"))
+        ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"]
+
+        for template_fn_path_obj in template_files:
+
+            template_fn_path = str(template_fn_path_obj)
+            if os.path.isdir(template_fn_path):
+                continue
+            if any([s in template_fn_path for s in ignore_strs]):
+                log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation")
+                continue
+
+            # Set up vars and directories
+            template_fn = os.path.relpath(template_fn_path, template_dir)
+            output_path = os.path.join(self.outdir, template_fn)
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+            # Just copy binary files
+            (ftype, encoding) = mimetypes.guess_type(template_fn_path)
+            if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in copy_ftypes])):
+                log.debug(f"Copying binary file: '{output_path}'")
+                shutil.copy(template_fn_path, output_path)
+                continue
+
+            # Render the template
+            log.debug(f"Rendering template file: '{template_fn}'")
+            j_template = env.get_template(template_fn)
+            rendered_output = j_template.render(object_attrs)
+
+            # Write to the pipeline output file
+            with open(output_path, "w") as fh:
+                log.debug(f"Writing to output file: '{output_path}'")
+                fh.write(rendered_output)
 
         # Make a logo and save it
         self.make_pipeline_logo()
 
-        # Move the template to the output directory
-        for f in os.listdir(os.path.join(self.tmpdir, self.name_noslash)):
-            shutil.move(os.path.join(self.tmpdir, self.name_noslash, f), self.outdir)
-
-        # Delete the temporary directory
-        shutil.rmtree(self.tmpdir)
-
     def make_pipeline_logo(self):
         """Fetch a logo for the new pipeline from the nf-core website"""
 
-        logo_url = "https://nf-co.re/logo/{}".format(self.short_name)
-        log.debug("Fetching logo from {}".format(logo_url))
+        logo_url = f"https://nf-co.re/logo/{self.short_name}"
+        log.debug(f"Fetching logo from {logo_url}")
 
-        email_logo_path = "{}/{}/assets/{}_logo.png".format(self.tmpdir, self.name_noslash, self.name_noslash)
-        log.debug("Writing logo to {}".format(email_logo_path))
-        r = requests.get("{}?w=400".format(logo_url))
+        email_logo_path = f"{self.outdir}/assets/{self.name_noslash}_logo.png"
+        os.makedirs(os.path.dirname(email_logo_path), exist_ok=True)
+        log.debug(f"Writing logo to '{email_logo_path}'")
+        r = requests.get(f"{logo_url}?w=400")
         with open(email_logo_path, "wb") as fh:
             fh.write(r.content)
 
-        readme_logo_path = "{}/{}/docs/images/{}_logo.png".format(self.tmpdir, self.name_noslash, self.name_noslash)
+        readme_logo_path = f"{self.outdir}/docs/images/{self.name_noslash}_logo.png"
 
-        log.debug("Writing logo to {}".format(readme_logo_path))
-        if not os.path.exists(os.path.dirname(readme_logo_path)):
-            os.makedirs(os.path.dirname(readme_logo_path))
-        r = requests.get("{}?w=600".format(logo_url))
+        log.debug(f"Writing logo to '{readme_logo_path}'")
+        os.makedirs(os.path.dirname(readme_logo_path), exist_ok=True)
+        r = requests.get(f"{logo_url}?w=600")
         with open(readme_logo_path, "wb") as fh:
             fh.write(r.content)
 
@@ -137,14 +152,14 @@ def git_init_pipeline(self):
         log.info("Initialising pipeline git repository")
         repo = git.Repo.init(self.outdir)
         repo.git.add(A=True)
-        repo.index.commit("initial template build from nf-core/tools, version {}".format(nf_core.__version__))
+        repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}")
         # Add TEMPLATE branch to git repository
         repo.git.branch("TEMPLATE")
         repo.git.branch("dev")
         log.info(
             "Done. Remember to add a remote and push to GitHub:\n"
-            + "[white on grey23] cd {} \n".format(self.outdir)
-            + " git remote add origin [email protected]:USERNAME/REPO_NAME.git \n"
-            + " git push --all origin                                       "
+            f"[white on grey23] cd {self.outdir} \n"
+            " git remote add origin [email protected]:USERNAME/REPO_NAME.git \n"
+            " git push --all origin                                       "
         )
         log.info("This will also push your newly created dev branch and the TEMPLATE branch for syncing.")

nf_core/lint/__init__.py

@@ -108,7 +108,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     from .conda_dockerfile import conda_dockerfile
     from .pipeline_todos import pipeline_todos
     from .pipeline_name_conventions import pipeline_name_conventions
-    from .cookiecutter_strings import cookiecutter_strings
+    from .template_strings import template_strings
     from .schema_lint import schema_lint
     from .schema_params import schema_params
     from .actions_schema_validation import actions_schema_validation
@@ -140,7 +140,7 @@ def __init__(self, wf_path, release_mode=False, fix=()):
             "conda_dockerfile",
             "pipeline_todos",
             "pipeline_name_conventions",
-            "cookiecutter_strings",
+            "template_strings",
             "schema_lint",
             "schema_params",
             "actions_schema_validation",

nf_core/lint/cookiecutter_strings.py → nf_core/lint/template_strings.py

@@ -5,17 +5,20 @@
 import re
 
 
-def cookiecutter_strings(self):
-    """Check for 'cookiecutter' placeholders.
+def template_strings(self):
+    """Check for template placeholders.
 
     The ``nf-core create`` pipeline template uses
-    `cookiecutter <https://github.com/cookiecutter/cookiecutter>`_ behind the scenes.
+    `Jinja <https://jinja.palletsprojects.com/en/2.11.x/>`_ behind the scenes.
 
-    This lint test fails if any cookiecutter template variables such as
-    ``{{ cookiecutter.pipeline_name }}`` are found in your pipeline code.
+    This lint test fails if any Jinja template variables such as
+    ``{{ pipeline_name }}`` are found in your pipeline code.
 
     Finding a placeholder like this means that something was probably copied and pasted
     from the template without being properly rendered for your pipeline.
+
+    This test ignores any double-brackets prefixed with a dollar sign, such as
+    ``${{ secrets.AWS_ACCESS_KEY_ID }}`` as these placeholders are used in GitHub Actions workflows.
     """
     passed = []
     failed = []
@@ -27,12 +30,12 @@ def cookiecutter_strings(self):
             lnum = 0
             for l in fh:
                 lnum += 1
-                cc_matches = re.findall(r"{{\s*cookiecutter[^}]*}}", l)
+                cc_matches = re.findall(r"[^$]{{[^}]*}}", l)
                 if len(cc_matches) > 0:
                     for cc_match in cc_matches:
-                        failed.append("Found a cookiecutter template string in `{}` L{}: {}".format(fn, lnum, cc_match))
+                        failed.append("Found a Jinja template string in `{}` L{}: {}".format(fn, lnum, cc_match))
                         num_matches += 1
     if num_matches == 0:
-        passed.append("Did not find any cookiecutter template strings ({} files)".format(len(self.files)))
+        passed.append("Did not find any Jinja template strings ({} files)".format(len(self.files)))
 
     return {"passed": passed, "failed": failed}

nf_core/modules/create.py

@@ -162,7 +162,7 @@ def create(self):
         if self.process_label is None:
             log.info(
                 "Provide an appropriate resource label for the process, taken from the "
-                "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/%7B%7Bcookiecutter.name_noslash%7D%7D/conf/base.config#L29]nf-core pipeline template[/link].\n"
+                "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n"
                 "For example: 'process_low', 'process_medium', 'process_high', 'process_long'"
             )
         while self.process_label is None:
@@ -208,12 +208,10 @@ def create(self):
 
     def render_template(self):
         """
-        Create new module files with cookiecutter in a temporyary directory.
-
-        Returns: Path to generated files.
+        Create new module files with Jinja2.
         """
         # Run jinja2 for each file in the template folder
-        env = jinja2.Environment(loader=jinja2.PackageLoader("nf_core", "module-template"))
+        env = jinja2.Environment(loader=jinja2.PackageLoader("nf_core", "module-template"), keep_trailing_newline=True)
         for template_fn, dest_fn in self.file_paths.items():
             log.debug(f"Rendering template file: '{template_fn}'")
             j_template = env.get_template(template_fn)
@@ -247,7 +245,7 @@ def get_repo_type(self, directory):
     def get_module_dirs(self):
         """Given a directory and a tool/subtool, set the file paths and check if they already exist
 
-        Returns dict: keys are file paths in cookiecutter output, vals are target paths.
+        Returns dict: keys are relative paths to template files, vals are target paths.
         """
 
         file_paths = {}
@@ -272,7 +270,7 @@ def get_module_dirs(self):
             if os.path.exists(test_dir) and not self.force_overwrite:
                 raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite")
 
-            # Set file paths - can be tool/ or tool/subtool/ so can't do in cookiecutter template
+            # Set file paths - can be tool/ or tool/subtool/ so can't do in template directory structure
             file_paths[os.path.join("software", "functions.nf")] = os.path.join(software_dir, "functions.nf")
             file_paths[os.path.join("software", "main.nf")] = os.path.join(software_dir, "main.nf")
             file_paths[os.path.join("software", "meta.yml")] = os.path.join(software_dir, "meta.yml")