From 6536ed7fce9aa97af3bc260ecee51c4b42d69a91 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 16:57:40 +0100 Subject: [PATCH 1/3] started with more robust jinja2 templating --- nf_core/create.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 6f23c99478..e29af0db39 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -83,7 +83,7 @@ def render_template(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") - binary_ftypes = ["image", "application/java-archive"] + binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] object_attrs = vars(self) object_attrs["nf_core_version"] = nf_core.__version__ @@ -107,16 +107,20 @@ def render_template(self): os.makedirs(os.path.dirname(output_path), exist_ok=True) # Just copy binary files - (ftype, encoding) = mimetypes.guess_type(template_fn_path) + (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): log.debug(f"Copying binary file: '{output_path}'") shutil.copy(template_fn_path, output_path) continue # Render the template - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - rendered_output = j_template.render(object_attrs) + try: + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + except UnicodeDecodeError as e: + log.error(f"Could not decode file {template_fn_path} {e}") + sys.exit(1) # Write to the pipeline output file with open(output_path, "w") as fh: From 676cf8fd20ad001718dec25f4b67e22f1f076f7d Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 18:20:57 +0100 Subject: [PATCH 2/3] added check for binary extensions --- nf_core/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index e29af0db39..40d3b366aa 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -3,6 +3,7 @@ organization's specification based on a template. """ from genericpath import exists +from pathlib import Path import git import jinja2 import logging @@ -84,6 +85,7 @@ def render_template(self): ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] + binary_extensions = [".jpeg", ".jpg", ".png", ".zip", ".gz", ".jar", ".tar"] object_attrs = vars(self) object_attrs["nf_core_version"] = nf_core.__version__ @@ -113,6 +115,11 @@ def render_template(self): shutil.copy(template_fn_path, output_path) continue + if Path(template_fn_path).suffix in binary_extensions: + log.debug(f"Copying binary file: '{output_path}'") + shutil.copy(template_fn_path, output_path) + continue + # Render the template try: log.debug(f"Rendering template file: '{template_fn}'") From 3261493f89234eb8da1f15a233d54682089f47f0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 22:01:03 +0100 Subject: [PATCH 3/3] Refactor code for tidyness points --- nf_core/create.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 40d3b366aa..2492169486 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -3,7 +3,6 @@ organization's specification based on a template. """ from genericpath import exists -from pathlib import Path import git import jinja2 import logging @@ -108,31 +107,32 @@ def render_template(self): output_path = os.path.join(self.outdir, template_fn) os.makedirs(os.path.dirname(output_path), exist_ok=True) - # Just copy binary files - (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) - if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): - log.debug(f"Copying binary file: '{output_path}'") - shutil.copy(template_fn_path, output_path) - continue + try: + # Just copy certain file extensions + filename, file_extension = os.path.splitext(template_fn_path) + if file_extension in binary_extensions: + raise AttributeError(f"File extension: {file_extension}") - if Path(template_fn_path).suffix in binary_extensions: - log.debug(f"Copying binary file: '{output_path}'") - shutil.copy(template_fn_path, output_path) - continue + # Try to detect binary files + (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) + if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + raise AttributeError(f"Encoding: {encoding}") - # Render the template - try: + # Got this far - render the template log.debug(f"Rendering template file: '{template_fn}'") j_template = env.get_template(template_fn) rendered_output = j_template.render(object_attrs) - except UnicodeDecodeError as e: - log.error(f"Could not decode file {template_fn_path} {e}") - sys.exit(1) - # Write to the pipeline output file - with open(output_path, "w") as fh: - log.debug(f"Writing to output file: '{output_path}'") - fh.write(rendered_output) + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) + + # Copy the file directly instead of using Jinja + except (AttributeError, UnicodeDecodeError) as e: + log.debug(f"Copying file without Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + continue # Make a logo and save it self.make_pipeline_logo()