Merge pull request #1164 from ErikDanielsson/refactor-modules-lint

Refactor `modules lint` and added modules test to `nf-core lint`

nf_core/__main__.py

@@ -333,8 +333,10 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json):
 
     # Run the lint tests!
     try:
-        lint_obj = nf_core.lint.run_linting(dir, release, fix, key, show_passed, fail_ignored, markdown, json)
-        if len(lint_obj.failed) > 0:
+        lint_obj, module_lint_obj = nf_core.lint.run_linting(
+            dir, release, fix, key, show_passed, fail_ignored, markdown, json
+        )
+        if len(lint_obj.failed) + len(module_lint_obj.failed) > 0:
             sys.exit(1)
     except AssertionError as e:
         log.critical(e)
@@ -541,9 +543,9 @@ def lint(ctx, tool, dir, key, all, local, passed):
     nf-core/modules repository.
     """
     try:
-        module_lint = nf_core.modules.ModuleLint(dir=dir, key=key)
+        module_lint = nf_core.modules.ModuleLint(dir=dir)
         module_lint.modules_repo = ctx.obj["modules_repo_obj"]
-        module_lint.lint(module=tool, all_modules=all, print_results=True, local=local, show_passed=passed)
+        module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed)
         if len(module_lint.failed) > 0:
             sys.exit(1)
     except nf_core.modules.lint.ModuleLintException as e:

nf_core/lint/__init__.py

@@ -5,9 +5,9 @@
 the nf-core community guidelines.
 """
 
-from rich.console import Console
 from rich.markdown import Markdown
 from rich.table import Table
+from rich.panel import Panel
 import datetime
 import git
 import json
@@ -19,6 +19,9 @@
 import yaml
 
 import nf_core.utils
+import nf_core.lint_utils
+from nf_core.lint_utils import console
+from nf_core.modules.lint import ModuleLint
 
 log = logging.getLogger(__name__)
 
@@ -47,16 +50,35 @@ def run_linting(
     lint_obj._load_pipeline_config()
     lint_obj._list_files()
 
-    # Run the linting tests
+    # Create the modules lint object
+    module_lint_obj = ModuleLint(pipeline_dir)
+
+    # Run only the tests we want
+    module_lint_tests = ("module_changes", "module_version")
+    module_lint_obj.filter_tests_by_key(module_lint_tests)
+
+    # Set up files for modules linting test
+    module_lint_obj.set_up_pipeline_files()
+
+    # Run the pipeline linting tests
     try:
         lint_obj._lint_pipeline()
     except AssertionError as e:
         log.critical("Critical error: {}".format(e))
         log.info("Stopping tests...")
-        return lint_obj
+        return lint_obj, module_lint_obj
+
+    # Run the module lint tests
+    if len(module_lint_obj.all_local_modules) > 0:
+        module_lint_obj.lint_modules(module_lint_obj.all_local_modules, local=True)
+    if len(module_lint_obj.all_nfcore_modules) > 0:
+        module_lint_obj.lint_modules(module_lint_obj.all_nfcore_modules, local=False)
 
     # Print the results
     lint_obj._print_results(show_passed)
+    module_lint_obj._print_results(show_passed)
+    nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj)
+    nf_core.lint_utils.print_fixes(lint_obj, module_lint_obj)
 
     # Save results to Markdown file
     if md_fn is not None:
@@ -74,7 +96,7 @@ def run_linting(
         if release_mode:
             log.info("Reminder: Lint tests were run in --release mode.")
 
-    return lint_obj
+    return lint_obj, module_lint_obj
 
 
 class PipelineLint(nf_core.utils.Pipeline):
@@ -264,15 +286,14 @@ def _lint_pipeline(self):
                 if test_results.get("could_fix", False):
                     self.could_fix.append(test_name)
 
-    def _print_results(self, show_passed=False):
+    def _print_results(self, show_passed):
         """Print linting results to the command line.
 
         Uses the ``rich`` library to print a set of formatted tables to the command line
         summarising the linting results.
         """
 
         log.debug("Printing final results")
-        console = Console(force_terminal=nf_core.utils.rich_force_colors())
 
         # Helper function to format test links nicely
         def format_result(test_results, table):
@@ -289,6 +310,9 @@ def _s(some_list):
                 return "s"
             return ""
 
+        # Print lint results header
+        console.print(Panel("[magenta]General lint results"))
+
         # Table of passed tests
         if len(self.passed) > 0 and show_passed:
             table = Table(style="green", box=rich.box.ROUNDED)
@@ -324,6 +348,12 @@ def _s(some_list):
             table = format_result(self.failed, table)
             console.print(table)
 
+    def _print_summary(self):
+        def _s(some_list):
+            if len(some_list) != 1:
+                return "s"
+            return ""
+
         # Summary table
         summary_colour = "red" if len(self.failed) > 0 else "green"
         table = Table(box=rich.box.ROUNDED, style=summary_colour)
@@ -336,16 +366,6 @@ def _s(some_list):
         table.add_row(r"[red][✗] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)))
         console.print(table)
 
-        if len(self.could_fix):
-            fix_cmd = "nf-core lint {} --fix {}".format(self.wf_path, " --fix ".join(self.could_fix))
-            console.print(
-                f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue]    {fix_cmd}\n"
-            )
-        if len(self.fix):
-            console.print(
-                "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout <file>'."
-            )
-
     def _get_results_md(self):
         """
         Create a markdown file suitable for posting in a GitHub comment.

nf_core/lint_utils.py

@@ -0,0 +1,48 @@
+import rich
+from rich.console import Console
+from rich.table import Table
+import logging
+
+import nf_core.utils
+
+log = logging.getLogger(__name__)
+
+# Create a console used by all lint tests
+console = Console(force_terminal=nf_core.utils.rich_force_colors())
+
+
+def print_joint_summary(lint_obj, module_lint_obj):
+    """Print a joint summary of the general pipe lint tests and the module lint tests"""
+    nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed)
+    nbr_ignored = len(lint_obj.ignored)
+    nbr_fixed = len(lint_obj.fixed)
+    nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned)
+    nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed)
+
+    def _s(some_length):
+        return "" if some_length == 1 else "s"
+
+    summary_colour = "red" if nbr_failed > 0 else "green"
+    table = Table(box=rich.box.ROUNDED, style=summary_colour)
+    table.add_column(f"LINT RESULTS SUMMARY".format(nbr_passed), no_wrap=True)
+    table.add_row(r"[green][✔] {:>3} Test{} Passed".format(nbr_passed, _s(nbr_passed)))
+    if nbr_fixed:
+        table.add_row(r"[bright blue][?] {:>3} Test{} Fixed".format(nbr_fixed, _s(nbr_fixed)))
+    table.add_row(r"[grey58][?] {:>3} Test{} Ignored".format(nbr_ignored, _s(nbr_ignored)))
+    table.add_row(r"[yellow][!] {:>3} Test Warning{}".format(nbr_warned, _s(nbr_warned)))
+    table.add_row(r"[red][✗] {:>3} Test{} Failed".format(nbr_failed, _s(nbr_failed)))
+    console.print(table)
+
+
+def print_fixes(lint_obj, module_lint_obj):
+    """Prints available and applied fixes"""
+
+    if len(lint_obj.could_fix):
+        fix_cmd = "nf-core lint {} --fix {}".format(lint_obj.wf_path, " --fix ".join(lint_obj.could_fix))
+        console.print(
+            f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue]    {fix_cmd}\n"
+        )
+    if len(lint_obj.fix):
+        console.print(
+            "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout <file>'."
+        )