Added TestStats

anta/cli/nrfu/__init__.py

@@ -92,7 +92,7 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
     default=None,
     type=click.Choice(HIDE_STATUS, case_sensitive=False),
     multiple=True,
-    help="Group result by test or device.",
+    help="Hide tests with specific status. Can be provided multiple times.",
     required=False,
 )
 @click.option(

anta/cli/nrfu/commands.py

@@ -91,12 +91,11 @@ def tpl_report(ctx: click.Context, template: pathlib.Path, output: pathlib.Path
 @click.command()
 @click.pass_context
 @click.option(
-    "--output",
-    "-o",
+    "--md-output",
     type=click.Path(file_okay=True, dir_okay=False, exists=False, writable=True, path_type=pathlib.Path),
     show_envvar=True,
     required=True,
-    help="Path to save the Markdown report in a file",
+    help="Path to save the report as a Markdown file. It only saves test results and not the output from the --group-by option.",
 )
 @click.option(
     "--only_failed_tests",
@@ -105,8 +104,8 @@ def tpl_report(ctx: click.Context, template: pathlib.Path, output: pathlib.Path
     show_envvar=True,
     help="Flag to determine if only failed tests should be saved in the report",
 )
-def md_report(ctx: click.Context, output: pathlib.Path, *, only_failed_tests: bool = False) -> None:
+def md_report(ctx: click.Context, md_output: pathlib.Path, *, only_failed_tests: bool = False) -> None:
     """ANTA command to check network state with Markdown report."""
     run_tests(ctx)
-    save_markdown_report(results=ctx.obj["result_manager"], output=output, only_failed_tests=only_failed_tests)
+    save_markdown_report(ctx, md_output=md_output, only_failed_tests=only_failed_tests)
     exit_with_code(ctx)

anta/cli/nrfu/utils.py

@@ -18,7 +18,7 @@
 from anta.cli.console import console
 from anta.models import AntaTest
 from anta.reporter import ReportJinja, ReportTable
-from anta.reporter.md_report import MDReportFactory
+from anta.reporter.md_reporter import MDReportGenerator
 from anta.runner import main
 
 if TYPE_CHECKING:
@@ -124,13 +124,12 @@ def print_jinja(results: ResultManager, template: pathlib.Path, output: pathlib.
             file.write(report)
 
 
-def save_markdown_report(results: ResultManager, output: pathlib.Path, *, only_failed_tests: bool = False) -> None:
+def save_markdown_report(ctx: click.Context, md_output: pathlib.Path, *, only_failed_tests: bool = False) -> None:
     """Save the markdown report."""
     console.print()
-    with output.open(mode="w", encoding="utf-8") as report_file:
-        MDReportFactory.generate_report(report_file, results, only_failed_tests=only_failed_tests)
+    MDReportGenerator.generate(results=_get_result_manager(ctx), md_filename=md_output, only_failed_tests=only_failed_tests)
     checkmark = Emoji("white_check_mark")
-    console.print(f"Markdown report saved to {output} {checkmark}", style="cyan")
+    console.print(f"Markdown report saved to {md_output} {checkmark}", style="cyan")
 
 
 # Adding our own ANTA spinner - overriding rich SPINNERS for our own

anta/reporter/__init__.py

@@ -140,24 +140,18 @@ def report_summary_tests(
             "# of skipped",
             "# of failure",
             "# of errors",
-            "List of failed or error nodes",
+            "List of failed or errored devices",
         ]
         table = self._build_headers(headers=headers, table=table)
-        for test in manager.get_tests():
+        for test, stats in sorted(manager.test_stats.items()):
             if tests is None or test in tests:
-                results = manager.filter_by_tests({test}).results
-                nb_failure = len([result for result in results if result.result == "failure"])
-                nb_error = len([result for result in results if result.result == "error"])
-                list_failure = [result.name for result in results if result.result in ["failure", "error"]]
-                nb_success = len([result for result in results if result.result == "success"])
-                nb_skipped = len([result for result in results if result.result == "skipped"])
                 table.add_row(
                     test,
-                    str(nb_success),
-                    str(nb_skipped),
-                    str(nb_failure),
-                    str(nb_error),
-                    str(list_failure),
+                    str(stats.devices_success_count),
+                    str(stats.devices_skipped_count),
+                    str(stats.devices_failure_count),
+                    str(stats.devices_error_count),
+                    ", ".join(stats.devices_failure),
                 )
         return table
 
@@ -188,24 +182,18 @@ def report_summary_devices(
             "# of skipped",
             "# of failure",
             "# of errors",
-            "List of failed or error test cases",
+            "List of failed or errored test cases",
         ]
         table = self._build_headers(headers=headers, table=table)
-        for device in manager.get_devices():
+        for device, stats in sorted(manager.dut_stats.items()):
             if devices is None or device in devices:
-                results = manager.filter_by_devices({device}).results
-                nb_failure = len([result for result in results if result.result == "failure"])
-                nb_error = len([result for result in results if result.result == "error"])
-                list_failure = [result.test for result in results if result.result in ["failure", "error"]]
-                nb_success = len([result for result in results if result.result == "success"])
-                nb_skipped = len([result for result in results if result.result == "skipped"])
                 table.add_row(
                     device,
-                    str(nb_success),
-                    str(nb_skipped),
-                    str(nb_failure),
-                    str(nb_error),
-                    str(list_failure),
+                    str(stats.tests_success_count),
+                    str(stats.tests_skipped_count),
+                    str(stats.tests_failure_count),
+                    str(stats.tests_error_count),
+                    ", ".join(stats.tests_failure),
                 )
         return table
 

anta/reporter/md_report.py → anta/reporter/md_reporter.py

@@ -12,43 +12,45 @@
 if TYPE_CHECKING:
     from collections.abc import Generator
     from io import TextIOWrapper
+    from pathlib import Path
 
     from anta.result_manager import ResultManager
 
 
 # pylint: disable=too-few-public-methods
-class MDReportFactory:
-    """Factory class responsible for generating a Markdown report based on the provided `ResultManager` object.
+class MDReportGenerator:
+    """Class responsible for generating a Markdown report based on the provided `ResultManager` object.
 
     It aggregates different report sections, each represented by a subclass of `MDReportBase`,
     and sequentially generates their content into a markdown file.
 
-    The `generate_report` method will loop over all the section subclasses and call their `generate_section` method.
+    The `generate` class method will loop over all the section subclasses and call their `generate_section` method.
     The final report will be generated in the same order as the `sections` list of the method.
 
-    The factory method also accepts an optional `only_failed_tests` flag to generate a report with only failed tests.
+    The class method also accepts an optional `only_failed_tests` flag to generate a report with only failed tests.
 
     By default, the report will include all test results.
     """
 
     @classmethod
-    def generate_report(cls, mdfile: TextIOWrapper, manager: ResultManager, *, only_failed_tests: bool = False) -> None:
+    def generate(cls, results: ResultManager, md_filename: Path, *, only_failed_tests: bool = False) -> None:
         """Generate and write the various sections of the markdown report."""
-        sections: list[MDReportBase] = [
-            ANTAReport(mdfile, manager),
-            TestResultsSummary(mdfile, manager),
-            SummaryTotals(mdfile, manager),
-            SummaryTotalsDeviceUnderTest(mdfile, manager),
-            SummaryTotalsPerCategory(mdfile, manager),
-            FailedTestResultsSummary(mdfile, manager),
-            AllTestResults(mdfile, manager),
-        ]
+        with md_filename.open("w", encoding="utf-8") as mdfile:
+            sections: list[MDReportBase] = [
+                ANTAReport(mdfile, results),
+                TestResultsSummary(mdfile, results),
+                SummaryTotals(mdfile, results),
+                SummaryTotalsDeviceUnderTest(mdfile, results),
+                SummaryTotalsPerCategory(mdfile, results),
+                FailedTestResultsSummary(mdfile, results),
+                AllTestResults(mdfile, results),
+            ]
 
-        if only_failed_tests:
-            sections.pop()
+            if only_failed_tests:
+                sections.pop()
 
-        for section in sections:
-            section.generate_section()
+            for section in sections:
+                section.generate_section()
 
 
 class MDReportBase(ABC):
@@ -58,7 +60,7 @@ class MDReportBase(ABC):
     to generate and write content to the provided markdown file.
     """
 
-    def __init__(self, mdfile: TextIOWrapper, manager: ResultManager) -> None:
+    def __init__(self, mdfile: TextIOWrapper, results: ResultManager) -> None:
         """Initialize the MDReportBase with an open markdown file object to write to and a ResultManager instance.
 
         Args:
@@ -67,7 +69,7 @@ def __init__(self, mdfile: TextIOWrapper, manager: ResultManager) -> None:
             results (ResultManager): The ResultsManager instance containing all test results.
         """
         self.mdfile = mdfile
-        self.manager = manager
+        self.results = results
 
     @abstractmethod
     def generate_section(self) -> None:
@@ -101,7 +103,7 @@ def generate_heading_name(self) -> str:
         class_name = self.__class__.__name__
 
         # Split the class name into words, keeping acronyms together
-        words = re.findall(r"([A-Z]+(?=[A-Z][a-z]|\d|\W|$|\s)|[A-Z]?[a-z]+|\d+)", class_name)
+        words = re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+", class_name)
 
         # Capitalize each word, but keep acronyms in all caps
         formatted_words = [word if word.isupper() else word.capitalize() for word in words]
@@ -185,17 +187,18 @@ class SummaryTotals(MDReportBase):
     """Generate the `### Summary Totals` section of the markdown report."""
 
     TABLE_HEADING: ClassVar[list[str]] = [
-        "| Total Tests | Total Tests Passed | Total Tests Failed | Total Tests Skipped |",
-        "| ----------- | ------------------ | ------------------ | ------------------- |",
+        "| Total Tests | Total Tests Success | Total Tests Skipped | Total Tests Failure | Total Tests Error |",
+        "| ----------- | ------------------- | ------------------- | ------------------- | ------------------|",
     ]
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the summary totals table."""
         yield (
-            f"| {self.manager.get_total_results()} "
-            f"| {self.manager.get_total_results('success')} "
-            f"| {self.manager.get_total_results({'failure', 'error', 'unset'})} "
-            f"| {self.manager.get_total_results('skipped')} |\n"
+            f"| {self.results.get_total_results()} "
+            f"| {self.results.get_total_results('success')} "
+            f"| {self.results.get_total_results('skipped')} "
+            f"| {self.results.get_total_results('failure')} "
+            f"| {self.results.get_total_results('error')} |\n"
         )
 
     def generate_section(self) -> None:
@@ -208,19 +211,19 @@ class SummaryTotalsDeviceUnderTest(MDReportBase):
     """Generate the `### Summary Totals Devices Under Tests` section of the markdown report."""
 
     TABLE_HEADING: ClassVar[list[str]] = [
-        "| Device Under Test | Total Tests | Tests Passed | Tests Failed | Tests Skipped | Categories Failed | Categories Skipped |",
-        "| ------------------| ----------- | ------------ | ------------ | ------------- | ----------------- | ------------------ |",
+        "| Device Under Test | Total Tests | Tests Success | Tests Skipped | Tests Failure | Tests Error | Categories Skipped | Categories Failed |",
+        "| ------------------| ----------- | ------------- | ------------- | ------------- | ----------- | -------------------| ------------------|",
     ]
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the summary totals dut table."""
-        for dut, stat in self.manager.dut_stats.items():
-            total_tests = stat.tests_passed + stat.tests_failed + stat.tests_skipped
-            categories_failed = ", ".join(sorted(stat.categories_failed))
+        for dut, stat in self.results.dut_stats.items():
+            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count
             categories_skipped = ", ".join(sorted(stat.categories_skipped))
+            categories_failed = ", ".join(sorted(stat.categories_failed))
             yield (
-                f"| {dut} | {total_tests} | {stat.tests_passed} | {stat.tests_failed} | {stat.tests_skipped} | {categories_failed or '-'} "
-                f"| {categories_skipped or '-'} |\n"
+                f"| {dut} | {total_tests} | {stat.tests_success_count} | {stat.tests_skipped_count} | {stat.tests_failure_count} | {stat.tests_error_count} "
+                f"| {categories_skipped or '-'} | {categories_failed or '-'} |\n"
             )
 
     def generate_section(self) -> None:
@@ -233,15 +236,15 @@ class SummaryTotalsPerCategory(MDReportBase):
     """Generate the `### Summary Totals Per Category` section of the markdown report."""
 
     TABLE_HEADING: ClassVar[list[str]] = [
-        "| Test Category | Total Tests | Tests Passed | Tests Failed | Tests Skipped |",
-        "| ------------- | ----------- | ------------ | ------------ | ------------- |",
+        "| Test Category | Total Tests | Tests Success | Tests Skipped | Tests Failure | Tests Error |",
+        "| ------------- | ----------- | ------------- | ------------- | ------------- | ----------- |",
     ]
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the summary totals per category table."""
-        for category, stat in self.manager.sorted_category_stats.items():
-            total_tests = stat.tests_passed + stat.tests_failed + stat.tests_skipped
-            yield f"| {category} | {total_tests} | {stat.tests_passed} | {stat.tests_failed} | {stat.tests_skipped} |\n"
+        for category, stat in self.results.sorted_category_stats.items():
+            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count
+            yield f"| {category} | {total_tests} | {stat.tests_success_count} | {stat.tests_skipped_count} | {stat.tests_failure_count} | {stat.tests_error_count}\n"
 
     def generate_section(self) -> None:
         """Generate the `### Summary Totals Per Category` section of the markdown report."""
@@ -254,12 +257,12 @@ class FailedTestResultsSummary(MDReportBase):
 
     TABLE_HEADING: ClassVar[list[str]] = [
         "| ID | Device Under Test | Categories | Test | Description | Custom Field | Result | Messages |",
-        "| -- | ----------------- | ---------- | ---- | ----------- | ------------ | -------| -------- |",
+        "| -- | ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- |",
     ]
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the failed test results table."""
-        for result in self.manager.get_results({"failure", "error", "unset"}):
+        for result in self.results.get_results({"failure", "error"}):
             messages = self.safe_markdown(", ".join(result.messages))
             categories = ", ".join(result.categories)
             yield (
@@ -281,12 +284,12 @@ class AllTestResults(MDReportBase):
 
     TABLE_HEADING: ClassVar[list[str]] = [
         "| ID | Device Under Test | Categories | Test | Description | Custom Field | Result | Messages |",
-        "| -- | ----------------- | ---------- | ---- | ----------- | ------------ | -------| -------- |",
+        "| -- | ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- |",
     ]
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the all test results table."""
-        for result in self.manager.results:
+        for result in self.results.get_results():
             messages = self.safe_markdown(", ".join(result.messages))
             categories = ", ".join(result.categories)
             yield (