Merge branch 'add-line-coverage-to-report' into 'main'

Add line coverage to coverage report

See merge request se2/pynguin/pynguin!131

pynguin/coverage/branchgoals.py

@@ -73,7 +73,7 @@ def line_id(self) -> int:
         return self._line_id
 
     def is_covered(self, result: ExecutionResult) -> bool:
-        return self._line_id in result.execution_trace.covered_lines
+        return self._line_id in result.execution_trace.covered_line_ids
 
     def __str__(self) -> str:
         return f"Line Coverage Goal{self._line_id}"

pynguin/ga/computations.py

@@ -234,7 +234,7 @@ def compute_fitness(self, individual) -> float:
         results = self._run_test_suite_chromosome(individual)
         merged_trace = analyze_results(results)
         existing_lines = self._executor.tracer.get_known_data().existing_lines
-        return len(existing_lines) - len(merged_trace.covered_lines)
+        return len(existing_lines) - len(merged_trace.covered_line_ids)
 
     def compute_is_covered(self, individual) -> bool:
         results = self._run_test_suite_chromosome(individual)
@@ -712,7 +712,7 @@ def compute_line_coverage_fitness_is_covered(
     Returns:
         True, if all lines were covered, false otherwise
     """
-    return len(trace.covered_lines) == len(known_data.existing_lines)
+    return len(trace.covered_line_ids) == len(known_data.existing_lines)
 
 
 def compute_branch_coverage(trace: ExecutionTrace, known_data: KnownData) -> float:
@@ -765,7 +765,7 @@ def compute_line_coverage(trace: ExecutionTrace, known_data: KnownData) -> float
         # Nothing to cover => everything is covered.
         coverage = 1.0
     else:
-        covered = len(trace.covered_lines)
+        covered = len(trace.covered_line_ids)
         coverage = covered / existing
     assert 0.0 <= coverage <= 1.0, "Coverage must be in [0,1]"
     return coverage

pynguin/generator.py

@@ -350,7 +350,11 @@ def _run() -> ReturnCode:
 
     if config.configuration.statistics_output.create_coverage_report:
         render_coverage_report(
-            get_coverage_report(generation_result, executor),
+            get_coverage_report(
+                generation_result,
+                executor,
+                config.configuration.statistics_output.coverage_metrics,
+            ),
             Path(config.configuration.statistics_output.report_dir) / "cov_report.html",
             datetime.datetime.now(),
         )

pynguin/instrumentation/instrumentation.py

@@ -14,7 +14,6 @@
 
 from bytecode import BasicBlock, Bytecode, Compare, ControlFlowGraph, Instr
 
-import pynguin.utils.opcodes as op
 from pynguin.analyses.controlflow import CFG, ControlDependenceGraph
 from pynguin.analyses.seeding import DynamicConstantSeeding
 from pynguin.testcase.execution import (
@@ -592,22 +591,19 @@ def visit_node(
         node: ProgramGraphNode,
         basic_block: BasicBlock,
     ) -> None:
-        if not is_return_none_basic_block(basic_block):
-            #  iterate over instructions after the fist one in BB,
-            #  put new instructions in the block for each line
-            file_name = cfg.bytecode_cfg().filename
-            lineno = -1
-            instr_index = 0
-            while instr_index < len(basic_block):
-                if basic_block[instr_index].lineno != lineno:
-                    lineno = basic_block[instr_index].lineno
-                    line_id = self._tracer.register_line(
-                        code_object_id, file_name, lineno
-                    )
-                    instr_index += (  # increment by the amount of instructions inserted
-                        self.instrument_line(basic_block, instr_index, line_id, lineno)
-                    )
-                instr_index += 1
+        #  iterate over instructions after the fist one in BB,
+        #  put new instructions in the block for each line
+        file_name = cfg.bytecode_cfg().filename
+        lineno = None
+        instr_index = 0
+        while instr_index < len(basic_block):
+            if basic_block[instr_index].lineno != lineno:
+                lineno = basic_block[instr_index].lineno
+                line_id = self._tracer.register_line(code_object_id, file_name, lineno)
+                instr_index += (  # increment by the amount of instructions inserted
+                    self.instrument_line(basic_block, instr_index, line_id, lineno)
+                )
+            instr_index += 1
 
     def instrument_line(
         self, block: BasicBlock, instr_index: int, line_id: int, lineno: int
@@ -854,21 +850,3 @@ def _instrument_compare_op(self, block: BasicBlock) -> None:
             Instr("POP_TOP", lineno=lineno),
         ]
         self._logger.debug("Instrumented compare_op")
-
-
-def is_return_none_basic_block(basic_block: BasicBlock) -> bool:
-    """Checks if a node is a "return None" line.
-
-    Args:
-        basic_block: The basic block that needs to be checked
-
-    Returns:
-        True, if the node is a "return None" line, false otherwise.
-    """
-    # TODO(fk) there seem to be cases where this check is not sufficient.
-    # Not sure how to detect those.
-    return (
-        len(basic_block) == 2
-        and basic_block[0] == Instr("LOAD_CONST", None, lineno=basic_block[0].lineno)
-        and basic_block[1].opcode == op.RETURN_VALUE
-    )

pynguin/resources/coverage-template.html

@@ -28,25 +28,31 @@
 }
 
 .notCovered{
-    border-right: 3px solid darkred;
+    border-right: 5px solid darkred;
 }
 .partiallyCovered{
-    border-right: 3px solid orangered;
+    border-right: 5px solid orangered;
 }
 .fullyCovered{
-    border-right: 3px solid darkgreen;
+    border-right: 5px solid darkgreen;
 }
 .notRelevant{
-    border-right: 3px solid transparent;
+    border-right: 5px solid transparent;
 }
 
 </style>
 </head>
 <body>
 <h1>Pynguin coverage report for module '{{cov_report.module}}'</h1>
-<p>Achieved {{"{:.2%}".format(cov_report.branch_coverage)}} branch coverage.
-<p>{{cov_report.branchless_code_objects.covered}}/{{cov_report.branchless_code_objects.existing}} branchless code objects covered.</p>
-<p>{{cov_report.branches.covered}}/{{cov_report.branches.existing}} branches covered.</p>
+{% if cov_report.branch_coverage != None -%}
+<p>Achieved {{"{:.2%}".format(cov_report.branch_coverage)}} branch coverage:
+{{cov_report.branchless_code_objects.covered}}/{{cov_report.branchless_code_objects.existing}} branchless code objects covered.
+{{cov_report.branches.covered}}/{{cov_report.branches.existing}} branches covered.</p>
+{% endif -%}
+{% if cov_report.line_coverage != None -%}
+<p>Achieved {{"{:.2%}".format(cov_report.line_coverage)}} line coverage:
+{{cov_report.lines.covered}}/{{cov_report.lines.existing}} lines covered. </p>
+{% endif -%}
   <table>
     <tbody>
         <tr>

pynguin/testcase/execution.py

@@ -314,7 +314,7 @@ class ExecutionTrace:
     executed_predicates: dict[int, int] = field(default_factory=dict)
     true_distances: dict[int, float] = field(default_factory=dict)
     false_distances: dict[int, float] = field(default_factory=dict)
-    covered_lines: OrderedSet[int] = field(default_factory=OrderedSet)
+    covered_line_ids: OrderedSet[int] = field(default_factory=OrderedSet)
 
     def merge(self, other: ExecutionTrace) -> None:
         """Merge the values from the other trace.
@@ -327,7 +327,7 @@ def merge(self, other: ExecutionTrace) -> None:
             self.executed_predicates[key] = self.executed_predicates.get(key, 0) + value
         self._merge_min(self.true_distances, other.true_distances)
         self._merge_min(self.false_distances, other.false_distances)
-        self.covered_lines.update(other.covered_lines)
+        self.covered_line_ids.update(other.covered_line_ids)
 
     @staticmethod
     def _merge_min(target: dict[int, float], source: dict[int, float]) -> None:
@@ -720,7 +720,7 @@ def track_line_visit(self, line_id: int) -> None:
         Args:
             line_id: the if of the line that was visited
         """
-        self._trace.covered_lines.add(line_id)
+        self._trace.covered_line_ids.add(line_id)
 
     def register_line(
         self, code_object_id: int, file_name: str, line_number: int
@@ -768,6 +768,22 @@ def _update_metrics(
     def __repr__(self) -> str:
         return "ExecutionTracer"
 
+    def lineids_to_linenos(self, line_ids: OrderedSet[int]) -> OrderedSet[int]:
+        """Convenience method to translate line ids to line numbers.
+
+        Args:
+            line_ids: The ids that should be translated.
+
+        Returns:
+            The line numbers.
+        """
+        return OrderedSet(
+            [
+                self._known_data.existing_lines[line_id].line_number
+                for line_id in line_ids
+            ]
+        )
+
 
 def _eq(val1, val2) -> float:
     """Distance computation for '=='