Distinguishing between JUnit final results and XUnit2 interim (rerun …

…& flaky) results (#144)

* Rework types of XUnit2 interim and JUnit final results

* Simplify type checking in TestCase.result, enforce input types

* Update CHANGELOG.md

* Fix README.md

* Simplify FINAL_RESULTS constant

Co-authored-by: Jan Wille <[email protected]>

---------

Co-authored-by: Jan Wille <[email protected]>

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [4.0.0]
+### Breaking
+- Setter method `TestCase.result` used to ignore values of invalid types. This method now throws a `ValueError` instead.
+- Method `xunit2.TestCase.add_rerun_result` has been renamed to `add_interim_result` result to better reflect class hierarchy
+  of interim (rerun and flaky) results.
+
 ## [3.1.2] - 2024-08-31
 ### Fixed
 - the `TestCase.result` type annotation
@@ -167,4 +173,4 @@ This release addresses issues and PRs by @markgras.
 ## [0.9.0]
 ### Changed
 * Supports xmls with ``testcase`` as root node.
-* First beta release.
+* First beta release.

README.rst

@@ -152,7 +152,7 @@ Junitparser also support extra schemas:
     rerun_failure.stack_trace = "Stack"
     rerun_failure.system_err = "E404"
     rerun_failure.system_out = "NOT FOUND"
-    case.add_rerun_result(rerun_failure)
+    case.add_interim_result(rerun_failure)
 
 Currently supported schemas including:
 

junitparser/junitparser.py

@@ -241,7 +241,12 @@ def text(self, value: str):
         self._elem.text = value
 
 
-class Skipped(Result):
+class FinalResult(Result):
+    """Base class for final test result (in contrast to XUnit2 InterimResult)."""
+
+    _tag = None
+
+class Skipped(FinalResult):
     """Test result when the case is skipped."""
 
     _tag = "skipped"
@@ -250,7 +255,7 @@ def __eq__(self, other):
         return super().__eq__(other)
 
 
-class Failure(Result):
+class Failure(FinalResult):
     """Test result when the case failed."""
 
     _tag = "failure"
@@ -259,7 +264,7 @@ def __eq__(self, other):
         return super().__eq__(other)
 
 
-class Error(Result):
+class Error(FinalResult):
     """Test result when the case has errors during execution."""
 
     _tag = "error"
@@ -268,9 +273,6 @@ def __eq__(self, other):
         return super().__eq__(other)
 
 
-POSSIBLE_RESULTS = {Failure, Error, Skipped}
-
-
 class System(Element):
     """Parent class for :class:`SystemOut` and :class:`SystemErr`.
 
@@ -329,7 +331,7 @@ def __hash__(self):
         return super().__hash__()
 
     def __iter__(self) -> Iterator[Union[Result, System]]:
-        all_types = set.union(POSSIBLE_RESULTS, {SystemOut}, {SystemErr})
+        all_types = {Failure, Error, Skipped, SystemOut, SystemErr}
         for elem in self._elem.iter():
             for entry_type in all_types:
                 if elem.tag == entry_type._tag:
@@ -353,27 +355,30 @@ def is_skipped(self):
         return False
 
     @property
-    def result(self):
+    def result(self) -> List[FinalResult]:
         """A list of :class:`Failure`, :class:`Skipped`, or :class:`Error` objects."""
         results = []
         for entry in self:
-            if isinstance(entry, tuple(POSSIBLE_RESULTS)):
+            if isinstance(entry, FinalResult):
                 results.append(entry)
 
         return results
 
     @result.setter
-    def result(self, value: Union[Result, List[Result]]):
+    def result(self, value: Union[FinalResult, List[FinalResult]]):
+        # Check typing
+        if not (isinstance(value, FinalResult) or
+                isinstance(value, list) and all(isinstance(item, FinalResult) for item in value)):
+            raise ValueError("Value must be either FinalResult or list of FinalResult")
+
         # First remove all existing results
         for entry in self.result:
-            if any(isinstance(entry, r) for r in POSSIBLE_RESULTS):
-                self.remove(entry)
-        if isinstance(value, Result):
+            self.remove(entry)
+        if isinstance(value, FinalResult):
             self.append(value)
-        elif isinstance(value, list):
+        else:
             for entry in value:
-                if any(isinstance(entry, r) for r in POSSIBLE_RESULTS):
-                    self.append(entry)
+                self.append(entry)
 
     @property
     def system_out(self):

junitparser/xunit2.py

@@ -23,8 +23,10 @@ class StackTrace(junitparser.System):
     _tag = "stackTrace"
 
 
-class RerunType(junitparser.Result):
-    _tag = "rerunType"
+class InterimResult(junitparser.Result):
+    """Base class for intermediate (rerun and flaky) test result (in contrast to JUnit FinalResult)."""
+
+    _tag = None
 
     @property
     def stack_trace(self):
@@ -81,19 +83,19 @@ def system_err(self, value: str):
             self.append(err)
 
 
-class RerunFailure(RerunType):
+class RerunFailure(InterimResult):
     _tag = "rerunFailure"
 
 
-class RerunError(RerunType):
+class RerunError(InterimResult):
     _tag = "rerunError"
 
 
-class FlakyFailure(RerunType):
+class FlakyFailure(InterimResult):
     _tag = "flakyFailure"
 
 
-class FlakyError(RerunType):
+class FlakyError(InterimResult):
     _tag = "flakyError"
 
 
@@ -123,8 +125,8 @@ def flaky_errors(self):
         """<flakyError>"""
         return self._rerun_results(FlakyError)
 
-    def add_rerun_result(self, result: RerunType):
-        """Append a rerun result to the testcase. A testcase can have multiple rerun results."""
+    def add_interim_result(self, result: InterimResult):
+        """Append an interim (rerun or flaky) result to the testcase. A testcase can have multiple interim results."""
         self.append(result)
 
 

tests/test_xunit2.py

@@ -45,14 +45,14 @@ def test_case_rerun(self):
         rerun_failure.stack_trace = "Stack"
         rerun_failure.system_err = "E404"
         rerun_failure.system_out = "NOT FOUND"
-        case.add_rerun_result(rerun_failure)
+        case.add_interim_result(rerun_failure)
         assert len(case.rerun_failures()) == 1
         # Interesting, same object is only added once by xml libs
         failure2 = deepcopy(rerun_failure)
         failure2.stack_trace = "Stack2"
         failure2.system_err = "E401"
         failure2.system_out = "401 Error"
-        case.add_rerun_result(failure2)
+        case.add_interim_result(failure2)
         assert len(case.rerun_failures()) == 2