More test result methods (#145)

* Add more is_* properties to JUnit and XUnit2 TestCase

- Add is_failure and is_error to JUnit TestCase
- Add is_rerun and is_flaky to XUnit2 TestCase
- Provide access to XUnit2 interim (rerun & flaky) results

---------

Co-authored-by: Jan Wille <[email protected]>

junitparser/junitparser.py

@@ -312,6 +312,9 @@ class TestCase(Element):
     time = FloatAttr()
     __test__ = False
 
+    # JUnit TestCase children are final results, SystemOut and SystemErr
+    ITER_TYPES = {t._tag: t for t in (Failure, Error, Skipped, SystemOut, SystemErr)}
+
     def __init__(self, name: str = None, classname: str = None, time: float = None):
         super().__init__(self._tag)
         if name is not None:
@@ -325,11 +328,9 @@ def __hash__(self):
         return super().__hash__()
 
     def __iter__(self) -> Iterator[Union[Result, System]]:
-        all_types = {Failure, Error, Skipped, SystemOut, SystemErr}
         for elem in self._elem.iter():
-            for entry_type in all_types:
-                if elem.tag == entry_type._tag:
-                    yield entry_type.fromelem(elem)
+            if elem.tag in self.ITER_TYPES:
+                yield self.ITER_TYPES[elem.tag].fromelem(elem)
 
     def __eq__(self, other):
         # TODO: May not work correctly if unreliable hash method is used.
@@ -340,23 +341,25 @@ def is_passed(self):
         """Whether this testcase was a success (i.e. if it isn't skipped, failed, or errored)."""
         return not self.result
 
+    @property
+    def is_failure(self):
+        """Whether this testcase failed."""
+        return any(isinstance(r, Failure) for r in self.result)
+
+    @property
+    def is_error(self):
+        """Whether this testcase errored."""
+        return any(isinstance(r, Error) for r in self.result)
+
     @property
     def is_skipped(self):
         """Whether this testcase was skipped."""
-        for r in self.result:
-            if isinstance(r, Skipped):
-                return True
-        return False
+        return any(isinstance(r, Skipped) for r in self.result)
 
     @property
     def result(self) -> List[FinalResult]:
         """A list of :class:`Failure`, :class:`Skipped`, or :class:`Error` objects."""
-        results = []
-        for entry in self:
-            if isinstance(entry, FinalResult):
-                results.append(entry)
-
-        return results
+        return [entry for entry in self if isinstance(entry, FinalResult)]
 
     @result.setter
     def result(self, value: Union[FinalResult, List[FinalResult]]):

junitparser/xunit2.py

@@ -12,11 +12,10 @@
 There may be many others that I'm not aware of.
 """
 
-from typing import List, TypeVar
+import itertools
+from typing import List, Type, TypeVar
 from . import junitparser
 
-T = TypeVar("T")
-
 
 class StackTrace(junitparser.System):
     _tag = "stackTrace"
@@ -98,31 +97,58 @@ class FlakyError(InterimResult):
     _tag = "flakyError"
 
 
+R = TypeVar("R", bound=InterimResult)
+
+
 class TestCase(junitparser.TestCase):
     group = junitparser.Attr()
 
-    def _rerun_results(self, _type: T) -> List[T]:
-        elems = self.iterchildren(_type)
-        results = []
-        for elem in elems:
-            results.append(_type.fromelem(elem))
-        return results
+    # XUnit2 TestCase children are JUnit children and intermediate results
+    ITER_TYPES = {
+        t._tag: t
+        for t in itertools.chain(
+            junitparser.TestCase.ITER_TYPES.values(),
+            (RerunFailure, RerunError, FlakyFailure, FlakyError),
+        )
+    }
 
-    def rerun_failures(self):
+    def _interim_results(self, _type: Type[R]) -> List[R]:
+        return [entry for entry in self if isinstance(entry, _type)]
+
+    @property
+    def interim_result(self) -> List[InterimResult]:
+        """
+        A list of interim results: :class:`RerunFailure`, :class:`RerunError`,
+        :class:`FlakyFailure`, or :class:`FlakyError` objects.
+        This is complementary to the result property returning final results.
+        """
+        return self._interim_results(InterimResult)
+
+    def rerun_failures(self) -> List[RerunFailure]:
         """<rerunFailure>"""
-        return self._rerun_results(RerunFailure)
+        return self._interim_results(RerunFailure)
 
-    def rerun_errors(self):
+    def rerun_errors(self) -> List[RerunError]:
         """<rerunError>"""
-        return self._rerun_results(RerunError)
+        return self._interim_results(RerunError)
 
-    def flaky_failures(self):
+    def flaky_failures(self) -> List[FlakyFailure]:
         """<flakyFailure>"""
-        return self._rerun_results(FlakyFailure)
+        return self._interim_results(FlakyFailure)
 
-    def flaky_errors(self):
+    def flaky_errors(self) -> List[FlakyError]:
         """<flakyError>"""
-        return self._rerun_results(FlakyError)
+        return self._interim_results(FlakyError)
+
+    @property
+    def is_rerun(self) -> bool:
+        """Whether this testcase is rerun, i.e., there are rerun failures or errors."""
+        return any(self.rerun_failures()) or any(self.rerun_errors())
+
+    @property
+    def is_flaky(self) -> bool:
+        """Whether this testcase is flaky, i.e., there are flaky failures or errors."""
+        return any(self.flaky_failures()) or any(self.flaky_errors())
 
     def add_interim_result(self, result: InterimResult):
         """Append an interim (rerun or flaky) result to the testcase. A testcase can have multiple interim results."""

tests/test_general.py

@@ -693,18 +693,32 @@ def test_case_is_skipped(self):
         case.result = [Skipped()]
         assert case.is_skipped
         assert not case.is_passed
+        assert not case.is_failure
+        assert not case.is_error
 
     def test_case_is_passed(self):
         case = TestCase()
         case.result = []
         assert not case.is_skipped
         assert case.is_passed
+        assert not case.is_failure
+        assert not case.is_error
 
     def test_case_is_failed(self):
         case = TestCase()
         case.result = [Failure()]
         assert not case.is_skipped
         assert not case.is_passed
+        assert case.is_failure
+        assert not case.is_error
+
+    def test_case_is_error(self):
+        case = TestCase()
+        case.result = [Error()]
+        assert not case.is_skipped
+        assert not case.is_passed
+        assert not case.is_failure
+        assert case.is_error
 
 
 class Test_Properties:

tests/test_xunit2.py

@@ -1,10 +1,10 @@
-from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure
+from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure, RerunError, FlakyFailure, FlakyError
 from junitparser import Failure
 from copy import deepcopy
 
 
 class Test_TestCase:
-    def test_case_fromstring(self):
+    def test_case_rerun_fromstring(self):
         text = """<testcase name="testname">
         <failure message="failure message" type="FailureType"/>
         <rerunFailure message="Not found" type="404">
@@ -14,29 +14,111 @@ def test_case_fromstring(self):
             <system-err>Error del servidor</system-err>
             <stackTrace>Stacktrace</stackTrace>
         </rerunFailure>
+        <rerunError message="Setup error"/>
         <system-out>System out</system-out>
         <system-err>System err</system-err>
         </testcase>"""
         case = TestCase.fromstring(text)
         assert isinstance(case, TestCase)
         assert case.name == "testname"
+        assert len(case.result) == 1
         assert isinstance(case.result[0], Failure)
         assert case.system_out == "System out"
         assert case.system_err == "System err"
+        assert case.is_passed is False
+        assert case.is_failure is True
+        assert case.is_error is False
+        assert case.is_skipped is False
+        assert case.is_rerun is True
+        assert case.is_flaky is False
+
+        interim_results = case.interim_result
+        assert len(interim_results) == 3
+        assert isinstance(interim_results[0], RerunFailure)
+        assert isinstance(interim_results[1], RerunFailure)
+        assert isinstance(interim_results[2], RerunError)
+
         rerun_failures = case.rerun_failures()
         assert len(rerun_failures) == 2
+        assert isinstance(rerun_failures[0], RerunFailure)
         assert rerun_failures[0].message == "Not found"
         assert rerun_failures[0].stack_trace is None
         assert rerun_failures[0].system_out == "No ha encontrado"
         assert rerun_failures[0].system_err is None
+        assert isinstance(rerun_failures[1], RerunFailure)
         assert rerun_failures[1].message == "Server error"
         assert rerun_failures[1].stack_trace == "Stacktrace"
         assert rerun_failures[1].system_out is None
         assert rerun_failures[1].system_err == "Error del servidor"
-        assert len(case.rerun_errors()) == 0
+
+        rerun_errors = case.rerun_errors()
+        assert len(rerun_errors) == 1
+        assert isinstance(rerun_errors[0], RerunError)
+        assert rerun_errors[0].message == "Setup error"
+        assert rerun_errors[0].stack_trace is None
+        assert rerun_errors[0].system_out is None
+        assert rerun_errors[0].system_err is None
+
         assert len(case.flaky_failures()) == 0
+
         assert len(case.flaky_errors()) == 0
 
+    def test_case_flaky_fromstring(self):
+        text = """<testcase name="testname">
+        <flakyFailure message="Not found" type="404">
+            <system-out>No ha encontrado</system-out>
+        </flakyFailure>
+        <flakyFailure message="Server error" type="500">
+            <system-err>Error del servidor</system-err>
+            <stackTrace>Stacktrace</stackTrace>
+        </flakyFailure>
+        <flakyError message="Setup error"/>
+        <system-out>System out</system-out>
+        <system-err>System err</system-err>
+        </testcase>"""
+        case = TestCase.fromstring(text)
+        assert case.name == "testname"
+        assert len(case.result) == 0
+        assert case.system_out == "System out"
+        assert case.system_err == "System err"
+        assert case.is_passed is True
+        assert case.is_failure is False
+        assert case.is_error is False
+        assert case.is_skipped is False
+        assert case.is_rerun is False
+        assert case.is_flaky is True
+
+        interim_results = case.interim_result
+        assert len(interim_results) == 3
+        assert isinstance(interim_results[0], FlakyFailure)
+        assert isinstance(interim_results[1], FlakyFailure)
+        assert isinstance(interim_results[2], FlakyError)
+
+        assert len(case.rerun_failures()) == 0
+
+        assert len(case.rerun_errors()) == 0
+
+        flaky_failures = case.flaky_failures()
+        assert len(flaky_failures) == 2
+        assert isinstance(flaky_failures[0], FlakyFailure)
+        assert flaky_failures[0].message == "Not found"
+        assert flaky_failures[0].stack_trace is None
+        assert flaky_failures[0].system_out == "No ha encontrado"
+        assert flaky_failures[0].system_err is None
+        assert isinstance(flaky_failures[1], FlakyFailure)
+        assert flaky_failures[1].message == "Server error"
+        assert flaky_failures[1].stack_trace == "Stacktrace"
+        assert flaky_failures[1].system_out is None
+        assert flaky_failures[1].system_err == "Error del servidor"
+
+        flaky_errors = case.flaky_errors()
+        assert len(flaky_errors) == 1
+        assert isinstance(flaky_errors[0], FlakyError)
+        assert flaky_errors[0].message == "Setup error"
+        assert flaky_errors[0].stack_trace is None
+        assert flaky_errors[0].system_out is None
+        assert flaky_errors[0].system_err is None
+
     def test_case_rerun(self):
         case = TestCase("testname")
         rerun_failure = RerunFailure("Not found", "404")