Consider app crash tests as non-fatal if they pass on retry

README.md

@@ -89,6 +89,7 @@ A full list supported options are listed here.
 |       video-paths      |           -V           | A list of videos that will be saved in the simulators.                              |     N    | n/a              |
 |       image-paths      |           -I           | A list of images that will be saved in the simulators.                              |     N    | n/a              |
 | unsafe-skip-xcode-version-check |               | Skip Xcode version check                                                            |     N    | NO               |
+|  retry-app-crash-tests |                        | Retry tests that crashed app and consider it non-fatal if it passes on retry.       |     N    | false            |
 
 
 ## Exit Status

bp/src/BPConfiguration.h

@@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 @property (nonatomic) BOOL saveDiagnosticsOnError;
 @property (nonatomic, strong) NSNumber *failureTolerance;
 @property (nonatomic) BOOL onlyRetryFailed;
+@property (nonatomic) BOOL retryAppCrashTests;
 @property (nonatomic, strong) NSArray *testCasesToSkip;
 @property (nonatomic, strong) NSArray *testCasesToRun;
 @property (nonatomic, strong) NSArray *allTestCases;

bp/src/BPConfiguration.m

@@ -103,7 +103,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
     {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet",
         "Turn off all output except fatal errors."},
     {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed",
-        "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."},
+        "Only retry failed tests instead of all. Also retry test that timed-out/crashed."},
     {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly",
         "Only list tests and exit without executing tests."},
     {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging",
@@ -143,7 +143,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
     {364, "test-plan-path", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testPlanPath",
         "The path of a json file which describes the test plan. It is equivalent to the .xctestrun file generated by Xcode, but it can be generated by a different build system, e.g. Bazel"},
     {365, "unsafe-skip-xcode-version-check", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL , "unsafeSkipXcodeVersionCheck",
-    "   "},
+        "Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."},
+    {366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
+        "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
 
     {0, 0, 0, 0, 0, 0, 0}
 };

bp/src/Bluepill.m

@@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context {
             return;
 
         case BPExitStatusAppCrashed:
-            // Crashed test is considered fatal and shall not be retried
-            self.finalExitStatus |= context.exitStatus;
+            if (!self.config.retryAppCrashTests) {
+                // Crashed test is considered fatal when retry is disabled
+                self.finalExitStatus |= context.exitStatus;
+            }
             NEXT([self proceed]);
             return;
 

bp/src/SimulatorMonitor.m

@@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
             if (__self.testsState == Running) {
-                [self updateExecutedTestCaseList:testName inClass:testClass];
-                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
+                if (self.config.retryAppCrashTests) {
+                    [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName];
+                } else {
+                    [self updateExecutedTestCaseList:testName inClass:testClass];
+                    [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName];
+                }
                 [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
             } else {
                 assert(__self.testsState == Idle);

bp/tests/BluepillTests.m

@@ -35,7 +35,7 @@ @implementation BluepillTests
 
 - (void)setUp {
     [super setUp];
-    
+
     self.continueAfterFailure = NO;
     NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
     NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath];
@@ -222,11 +222,11 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet {
     self.config.outputDirectory = outputDir;
     self.config.errorRetriesCount = @1;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
-    
+    self.config.onlyRetryFailed = TRUE;
+
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
-    
+
     NSString *junitReportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppCrashingTests-1-results.xml"];
     NSLog(@"JUnit file: %@", junitReportPath);
     NSString *expectedFilePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:@"crash_tests_with_retry_attempt_1.xml"];
@@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode {
     self.config.testing_crashOnAttempt = @1;
     self.config.errorRetriesCount = @2;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
 
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
@@ -326,11 +326,11 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure {
 }
 
 /**
- Execution plan: TIMEOUT, CRASH, PASS
+ Execution plan: TIMEOUT, CRASH (not retried)
  */
 - (void)testReportFailureOnTimeoutCrashAndPass {
     self.config.stuckTimeout = @6;
-    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH";
     self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = TRUE;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
@@ -345,6 +345,48 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+/**
+ Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal
+ */
+- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH";
+    self.config.errorRetriesCount = @2;
+    self.config.onlyRetryFailed = TRUE;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed));
+}
+
+/**
+ Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal
+ */
+- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
+    self.config.errorRetriesCount = @4;
+    self.config.onlyRetryFailed = TRUE;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
+}
+
 /**
  Execution plan: CRASH
  */
@@ -387,6 +429,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+/**
+ Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set
+ */
+- (void)testReportSuccessOnAppCrashTestPassesOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
+}
+
 /**
  Execution plan: One test CRASHes and another one keeps timing out
  */
@@ -457,7 +521,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry {
     self.config.stuckTimeout = @6;
     self.config.testing_ExecutionPlan = @"TIMEOUT PASS";
     self.config.errorRetriesCount = @4;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
     self.config.failureTolerance = @0;  // Not relevant
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -478,7 +542,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry {
     self.config.stuckTimeout = @6;
     self.config.testing_ExecutionPlan = @"TIMEOUT";
     self.config.errorRetriesCount = @2;
-    self.config.onlyRetryFailed = NO;
+    self.config.onlyRetryFailed = FALSE;
     self.config.failureTolerance = @1;  // Not relevant since it's not a test failure
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -500,7 +564,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll {
     self.config.testing_ExecutionPlan = @"FAIL PASS";
     self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = NO;  // Indicates to retry all tests when a test fails
-    self.config.failureTolerance = @1;  // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO
+    self.config.failureTolerance = @1;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     NSString *tempDir = NSTemporaryDirectory();
@@ -578,7 +642,7 @@ - (void)testRetryOnlyFailures {
     self.config.outputDirectory = outputDir;
     self.config.errorRetriesCount = @100;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssert(exitCode == BPExitStatusTestsFailed);
     // Make sure all tests started on the first run
@@ -626,7 +690,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet  {
     NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusAppCrashed);
@@ -639,7 +703,7 @@ - (void)testKeepSimulatorWithAppHangingTestsSet  {
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
     self.config.testing_ExecutionPlan = @"TIMEOUT";
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusTestTimeout);
@@ -649,15 +713,15 @@ - (void)testDeleteSimulatorOnly {
     NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusAllTestsPassed);
     XCTAssertNotNil(bp.test_simulatorUDID);
-    
+
     self.config.deleteSimUDID = bp.test_simulatorUDID;
     XCTAssertNotNil(self.config.deleteSimUDID);
-    
+
     Bluepill *bp2 = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode2 = [bp2 run];
     XCTAssert(exitCode2 == BPExitStatusSimulatorDeleted);