Do not retry crashed tests

Based on recent change crashed tests are fatal and would always return a non-zero exit code. After this change, the crashed tests are NOT retried to make sure the app crash errors are surfaced without confusion. Also added new tests and fixed existing ones as per the new no-retry behavior.

bp/src/SimulatorMonitor.m

@@ -195,7 +195,8 @@ - (void)onOutputReceived:(NSString *)output {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
             if (__self.testsState == Running) {
-                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app.", testClass, testName];
+                [self updateExecutedTestCaseList:testName inClass:testClass];
+                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
                 [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
             } else {
                 assert(__self.testsState == Idle);

bp/tests/BluepillTests.m

@@ -346,11 +346,12 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
 }
 
 /**
- Execution plan: CRASH, TIMEOUT, PASS
+ Execution plan: CRASH
  */
-- (void)testReportFailureOnCrashTimeoutAndPass {
+- (void)testNoRetryOnCrash {
     self.config.stuckTimeout = @6;
-    self.config.testing_ExecutionPlan = @"CRASH TIMEOUT PASS";
+    self.config.testing_ExecutionPlan = @"CRASH";  // No retry
+    self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = TRUE;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -364,6 +365,49 @@ - (void)testReportFailureOnCrashTimeoutAndPass {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+
+/**
+ Execution plan: One test CRASHes and another one TIMEs OUT and PASSes on retry
+ */
+- (void)testReportFailureOnCrashAndTimeoutTests {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT PASS";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
+}
+
+/**
+ Execution plan: One test CRASHes and another one keeps timing out
+ */
+- (void)testReportBothCrashAndTimeout {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT TIMEOUT";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout));
+}
+
 /**
  Execution plan: FAIL, TIMEOUT, PASS
  */