diff --git a/README.md b/README.md index 291d8589..673599d9 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ A full list supported options are listed here. | video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a | | image-paths | -I | A list of images that will be saved in the simulators. | N | n/a | | unsafe-skip-xcode-version-check | | Skip Xcode version check | N | NO | +| retry-app-crash-tests | | Retry tests that crashed app and consider it non-fatal if it passes on retry. | N | false | ## Exit Status diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index ac8cf6ae..29ebc422 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic) BOOL saveDiagnosticsOnError; @property (nonatomic, strong) NSNumber *failureTolerance; @property (nonatomic) BOOL onlyRetryFailed; +@property (nonatomic) BOOL retryAppCrashTests; @property (nonatomic, strong) NSArray *testCasesToSkip; @property (nonatomic, strong) NSArray *testCasesToRun; @property (nonatomic, strong) NSArray *allTestCases; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 3e0c880f..f6edddbd 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -103,7 +103,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet", "Turn off all output except fatal errors."}, {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed", - "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."}, + "Only retry failed tests instead of all. Also retry test that timed-out/crashed."}, {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly", "Only list tests and exit without executing tests."}, {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging", @@ -143,7 +143,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { {364, "test-plan-path", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testPlanPath", "The path of a json file which describes the test plan. It is equivalent to the .xctestrun file generated by Xcode, but it can be generated by a different build system, e.g. Bazel"}, {365, "unsafe-skip-xcode-version-check", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL , "unsafeSkipXcodeVersionCheck", - " "}, + "Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."}, + {366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests", + "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."}, {0, 0, 0, 0, 0, 0, 0} }; diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index a1f85b49..26bb4eb9 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context { return; case BPExitStatusAppCrashed: - // Crashed test is considered fatal and shall not be retried - self.finalExitStatus |= context.exitStatus; + if (!self.config.retryAppCrashTests) { + // Crashed test is considered fatal when retry is disabled + self.finalExitStatus |= context.exitStatus; + } NEXT([self proceed]); return; diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m index 83341d2b..c8a5fad7 100644 --- a/bp/src/SimulatorMonitor.m +++ b/bp/src/SimulatorMonitor.m @@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output { NSString *testClass = (__self.currentClassName ?: __self.previousClassName); NSString *testName = (__self.currentTestName ?: __self.previousTestName); if (__self.testsState == Running) { - [self updateExecutedTestCaseList:testName inClass:testClass]; - [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName]; + if (self.config.retryAppCrashTests) { + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName]; + } else { + [self updateExecutedTestCaseList:testName inClass:testClass]; + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName]; + } [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"]; } else { assert(__self.testsState == Idle); diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index b23f8528..e63d2ed0 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -35,7 +35,7 @@ @implementation BluepillTests - (void)setUp { [super setUp]; - + self.continueAfterFailure = NO; NSString *hostApplicationPath = [BPTestHelper sampleAppPath]; NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath]; @@ -222,11 +222,11 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet { self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @1; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; - + self.config.onlyRetryFailed = TRUE; + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAppCrashed); - + NSString *junitReportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppCrashingTests-1-results.xml"]; NSLog(@"JUnit file: %@", junitReportPath); NSString *expectedFilePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:@"crash_tests_with_retry_attempt_1.xml"]; @@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode { self.config.testing_crashOnAttempt = @1; self.config.errorRetriesCount = @2; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); @@ -326,11 +326,11 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure { } /** - Execution plan: TIMEOUT, CRASH, PASS + Execution plan: TIMEOUT, CRASH (not retried) */ - (void)testReportFailureOnTimeoutCrashAndPass { self.config.stuckTimeout = @6; - self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS"; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH"; self.config.errorRetriesCount = @4; self.config.onlyRetryFailed = TRUE; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; @@ -345,6 +345,48 @@ - (void)testReportFailureOnTimeoutCrashAndPass { XCTAssertTrue(exitCode == BPExitStatusAppCrashed); } +/** + Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH"; + self.config.errorRetriesCount = @2; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed)); +} + +/** + Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + /** Execution plan: CRASH */ @@ -387,6 +429,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests { XCTAssertTrue(exitCode == BPExitStatusAppCrashed); } +/** + Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set + */ +- (void)testReportSuccessOnAppCrashTestPassesOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS"; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + self.config.errorRetriesCount = @2; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + /** Execution plan: One test CRASHes and another one keeps timing out */ @@ -457,7 +521,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry { self.config.stuckTimeout = @6; self.config.testing_ExecutionPlan = @"TIMEOUT PASS"; self.config.errorRetriesCount = @4; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; self.config.failureTolerance = @0; // Not relevant NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; @@ -478,7 +542,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry { self.config.stuckTimeout = @6; self.config.testing_ExecutionPlan = @"TIMEOUT"; self.config.errorRetriesCount = @2; - self.config.onlyRetryFailed = NO; + self.config.onlyRetryFailed = FALSE; self.config.failureTolerance = @1; // Not relevant since it's not a test failure NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; @@ -500,7 +564,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll { self.config.testing_ExecutionPlan = @"FAIL PASS"; self.config.errorRetriesCount = @4; self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails - self.config.failureTolerance = @1; // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO + self.config.failureTolerance = @1; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -578,7 +642,7 @@ - (void)testRetryOnlyFailures { self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @100; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssert(exitCode == BPExitStatusTestsFailed); // Make sure all tests started on the first run @@ -626,7 +690,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; - + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; XCTAssert(exitCode == BPExitStatusAppCrashed); @@ -639,7 +703,7 @@ - (void)testKeepSimulatorWithAppHangingTestsSet { self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; self.config.testing_ExecutionPlan = @"TIMEOUT"; - + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; XCTAssert(exitCode == BPExitStatusTestTimeout); @@ -649,15 +713,15 @@ - (void)testDeleteSimulatorOnly { NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; - + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); - + self.config.deleteSimUDID = bp.test_simulatorUDID; XCTAssertNotNil(self.config.deleteSimUDID); - + Bluepill *bp2 = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode2 = [bp2 run]; XCTAssert(exitCode2 == BPExitStatusSimulatorDeleted);