Skip to content

Commit

Permalink
Retry app crash tests and consider then non-fatal if they pass
Browse files Browse the repository at this point in the history
  • Loading branch information
@ravimandala
ravimandala committed Aug 11, 2020
1 parent 08d3951 commit f7159fc
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 12 deletions.
1 change: 1 addition & 0 deletions bp/src/BPConfiguration.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
@property (nonatomic) BOOL saveDiagnosticsOnError;
@property (nonatomic, strong) NSNumber *failureTolerance;
@property (nonatomic) BOOL onlyRetryFailed;
@property (nonatomic) BOOL retryAppCrashTests;
@property (nonatomic, strong) NSArray *testCasesToSkip;
@property (nonatomic, strong) NSArray *testCasesToRun;
@property (nonatomic, strong) NSArray *allTestCases;
Expand Down
4 changes: 3 additions & 1 deletion bp/src/BPConfiguration.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
{'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet",
"Turn off all output except fatal errors."},
{'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed",
"Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."},
"Only retry failed tests instead of all. Also retry test that timed-out/crashed."},
{'s', "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
"Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
{'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly",
"Only list tests and exit without executing tests."},
{'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging",
Expand Down
6 changes: 4 additions & 2 deletions bp/src/Bluepill.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context {
return;

case BPExitStatusAppCrashed:
// Crashed test is considered fatal and shall not be retried
self.finalExitStatus |= context.exitStatus;
if (!self.config.retryAppCrashTests) {
// Crashed test is considered fatal when retry is disabled
self.finalExitStatus |= context.exitStatus;
}
NEXT([self proceed]);
return;

Expand Down
8 changes: 6 additions & 2 deletions bp/src/SimulatorMonitor.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output {
NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
NSString *testName = (__self.currentTestName ?: __self.previousTestName);
if (__self.testsState == Running) {
[self updateExecutedTestCaseList:testName inClass:testClass];
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
if (self.config.retryAppCrashTests) {
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName];
} else {
[self updateExecutedTestCaseList:testName inClass:testClass];
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName];
}
[[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
} else {
assert(__self.testsState == Idle);
Expand Down
79 changes: 72 additions & 7 deletions bp/tests/BluepillTests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet {
self.config.outputDirectory = outputDir;
self.config.errorRetriesCount = @1;
self.config.failureTolerance = @1;
self.config.onlyRetryFailed = YES;
self.config.onlyRetryFailed = TRUE;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
Expand All @@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode {
self.config.testing_crashOnAttempt = @1;
self.config.errorRetriesCount = @2;
self.config.failureTolerance = @1;
self.config.onlyRetryFailed = YES;
self.config.onlyRetryFailed = TRUE;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
Expand Down Expand Up @@ -342,7 +342,50 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
self.config.outputDirectory = outputDir;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout));
}

/**
Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal
*/
- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry {
self.config.stuckTimeout = @6;
self.config.retryAppCrashTests = TRUE;
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH";
self.config.errorRetriesCount = @2;
self.config.onlyRetryFailed = TRUE;
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
NSString *tempDir = NSTemporaryDirectory();
NSError *error;
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
NSLog(@"output directory is %@", outputDir);
self.config.outputDirectory = outputDir;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed));
}


/**
Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal
*/
- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry {
self.config.stuckTimeout = @6;
self.config.retryAppCrashTests = TRUE;
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
self.config.errorRetriesCount = @4;
self.config.onlyRetryFailed = TRUE;
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
NSString *tempDir = NSTemporaryDirectory();
NSError *error;
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
NSLog(@"output directory is %@", outputDir);
self.config.outputDirectory = outputDir;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
}

/**
Expand Down Expand Up @@ -387,6 +430,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests {
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
}

/**
Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set
*/
- (void)testReportSuccessOnAppCrashTestPassesOnRetry {
self.config.stuckTimeout = @6;
self.config.retryAppCrashTests = TRUE;
self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS";
self.config.onlyRetryFailed = TRUE;
self.config.failureTolerance = @1;
self.config.errorRetriesCount = @2;
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
NSString *tempDir = NSTemporaryDirectory();
NSError *error;
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
NSLog(@"output directory is %@", outputDir);
self.config.outputDirectory = outputDir;

BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
}

/**
Execution plan: One test CRASHes and another one keeps timing out
*/
Expand Down Expand Up @@ -457,7 +522,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry {
self.config.stuckTimeout = @6;
self.config.testing_ExecutionPlan = @"TIMEOUT PASS";
self.config.errorRetriesCount = @4;
self.config.onlyRetryFailed = YES;
self.config.onlyRetryFailed = TRUE;
self.config.failureTolerance = @0; // Not relevant
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
Expand All @@ -478,7 +543,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry {
self.config.stuckTimeout = @6;
self.config.testing_ExecutionPlan = @"TIMEOUT";
self.config.errorRetriesCount = @2;
self.config.onlyRetryFailed = NO;
self.config.onlyRetryFailed = FALSE;
self.config.failureTolerance = @1; // Not relevant since it's not a test failure
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
Expand All @@ -500,7 +565,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll {
self.config.testing_ExecutionPlan = @"FAIL PASS";
self.config.errorRetriesCount = @4;
self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails
self.config.failureTolerance = @1; // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO
self.config.failureTolerance = @1;
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
self.config.testBundlePath = testBundlePath;
NSString *tempDir = NSTemporaryDirectory();
Expand Down Expand Up @@ -578,7 +643,7 @@ - (void)testRetryOnlyFailures {
self.config.outputDirectory = outputDir;
self.config.errorRetriesCount = @100;
self.config.failureTolerance = @1;
self.config.onlyRetryFailed = YES;
self.config.onlyRetryFailed = TRUE;
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
XCTAssert(exitCode == BPExitStatusTestsFailed);
// Make sure all tests started on the first run
Expand Down

0 comments on commit f7159fc

Please sign in to comment.