From f7159fcf49b5053f045e47ebd2ba0f217460caaa Mon Sep 17 00:00:00 2001 From: Ravi Mandala <3135049+ravimandala@users.noreply.github.com> Date: Tue, 11 Aug 2020 09:50:42 -0700 Subject: [PATCH] Retry app crash tests and consider then non-fatal if they pass --- bp/src/BPConfiguration.h | 1 + bp/src/BPConfiguration.m | 4 +- bp/src/Bluepill.m | 6 ++- bp/src/SimulatorMonitor.m | 8 +++- bp/tests/BluepillTests.m | 79 +++++++++++++++++++++++++++++++++++---- 5 files changed, 86 insertions(+), 12 deletions(-) diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index ac8cf6ae..29ebc422 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic) BOOL saveDiagnosticsOnError; @property (nonatomic, strong) NSNumber *failureTolerance; @property (nonatomic) BOOL onlyRetryFailed; +@property (nonatomic) BOOL retryAppCrashTests; @property (nonatomic, strong) NSArray *testCasesToSkip; @property (nonatomic, strong) NSArray *testCasesToRun; @property (nonatomic, strong) NSArray *allTestCases; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 3e0c880f..0ab2beb9 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -103,7 +103,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet", "Turn off all output except fatal errors."}, {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed", - "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."}, + "Only retry failed tests instead of all. Also retry test that timed-out/crashed."}, + {'s', "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests", + "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."}, {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly", "Only list tests and exit without executing tests."}, {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging", diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index a1f85b49..26bb4eb9 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context { return; case BPExitStatusAppCrashed: - // Crashed test is considered fatal and shall not be retried - self.finalExitStatus |= context.exitStatus; + if (!self.config.retryAppCrashTests) { + // Crashed test is considered fatal when retry is disabled + self.finalExitStatus |= context.exitStatus; + } NEXT([self proceed]); return; diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m index 83341d2b..c8a5fad7 100644 --- a/bp/src/SimulatorMonitor.m +++ b/bp/src/SimulatorMonitor.m @@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output { NSString *testClass = (__self.currentClassName ?: __self.previousClassName); NSString *testName = (__self.currentTestName ?: __self.previousTestName); if (__self.testsState == Running) { - [self updateExecutedTestCaseList:testName inClass:testClass]; - [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName]; + if (self.config.retryAppCrashTests) { + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName]; + } else { + [self updateExecutedTestCaseList:testName inClass:testClass]; + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName]; + } [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"]; } else { assert(__self.testsState == Idle); diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index b23f8528..41affe00 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -222,7 +222,7 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet { self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @1; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAppCrashed); @@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode { self.config.testing_crashOnAttempt = @1; self.config.errorRetriesCount = @2; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); @@ -342,7 +342,50 @@ - (void)testReportFailureOnTimeoutCrashAndPass { self.config.outputDirectory = outputDir; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssertTrue(exitCode == BPExitStatusAppCrashed); + XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout)); +} + +/** + Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH"; + self.config.errorRetriesCount = @2; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed)); +} + + +/** + Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); } /** @@ -387,6 +430,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests { XCTAssertTrue(exitCode == BPExitStatusAppCrashed); } +/** + Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set + */ +- (void)testReportSuccessOnAppCrashTestPassesOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS"; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + self.config.errorRetriesCount = @2; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + /** Execution plan: One test CRASHes and another one keeps timing out */ @@ -457,7 +522,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry { self.config.stuckTimeout = @6; self.config.testing_ExecutionPlan = @"TIMEOUT PASS"; self.config.errorRetriesCount = @4; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; self.config.failureTolerance = @0; // Not relevant NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; @@ -478,7 +543,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry { self.config.stuckTimeout = @6; self.config.testing_ExecutionPlan = @"TIMEOUT"; self.config.errorRetriesCount = @2; - self.config.onlyRetryFailed = NO; + self.config.onlyRetryFailed = FALSE; self.config.failureTolerance = @1; // Not relevant since it's not a test failure NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; @@ -500,7 +565,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll { self.config.testing_ExecutionPlan = @"FAIL PASS"; self.config.errorRetriesCount = @4; self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails - self.config.failureTolerance = @1; // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO + self.config.failureTolerance = @1; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -578,7 +643,7 @@ - (void)testRetryOnlyFailures { self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @100; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssert(exitCode == BPExitStatusTestsFailed); // Make sure all tests started on the first run