From 7df9740ac37c5adf7abdbfcd54bc1bffdbd15345 Mon Sep 17 00:00:00 2001 From: manuroe Date: Thu, 3 Nov 2022 18:18:24 +0100 Subject: [PATCH 1/6] Tests: Fix/comment some push tests due to default push rules changes on server --- MatrixSDKTests/MXNotificationCenterTests.m | 4 +-- MatrixSDKTests/MXRestClientTests.m | 37 +--------------------- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/MatrixSDKTests/MXNotificationCenterTests.m b/MatrixSDKTests/MXNotificationCenterTests.m index d4366425ae..06127778f3 100644 --- a/MatrixSDKTests/MXNotificationCenterTests.m +++ b/MatrixSDKTests/MXNotificationCenterTests.m @@ -288,8 +288,8 @@ - (void)testDefaultRoomMemberCountCondition XCTAssert(rule.isDefault, @"The rule must be the server default rule. Rule: %@", rule); MXPushRuleCondition *condition = rule.conditions[0]; - XCTAssertEqualObjects(condition.kind, kMXPushRuleConditionStringRoomMemberCount, @"The default content rule with room_member_count condition must fire first"); - XCTAssertEqual(condition.kindType, MXPushRuleConditionTypeRoomMemberCount); + XCTAssertEqualObjects(condition.kind, kMXPushRuleConditionStringEventMatch, @"The default content rule with room_member_count condition must fire first"); + XCTAssertEqual(condition.kindType, MXPushRuleConditionTypeEventMatch); // Check the right event has been notified XCTAssertEqualObjects(event.content[kMXMessageBodyKey], messageFromAlice, @"The wrong messsage has been caught. event: %@", event); diff --git a/MatrixSDKTests/MXRestClientTests.m b/MatrixSDKTests/MXRestClientTests.m index 2406d6445d..d7ff6c3060 100644 --- a/MatrixSDKTests/MXRestClientTests.m +++ b/MatrixSDKTests/MXRestClientTests.m @@ -1333,42 +1333,7 @@ - (void)testPushRules // Check data sent by the home server has been correcltly modelled XCTAssertTrue([pushRules.global isKindOfClass:[MXPushRulesSet class]]); - XCTAssertNotNil(pushRules.global.content); - XCTAssertTrue([pushRules.global.content isKindOfClass:[NSArray class]]); - - MXPushRule *pushRule = pushRules.global.content[0]; - XCTAssertTrue([pushRule isKindOfClass:[MXPushRule class]]); - - XCTAssertNotNil(pushRule.actions); - - MXPushRuleAction *pushAction = pushRule.actions[0]; - XCTAssertTrue([pushAction isKindOfClass:[MXPushRuleAction class]]); - - // Test a rule with room_member_count condition. There must be one for 1:1 in underride rules - MXPushRule *roomMemberCountRule; - for (MXPushRule *pushRule in pushRules.global.underride) - { - if (pushRule.conditions.count) - { - MXPushRuleCondition *condition = pushRule.conditions[0]; - if (condition.kindType == MXPushRuleConditionTypeRoomMemberCount) - { - roomMemberCountRule = pushRule; - break; - } - } - } - XCTAssertNotNil(roomMemberCountRule); - - MXPushRuleCondition *condition = roomMemberCountRule.conditions[0]; - XCTAssertNotNil(condition); - XCTAssertEqualObjects(condition.kind, kMXPushRuleConditionStringRoomMemberCount); - - XCTAssertEqual(condition.kindType, MXPushRuleConditionTypeRoomMemberCount); - - XCTAssertNotNil(condition.parameters); - NSNumber *number= condition.parameters[@"is"]; - XCTAssertEqual(number.intValue, 2); + // TODO: Check new default push rules [expectation fulfill]; From 78105975c4dcba1146e08b5717d1d5b19f18168e Mon Sep 17 00:00:00 2001 From: manuroe Date: Thu, 3 Nov 2022 18:19:51 +0100 Subject: [PATCH 2/6] Tests: Update MXRestClientTests.testFilter() to match server expectation --- MatrixSDKTests/MXRestClientTests.m | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/MatrixSDKTests/MXRestClientTests.m b/MatrixSDKTests/MXRestClientTests.m index d7ff6c3060..7cd11d8b54 100644 --- a/MatrixSDKTests/MXRestClientTests.m +++ b/MatrixSDKTests/MXRestClientTests.m @@ -1112,7 +1112,8 @@ - (void)testAddAndRemoveTag #pragma mark - Filter operations - (void)testFilter { - [self.matrixSDKTestsData doMXRestClientTestWithAlice:self readyToTest:^(MXRestClient *aliceRestClient, XCTestExpectation *expectation) { + [self.matrixSDKTestsData doMXRestClientTestWithBobAndAliceInARoom:self + readyToTest:^(MXRestClient *bobRestClient, MXRestClient *aliceRestClient, NSString *roomId, XCTestExpectation *expectation) { MXFilterJSONModel *filter = [[MXFilterJSONModel alloc] init]; @@ -1120,15 +1121,13 @@ - (void)testFilter filter.eventFormat = @"federation"; filter.room = [[MXRoomFilter alloc] init]; - filter.room.rooms = @[@"!aroom:matrix:org"]; - filter.room.notRooms = @[@"!notaroom:matrix:org"]; + filter.room.rooms = @[roomId]; filter.room.ephemeral = [[MXRoomEventFilter alloc] init]; filter.room.ephemeral.containsURL = NO; filter.room.ephemeral.types = @[@"atype"]; filter.room.ephemeral.notTypes = @[@"notatype"]; - filter.room.ephemeral.rooms = @[@"!aroom_ephemeral:matrix:org"]; - filter.room.ephemeral.notRooms = @[@"!notaroom_ephemeral:matrix:org"]; + filter.room.ephemeral.rooms = @[roomId];; filter.room.ephemeral.senders = @[@"@asender:matrix.org"]; filter.room.ephemeral.notSenders = @[@"@notasender:matrix.org"]; From ce1bcda1074ad39f8610a8844726006854c53ca6 Mon Sep 17 00:00:00 2001 From: manuroe Date: Thu, 3 Nov 2022 18:29:59 +0100 Subject: [PATCH 3/6] Tests: Disable MXSpaceChildContentTests.testUpdateChildSuggestion() and testUpgradeSpaceChild() It seems we have race conditions that we could improve by adding some sleep(1) before REST request but they are still not 100% reliable. So skip them. testUpdateChildSuggestion is broken since 2022-09-16 without good reasons. testUpgradeSpaceChild() is bit more random --- .../TestPlans/AllWorkingTests.xctestplan | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan b/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan index 24fd2e67a5..3956471054 100644 --- a/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan +++ b/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan @@ -24,23 +24,23 @@ "MXAggregatedEditsTests\/testEditsFromInitialSync", "MXAggregatedEditsTests\/testFormatedEditsFromInitialSync", "MXAggregatedEditsTests\/testFormattedEditSendAndReceive", - "MXAggregatedEditsUnitTests", - "MXAggregatedReferenceUnitTests", - "MXAsyncTaskQueueUnitTests", - "MXAuthenticationSessionUnitTests", "MXAggregatedEditsTests\/testFormattedEditServerSide", + "MXAggregatedEditsUnitTests", "MXAggregatedReactionTests\/testAggregationsFromInitialSync", "MXAggregatedReactionTests\/testReactionsWhenPaginatingFromAGappyInitialSync", "MXAggregatedReactionTests\/testReactionsWhenPaginatingFromAGappySync", "MXAggregatedReactionTests\/testUnreactAfterInitialSync", "MXAggregatedReferenceTests\/testReferenceFromInitialSync", + "MXAggregatedReferenceUnitTests", + "MXAsyncTaskQueueUnitTests", + "MXAuthenticationSessionUnitTests", "MXAutoDiscoveryTests\/testAutoDiscoveryNotJSON", "MXAutoDiscoveryTests\/testAutoDiscoverySuccessful", "MXAutoDiscoveryTests\/testAutoDiscoverySuccessfulWithNoContentType", + "MXBackgroundSyncServiceTests\/testStoreWithGappyAndOutdatedSync()", "MXBackgroundTaskUnitTests", "MXBeaconInfoUnitTests", "MXCredentialsUnitTests", - "MXBackgroundSyncServiceTests\/testStoreWithGappyAndOutdatedSync()", "MXCrossSigningTests", "MXCrossSigningVerificationTests", "MXCrossSigningVerificationTests\/testVerificationByDMFullFlow", @@ -80,8 +80,8 @@ "MXMyUserTests\/testIdenticon", "MXOlmDeviceUnitTests", "MXPeekingRoomTests\/testPeekingOnNonWorldReadable", - "MXPollRelationTests\/testBobAndAliceAnswer", "MXPollAggregatorTest\/testEditing()", + "MXPollRelationTests\/testBobAndAliceAnswer", "MXPushRuleUnitTests", "MXQRCodeDataUnitTests", "MXReplyEventParserUnitTests", @@ -106,6 +106,8 @@ "MXSelfSignedHomeserverTests\/testTrustedCertificate", "MXSessionTests", "MXSharedHistoryKeyManagerUnitTests", + "MXSpaceChildContentTests\/testUpdateChildSuggestion()", + "MXSpaceChildContentTests\/testUpgradeSpaceChild()", "MXStoreFileStoreTests", "MXStoreMemoryStoreTests", "MXStoreNoStoreTests\/testMXNoStoreSeveralPaginateBacks", From b20ef03f06305369b9093f5f09d9770410d1dfd2 Mon Sep 17 00:00:00 2001 From: manuroe Date: Thu, 3 Nov 2022 18:40:46 +0100 Subject: [PATCH 4/6] Tests: Disable tests that randomly fail Failure on the 3 last months: MXAggregatedReactionTests testUnreactOnEventReactedByOther: 5 MXCryptoSecretStorageTests testDeleteSecret: 4 MXAggregatedReactionTests testAggregationsListener: 2 --- MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan b/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan index 3956471054..fb12ca99dc 100644 --- a/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan +++ b/MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan @@ -30,6 +30,8 @@ "MXAggregatedReactionTests\/testReactionsWhenPaginatingFromAGappyInitialSync", "MXAggregatedReactionTests\/testReactionsWhenPaginatingFromAGappySync", "MXAggregatedReactionTests\/testUnreactAfterInitialSync", + "MXAggregatedReactionTests\/testUnreactOnEventReactedByOther", + "MXAggregatedReferenceTests\/testE2EFetchAllReferenceEvents", "MXAggregatedReferenceTests\/testReferenceFromInitialSync", "MXAggregatedReferenceUnitTests", "MXAsyncTaskQueueUnitTests", @@ -53,6 +55,7 @@ "MXCryptoRecoveryServiceTests", "MXCryptoRequestsUnitTests", "MXCryptoSecretShareTests\/testSecretRequestCancellation", + "MXCryptoSecretStorageTests\/testDeleteSecret", "MXCryptoShareTests", "MXCryptoTests", "MXErrorUnitTests", From 4ff7392ccbb8e268e1f00845f77a496e1241919d Mon Sep 17 00:00:00 2001 From: manuroe Date: Fri, 4 Nov 2022 10:23:41 +0100 Subject: [PATCH 5/6] changelog --- changelog.d/pr-1628.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/pr-1628.bugfix diff --git a/changelog.d/pr-1628.bugfix b/changelog.d/pr-1628.bugfix new file mode 100644 index 0000000000..8ee0cac862 --- /dev/null +++ b/changelog.d/pr-1628.bugfix @@ -0,0 +1 @@ +Tests: Fix or disable flakey integration tests \ No newline at end of file From e25224cf5912b66f59d8db025b54143a559ed413 Mon Sep 17 00:00:00 2001 From: manuroe Date: Fri, 4 Nov 2022 13:07:27 +0100 Subject: [PATCH 6/6] Tests: Rename testDefaultRoomMemberCountCondition into testDefaultEventMatchCondition as we test a different thing now --- MatrixSDKTests/MXNotificationCenterTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MatrixSDKTests/MXNotificationCenterTests.m b/MatrixSDKTests/MXNotificationCenterTests.m index 06127778f3..a22465f6b4 100644 --- a/MatrixSDKTests/MXNotificationCenterTests.m +++ b/MatrixSDKTests/MXNotificationCenterTests.m @@ -275,7 +275,7 @@ - (void)testDefaultDisplayNameCondition }]; } -- (void)testDefaultRoomMemberCountCondition +- (void)testDefaultEventMatchCondition { [matrixSDKTestsData doMXSessionTestWithBobAndAliceInARoom:self readyToTest:^(MXSession *bobSession, MXRestClient *aliceRestClient, NSString *roomId, XCTestExpectation *expectation) {