Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deflake macos MixerFaultTest by broadening assertion ranges. #2126

Merged
merged 3 commits into from
Mar 1, 2019
+175 −58
Merged

Deflake macos MixerFaultTest by broadening assertion ranges. #2126

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3bf6de2
Deflake macos MixerFaultTest by broadening assertion ranges.
Feb 22, 2019
d98ed7c
Cleanup a few readability issues and add an assertion.
Mar 1, 2019
99d1b10
More redability changes.
Mar 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 175 additions & 58 deletions test/integration/mixer_fault_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,19 +362,19 @@ TEST_F(MixerFaultTest, HappyPath) {

// All client connections are successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 2xx class
EXPECT_EQ(client->class2xxResponses(), requests_to_send);
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// assert that the origin request callback is called for every client request
// sent
Expand Down Expand Up @@ -449,32 +449,32 @@ TEST_F(MixerFaultTest, FailClosedAndClosePolicySocketAfterAccept) {

// All client connections are successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 5xx class
EXPECT_EQ(0, client->class2xxResponses());
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(requests_to_send, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class2xxResponses(), 0);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), requests_to_send);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// Origin server should see no requests since the mixer filter is configured
// to fail closed.
EXPECT_EQ(0, origin_callbacks.requestsReceived());
EXPECT_EQ(origin_callbacks.requestsReceived(), 0);

// Policy server accept callback is called for every client connection
// initiated.
EXPECT_GE(policy_cluster.connectionsAccepted(), connections_to_initiate);
// Policy server request callback is never called
EXPECT_EQ(0, policy_cluster.requestsReceived());
EXPECT_EQ(policy_cluster.requestsReceived(), 0);
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(), policy_cluster.localCloses());
EXPECT_EQ(0, policy_cluster.remoteCloses());
EXPECT_EQ(policy_cluster.remoteCloses(), 0);
}

TEST_F(MixerFaultTest, FailClosedAndSendPolicyResponseSlowly) {
Expand Down Expand Up @@ -535,25 +535,22 @@ TEST_F(MixerFaultTest, FailClosedAndSendPolicyResponseSlowly) {
// Evaluate test
//

// All client connections are successfully established.
#ifndef __APPLE__
// All connections are successfully established
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 5xx class
EXPECT_EQ(0, client->class2xxResponses());
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(requests_to_send, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class2xxResponses(), 0);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), requests_to_send);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());

// Origin server should see no requests since the mixer filter is configured
// to fail closed.
EXPECT_EQ(0, origin_callbacks.requestsReceived());
EXPECT_EQ(client->remoteCloses(), 0);

// Policy server accept callback is called at least once (h2 socket reuse
// means may only be called once)
Expand All @@ -563,6 +560,46 @@ TEST_F(MixerFaultTest, FailClosedAndSendPolicyResponseSlowly) {
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(),
policy_cluster.localCloses() + policy_cluster.remoteCloses());
#else
// MacOS is a bit flakier than Linux, so broaden assetion ranges to reduce
// test flakes.

// Most connections are successfully established.
EXPECT_IN_RANGE(client->connectSuccesses(), 0.8 * connections_to_initiate,
connections_to_initiate);
EXPECT_IN_RANGE(client->connectFailures(), 0, 0.2 * connections_to_initiate);
EXPECT_EQ(client->connectSuccesses() + client->connectFailures(),
connections_to_initiate);
// Client close callback usually called for every client connection.
EXPECT_IN_RANGE(client->localCloses(), 0.8 * connections_to_initiate,
connections_to_initiate);
// Client response callback is usually called for every request sent
EXPECT_IN_RANGE(client->responsesReceived(), 0.8 * requests_to_send,
requests_to_send);
// Most responses are a 5xx class and none are successful
EXPECT_EQ(client->class2xxResponses(), 0);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_IN_RANGE(client->class5xxResponses(), 0.8 * requests_to_send,
requests_to_send);
EXPECT_EQ(client->responseTimeouts(), 0);
// Almost no client sockets are rudely closed by server / almost no client
// sockets are reset.
EXPECT_IN_RANGE(client->remoteCloses(), 0, 0.2 * connections_to_initiate);

// Policy server accept callback is called at least once (h2 socket reuse
// means may only be called once)
EXPECT_GE(policy_cluster.connectionsAccepted(), 1);
// Policy server request callback sees most policy checks
EXPECT_IN_RANGE(policy_cluster.requestsReceived(), 0.8 * requests_to_send,
requests_to_send);
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(),
policy_cluster.localCloses() + policy_cluster.remoteCloses());
#endif

// Origin server should see no requests since the mixer filter is
// configured to fail closed.
EXPECT_EQ(origin_callbacks.requestsReceived(), 0);
}

TEST_F(MixerFaultTest, TolerateTelemetryBlackhole) {
Expand Down Expand Up @@ -642,31 +679,64 @@ TEST_F(MixerFaultTest, TolerateTelemetryBlackhole) {
// Evaluate test
//

// All client connections are successfully established.
#ifndef __APPLE__
// On Linux every connection will be successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 2xx class
EXPECT_EQ(client->class2xxResponses(), requests_to_send);
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// assert that the origin request callback is called for every client request
// sent
// Origin server should see all requests
EXPECT_EQ(origin_callbacks.requestsReceived(), requests_to_send);

// Policy server request callback sees every policy check
EXPECT_EQ(requests_to_send, policy_cluster.requestsReceived());
#else
// MacOS is a bit flakier than Linux, so broaden assetion ranges to reduce
// test flakes.

// Most connections are successfully established.
EXPECT_IN_RANGE(client->connectSuccesses(), 0.8 * connections_to_initiate,
connections_to_initiate);
EXPECT_IN_RANGE(client->connectFailures(), 0, 0.2 * connections_to_initiate);
// Client close callback usually called for every client connection.
EXPECT_IN_RANGE(client->localCloses(), 0.8 * connections_to_initiate,
connections_to_initiate);
// Client response callback is usually called for every request sent
EXPECT_IN_RANGE(client->responsesReceived(), 0.8 * requests_to_send,
requests_to_send);
// Most responses were a 2xx class
EXPECT_IN_RANGE(client->class2xxResponses(), 0.8 * requests_to_send,
requests_to_send);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// Almost no client sockets are rudely closed by server / almost no client
// sockets are reset.
EXPECT_IN_RANGE(client->remoteCloses(), 0, 0.2 * connections_to_initiate);

// Origin server should see most requests
EXPECT_IN_RANGE(origin_callbacks.requestsReceived(), 0.8 * requests_to_send,
requests_to_send);

// Policy server request callback sees most policy checks
EXPECT_IN_RANGE(policy_cluster.requestsReceived(), 0.8 * requests_to_send,
requests_to_send);
#endif

// Policy server accept callback is called at least once (h2 socket reuse
// means may only be called once)
EXPECT_GE(policy_cluster.connectionsAccepted(), 1);
// Policy server request callback sees every policy check
EXPECT_EQ(requests_to_send, policy_cluster.requestsReceived());
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(),
policy_cluster.localCloses() + policy_cluster.remoteCloses());
Expand Down Expand Up @@ -736,21 +806,22 @@ TEST_F(MixerFaultTest, FailOpenAndSendPolicyResponseSlowly) {
// Evaluate test
//

// All client connections are successfully established.
#ifndef __APPLE__
// All connections are successfully established
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 2xx class
EXPECT_EQ(client->class2xxResponses(), requests_to_send);
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// Origin server should see every requests since the mixer filter is
// configured to fail open.
Expand All @@ -764,6 +835,45 @@ TEST_F(MixerFaultTest, FailOpenAndSendPolicyResponseSlowly) {
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(),
policy_cluster.localCloses() + policy_cluster.remoteCloses());
#else
// MacOS is a bit flakier than Linux, so broaden assetion ranges to reduce
// test flakes.

// Most connections are successfully established.
EXPECT_IN_RANGE(client->connectSuccesses(), 0.8 * connections_to_initiate,
connections_to_initiate);
EXPECT_IN_RANGE(client->connectFailures(), 0, 0.2 * connections_to_initiate);
// Client close callback usually called for every client connection.
EXPECT_IN_RANGE(client->localCloses(), 0.8 * connections_to_initiate,
connections_to_initiate);
// Client response callback is usually called for every request sent
EXPECT_IN_RANGE(client->responsesReceived(), 0.8 * requests_to_send,
requests_to_send);
// Most responses were a 2xx class
EXPECT_IN_RANGE(client->class2xxResponses(), 0.8 * requests_to_send,
requests_to_send);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// Almost no client sockets are rudely closed by server / almost no client
// sockets are reset.
EXPECT_IN_RANGE(client->remoteCloses(), 0, 0.2 * connections_to_initiate);

// Origin server should see most requests since the mixer filter is
// configured to fail open.
EXPECT_IN_RANGE(origin_callbacks.requestsReceived(), 0.8 * requests_to_send,
requests_to_send);

// Policy server accept callback is called at least once (h2 socket reuse
// means may only be called once)
EXPECT_GE(policy_cluster.connectionsAccepted(), 1);
// Policy server request callback sees most policy checks
EXPECT_IN_RANGE(policy_cluster.requestsReceived(), 0.8 * requests_to_send,
requests_to_send);
// Policy server closes every connection
EXPECT_EQ(policy_cluster.connectionsAccepted(),
policy_cluster.localCloses() + policy_cluster.remoteCloses());
#endif
}

TEST_F(MixerFaultTest, RetryOnTransportError) {
Expand Down Expand Up @@ -854,19 +964,19 @@ TEST_F(MixerFaultTest, RetryOnTransportError) {

// All client connections are successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client response callback is called for every request sent
EXPECT_EQ(client->responsesReceived(), requests_to_send);
// Every response was a 2xx class
EXPECT_EQ(client->class2xxResponses(), requests_to_send);
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(0, client->responseTimeouts());
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
EXPECT_EQ(client->responseTimeouts(), 0);
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// assert that the origin request callback is called for every client request
// sent
Expand Down Expand Up @@ -1005,21 +1115,21 @@ TEST_F(MixerFaultTest, CancelCheck) {

// All client connections are successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Not all responses are received due to timeouts
EXPECT_LE(client->responsesReceived(), requests_to_send);
EXPECT_GE(client->responsesReceived(), 1);
// Every response was a 2xx class
EXPECT_EQ(client->class2xxResponses(), client->responsesReceived());
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
// Or a timeout. Implementational artifact: timeouts kill the connection and
// new connections are not created to take their place.
EXPECT_EQ(connections_to_initiate, client->responseTimeouts());
// No client sockets are rudely closed by server. They timeout instead.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// assert that the origin request callback is called for every response
// received by the client.
Expand All @@ -1029,6 +1139,13 @@ TEST_F(MixerFaultTest, CancelCheck) {
// received by the client.
EXPECT_GE(policy_cluster.requestsReceived(), client->responsesReceived());

#ifdef __APPLE__
// Envoy doesn't detect client disconnects on MacOS so any outstanding
// requests to the policy server won't be cancelled. See
// https://github.com/envoyproxy/envoy/issues/4294
return;
#endif

// Assertions against the mixer filter's internal counters. Many of these
// assertions rely on an implementational artifact of the load generator
// client - when a request is cancelled due to timeout the connection is
Expand Down Expand Up @@ -1150,26 +1267,26 @@ TEST_F(MixerFaultTest, CancelRetry) {

// All client connections are successfully established.
EXPECT_EQ(client->connectSuccesses(), connections_to_initiate);
EXPECT_EQ(0, client->connectFailures());
EXPECT_EQ(client->connectFailures(), 0);
// Client close callback called for every client connection.
EXPECT_EQ(client->localCloses(), connections_to_initiate);
// Client doesn't receive any responses
EXPECT_EQ(0, client->responsesReceived());
EXPECT_EQ(0, client->class2xxResponses());
EXPECT_EQ(0, client->class4xxResponses());
EXPECT_EQ(0, client->class5xxResponses());
EXPECT_EQ(client->responsesReceived(), 0);
EXPECT_EQ(client->class2xxResponses(), 0);
EXPECT_EQ(client->class4xxResponses(), 0);
EXPECT_EQ(client->class5xxResponses(), 0);
// All requests timeout. Implementational artifact: timeouts kill the
// connection and new connections are not created to take their place.
EXPECT_EQ(connections_to_initiate, client->responseTimeouts());
// No client sockets are rudely closed by server / no client sockets are
// reset.
EXPECT_EQ(0, client->remoteCloses());
EXPECT_EQ(client->remoteCloses(), 0);

// The origin server receives no requests
EXPECT_EQ(0, origin_callbacks.requestsReceived());
EXPECT_EQ(origin_callbacks.requestsReceived(), 0);

// The policy server receives no requests
EXPECT_EQ(0, policy_cluster.requestsReceived());
EXPECT_EQ(policy_cluster.requestsReceived(), 0);

// Assertions against the mixer filter's internal counters. Many of these
// assertions rely on an implementational artifact of the load generator
Expand Down