ClientV3: Add integration test for server timeout

.words

@@ -104,3 +104,4 @@ PermitWithoutStream
 __lostleader
 ErrConnClosing
 unfreed
+ServerParameters

clientv3/integration/black_hole_test.go

@@ -26,8 +26,69 @@ import (
 	"go.etcd.io/etcd/integration"
 	"go.etcd.io/etcd/pkg/testutil"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
+// TestServerGRPCKeepAliveTimeout tests ServerParameters keepalive ping on server-side that
+// after having pinged for keepalive check, the server waits for a duration of Timeout and
+// if no activity is seen even after that the connection is closed on server-side. The keepalive
+// takes few extra seconds to get in impact so typically the time it takes to close the connection
+// is somewhat higher than specified Timeout value.
+func TestServerGRPCKeepAliveTimeout(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{
+		Size:                  1,
+		GRPCKeepAliveInterval: 1 * time.Second,
+		GRPCKeepAliveTimeout:  500 * time.Millisecond,
+	})
+	defer clus.Terminate(t)
+
+	eps := []string{clus.Members[0].GRPCAddr()}
+	ccfg := clientv3.Config{
+		Endpoints:   []string{eps[0]},
+		DialTimeout: 12 * time.Second,
+	}
+	cli, err := clientv3.New(ccfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	// give keepalive some time
+	time.Sleep(4 * time.Second)
+
+	if _, err = cli.Put(context.TODO(), "foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+	// TODO: keepalive sometimes doesn't work on first attempt.
+	for i := 0; i < 5; i++ {
+		clus.Members[0].Blackhole()
+		time.Sleep(10 * time.Second)
+		// remove blackhole but by now the keepalive ping should have triggered server to
+		// close server-to-client connection.
+		clus.Members[0].Unblackhole()
+		_, err = cli.Put(context.TODO(), "foo1", "bar1")
+		if err != nil {
+			ev, ok := status.FromError(err)
+			if !ok {
+				t.Fatal(err)
+			}
+			if ev.Code() != codes.Unavailable {
+				t.Fatal(err)
+			}
+			break
+		} else {
+			t.Logf("info: expected an error, received none.")
+		}
+	}
+
+	if err == nil {
+		t.Error("rpc error expected")
+	}
+}
+
 // TestBalancerUnderBlackholeKeepAliveWatch tests when watch discovers it cannot talk to
 // blackholed endpoint, client balancer switches to healthy one.
 // TODO: test server-to-client keepalive ping