fix: gRPC connection bug (#329)

* try fix bug Signed-off-by: xiafeng <[email protected]> * try fix grpc connection Signed-off-by: xiafeng <[email protected]> * try fix grpc connection Signed-off-by: xiafeng <[email protected]> * try fix grpc connection Signed-off-by: xiafeng <[email protected]> --------- Signed-off-by: xiafeng <[email protected]>
PKUHPC · Oct 11, 2024 · 258acd4 · 258acd4
1 parent a1ace47
commit 258acd4
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 23 deletions.
diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/CranedKeeper.cpp
@@ -593,6 +593,7 @@ CranedKeeper::CqTag *CranedKeeper::EstablishedCranedStateMachine_(
     // prev     current
     // READY -> IDLE (the only edge)
 
+    // CRANE_TRACE("READY -> IDLE");
     craned->m_invalid_ = true;
 
     next_tag_type = CqTag::kEstablishedCraned;
@@ -700,7 +701,7 @@ void CranedKeeper::ConnectCranedNode_(CranedId const &craned_id) {
    * https://grpc.github.io/grpc/cpp/md_doc_connection-backoff.html
    */
   grpc::ChannelArguments channel_args;
-  SetKeepAliveChannelArgs(&channel_args);
+  SetGrpcClientKeepAliveChannelArgs(&channel_args);
 
   if (g_config.CompressedRpc)
     channel_args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);

diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp
@@ -33,11 +33,14 @@ void ServerBuilderSetCompression(grpc::ServerBuilder* builder) {
 void ServerBuilderSetKeepAliveArgs(grpc::ServerBuilder* builder) {
   builder->AddChannelArgument(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA,
                               0 /*no limit*/);
-  builder->AddChannelArgument(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS,
-                              1 /*true*/);
   builder->AddChannelArgument(
       GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS,
-      kCranedGrpcServerPingRecvMinIntervalSec * 1000 /*ms*/);
+      10 * 1000 /*10 sec*/);
+  builder->AddChannelArgument(GRPC_ARG_KEEPALIVE_TIME_MS, 10 * 60 * 1000);
+  builder->AddChannelArgument(GRPC_ARG_KEEPALIVE_TIMEOUT_MS,
+                              20 * 1000 /*20 sec*/);
+  builder->AddChannelArgument(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS,
+                              1 /*true*/);
   builder->AddChannelArgument(GRPC_ARG_HTTP2_MAX_PING_STRIKES,
                               0 /* unlimited */);
 }
@@ -81,7 +84,7 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder,
                             grpc::SslServerCredentials(ssl_opts));
 }
 
-void SetKeepAliveChannelArgs(grpc::ChannelArguments* args) {
+void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args) {
   args->SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, 1000 /*ms*/);
   args->SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS, 2 /*s*/ * 1000
                /*ms*/);
@@ -91,11 +94,11 @@ void SetKeepAliveChannelArgs(grpc::ChannelArguments* args) {
   // the socket will remain ESTABLISHED state even if that craned has died.
   // Open KeepAlive option in case of such situation.
   // See https://grpc.github.io/grpc/cpp/md_doc_keepalive.html
-  args->SetInt(GRPC_ARG_KEEPALIVE_TIME_MS,
-               kCraneCtldGrpcClientPingSendIntervalSec /*s*/ * 1000 /*ms*/);
-  args->SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 10 /*s*/ * 1000 /*ms*/);
-  args->SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1 /*true*/);
-  args->SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0 /*no limit*/);
+  args->SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, 20 * 1000);
+  args->SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 10 * 1000);
+  args->SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1);
+
+  args->SetInt(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS, INT_MAX);
 }
 
 void SetTlsHostnameOverride(grpc::ChannelArguments* args,

diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h
@@ -19,18 +19,6 @@
 #include <grpc++/grpc++.h>
 #include <spdlog/fmt/bundled/format.h>
 
-// gRPC Doc: If smaller than 10 seconds, ten seconds will be used instead.
-// See https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md
-constexpr int64_t kCraneCtldGrpcClientPingSendIntervalSec = 10;
-
-// Server MUST have a high value of
-// GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS than the value of
-// GRPC_ARG_KEEPALIVE_TIME_MS. We set server's value to the multiple times of
-// the client's value plus 1s to tolerate 1 time packet dropping. See
-// https://github.com/grpc/grpc/blob/master/doc/keepalive.md
-constexpr int64_t kCranedGrpcServerPingRecvMinIntervalSec =
-    3 * kCraneCtldGrpcClientPingSendIntervalSec + 1;
-
 struct TlsCertificates {
   std::string DomainSuffix;
   std::string ServerCertFilePath;
@@ -55,7 +43,7 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder,
                                          const std::string& port,
                                          const TlsCertificates& certs);
 
-void SetKeepAliveChannelArgs(grpc::ChannelArguments* args);
+void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args);
 
 void SetTlsHostnameOverride(grpc::ChannelArguments* args,
                             const std::string& hostname,