diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go
index c87f92f615c92..5edcd76b3b497 100644
--- a/br/pkg/lightning/backend/local/local.go
+++ b/br/pkg/lightning/backend/local/local.go
@@ -167,7 +167,6 @@ func (f *importClientFactoryImpl) makeConn(ctx context.Context, storeID uint64)
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
 			Time:                gRPCKeepAliveTime,
 			Timeout:             gRPCKeepAliveTimeout,
-			PermitWithoutStream: true,
 		}),
 	)
 	cancel()
diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go
index 4cec79e7acf62..84eb8c479241c 100644
--- a/br/pkg/task/restore.go
+++ b/br/pkg/task/restore.go
@@ -430,7 +430,6 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		}
 	}
 
-	keepaliveCfg.PermitWithoutStream = true
 	client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetTLSConfig(), keepaliveCfg, false)
 	err = configureRestoreClient(ctx, client, cfg)
 	if err != nil {
diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go
index 6c15cd9989512..22cb4a60db0ae 100644
--- a/br/pkg/task/restore_raw.go
+++ b/br/pkg/task/restore_raw.go
@@ -87,9 +87,6 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
 	}
 
 	keepaliveCfg := GetKeepalive(&cfg.Config)
-	// sometimes we have pooled the connections.
-	// sending heartbeats in idle times is useful.
-	keepaliveCfg.PermitWithoutStream = true
 	client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetTLSConfig(), keepaliveCfg, true)
 	client.SetRateLimit(cfg.RateLimit)
 	client.SetCrypter(&cfg.CipherInfo)
diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go
index 3c3d20967f58e..e230c5c19bf53 100644
--- a/br/pkg/task/stream.go
+++ b/br/pkg/task/stream.go
@@ -1173,7 +1173,6 @@ func restoreStream(
 func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, mgr *conn.Mgr) (*restore.Client, error) {
 	var err error
 	keepaliveCfg := GetKeepalive(&cfg.Config)
-	keepaliveCfg.PermitWithoutStream = true
 	client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetTLSConfig(), keepaliveCfg, false)
 	err = client.Init(g, mgr.GetStorage())
 	if err != nil {