From eb2f77a0119cb5f5ee95625077d04dfbb5368827 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 14 May 2020 15:44:03 -0400 Subject: [PATCH] csi: use a blocking initial connection with timeout The plugin supervisor lazily connects to plugins, but this means we only get "Unavailable" back from the gRPC call in cases where the plugin can never be reached (for example, if the Nomad client has the wrong permissions for the socket). This changeset improves the operator experience by switching to a blocking `DialWithContext`. It eagerly connects so that we can validate the connection is real and get a "failed to open" error in case where Nomad can't establish the initial connection. --- client/allocrunner/taskrunner/plugin_supervisor_hook.go | 2 +- plugins/csi/client.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client/allocrunner/taskrunner/plugin_supervisor_hook.go b/client/allocrunner/taskrunner/plugin_supervisor_hook.go index 7d730ead7aa..b95daa85769 100644 --- a/client/allocrunner/taskrunner/plugin_supervisor_hook.go +++ b/client/allocrunner/taskrunner/plugin_supervisor_hook.go @@ -335,10 +335,10 @@ func (h *csiPluginSupervisorHook) supervisorLoopOnce(ctx context.Context, socket } client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type)) - defer client.Close() if err != nil { return false, fmt.Errorf("failed to create csi client: %v", err) } + defer client.Close() healthy, err := client.PluginProbe(ctx) if err != nil { diff --git a/plugins/csi/client.go b/plugins/csi/client.go index 99c0cad3848..0305b04d95d 100644 --- a/plugins/csi/client.go +++ b/plugins/csi/client.go @@ -114,8 +114,12 @@ func NewClient(addr string, logger hclog.Logger) (CSIPlugin, error) { } func newGrpcConn(addr string, logger hclog.Logger) (*grpc.ClientConn, error) { - conn, err := grpc.Dial( + ctx, cancel := context.WithTimeout(context.Background(), time.Second*1) + defer cancel() + conn, err := grpc.DialContext( + ctx, addr, + grpc.WithBlock(), grpc.WithInsecure(), grpc.WithUnaryInterceptor(logging.UnaryClientInterceptor(logger)), grpc.WithStreamInterceptor(logging.StreamClientInterceptor(logger)),