500 for grpc context canceled errors during login (#28866)

vault/request_handling.go

@@ -2121,11 +2121,14 @@ func isRetryableRPCError(ctx context.Context, err error) bool {
 	case codes.Unavailable:
 		return true
 	case codes.Canceled:
-		// if the request context is canceled, we want to return false
-		// but if the request context hasn't been canceled, then there was
-		// either an EOF or the RPC client context has been canceled which
-		// should be retried
-		return ctx.Err() == nil
+		// if the request context is canceled through a deadline exceeded, we
+		// want to return false. But otherwise, there could have been an EOF or
+		// the RPC client context has been canceled which should be retried
+		ctxErr := ctx.Err()
+		if ctxErr == nil {
+			return true
+		}
+		return !errors.Is(ctxErr, context.DeadlineExceeded)
 	case codes.Unknown:
 		// sometimes a missing HTTP content-type error can happen when multiple
 		// HTTP statuses have been written. This can happen when the error

vault/request_handling_test.go

@@ -489,16 +489,25 @@ func TestRequestHandling_SecretLeaseMetric(t *testing.T) {
 func TestRequestHandling_isRetryableRPCError(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
+
+	deadlineCtx, deadlineCancel := context.WithDeadline(context.Background(), time.Now().Add(-1*time.Second))
+	defer deadlineCancel()
 	testCases := []struct {
 		name string
 		ctx  context.Context
 		err  error
 		want bool
 	}{
 		{
-			name: "req context canceled",
+			name: "req context canceled, not deadline",
 			ctx:  ctx,
 			err:  status.Error(codes.Canceled, "context canceled"),
+			want: true,
+		},
+		{
+			name: "req context deadline exceeded",
+			ctx:  deadlineCtx,
+			err:  status.Error(codes.Canceled, "context canceled"),
 			want: false,
 		},
 		{