VAULT-31393: Return 500 for some gRPC errors during login on perf sta…

…ndbys (#28807)

* return 500 for RPC failures during perf standby logins

* godoc

* changelog

changelog/28807.txt

@@ -0,0 +1,3 @@
+```release-note:change
+login (enterprise): Return a 500 error during logins when performance standby nodes make failed gRPC requests to the active node. 
+```

vault/request_handling.go

@@ -42,6 +42,8 @@ import (
 	"github.com/hashicorp/vault/vault/quotas"
 	"github.com/hashicorp/vault/vault/tokens"
 	uberAtomic "go.uber.org/atomic"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 const (
@@ -2079,7 +2081,7 @@ func (c *Core) LoginCreateToken(ctx context.Context, ns *namespace.Namespace, re
 		if auth.TokenType != logical.TokenTypeBatch {
 			leaseGenerated = true
 		}
-	case err == ErrInternalError:
+	case errors.Is(err, ErrInternalError), isRetryableRPCError(ctx, err):
 		return false, nil, err
 	default:
 		return false, logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest
@@ -2109,6 +2111,31 @@ func (c *Core) LoginCreateToken(ctx context.Context, ns *namespace.Namespace, re
 	return leaseGenerated, resp, nil
 }
 
+func isRetryableRPCError(ctx context.Context, err error) bool {
+	stat, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+
+	switch stat.Code() {
+	case codes.Unavailable:
+		return true
+	case codes.Canceled:
+		// if the request context is canceled, we want to return false
+		// but if the request context hasn't been canceled, then there was
+		// either an EOF or the RPC client context has been canceled which
+		// should be retried
+		return ctx.Err() == nil
+	case codes.Unknown:
+		// sometimes a missing HTTP content-type error can happen when multiple
+		// HTTP statuses have been written. This can happen when the error
+		// occurs in the middle of a response. This should be retried.
+		return strings.Contains(err.Error(), "malformed header: missing HTTP content-type")
+	default:
+		return false
+	}
+}
+
 // failedUserLoginProcess updates the userFailedLoginMap with login count and  last failed
 // login time for users with failed login attempt
 // If the user gets locked for current login attempt, it updates the storage entry too

vault/request_handling_test.go

@@ -4,6 +4,8 @@
 package vault
 
 import (
+	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -15,6 +17,9 @@ import (
 	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func TestRequestHandling_Wrapping(t *testing.T) {
@@ -478,3 +483,62 @@ func TestRequestHandling_SecretLeaseMetric(t *testing.T) {
 		},
 	)
 }
+
+// TestRequestHandling_isRetryableRPCError tests that a retryable RPC error
+// can be distinguished from a normal error
+func TestRequestHandling_isRetryableRPCError(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	testCases := []struct {
+		name string
+		ctx  context.Context
+		err  error
+		want bool
+	}{
+		{
+			name: "req context canceled",
+			ctx:  ctx,
+			err:  status.Error(codes.Canceled, "context canceled"),
+			want: false,
+		},
+		{
+			name: "server context canceled",
+			err:  status.Error(codes.Canceled, "context canceled"),
+			want: true,
+		},
+		{
+			name: "unavailable",
+			err:  status.Error(codes.Unavailable, "unavailable"),
+			want: true,
+		},
+		{
+			name: "other status",
+			err:  status.Error(codes.FailedPrecondition, "failed"),
+			want: false,
+		},
+		{
+			name: "other unknown",
+			err:  status.Error(codes.Unknown, "unknown"),
+			want: false,
+		},
+		{
+			name: "malformed header unknown",
+			err:  status.Error(codes.Unknown, "malformed header: missing HTTP content-type"),
+			want: true,
+		},
+		{
+			name: "other error",
+			err:  errors.New("other type of error"),
+			want: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			useCtx := tc.ctx
+			if tc.ctx == nil {
+				useCtx = context.Background()
+			}
+			require.Equal(t, tc.want, isRetryableRPCError(useCtx, tc.err))
+		})
+	}
+}