refactor(server/v2): auto-gateway improvements and doc

server/v2/api/grpcgateway/doc.go

@@ -0,0 +1,11 @@
+// Package grpcgateway provides a custom http mux that utilizes the global gogoproto registry to match
+// grpc gateway requests to query handlers. POST requests with JSON bodies and GET requests with query params are supported.
+// Wildcard endpoints (i.e. foo/bar/{baz}), as well as catch-all endpoints (i.e. foo/bar/{baz=**} are supported. Using
+// header `x-cosmos-block-height` allows you to specify a height for the query.
+//
+// The URL matching logic is achieved by building regular expressions from the gateway HTTP annotations. These regular expressions
+// are then used to match against incoming requests to the HTTP server.
+//
+// In cases where the custom http mux is unable to handle the query (i.e. no match found), the request will fall back to the
+// ServeMux from github.com/grpc-ecosystem/grpc-gateway/runtime.
+package grpcgateway

server/v2/api/grpcgateway/interceptor.go

@@ -1,12 +1,19 @@
 package grpcgateway
 
 import (
+	"context"
+	"errors"
+	"io"
 	"net/http"
+	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 
 	gogoproto "github.com/cosmos/gogoproto/proto"
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"github.com/grpc-ecosystem/grpc-gateway/utilities"
+	"github.com/mitchellh/mapstructure"
 	"google.golang.org/genproto/googleapis/api/annotations"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -18,18 +25,32 @@ import (
 	"cosmossdk.io/server/v2/appmanager"
 )
 
+const MaxBodySize = 1 << 20 // 1 MB
+
 var _ http.Handler = &gatewayInterceptor[transaction.Tx]{}
 
+// queryMetadata holds information related to handling gateway queries.
+type queryMetadata struct {
+	// queryInputProtoName is the proto name of the query's input type.
+	queryInputProtoName string
+	// wildcardKeyNames are the wildcard key names from the query's HTTP annotation.
+	// for example /foo/bar/{baz}/{qux} would produce []string{"baz", "qux"}
+	// this is used for building the query's parameter map.
+	wildcardKeyNames []string
+}
+
 // gatewayInterceptor handles routing grpc-gateway queries to the app manager's query router.
 type gatewayInterceptor[T transaction.Tx] struct {
 	logger log.Logger
 	// gateway is the fallback grpc gateway mux handler.
 	gateway *runtime.ServeMux
 
-	// customEndpointMapping is a mapping of custom GET options on proto RPC handlers, to the fully qualified method name.
+	// regexpToQueryMetadata is a mapping of regular expressions of HTTP annotations to metadata for the query.
+	// it is built from parsing the HTTP annotations obtained from the gogoproto global registry.'
 	//
-	// example: /cosmos/bank/v1beta1/denoms_metadata -> cosmos.bank.v1beta1.Query.DenomsMetadata
-	customEndpointMapping map[string]string
+	// TODO: it might be interesting to make this a 'most frequently used' data structure, so frequently used regexp's are
+	// iterated over first.
+	regexpToQueryMetadata map[*regexp.Regexp]queryMetadata
 
 	// appManager is used to route queries to the application.
 	appManager appmanager.AppManager[T]
@@ -41,57 +62,73 @@ func newGatewayInterceptor[T transaction.Tx](logger log.Logger, gateway *runtime
 	if err != nil {
 		return nil, err
 	}
+	// convert the mapping to regular expressions for URL matching.
+	regexQueryMD := createRegexMapping(logger, getMapping)
+	if err != nil {
+		return nil, err
+	}
 	return &gatewayInterceptor[T]{
 		logger:                logger,
 		gateway:               gateway,
-		customEndpointMapping: getMapping,
+		regexpToQueryMetadata: regexQueryMD,
 		appManager:            am,
 	}, nil
 }
 
-// ServeHTTP implements the http.Handler interface. This function will attempt to match http requests to the
-// interceptors internal mapping of http annotations to query request type names.
-// If no match can be made, it falls back to the runtime gateway server mux.
+// ServeHTTP implements the http.Handler interface. This method will attempt to match request URIs to its internal mapping
+// of gateway HTTP annotations. If no match can be made, it falls back to the runtime gateway server mux.
 func (g *gatewayInterceptor[T]) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
 	g.logger.Debug("received grpc-gateway request", "request_uri", request.RequestURI)
-	match := matchURL(request.URL, g.customEndpointMapping)
+	match := matchURL(request.URL, g.regexpToQueryMetadata)
 	if match == nil {
 		// no match cases fall back to gateway mux.
 		g.gateway.ServeHTTP(writer, request)
 		return
 	}
+
 	g.logger.Debug("matched request", "query_input", match.QueryInputName)
-	_, out := runtime.MarshalerForRequest(g.gateway, request)
-	var msg gogoproto.Message
-	var err error
 
+	in, out := runtime.MarshalerForRequest(g.gateway, request)
+
+	// extract the proto message type.
+	msgType := gogoproto.MessageType(match.QueryInputName)
+	msg, ok := reflect.New(msgType.Elem()).Interface().(gogoproto.Message)
+	if !ok {
+		runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, status.Errorf(codes.Internal, "unable to to create gogoproto message from query input name %s", match.QueryInputName))
+		return
+	}
+
+	// msg population based on http method.
+	var inputMsg gogoproto.Message
+	var err error
 	switch request.Method {
-	case http.MethodPost:
-		msg, err = createMessageFromJSON(match, request)
 	case http.MethodGet:
-		msg, err = createMessage(match)
+		inputMsg, err = g.createMessageFromGetRequest(request.Context(), in, request, msg, match.Params)
+	case http.MethodPost:
+		inputMsg, err = g.createMessageFromPostRequest(request.Context(), in, request, msg)
 	default:
-		runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, status.Error(codes.Unimplemented, "HTTP method must be POST or GET"))
+		runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, status.Error(codes.InvalidArgument, "HTTP method was not POST or GET"))
 		return
 	}
 	if err != nil {
+		// the errors returned from the message creation methods return status errors. no need to make one here.
 		runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, err)
 		return
 	}
 
-	// extract block height header
+	// get the height from the header.
 	var height uint64
 	heightStr := request.Header.Get(GRPCBlockHeightHeader)
-	if heightStr != "" {
+	heightStr = strings.Trim(heightStr, `\"`)
+	if heightStr != "" && heightStr != "latest" {
 		height, err = strconv.ParseUint(heightStr, 10, 64)
 		if err != nil {
-			err = status.Errorf(codes.InvalidArgument, "invalid height: %s", heightStr)
-			runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, err)
+			runtime.DefaultHTTPProtoErrorHandler(request.Context(), g.gateway, out, writer, request, status.Errorf(codes.InvalidArgument, "invalid height in header: %s", heightStr))
 			return
 		}
 	}
 
-	query, err := g.appManager.Query(request.Context(), height, msg)
+	responseMsg, err := g.appManager.Query(request.Context(), height, inputMsg)
 	if err != nil {
 		// if we couldn't find a handler for this request, just fall back to the gateway mux.
 		if strings.Contains(err.Error(), "no handler") {
@@ -102,8 +139,54 @@ func (g *gatewayInterceptor[T]) ServeHTTP(writer http.ResponseWriter, request *h
 		}
 		return
 	}
+
 	// for no errors, we forward the response.
-	runtime.ForwardResponseMessage(request.Context(), g.gateway, out, writer, request, query)
+	runtime.ForwardResponseMessage(request.Context(), g.gateway, out, writer, request, responseMsg)
+}
+
+func (g *gatewayInterceptor[T]) createMessageFromPostRequest(_ context.Context, marshaler runtime.Marshaler, req *http.Request, input gogoproto.Message) (gogoproto.Message, error) {
+	if req.ContentLength > MaxBodySize {
+		return nil, status.Errorf(codes.InvalidArgument, "request body too large: %d bytes, max=%d", req.ContentLength, MaxBodySize)
+	}
+	newReader, err := utilities.IOReaderFactory(req.Body)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	if err = marshaler.NewDecoder(newReader()).Decode(input); err != nil && !errors.Is(err, io.EOF) {
+		return nil, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	return input, nil
+}
+
+func (g *gatewayInterceptor[T]) createMessageFromGetRequest(_ context.Context, _ runtime.Marshaler, req *http.Request, input gogoproto.Message, wildcardValues map[string]string) (gogoproto.Message, error) {
+	// decode the path wildcards into the message.
+	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		Result:           input,
+		TagName:          "json",
+		WeaklyTypedInput: true,
+	})
+	if err != nil {
+		return nil, status.Error(codes.Internal, "failed to create message decoder")
+	}
+	if err := decoder.Decode(wildcardValues); err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	if err = req.ParseForm(); err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	// im not really sure what this filter is for, but defaulting it like so doesn't seem to break anything.
+	// pb.gw.go code uses it in a few queries, but it's not clear what actual behavior is gained from this.
+	filter := &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)}
+	err = runtime.PopulateQueryParameters(input, req.Form, filter)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
+	return input, err
 }
 
 // getHTTPGetAnnotationMapping returns a mapping of RPC Method HTTP GET annotation to the RPC Handler's Request Input type full name.
@@ -121,7 +204,6 @@ func getHTTPGetAnnotationMapping() (map[string]string, error) {
 			serviceDesc := fd.Services().Get(i)
 			for j := 0; j < serviceDesc.Methods().Len(); j++ {
 				methodDesc := serviceDesc.Methods().Get(j)
-
 				httpAnnotation := proto.GetExtension(methodDesc.Options(), annotations.E_Http)
 				if httpAnnotation == nil {
 					continue
@@ -143,3 +225,26 @@ func getHTTPGetAnnotationMapping() (map[string]string, error) {
 
 	return httpGets, nil
 }
+
+// createRegexMapping converts the annotationMapping (HTTP annotation -> query input type name) to a
+// map of regular expressions for that HTTP annotation pattern, to queryMetadata.
+func createRegexMapping(logger log.Logger, annotationMapping map[string]string) map[*regexp.Regexp]queryMetadata {
+	regexQueryMD := make(map[*regexp.Regexp]queryMetadata)
+	seenPatterns := make(map[string]string)
+	for annotation, queryInputName := range annotationMapping {
+		pattern, wildcardNames := patternToRegex(annotation)
+		reg := regexp.MustCompile(pattern)
+		if otherAnnotation, ok := seenPatterns[pattern]; !ok {
+			seenPatterns[pattern] = annotation
+		} else {
+			// TODO: eventually we want this to error, but there is currently a duplicate in the protobuf.
+			// see: https://github.com/cosmos/cosmos-sdk/issues/23281
+			logger.Warn("duplicate HTTP annotation found", "annotation1", annotation, "annotation2", otherAnnotation, "query_input_name", queryInputName)
+		}
+		regexQueryMD[reg] = queryMetadata{
+			queryInputProtoName: queryInputName,
+			wildcardKeyNames:    wildcardNames,
+		}
+	}
+	return regexQueryMD
+}