Enable DefaultHTTPErrorHandler and Upgrade grpc-gateway to v2 (#369)

* Upgrade grpc-gateway to v2

* Customize error and enable DefaultHTTPErrorHandler

1. Enable `runtime.DefaultHTTPErrorHandler` to convert gRPC status code to HTTP code.
2. Since we use custom error struct, we need to implement `GRPCStatus` to make sure grpc-go can recognize the error code in our struct,
Otherwise, it will always return codes.Unknown(2), that's the reason we always see code 2 and HTTP 500 in the past.

apiserver/cmd/main.go

@@ -3,18 +3,17 @@ package main
 import (
 	"context"
 	"flag"
-
 	"math"
 	"net"
 	"net/http"
 	"path"
 	"strings"
 
 	assetfs "github.com/elazarl/go-bindata-assetfs"
-	"github.com/grpc-ecosystem/grpc-gateway/runtime"
-
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
+	"google.golang.org/protobuf/encoding/protojson"
 	"k8s.io/klog/v2"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
@@ -82,7 +81,15 @@ func startHttpProxy() {
 	defer cancel()
 
 	// Create gRPC HTTP MUX and register services.
-	runtimeMux := runtime.NewServeMux(runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{OrigName: true, EmitDefaults: true}))
+	runtimeMux := runtime.NewServeMux(
+		runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{
+			MarshalOptions: protojson.MarshalOptions{
+				UseProtoNames:  true,
+				UseEnumNumbers: true,
+			},
+		}),
+		runtime.WithErrorHandler(runtime.DefaultHTTPErrorHandler),
+	)
 	registerHttpHandlerFromEndpoint(api.RegisterClusterServiceHandlerFromEndpoint, "ClusterService", ctx, runtimeMux)
 	registerHttpHandlerFromEndpoint(api.RegisterComputeTemplateServiceHandlerFromEndpoint, "ComputeTemplateService", ctx, runtimeMux)
 

apiserver/pkg/manager/resource_manager.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 
 	"github.com/ray-project/kuberay/apiserver/pkg/model"
-
 	"github.com/ray-project/kuberay/apiserver/pkg/util"
 	api "github.com/ray-project/kuberay/proto/go_client"
 	"github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
 	rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -264,6 +264,10 @@ func (r *ResourceManager) DeleteComputeTemplate(ctx context.Context, name string
 func getClusterByName(ctx context.Context, client rayiov1alpha1.RayClusterInterface, name string) (*v1alpha1.RayCluster, error) {
 	cluster, err := client.Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil, util.NewNotFoundError(err, "Cluster %s not found", name)
+		}
+
 		return nil, util.Wrap(err, "Get Cluster failed")
 	}
 	if managedBy, ok := cluster.Labels[util.KubernetesManagedByLabelKey]; !ok || managedBy != util.ComponentName {
@@ -277,6 +281,10 @@ func getClusterByName(ctx context.Context, client rayiov1alpha1.RayClusterInterf
 func getComputeTemplateByName(ctx context.Context, client clientv1.ConfigMapInterface, name string) (*v1.ConfigMap, error) {
 	runtime, err := client.Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil, util.NewNotFoundError(err, "Cluster %s not found", name)
+		}
+
 		return nil, util.Wrap(err, "Get compute template failed")
 	}
 

apiserver/pkg/util/error.go

@@ -7,6 +7,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	k8errors "k8s.io/apimachinery/pkg/api/errors"
 	k8metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -215,6 +216,13 @@ func (e *UserError) String() string {
 		e.internalError)
 }
 
+// GRPCStatus implements `GRPCStatus` to make sure `FromError` in grpc-go can honor the code.
+// Otherwise, it will always return codes.Unknown(2).
+// https://github.com/grpc/grpc-go/blob/2c0949c22d46095edc579d9e66edcd025192b98c/status/status.go#L91-L92
+func (e *UserError) GRPCStatus() *status.Status {
+	return status.New(e.externalStatusCode, e.externalMessage)
+}
+
 func (e *UserError) wrapf(format string, args ...interface{}) *UserError {
 	return newUserError(errors.Wrapf(e.internalError, format, args...),
 		e.externalMessage, e.externalStatusCode)

proto/Dockerfile

@@ -1,5 +1,5 @@
 # Generate client code (go & json) from API protocol buffers
-FROM golang:1.16.6 as generator
+FROM golang:1.17.8 as generator
 
 ENV PROTOC_VERSION 3.17.3
 ENV GOLANG_PROTOBUF_VERSION v1.5.2

proto/Makefile

@@ -2,7 +2,7 @@
 
 IMAGE_NAME=kuberay/proto-generator
 IMAGE_TAG=$(shell git rev-parse HEAD)
-PREBUILT_REMOTE_IMAGE=kuberay/proto-generator:550d4af484e4e1d2f92117357cc49ffcdc2e41ce
+PREBUILT_REMOTE_IMAGE=kuberay/proto-generator:8210e17c28b3b373449f99561f10534e13d35969
 
 OUTPUT_MODE=import
 TMP_OUTPUT=/tmp

proto/go.mod

@@ -3,15 +3,14 @@ module github.com/ray-project/kuberay/proto
 go 1.17
 
 require (
-	github.com/golang/protobuf v1.5.2
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0
 	google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
 )
 
 require (
+	github.com/golang/protobuf v1.5.2 // indirect
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect

proto/go.sum

@@ -56,6 +56,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=