diff --git a/admin/admin.go b/admin/admin.go new file mode 100644 index 000000000000..5212250b7d4e --- /dev/null +++ b/admin/admin.go @@ -0,0 +1,57 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin provides a convenient method for registering a collection of +// administration services to a gRPC server. The services registered are: +// +// - Channelz: https://github.com/grpc/proposal/blob/master/A14-channelz.md +// - CSDS: https://github.com/grpc/proposal/blob/master/A40-csds-support.md +// +// Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package admin + +import ( + "google.golang.org/grpc" + channelzservice "google.golang.org/grpc/channelz/service" + internaladmin "google.golang.org/grpc/internal/admin" +) + +func init() { + // Add a list of default services to admin here. Optional services, like + // CSDS, will be added by other packages. + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + channelzservice.RegisterChannelzServiceToServer(registrar) + return nil, nil + }) +} + +// Register registers the set of admin services to the given server. +// +// The returned cleanup function should be called to clean up the resources +// allocated for the service handlers after the server is stopped. +// +// Note that if `s` is not a *grpc.Server or a *xds.GRPCServer, CSDS will not be +// registered because CSDS generated code is old and doesn't support interface +// `grpc.ServiceRegistrar`. +// https://github.com/envoyproxy/go-control-plane/issues/403 +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + return internaladmin.Register(s) +} diff --git a/admin/admin_test.go b/admin/admin_test.go new file mode 100644 index 000000000000..0ee4aade0f3c --- /dev/null +++ b/admin/admin_test.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package admin_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" +) + +func TestRegisterNoCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + // CSDS is not registered because xDS isn't imported. + CSDSCode: codes.Unimplemented, + }) +} diff --git a/admin/test/admin_test.go b/admin/test/admin_test.go new file mode 100644 index 000000000000..f0f784bfdf36 --- /dev/null +++ b/admin/test/admin_test.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file has the same content as admin_test.go, difference is that this is +// in another package, and it imports "xds", so we can test that csds is +// registered when xds is imported. + +package test_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" + _ "google.golang.org/grpc/xds" +) + +func TestRegisterWithCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + CSDSCode: codes.OK, + }) +} diff --git a/admin/test/utils.go b/admin/test/utils.go new file mode 100644 index 000000000000..069ffb3cadb7 --- /dev/null +++ b/admin/test/utils.go @@ -0,0 +1,113 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package test contains test only functions for package admin. It's used by +// admin/admin_test.go and admin/test/admin_test.go. +package test + +import ( + "context" + "net" + "testing" + "time" + + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/admin" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/status" +) + +const ( + defaultTestTimeout = 10 * time.Second +) + +// ExpectedStatusCodes contains the expected status code for each RPC (can be +// OK). +type ExpectedStatusCodes struct { + ChannelzCode codes.Code + CSDSCode codes.Code +} + +// RunRegisterTests makes a client, runs the RPCs, and compares the status +// codes. +func RunRegisterTests(t *testing.T, ec ExpectedStatusCodes) { + nodeID := uuid.New().String() + bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{ + Version: xds.TransportV3, + NodeID: nodeID, + ServerURI: "no.need.for.a.server", + }) + if err != nil { + t.Fatal(err) + } + defer bootstrapCleanup() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("cannot create listener: %v", err) + } + + server := grpc.NewServer() + defer server.Stop() + cleanup, err := admin.Register(server) + if err != nil { + t.Fatalf("failed to register admin: %v", err) + } + defer cleanup() + go func() { + server.Serve(lis) + }() + + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure()) + if err != nil { + t.Fatalf("cannot connect to server: %v", err) + } + + t.Run("channelz", func(t *testing.T) { + if err := RunChannelz(conn); status.Code(err) != ec.ChannelzCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "channelz", err, ec.ChannelzCode) + } + }) + t.Run("csds", func(t *testing.T) { + if err := RunCSDS(conn); status.Code(err) != ec.CSDSCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "CSDS", err, ec.CSDSCode) + } + }) +} + +// RunChannelz makes a channelz RPC. +func RunChannelz(conn *grpc.ClientConn) error { + c := channelzpb.NewChannelzClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}, grpc.WaitForReady(true)) + return err +} + +// RunCSDS makes a CSDS RPC. +func RunCSDS(conn *grpc.ClientConn) error { + c := v3statuspb.NewClientStatusDiscoveryServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.FetchClientStatus(ctx, &v3statuspb.ClientStatusRequest{}, grpc.WaitForReady(true)) + return err +} diff --git a/channelz/service/service.go b/channelz/service/service.go index 4d175fef823d..c60ab604e81b 100644 --- a/channelz/service/service.go +++ b/channelz/service/service.go @@ -43,7 +43,7 @@ func init() { var logger = grpclog.Component("channelz") // RegisterChannelzServiceToServer registers the channelz service to the given server. -func RegisterChannelzServiceToServer(s *grpc.Server) { +func RegisterChannelzServiceToServer(s grpc.ServiceRegistrar) { channelzgrpc.RegisterChannelzServer(s, newCZServer()) } diff --git a/internal/admin/admin.go b/internal/admin/admin.go new file mode 100644 index 000000000000..a9285ee74842 --- /dev/null +++ b/internal/admin/admin.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin contains internal implementation for admin service. +package admin + +import "google.golang.org/grpc" + +// services is a map from name to service register functions. +var services []func(grpc.ServiceRegistrar) (func(), error) + +// AddService adds a service to the list of admin services. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. +// +// If multiple services with the same service name are added (e.g. two services +// for `grpc.channelz.v1.Channelz`), the server will panic on `Register()`. +func AddService(f func(grpc.ServiceRegistrar) (func(), error)) { + services = append(services, f) +} + +// Register registers the set of admin services to the given server. +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + var cleanups []func() + for _, f := range services { + cleanup, err := f(s) + if err != nil { + callFuncs(cleanups) + return nil, err + } + if cleanup != nil { + cleanups = append(cleanups, cleanup) + } + } + return func() { + callFuncs(cleanups) + }, nil +} + +func callFuncs(fs []func()) { + for _, f := range fs { + f() + } +} diff --git a/internal/xds/bootstrap.go b/internal/xds/bootstrap.go new file mode 100644 index 000000000000..97ec8e17208e --- /dev/null +++ b/internal/xds/bootstrap.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/xds/env" +) + +var logger = grpclog.Component("internal/xds") + +// TransportAPI refers to the API version for xDS transport protocol. +type TransportAPI int + +const ( + // TransportV2 refers to the v2 xDS transport protocol. + TransportV2 TransportAPI = iota + // TransportV3 refers to the v3 xDS transport protocol. + TransportV3 +) + +// BootstrapOptions wraps the parameters passed to SetupBootstrapFile. +type BootstrapOptions struct { + // Version is the xDS transport protocol version. + Version TransportAPI + // NodeID is the node identifier of the gRPC client/server node in the + // proxyless service mesh. + NodeID string + // ServerURI is the address of the management server. + ServerURI string + // ServerListenerResourceNameTemplate is the Listener resource name to fetch. + ServerListenerResourceNameTemplate string + // CertificateProviders is the certificate providers configuration. + CertificateProviders map[string]json.RawMessage +} + +// SetupBootstrapFile creates a temporary file with bootstrap contents, based on +// the passed in options, and updates the bootstrap environment variable to +// point to this file. +// +// Returns a cleanup function which will be non-nil if the setup process was +// completed successfully. It is the responsibility of the caller to invoke the +// cleanup function at the end of the test. +func SetupBootstrapFile(opts BootstrapOptions) (func(), error) { + f, err := ioutil.TempFile("", "test_xds_bootstrap_*") + if err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + + cfg := &bootstrapConfig{ + XdsServers: []server{ + { + ServerURI: opts.ServerURI, + ChannelCreds: []creds{ + { + Type: "insecure", + }, + }, + }, + }, + Node: node{ + ID: opts.NodeID, + }, + CertificateProviders: opts.CertificateProviders, + ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, + } + switch opts.Version { + case TransportV2: + // TODO: Add any v2 specific fields. + case TransportV3: + cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3") + default: + return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version) + } + + bootstrapContents, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents) + + origBootstrapFileName := env.BootstrapFileName + env.BootstrapFileName = f.Name() + return func() { + os.Remove(f.Name()) + env.BootstrapFileName = origBootstrapFileName + }, nil +} + +type bootstrapConfig struct { + XdsServers []server `json:"xds_servers,omitempty"` + Node node `json:"node,omitempty"` + CertificateProviders map[string]json.RawMessage `json:"certificate_providers,omitempty"` + ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"` +} + +type server struct { + ServerURI string `json:"server_uri,omitempty"` + ChannelCreds []creds `json:"channel_creds,omitempty"` + ServerFeatures []string `json:"server_features,omitempty"` +} + +type creds struct { + Type string `json:"type,omitempty"` + Config interface{} `json:"config,omitempty"` +} + +type node struct { + ID string `json:"id,omitempty"` +} diff --git a/xds/internal/env/env.go b/internal/xds/env/env.go similarity index 100% rename from xds/internal/env/env.go rename to internal/xds/env/env.go diff --git a/xds/csds/csds_test.go b/xds/csds/csds_test.go index db919e138fbd..856894b4e4d4 100644 --- a/xds/csds/csds_test.go +++ b/xds/csds/csds_test.go @@ -37,6 +37,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds" "google.golang.org/grpc/xds/internal/client" "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/testutils/e2e" @@ -255,8 +256,8 @@ func commonSetup(t *testing.T) (xdsClientInterfaceWithWatch, *e2e.ManagementServ } // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{ - Version: e2e.TransportV3, + bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{ + Version: xds.TransportV3, NodeID: nodeID, ServerURI: fs.Address, }) diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go index 791a568ecb02..4ccec4ec4120 100644 --- a/xds/googledirectpath/googlec2p.go +++ b/xds/googledirectpath/googlec2p.go @@ -35,11 +35,11 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/googlecloud" internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/resolver" _ "google.golang.org/grpc/xds" // To register xds resolvers and balancers. xdsclient "google.golang.org/grpc/xds/internal/client" "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/version" "google.golang.org/protobuf/types/known/structpb" ) diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go index 86ea34599435..2dd31d754f3f 100644 --- a/xds/googledirectpath/googlec2p_test.go +++ b/xds/googledirectpath/googlec2p_test.go @@ -27,9 +27,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/resolver" "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/version" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/structpb" diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go index 894b9a0152bc..5318a5342e83 100644 --- a/xds/internal/balancer/edsbalancer/eds_impl.go +++ b/xds/internal/balancer/edsbalancer/eds_impl.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/xds/internal" @@ -39,7 +40,6 @@ import ( "google.golang.org/grpc/xds/internal/client" xdsclient "google.golang.org/grpc/xds/internal/client" "google.golang.org/grpc/xds/internal/client/load" - "google.golang.org/grpc/xds/internal/env" ) // TODO: make this a environment variable? diff --git a/xds/internal/balancer/edsbalancer/eds_impl_test.go b/xds/internal/balancer/edsbalancer/eds_impl_test.go index 3334376f4a9a..ebaea13cc88a 100644 --- a/xds/internal/balancer/edsbalancer/eds_impl_test.go +++ b/xds/internal/balancer/edsbalancer/eds_impl_test.go @@ -31,12 +31,12 @@ import ( "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal" "google.golang.org/grpc/xds/internal/balancer/balancergroup" "google.golang.org/grpc/xds/internal/client" xdsclient "google.golang.org/grpc/xds/internal/client" "google.golang.org/grpc/xds/internal/client/load" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/testutils" ) diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go index fab93f50f4eb..f32c698b4f55 100644 --- a/xds/internal/client/bootstrap/bootstrap.go +++ b/xds/internal/client/bootstrap/bootstrap.go @@ -35,7 +35,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/internal" - "google.golang.org/grpc/xds/internal/env" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal/version" ) diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go index bdba74099c71..501d62102d21 100644 --- a/xds/internal/client/bootstrap/bootstrap_test.go +++ b/xds/internal/client/bootstrap/bootstrap_test.go @@ -36,7 +36,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/internal" - "google.golang.org/grpc/xds/internal/env" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal/version" ) diff --git a/xds/internal/client/cds_test.go b/xds/internal/client/cds_test.go index bac0ef1aeaac..104260759b95 100644 --- a/xds/internal/client/cds_test.go +++ b/xds/internal/client/cds_test.go @@ -33,7 +33,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" xdsinternal "google.golang.org/grpc/internal/xds" - "google.golang.org/grpc/xds/internal/env" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal/version" "google.golang.org/protobuf/types/known/wrapperspb" ) diff --git a/xds/internal/client/lds_test.go b/xds/internal/client/lds_test.go index e2be3b4025ec..df8098df8368 100644 --- a/xds/internal/client/lds_test.go +++ b/xds/internal/client/lds_test.go @@ -31,7 +31,7 @@ import ( "github.com/golang/protobuf/ptypes" spb "github.com/golang/protobuf/ptypes/struct" "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/xds/internal/env" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/version" "google.golang.org/protobuf/types/known/durationpb" diff --git a/xds/internal/client/rds_test.go b/xds/internal/client/rds_test.go index 0c1e2b285388..2ca01dca9ca2 100644 --- a/xds/internal/client/rds_test.go +++ b/xds/internal/client/rds_test.go @@ -34,7 +34,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/xds/internal/env" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/version" "google.golang.org/protobuf/types/known/durationpb" diff --git a/xds/internal/client/xds.go b/xds/internal/client/xds.go index 13b4e7f76d49..2791603ce26e 100644 --- a/xds/internal/client/xds.go +++ b/xds/internal/client/xds.go @@ -41,8 +41,8 @@ import ( "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/version" ) diff --git a/xds/internal/httpfilter/fault/fault.go b/xds/internal/httpfilter/fault/fault.go index 98e05b51590f..639a08db8e3c 100644 --- a/xds/internal/httpfilter/fault/fault.go +++ b/xds/internal/httpfilter/fault/fault.go @@ -33,9 +33,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/grpcrand" iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/protobuf/types/known/anypb" diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go index 1ad955146b0c..61100e8c44f8 100644 --- a/xds/internal/httpfilter/fault/fault_test.go +++ b/xds/internal/httpfilter/fault/fault_test.go @@ -37,9 +37,10 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/testutils/e2e" @@ -104,8 +105,8 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) { } // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{ - Version: e2e.TransportV3, + bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{ + Version: xds.TransportV3, NodeID: nodeID, ServerURI: fs.Address, ServerListenerResourceNameTemplate: "grpc/server", diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go index 7c1ec853e4cd..ef7c37128c13 100644 --- a/xds/internal/resolver/serviceconfig.go +++ b/xds/internal/resolver/serviceconfig.go @@ -28,10 +28,10 @@ import ( "google.golang.org/grpc/codes" iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/status" "google.golang.org/grpc/xds/internal/balancer/clustermanager" xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/httpfilter/router" ) diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go index 53ea17042aa5..36c7416cb436 100644 --- a/xds/internal/resolver/xds_resolver_test.go +++ b/xds/internal/resolver/xds_resolver_test.go @@ -36,6 +36,7 @@ import ( iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/internal/xds/env" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/status" @@ -44,7 +45,6 @@ import ( "google.golang.org/grpc/xds/internal/client" xdsclient "google.golang.org/grpc/xds/internal/client" "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/env" "google.golang.org/grpc/xds/internal/httpfilter" "google.golang.org/grpc/xds/internal/httpfilter/router" xdstestutils "google.golang.org/grpc/xds/internal/testutils" diff --git a/xds/internal/test/xds_client_integration_test.go b/xds/internal/test/xds_client_integration_test.go index f950a881e508..f97e42af2a0a 100644 --- a/xds/internal/test/xds_client_integration_test.go +++ b/xds/internal/test/xds_client_integration_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds" "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/testutils/e2e" @@ -54,8 +55,8 @@ func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) { } // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{ - Version: e2e.TransportV3, + bootstrapCleanup, err := xds.SetupBootstrapFile(xds.BootstrapOptions{ + Version: xds.TransportV3, NodeID: nodeID, ServerURI: fs.Address, }) diff --git a/xds/internal/test/xds_server_integration_test.go b/xds/internal/test/xds_server_integration_test.go index 3b9156347e8f..f18d4e356aa6 100644 --- a/xds/internal/test/xds_server_integration_test.go +++ b/xds/internal/test/xds_server_integration_test.go @@ -38,6 +38,7 @@ import ( v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" wrapperspb "github.com/golang/protobuf/ptypes/wrappers" "github.com/google/uuid" + xds2 "google.golang.org/grpc/internal/xds" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -150,8 +151,8 @@ func commonSetup(t *testing.T) (*e2e.ManagementServer, string, net.Listener, fun cpc := e2e.DefaultFileWatcherConfig(path.Join(tmpdir, certFile), path.Join(tmpdir, keyFile), path.Join(tmpdir, rootFile)) // Create a bootstrap file in a temporary directory. - bootstrapCleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{ - Version: e2e.TransportV3, + bootstrapCleanup, err := xds2.SetupBootstrapFile(xds2.BootstrapOptions{ + Version: xds2.TransportV3, NodeID: nodeID, ServerURI: fs.Address, CertificateProviders: cpc, diff --git a/xds/internal/testutils/e2e/bootstrap.go b/xds/internal/testutils/e2e/bootstrap.go index c6fd2f9291fe..72a1a9900cfe 100644 --- a/xds/internal/testutils/e2e/bootstrap.go +++ b/xds/internal/testutils/e2e/bootstrap.go @@ -21,93 +21,8 @@ package e2e import ( "encoding/json" "fmt" - "io/ioutil" - "os" - - "google.golang.org/grpc/xds/internal/env" -) - -// TransportAPI refers to the API version for xDS transport protocol. -type TransportAPI int - -const ( - // TransportV2 refers to the v2 xDS transport protocol. - TransportV2 TransportAPI = iota - // TransportV3 refers to the v3 xDS transport protocol. - TransportV3 ) -// BootstrapOptions wraps the parameters passed to SetupBootstrapFile. -type BootstrapOptions struct { - // Version is the xDS transport protocol version. - Version TransportAPI - // NodeID is the node identifier of the gRPC client/server node in the - // proxyless service mesh. - NodeID string - // ServerURI is the address of the management server. - ServerURI string - // ServerListenerResourceNameTemplate is the Listener resource name to fetch. - ServerListenerResourceNameTemplate string - // CertificateProviders is the certificate providers configuration. - CertificateProviders map[string]json.RawMessage -} - -// SetupBootstrapFile creates a temporary file with bootstrap contents, based on -// the passed in options, and updates the bootstrap environment variable to -// point to this file. -// -// Returns a cleanup function which will be non-nil if the setup process was -// completed successfully. It is the responsibility of the caller to invoke the -// cleanup function at the end of the test. -func SetupBootstrapFile(opts BootstrapOptions) (func(), error) { - f, err := ioutil.TempFile("", "test_xds_bootstrap_*") - if err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - - cfg := &bootstrapConfig{ - XdsServers: []server{ - { - ServerURI: opts.ServerURI, - ChannelCreds: []creds{ - { - Type: "insecure", - }, - }, - }, - }, - Node: node{ - ID: opts.NodeID, - }, - CertificateProviders: opts.CertificateProviders, - ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, - } - switch opts.Version { - case TransportV2: - // TODO: Add any v2 specific fields. - case TransportV3: - cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3") - default: - return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version) - } - - bootstrapContents, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { - return nil, fmt.Errorf("failed to created bootstrap file: %v", err) - } - logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents) - - origBootstrapFileName := env.BootstrapFileName - env.BootstrapFileName = f.Name() - return func() { - os.Remove(f.Name()) - env.BootstrapFileName = origBootstrapFileName - }, nil -} - // DefaultFileWatcherConfig is a helper function to create a default certificate // provider plugin configuration. The test is expected to have setup the files // appropriately before this configuration is used to instantiate providers. @@ -125,25 +40,3 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) map[string]json. "google_cloud_private_spiffe": json.RawMessage(cfg), } } - -type bootstrapConfig struct { - XdsServers []server `json:"xds_servers,omitempty"` - Node node `json:"node,omitempty"` - CertificateProviders map[string]json.RawMessage `json:"certificate_providers,omitempty"` - ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"` -} - -type server struct { - ServerURI string `json:"server_uri,omitempty"` - ChannelCreds []creds `json:"channel_creds,omitempty"` - ServerFeatures []string `json:"server_features,omitempty"` -} - -type creds struct { - Type string `json:"type,omitempty"` - Config interface{} `json:"config,omitempty"` -} - -type node struct { - ID string `json:"id,omitempty"` -} diff --git a/xds/xds.go b/xds/xds.go index 17b84ec932cb..0af8f3d6d892 100644 --- a/xds/xds.go +++ b/xds/xds.go @@ -28,6 +28,13 @@ package xds import ( + "fmt" + + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "google.golang.org/grpc" + internaladmin "google.golang.org/grpc/internal/admin" + "google.golang.org/grpc/xds/csds" + _ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin. _ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers. _ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client. @@ -35,3 +42,32 @@ import ( _ "google.golang.org/grpc/xds/internal/httpfilter/fault" // Register the fault injection filter. _ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver. ) + +func init() { + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + var grpcServer *grpc.Server + switch ss := registrar.(type) { + case *grpc.Server: + grpcServer = ss + case *GRPCServer: + sss, ok := ss.gs.(*grpc.Server) + if !ok { + logger.Warningf("grpc server within xds.GRPCServer is not *grpc.Server, CSDS will not be registered") + return nil, nil + } + grpcServer = sss + default: + // Returning an error would cause the top level admin.Register() to + // fail. Log a warning instead. + logger.Warningf("server to register service on is neither a *grpc.Server or a *xds.GRPCServer, CSDS will not be registered") + return nil, nil + } + + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + return nil, fmt.Errorf("failed to create csds server: %v", err) + } + v3statuspb.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss) + return csdss.Close, nil + }) +}