diff --git a/server/embed/etcd.go b/server/embed/etcd.go index b4f4defe54e..b3b125dd2f9 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -747,7 +747,7 @@ func (e *Etcd) serveClients() (err error) { } else { mux := http.NewServeMux() etcdhttp.HandleBasic(e.cfg.logger, mux, e.Server) - etcdhttp.HandleMetricsHealthForV3(e.cfg.logger, mux, e.Server) + etcdhttp.HandleMetricsHealth(e.cfg.logger, mux, e.Server) h = mux } @@ -836,7 +836,7 @@ func (e *Etcd) serveMetrics() (err error) { if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() - etcdhttp.HandleMetricsHealthForV3(e.cfg.logger, metricsMux, e.Server) + etcdhttp.HandleMetricsHealth(e.cfg.logger, metricsMux, e.Server) for _, murl := range e.cfg.ListenMetricsUrls { tlsInfo := &e.cfg.ClientTLSInfo diff --git a/server/etcdserver/api/etcdhttp/metrics.go b/server/etcdserver/api/etcdhttp/metrics.go index 9d9ccec71b7..ec33bfde01e 100644 --- a/server/etcdserver/api/etcdhttp/metrics.go +++ b/server/etcdserver/api/etcdhttp/metrics.go @@ -24,8 +24,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.etcd.io/etcd/api/v3/etcdserverpb" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/server/v3/auth" + "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" ) @@ -37,8 +40,19 @@ const ( PathProxyHealth = "/proxy/health" ) -// HandleMetricsHealth registers metrics and health handlers. -func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) { +type ServerHealth interface { + serverHealthV2V3 + Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error) + Config() config.ServerConfig +} + +type serverHealthV2V3 interface { + Alarms() []*pb.AlarmMember + Leader() types.ID +} + +// HandleMetricsHealthForV2 registers metrics and health handlers for v2. +func HandleMetricsHealthForV2(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) { mux.Handle(PathMetrics, promhttp.Handler()) mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health { if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" { @@ -51,9 +65,9 @@ func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.Serv })) } -// HandleMetricsHealthForV3 registers metrics and health handlers. it checks health by using v3 range request +// HandleMetricsHealth registers metrics and health handlers. it checks health by using v3 range request // and its corresponding timeout. -func HandleMetricsHealthForV3(lg *zap.Logger, mux *http.ServeMux, srv *etcdserver.EtcdServer) { +func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv ServerHealth) { mux.Handle(PathMetrics, promhttp.Handler()) mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health { if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" { @@ -62,7 +76,7 @@ func HandleMetricsHealthForV3(lg *zap.Logger, mux *http.ServeMux, srv *etcdserve if h := checkLeader(lg, srv, serializable); h.Health != "true" { return h } - return checkV3API(lg, srv, serializable) + return checkAPI(lg, srv, serializable) })) } @@ -155,7 +169,7 @@ func getSerializableFlag(r *http.Request) bool { // TODO: etcdserver.ErrNoLeader in health API -func checkAlarms(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet) Health { +func checkAlarms(lg *zap.Logger, srv serverHealthV2V3, excludedAlarms AlarmSet) Health { h := Health{Health: "true"} as := srv.Alarms() if len(as) > 0 { @@ -168,9 +182,9 @@ func checkAlarms(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSe h.Health = "false" switch v.Alarm { - case etcdserverpb.AlarmType_NOSPACE: + case pb.AlarmType_NOSPACE: h.Reason = "ALARM NOSPACE" - case etcdserverpb.AlarmType_CORRUPT: + case pb.AlarmType_CORRUPT: h.Reason = "ALARM CORRUPT" default: h.Reason = "ALARM UNKNOWN" @@ -183,7 +197,7 @@ func checkAlarms(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSe return h } -func checkLeader(lg *zap.Logger, srv etcdserver.ServerV2, serializable bool) Health { +func checkLeader(lg *zap.Logger, srv serverHealthV2V3, serializable bool) Health { h := Health{Health: "true"} if !serializable && (uint64(srv.Leader()) == raft.None) { h.Health = "false" @@ -208,10 +222,11 @@ func checkV2API(lg *zap.Logger, srv etcdserver.ServerV2) Health { return h } -func checkV3API(lg *zap.Logger, srv *etcdserver.EtcdServer, serializable bool) Health { +func checkAPI(lg *zap.Logger, srv ServerHealth, serializable bool) Health { h := Health{Health: "true"} - ctx, cancel := context.WithTimeout(context.Background(), srv.Cfg.ReqTimeout()) - _, err := srv.Range(ctx, &etcdserverpb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) + cfg := srv.Config() + ctx, cancel := context.WithTimeout(context.Background(), cfg.ReqTimeout()) + _, err := srv.Range(ctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) cancel() if err != nil && err != auth.ErrUserEmpty && err != auth.ErrPermissionDenied { h.Health = "false" diff --git a/server/etcdserver/api/etcdhttp/metrics_test.go b/server/etcdserver/api/etcdhttp/metrics_test.go index 8b1638f7265..76ee04ca44c 100644 --- a/server/etcdserver/api/etcdhttp/metrics_test.go +++ b/server/etcdserver/api/etcdhttp/metrics_test.go @@ -14,9 +14,11 @@ import ( "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/server/v3/auth" + "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" - "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) type fakeStats struct{} @@ -25,25 +27,34 @@ func (s *fakeStats) SelfStats() []byte { return nil } func (s *fakeStats) LeaderStats() []byte { return nil } func (s *fakeStats) StoreStats() []byte { return nil } -type fakeServerV2 struct { +type fakeHealthServer struct { fakeServer stats.Stats - health string + health string + apiError error } -func (s *fakeServerV2) Leader() types.ID { +func (s *fakeHealthServer) Range(ctx context.Context, request *pb.RangeRequest) (*pb.RangeResponse, error) { + return nil, s.apiError +} + +func (s *fakeHealthServer) Config() config.ServerConfig { + return config.ServerConfig{} +} + +func (s *fakeHealthServer) Leader() types.ID { if s.health == "true" { return 1 } return types.ID(raft.None) } -func (s *fakeServerV2) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) { +func (s *fakeHealthServer) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) { if s.health == "true" { return etcdserver.Response{}, nil } return etcdserver.Response{}, fmt.Errorf("fail health check") } -func (s *fakeServerV2) ClientCertAuthEnabled() bool { return false } +func (s *fakeHealthServer) ClientCertAuthEnabled() bool { return false } func TestHealthHandler(t *testing.T) { // define the input and expected output @@ -52,6 +63,7 @@ func TestHealthHandler(t *testing.T) { name string alarms []*pb.AlarmMember healthCheckURL string + apiError error expectStatusCode int expectHealth string @@ -105,15 +117,34 @@ func TestHealthHandler(t *testing.T) { expectStatusCode: http.StatusOK, expectHealth: "true", }, + { + healthCheckURL: "/health", + apiError: auth.ErrUserEmpty, + expectStatusCode: http.StatusOK, + expectHealth: "true", + }, + { + healthCheckURL: "/health", + apiError: auth.ErrPermissionDenied, + expectStatusCode: http.StatusOK, + expectHealth: "true", + }, + { + healthCheckURL: "/health", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusServiceUnavailable, + expectHealth: "false", + }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() - HandleMetricsHealth(zap.NewExample(), mux, &fakeServerV2{ + HandleMetricsHealth(zaptest.NewLogger(t), mux, &fakeHealthServer{ fakeServer: fakeServer{alarms: tt.alarms}, Stats: &fakeStats{}, health: tt.expectHealth, + apiError: tt.apiError, }) ts := httptest.NewServer(mux) defer ts.Close() diff --git a/server/etcdserver/api/v2http/client.go b/server/etcdserver/api/v2http/client.go index 17b420732e6..e8cb1bff142 100644 --- a/server/etcdserver/api/v2http/client.go +++ b/server/etcdserver/api/v2http/client.go @@ -58,7 +58,7 @@ func NewClientHandler(lg *zap.Logger, server etcdserver.ServerPeer, timeout time } mux := http.NewServeMux() etcdhttp.HandleBasic(lg, mux, server) - etcdhttp.HandleMetricsHealth(lg, mux, server) + etcdhttp.HandleMetricsHealthForV2(lg, mux, server) handleV2(lg, mux, server, timeout) return requestLogger(lg, mux) } diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 4e1c2c041a7..5f62ddc2652 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -724,6 +724,10 @@ func (s *EtcdServer) Logger() *zap.Logger { return l } +func (s *EtcdServer) Config() config.ServerConfig { + return s.Cfg +} + func tickToDur(ticks int, tickMs uint) string { return fmt.Sprintf("%v", time.Duration(ticks)*time.Duration(tickMs)*time.Millisecond) }