From f37e97bfaf40e800d9ad223998d8afcbe1ea9d0d Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 10:22:35 -0500 Subject: [PATCH 1/8] Add "operator members" command to list nodes in the cluster. --- command/commands.go | 5 ++ command/operator_members.go | 107 ++++++++++++++++++++++++++++++++ http/handler.go | 3 +- http/sys_ha_status.go | 68 ++++++++++++++++++++ http/sys_ha_status_test.go | 61 ++++++++++++++++++ vault/core.go | 22 +++++++ vault/request_forwarding_rpc.go | 22 ++++++- 7 files changed, 283 insertions(+), 5 deletions(-) create mode 100644 command/operator_members.go create mode 100644 http/sys_ha_status.go create mode 100644 http/sys_ha_status_test.go diff --git a/command/commands.go b/command/commands.go index dc08c4a74675..bf5daa7474d4 100644 --- a/command/commands.go +++ b/command/commands.go @@ -461,6 +461,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { BaseCommand: getBaseCommand(), }, nil }, + "operator members": func() (cli.Command, error) { + return &OperatorMembersCommand{ + BaseCommand: getBaseCommand(), + }, nil + }, "path-help": func() (cli.Command, error) { return &PathHelpCommand{ BaseCommand: getBaseCommand(), diff --git a/command/operator_members.go b/command/operator_members.go new file mode 100644 index 000000000000..15d3d71a5564 --- /dev/null +++ b/command/operator_members.go @@ -0,0 +1,107 @@ +package command + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +var ( + _ cli.Command = (*OperatorMembersCommand)(nil) + _ cli.CommandAutocomplete = (*OperatorMembersCommand)(nil) +) + +type OperatorMembersCommand struct { + *BaseCommand +} + +func (c *OperatorMembersCommand) Synopsis() string { + return "Returns the list of nodes in the cluster" +} + +func (c *OperatorMembersCommand) Help() string { + helpText := ` +Usage: vault operator members + + Provides the details of all the nodes in the cluster. + + $ vault operator members + +` + c.Flags().Help() + + return strings.TrimSpace(helpText) +} + +func (c *OperatorMembersCommand) Flags() *FlagSets { + set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat) + + return set +} + +func (c *OperatorMembersCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictAnything +} + +func (c *OperatorMembersCommand) AutocompleteFlags() complete.Flags { + return c.Flags().Completions() +} + +func (c *OperatorMembersCommand) Run(args []string) int { + f := c.Flags() + + if err := f.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + client, err := c.Client() + if err != nil { + c.UI.Error(err.Error()) + return 2 + } + + r := client.NewRequest("GET", "/v1/sys/ha-status") + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + resp, err := client.RawRequestWithContext(ctx, r) + if err != nil { + return 1 + } + defer resp.Body.Close() + + var result HaStatusResponse + err = resp.DecodeJSON(&result) + + if err != nil { + c.UI.Error(err.Error()) + return 2 + } + + switch Format(c.UI) { + case "table": + out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last echo"} + for _, node := range result.Nodes { + out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho)) + } + c.UI.Output(tableOutput(out, nil)) + return 0 + default: + return OutputData(c.UI, result) + } +} + +type Node struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} + +type HaStatusResponse struct { + Nodes []Node +} diff --git a/http/handler.go b/http/handler.go index 01085884b74d..f4e1bb18135f 100644 --- a/http/handler.go +++ b/http/handler.go @@ -142,6 +142,7 @@ func Handler(props *vault.HandlerProperties) http.Handler { mux.Handle("/v1/sys/step-down", handleRequestForwarding(core, handleSysStepDown(core))) mux.Handle("/v1/sys/unseal", handleSysUnseal(core)) mux.Handle("/v1/sys/leader", handleSysLeader(core)) + mux.Handle("/v1/sys/ha-status", handleRequestForwarding(core, handleSysHaStatus(core))) mux.Handle("/v1/sys/health", handleSysHealth(core)) mux.Handle("/v1/sys/monitor", handleLogicalNoForward(core)) mux.Handle("/v1/sys/generate-root/attempt", handleRequestForwarding(core, @@ -213,7 +214,6 @@ func Handler(props *vault.HandlerProperties) http.Handler { return printablePathCheckHandler } - type copyResponseWriter struct { wrapped http.ResponseWriter statusCode int @@ -304,7 +304,6 @@ func wrapGenericHandler(core *vault.Core, h http.Handler, props *vault.HandlerPr hostname, _ := os.Hostname() return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // This block needs to be here so that upon sending SIGHUP, custom response // headers are also reloaded into the handlers. var customHeaders map[string][]*logical.CustomHeader diff --git a/http/sys_ha_status.go b/http/sys_ha_status.go new file mode 100644 index 000000000000..38e22183a63d --- /dev/null +++ b/http/sys_ha_status.go @@ -0,0 +1,68 @@ +package http + +import ( + "net/http" + "time" + + "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/vault" + "github.com/shirou/gopsutil/host" +) + +func handleSysHaStatus(core *vault.Core) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "GET": + handleSysHaStatusGet(core, w, r) + default: + respondError(w, http.StatusMethodNotAllowed, nil) + } + }) +} + +func handleSysHaStatusGet(core *vault.Core, w http.ResponseWriter, r *http.Request) { + _, address, clusterAddr, err := core.Leader() + if errwrap.Contains(err, vault.ErrHANotEnabled.Error()) { + err = nil + } + if err != nil { + respondError(w, http.StatusInternalServerError, err) + return + } + h, _ := host.Info() + nodes := []Node{ + { + Hostname: h.Hostname, + APIAddress: address, + ClusterAddress: clusterAddr, + ActiveNode: true, + }, + } + + for _, peerNode := range core.GetHAPeerNodesCached() { + lastEcho := peerNode.LastEcho + nodes = append(nodes, Node{ + Hostname: peerNode.Hostname, + APIAddress: peerNode.APIAddress, + ClusterAddress: peerNode.ClusterAddress, + LastEcho: &lastEcho, + }) + } + resp := &HaStatusResponse{ + Nodes: nodes, + } + + respondOk(w, resp) +} + +type Node struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} + +type HaStatusResponse struct { + Nodes []Node +} diff --git a/http/sys_ha_status_test.go b/http/sys_ha_status_test.go new file mode 100644 index 000000000000..433e87dbb82e --- /dev/null +++ b/http/sys_ha_status_test.go @@ -0,0 +1,61 @@ +package http + +import ( + "context" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/helper/testhelpers" + "github.com/hashicorp/vault/sdk/helper/logging" + "github.com/hashicorp/vault/sdk/physical" + "github.com/hashicorp/vault/sdk/physical/inmem" + "github.com/hashicorp/vault/vault" +) + +func TestSysHAStatus(t *testing.T) { + logger := logging.NewVaultLogger(hclog.Trace) + inm, err := inmem.NewTransactionalInmem(nil, logger) + if err != nil { + t.Fatal(err) + } + inmha, err := inmem.NewInmemHA(nil, logger) + if err != nil { + t.Fatal(err) + } + + conf := &vault.CoreConfig{ + Physical: inm, + HAPhysical: inmha.(physical.HABackend), + } + opts := &vault.TestClusterOptions{ + HandlerFunc: Handler, + } + cluster := vault.NewTestCluster(t, conf, opts) + cluster.Start() + defer cluster.Cleanup() + testhelpers.WaitForActiveNodeAndStandbys(t, cluster) + // Make sure standbys have time to echo and populate the cache + time.Sleep(6 * time.Second) + + // Use standby deliberately to make sure it forwards + client := cluster.Cores[1].Client + r := client.NewRequest("GET", "/v1/sys/ha-status") + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + resp, err := client.RawRequestWithContext(ctx, r) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + var result HaStatusResponse + err = resp.DecodeJSON(&result) + if err != nil { + t.Fatal(err) + } + + if len(result.Nodes) != len(cluster.Cores) { + t.Fatalf("expected %d nodes, got %d", len(cluster.Cores), len(result.Nodes)) + } +} diff --git a/vault/core.go b/vault/core.go index 210144a3b026..d1f3693c5e77 100644 --- a/vault/core.go +++ b/vault/core.go @@ -2946,3 +2946,25 @@ type LicenseState struct { ExpiryTime time.Time Terminated bool } + +type PeerNode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + LastEcho time.Time `json:"last_echo"` +} + +// GetHAPeerNodesCached returns the nodes that've sent us Echo requests recently. +func (c *Core) GetHAPeerNodesCached() []PeerNode { + var nodes []PeerNode + for itemClusterAddr, item := range c.clusterPeerClusterAddrsCache.Items() { + info := item.Object.(nodeHAConnectionInfo) + nodes = append(nodes, PeerNode{ + Hostname: info.nodeInfo.NodeID, + APIAddress: info.nodeInfo.ApiAddr, + ClusterAddress: itemClusterAddr, + LastEcho: info.lastHeartbeat, + }) + } + return nodes +} diff --git a/vault/request_forwarding_rpc.go b/vault/request_forwarding_rpc.go index 0388e4c67220..a1e28ee09e78 100644 --- a/vault/request_forwarding_rpc.go +++ b/vault/request_forwarding_rpc.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/vault/helper/forwarding" "github.com/hashicorp/vault/physical/raft" "github.com/hashicorp/vault/vault/replication" + "github.com/shirou/gopsutil/host" ) type forwardedRequestRPCServer struct { @@ -71,9 +72,18 @@ func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *fo return resp, nil } +type nodeHAConnectionInfo struct { + nodeInfo *NodeInformation + lastHeartbeat time.Time +} + func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) { + incomingNodeConnectionInfo := nodeHAConnectionInfo{ + nodeInfo: in.NodeInfo, + lastHeartbeat: time.Now(), + } if in.ClusterAddr != "" { - s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0) + s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0) } if in.RaftAppliedIndex > 0 && len(in.RaftNodeID) > 0 && s.raftFollowerStates != nil { @@ -106,12 +116,18 @@ type forwardingClient struct { // with these requests it's useful to keep this as well func (c *forwardingClient) startHeartbeat() { go func() { + clusterAddr := c.core.ClusterAddr() + h, _ := host.Info() + ni := NodeInformation{ + ApiAddr: c.core.redirectAddr, + NodeID: h.Hostname, + Mode: "standby", + } tick := func() { - clusterAddr := c.core.ClusterAddr() - req := &EchoRequest{ Message: "ping", ClusterAddr: clusterAddr, + NodeInfo: &ni, } if raftBackend := c.core.getRaftBackend(); raftBackend != nil { From cbd5bff883bc210aefb028b13fffeaa7f5bbcd16 Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 10:25:24 -0500 Subject: [PATCH 2/8] Add "operator members" command to list nodes in the cluster. --- changelog/13292.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/13292.txt diff --git a/changelog/13292.txt b/changelog/13292.txt new file mode 100644 index 000000000000..e3943389b01e --- /dev/null +++ b/changelog/13292.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core/ha: Add new mechanism for keeping track of peers talking to active node, and new 'operator members' command to view them. +``` From e75a4dfbbcf3102ee9239bfcc72e96adad92591a Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 10:31:33 -0500 Subject: [PATCH 3/8] Add API docs --- website/content/api-docs/system/ha-status.mdx | 56 +++++++++++++++++++ website/data/api-docs-nav-data.json | 4 ++ 2 files changed, 60 insertions(+) create mode 100644 website/content/api-docs/system/ha-status.mdx diff --git a/website/content/api-docs/system/ha-status.mdx b/website/content/api-docs/system/ha-status.mdx new file mode 100644 index 000000000000..e518f3f6f3a4 --- /dev/null +++ b/website/content/api-docs/system/ha-status.mdx @@ -0,0 +1,56 @@ +--- +layout: api +page_title: /sys/ha-status - HTTP API +description: The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster. +--- + +# `/sys/ha-status` + +The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster. + +## Seal Status + +This endpoint returns the HA status of the Vault cluster. + +| Method | Path | +| :----- | :----------------- | +| `GET` | `/sys/ha-status` | + +### Sample Request + +```shell-session +$ curl \ + --header "X-Vault-Token: ..." \ + http://127.0.0.1:8200/v1/sys/ha-status +``` + +### Sample Response + +```json +{ + "Nodes": [ + { + "hostname": "node1", + "api_address": "http://10.0.0.2:8200", + "cluster_address": "https://10.0.0.2:8201", + "active_node": true, + "last_echo": null + }, + { + "hostname": "node2", + "api_address": "http://10.0.0.3:8200", + "cluster_address": "https://10.0.0.3:8201", + "active_node": false, + "last_echo": "2021-11-29T10:29:09.202235-05:00" + }, + { + "hostname": "node3", + "api_address": "http://10.0.0.4:8200", + "cluster_address": "https://10.0.0.4:8201", + "active_node": false, + "last_echo": "2021-11-29T10:29:07.402548-05:00" + } + ] +} + +``` diff --git a/website/data/api-docs-nav-data.json b/website/data/api-docs-nav-data.json index 761cdb7ef16a..14a1c9a74ab5 100644 --- a/website/data/api-docs-nav-data.json +++ b/website/data/api-docs-nav-data.json @@ -429,6 +429,10 @@ "title": "/sys/key-status", "path": "system/key-status" }, + { + "title": "/sys/ha-status", + "path": "system/ha-status" + }, { "title": "/sys/leader", "path": "system/leader" From b09895b3c1b12825745a0f821985a0ec8d9dfecd Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 10:41:48 -0500 Subject: [PATCH 4/8] More docs --- website/content/api-docs/system/ha-status.mdx | 3 +- .../docs/commands/operator/members.mdx | 36 +++++++++++++++++++ website/data/docs-nav-data.json | 4 +++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 website/content/docs/commands/operator/members.mdx diff --git a/website/content/api-docs/system/ha-status.mdx b/website/content/api-docs/system/ha-status.mdx index e518f3f6f3a4..fe16a76f76b3 100644 --- a/website/content/api-docs/system/ha-status.mdx +++ b/website/content/api-docs/system/ha-status.mdx @@ -7,8 +7,9 @@ description: The `/sys/ha-status` endpoint is used to check the HA status of a V # `/sys/ha-status` The `/sys/ha-status` endpoint is used to check the HA status of a Vault cluster. +It lists the active node and the peers that it's heard from since it became active. -## Seal Status +## HA Status This endpoint returns the HA status of the Vault cluster. diff --git a/website/content/docs/commands/operator/members.mdx b/website/content/docs/commands/operator/members.mdx new file mode 100644 index 000000000000..119ae807f6ee --- /dev/null +++ b/website/content/docs/commands/operator/members.mdx @@ -0,0 +1,36 @@ +--- +layout: docs +page_title: operator members - Command +description: |- + The "operator members" command provides information about the nodes making up an HA cluster. +--- + +# operator members + +The `operator members` lists the active node and the peers that it's heard from +since it became active. + +## Examples + +Get the key status: + +```shell-session +$ vault operator members +Host Name API Address Cluster Address ActiveNode Last echo +--------- ----------- --------------- ---------- --------- +node1 http://10.0.0.2:8200 https://10.0.0.2:8201 true +node2 http://10.0.0.3:8200 https://10.0.0.3:8201 false 2021-11-29 10:19:39.236409 -0500 EST +node3 http://10.0.0.4:8200 https://10.0.0.4:8201 false 2021-11-29 10:19:37.436283 -0500 EST + +``` + +## Usage + +The following flags are available in addition to the [standard set of +flags](/docs/commands) included on all commands. + +### Output Options + +- `-format` `(string: "table")` - Print the output in the given format. Valid + formats are "table", "json", or "yaml". This can also be specified via the + `VAULT_FORMAT` environment variable. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index c3cc225eb1b3..adf19fc5864b 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -574,6 +574,10 @@ "title": "key-status", "path": "commands/operator/key-status" }, + { + "title": "members", + "path": "commands/operator/members" + }, { "title": "migrate", "path": "commands/operator/migrate" From fec01b7ac2487bb24c542978d432cea37bdc4666 Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 13:29:12 -0500 Subject: [PATCH 5/8] Address reviewer feedback. --- command/operator_members.go | 2 +- http/sys_ha_status_test.go | 42 +++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/command/operator_members.go b/command/operator_members.go index 15d3d71a5564..1c993eef253d 100644 --- a/command/operator_members.go +++ b/command/operator_members.go @@ -83,7 +83,7 @@ func (c *OperatorMembersCommand) Run(args []string) int { switch Format(c.UI) { case "table": - out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last echo"} + out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"} for _, node := range result.Nodes { out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho)) } diff --git a/http/sys_ha_status_test.go b/http/sys_ha_status_test.go index 433e87dbb82e..07e37a288b7c 100644 --- a/http/sys_ha_status_test.go +++ b/http/sys_ha_status_test.go @@ -2,6 +2,7 @@ package http import ( "context" + "fmt" "testing" "time" @@ -35,27 +36,28 @@ func TestSysHAStatus(t *testing.T) { cluster.Start() defer cluster.Cleanup() testhelpers.WaitForActiveNodeAndStandbys(t, cluster) - // Make sure standbys have time to echo and populate the cache - time.Sleep(6 * time.Second) - // Use standby deliberately to make sure it forwards - client := cluster.Cores[1].Client - r := client.NewRequest("GET", "/v1/sys/ha-status") - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - resp, err := client.RawRequestWithContext(ctx, r) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() + testhelpers.RetryUntil(t, 10*time.Second, func() error { + // Use standby deliberately to make sure it forwards + client := cluster.Cores[1].Client + r := client.NewRequest("GET", "/v1/sys/ha-status") + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + resp, err := client.RawRequestWithContext(ctx, r) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() - var result HaStatusResponse - err = resp.DecodeJSON(&result) - if err != nil { - t.Fatal(err) - } + var result HaStatusResponse + err = resp.DecodeJSON(&result) + if err != nil { + t.Fatal(err) + } - if len(result.Nodes) != len(cluster.Cores) { - t.Fatalf("expected %d nodes, got %d", len(cluster.Cores), len(result.Nodes)) - } + if len(result.Nodes) != len(cluster.Cores) { + return fmt.Errorf("expected %d nodes, got %d", len(cluster.Cores), len(result.Nodes)) + } + return nil + }) } From 458b65c64931232cd6825d01f1db73d5a3efe8cc Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 29 Nov 2021 15:04:01 -0500 Subject: [PATCH 6/8] Address reviewer feedback. --- vault/core.go | 2 +- vault/request_forwarding_rpc.go | 6 +- vault/request_forwarding_service.pb.go | 91 ++++++++++--------- vault/request_forwarding_service.proto | 1 + .../docs/commands/operator/members.mdx | 2 +- 5 files changed, 56 insertions(+), 46 deletions(-) diff --git a/vault/core.go b/vault/core.go index d1f3693c5e77..eb62e395a6af 100644 --- a/vault/core.go +++ b/vault/core.go @@ -2960,7 +2960,7 @@ func (c *Core) GetHAPeerNodesCached() []PeerNode { for itemClusterAddr, item := range c.clusterPeerClusterAddrsCache.Items() { info := item.Object.(nodeHAConnectionInfo) nodes = append(nodes, PeerNode{ - Hostname: info.nodeInfo.NodeID, + Hostname: info.nodeInfo.Hostname, APIAddress: info.nodeInfo.ApiAddr, ClusterAddress: itemClusterAddr, LastEcho: info.lastHeartbeat, diff --git a/vault/request_forwarding_rpc.go b/vault/request_forwarding_rpc.go index a1e28ee09e78..2cb04cb91955 100644 --- a/vault/request_forwarding_rpc.go +++ b/vault/request_forwarding_rpc.go @@ -119,9 +119,9 @@ func (c *forwardingClient) startHeartbeat() { clusterAddr := c.core.ClusterAddr() h, _ := host.Info() ni := NodeInformation{ - ApiAddr: c.core.redirectAddr, - NodeID: h.Hostname, - Mode: "standby", + ApiAddr: c.core.redirectAddr, + Hostname: h.Hostname, + Mode: "standby", } tick := func() { req := &EchoRequest{ diff --git a/vault/request_forwarding_service.pb.go b/vault/request_forwarding_service.pb.go index 9a8880e8d6c7..62962be0c670 100644 --- a/vault/request_forwarding_service.pb.go +++ b/vault/request_forwarding_service.pb.go @@ -225,6 +225,7 @@ type NodeInformation struct { Mode string `protobuf:"bytes,3,opt,name=mode,proto3" json:"mode,omitempty"` NodeID string `protobuf:"bytes,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` ReplicationState uint32 `protobuf:"varint,5,opt,name=replication_state,json=replicationState,proto3" json:"replication_state,omitempty"` + Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"` } func (x *NodeInformation) Reset() { @@ -294,6 +295,13 @@ func (x *NodeInformation) GetReplicationState() uint32 { return 0 } +func (x *NodeInformation) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + type ClientKey struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -534,7 +542,7 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{ 0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64, - 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xa9, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08, @@ -545,46 +553,47 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{ 0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, - 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, - 0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, - 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, - 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, - 0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, - 0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, - 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, - 0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, - 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, - 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, - 0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, - 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, - 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, - 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, - 0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, - 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, - 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, + 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c, + 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x50, 0x65, 0x72, 0x66, + 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, + 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, + 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, + 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x6f, 0x72, 0x77, + 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, + 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e, + 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x76, 0x61, 0x75, + 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x61, + 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, + 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vault/request_forwarding_service.proto b/vault/request_forwarding_service.proto index 1e54549e4282..5617242b9cc8 100644 --- a/vault/request_forwarding_service.proto +++ b/vault/request_forwarding_service.proto @@ -37,6 +37,7 @@ message NodeInformation { string mode = 3; string node_id = 4; uint32 replication_state = 5; + string hostname = 6; } message ClientKey { diff --git a/website/content/docs/commands/operator/members.mdx b/website/content/docs/commands/operator/members.mdx index 119ae807f6ee..2ee055293759 100644 --- a/website/content/docs/commands/operator/members.mdx +++ b/website/content/docs/commands/operator/members.mdx @@ -16,7 +16,7 @@ Get the key status: ```shell-session $ vault operator members -Host Name API Address Cluster Address ActiveNode Last echo +Host Name API Address Cluster Address ActiveNode Last Echo --------- ----------- --------------- ---------- --------- node1 http://10.0.0.2:8200 https://10.0.0.2:8201 true node2 http://10.0.0.3:8200 https://10.0.0.3:8201 false 2021-11-29 10:19:39.236409 -0500 EST From b8b0e9b3813fce29615f251a044e13165cdaa2b4 Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Tue, 30 Nov 2021 08:51:43 -0500 Subject: [PATCH 7/8] Make ha-status endpoint require authentication. --- api/sys_hastatus.go | 34 +++++++++++++++ command/operator_members.go | 31 ++------------ http/handler.go | 2 +- http/sys_ha_status.go | 68 ------------------------------ http/sys_ha_status_test.go | 63 --------------------------- vault/logical_system.go | 51 ++++++++++++++++++++++ vault/logical_system_integ_test.go | 42 +++++++++++++++++- vault/logical_system_paths.go | 33 ++++++++++----- vault/testing.go | 15 +++++++ 9 files changed, 168 insertions(+), 171 deletions(-) create mode 100644 api/sys_hastatus.go delete mode 100644 http/sys_ha_status.go delete mode 100644 http/sys_ha_status_test.go diff --git a/api/sys_hastatus.go b/api/sys_hastatus.go new file mode 100644 index 000000000000..408da0509109 --- /dev/null +++ b/api/sys_hastatus.go @@ -0,0 +1,34 @@ +package api + +import ( + "context" + "time" +) + +func (c *Sys) HAStatus() (*HAStatusResponse, error) { + r := c.c.NewRequest("GET", "/v1/sys/ha-status") + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + resp, err := c.c.RawRequestWithContext(ctx, r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result HAStatusResponse + err = resp.DecodeJSON(&result) + return &result, err +} + +type HAStatusResponse struct { + Nodes []HANode +} + +type HANode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} diff --git a/command/operator_members.go b/command/operator_members.go index 1c993eef253d..6b163d669f8a 100644 --- a/command/operator_members.go +++ b/command/operator_members.go @@ -1,10 +1,8 @@ package command import ( - "context" "fmt" "strings" - "time" "github.com/mitchellh/cli" "github.com/posener/complete" @@ -64,18 +62,7 @@ func (c *OperatorMembersCommand) Run(args []string) int { return 2 } - r := client.NewRequest("GET", "/v1/sys/ha-status") - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - resp, err := client.RawRequestWithContext(ctx, r) - if err != nil { - return 1 - } - defer resp.Body.Close() - - var result HaStatusResponse - err = resp.DecodeJSON(&result) - + resp, err := client.Sys().HAStatus() if err != nil { c.UI.Error(err.Error()) return 2 @@ -84,24 +71,12 @@ func (c *OperatorMembersCommand) Run(args []string) int { switch Format(c.UI) { case "table": out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"} - for _, node := range result.Nodes { + for _, node := range resp.Nodes { out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho)) } c.UI.Output(tableOutput(out, nil)) return 0 default: - return OutputData(c.UI, result) + return OutputData(c.UI, resp) } } - -type Node struct { - Hostname string `json:"hostname"` - APIAddress string `json:"api_address"` - ClusterAddress string `json:"cluster_address"` - ActiveNode bool `json:"active_node"` - LastEcho *time.Time `json:"last_echo"` -} - -type HaStatusResponse struct { - Nodes []Node -} diff --git a/http/handler.go b/http/handler.go index f4e1bb18135f..02a153eb4a0f 100644 --- a/http/handler.go +++ b/http/handler.go @@ -93,6 +93,7 @@ var ( "/v1/sys/capabilities", "/v1/sys/capabilities-accessor", "/v1/sys/capabilities-self", + "/v1/sys/ha-status", "/v1/sys/key-status", "/v1/sys/mounts", "/v1/sys/mounts/", @@ -142,7 +143,6 @@ func Handler(props *vault.HandlerProperties) http.Handler { mux.Handle("/v1/sys/step-down", handleRequestForwarding(core, handleSysStepDown(core))) mux.Handle("/v1/sys/unseal", handleSysUnseal(core)) mux.Handle("/v1/sys/leader", handleSysLeader(core)) - mux.Handle("/v1/sys/ha-status", handleRequestForwarding(core, handleSysHaStatus(core))) mux.Handle("/v1/sys/health", handleSysHealth(core)) mux.Handle("/v1/sys/monitor", handleLogicalNoForward(core)) mux.Handle("/v1/sys/generate-root/attempt", handleRequestForwarding(core, diff --git a/http/sys_ha_status.go b/http/sys_ha_status.go deleted file mode 100644 index 38e22183a63d..000000000000 --- a/http/sys_ha_status.go +++ /dev/null @@ -1,68 +0,0 @@ -package http - -import ( - "net/http" - "time" - - "github.com/hashicorp/errwrap" - "github.com/hashicorp/vault/vault" - "github.com/shirou/gopsutil/host" -) - -func handleSysHaStatus(core *vault.Core) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - handleSysHaStatusGet(core, w, r) - default: - respondError(w, http.StatusMethodNotAllowed, nil) - } - }) -} - -func handleSysHaStatusGet(core *vault.Core, w http.ResponseWriter, r *http.Request) { - _, address, clusterAddr, err := core.Leader() - if errwrap.Contains(err, vault.ErrHANotEnabled.Error()) { - err = nil - } - if err != nil { - respondError(w, http.StatusInternalServerError, err) - return - } - h, _ := host.Info() - nodes := []Node{ - { - Hostname: h.Hostname, - APIAddress: address, - ClusterAddress: clusterAddr, - ActiveNode: true, - }, - } - - for _, peerNode := range core.GetHAPeerNodesCached() { - lastEcho := peerNode.LastEcho - nodes = append(nodes, Node{ - Hostname: peerNode.Hostname, - APIAddress: peerNode.APIAddress, - ClusterAddress: peerNode.ClusterAddress, - LastEcho: &lastEcho, - }) - } - resp := &HaStatusResponse{ - Nodes: nodes, - } - - respondOk(w, resp) -} - -type Node struct { - Hostname string `json:"hostname"` - APIAddress string `json:"api_address"` - ClusterAddress string `json:"cluster_address"` - ActiveNode bool `json:"active_node"` - LastEcho *time.Time `json:"last_echo"` -} - -type HaStatusResponse struct { - Nodes []Node -} diff --git a/http/sys_ha_status_test.go b/http/sys_ha_status_test.go deleted file mode 100644 index 07e37a288b7c..000000000000 --- a/http/sys_ha_status_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package http - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/vault/helper/testhelpers" - "github.com/hashicorp/vault/sdk/helper/logging" - "github.com/hashicorp/vault/sdk/physical" - "github.com/hashicorp/vault/sdk/physical/inmem" - "github.com/hashicorp/vault/vault" -) - -func TestSysHAStatus(t *testing.T) { - logger := logging.NewVaultLogger(hclog.Trace) - inm, err := inmem.NewTransactionalInmem(nil, logger) - if err != nil { - t.Fatal(err) - } - inmha, err := inmem.NewInmemHA(nil, logger) - if err != nil { - t.Fatal(err) - } - - conf := &vault.CoreConfig{ - Physical: inm, - HAPhysical: inmha.(physical.HABackend), - } - opts := &vault.TestClusterOptions{ - HandlerFunc: Handler, - } - cluster := vault.NewTestCluster(t, conf, opts) - cluster.Start() - defer cluster.Cleanup() - testhelpers.WaitForActiveNodeAndStandbys(t, cluster) - - testhelpers.RetryUntil(t, 10*time.Second, func() error { - // Use standby deliberately to make sure it forwards - client := cluster.Cores[1].Client - r := client.NewRequest("GET", "/v1/sys/ha-status") - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - resp, err := client.RawRequestWithContext(ctx, r) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - var result HaStatusResponse - err = resp.DecodeJSON(&result) - if err != nil { - t.Fatal(err) - } - - if len(result.Nodes) != len(cluster.Cores) { - return fmt.Errorf("expected %d nodes, got %d", len(cluster.Cores), len(result.Nodes)) - } - return nil - }) -} diff --git a/vault/logical_system.go b/vault/logical_system.go index 417ad72408a9..e09927081fbc 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -40,6 +40,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/version" "github.com/mitchellh/mapstructure" + "github.com/shirou/gopsutil/host" ) const ( @@ -4119,6 +4120,49 @@ func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error { return nil } +func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + _, address, clusterAddr, err := b.Core.Leader() + if errwrap.Contains(err, ErrHANotEnabled.Error()) { + err = nil + } + if err != nil { + return nil, err + } + h, _ := host.Info() + nodes := []HAStatusNode{ + { + Hostname: h.Hostname, + APIAddress: address, + ClusterAddress: clusterAddr, + ActiveNode: true, + }, + } + + for _, peerNode := range b.Core.GetHAPeerNodesCached() { + lastEcho := peerNode.LastEcho + nodes = append(nodes, HAStatusNode{ + Hostname: peerNode.Hostname, + APIAddress: peerNode.APIAddress, + ClusterAddress: peerNode.ClusterAddress, + LastEcho: &lastEcho, + }) + } + + return &logical.Response{ + Data: map[string]interface{}{ + "nodes": nodes, + }, + }, nil +} + +type HAStatusNode struct { + Hostname string `json:"hostname"` + APIAddress string `json:"api_address"` + ClusterAddress string `json:"cluster_address"` + ActiveNode bool `json:"active_node"` + LastEcho *time.Time `json:"last_echo"` +} + func sanitizePath(path string) string { if !strings.HasSuffix(path, "/") { path += "/" @@ -4609,6 +4653,13 @@ Enable a new audit backend or disable an existing backend. `, }, + "ha-status": { + "Provides information about the nodes in an HA cluster.", + ` + Provides the list of hosts known to the active node and when they were last heard from. + `, + }, + "key-status": { "Provides information about the backend encryption key.", ` diff --git a/vault/logical_system_integ_test.go b/vault/logical_system_integ_test.go index 4e1b1aecf0cd..b76b8755bdd2 100644 --- a/vault/logical_system_integ_test.go +++ b/vault/logical_system_integ_test.go @@ -9,13 +9,17 @@ import ( "time" "github.com/go-test/deep" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/builtin/plugin" "github.com/hashicorp/vault/helper/namespace" vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/hashicorp/vault/sdk/helper/logging" "github.com/hashicorp/vault/sdk/helper/pluginutil" "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/sdk/physical" + "github.com/hashicorp/vault/sdk/physical/inmem" lplugin "github.com/hashicorp/vault/sdk/plugin" "github.com/hashicorp/vault/sdk/plugin/mock" "github.com/hashicorp/vault/vault" @@ -301,7 +305,6 @@ func testPlugin_continueOnError(t *testing.T, btype logical.BackendType, mismatc if err != nil { t.Fatalf("err:%v", err) } - } // Trigger a sha256 mismatch or missing plugin error @@ -856,3 +859,40 @@ func TestSystemBackend_InternalUIResultantACL(t *testing.T) { t.Fatal(diff) } } + +func TestSystemBackend_HAStatus(t *testing.T) { + logger := logging.NewVaultLogger(hclog.Trace) + inm, err := inmem.NewTransactionalInmem(nil, logger) + if err != nil { + t.Fatal(err) + } + inmha, err := inmem.NewInmemHA(nil, logger) + if err != nil { + t.Fatal(err) + } + + conf := &vault.CoreConfig{ + Physical: inm, + HAPhysical: inmha.(physical.HABackend), + } + opts := &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + } + cluster := vault.NewTestCluster(t, conf, opts) + cluster.Start() + defer cluster.Cleanup() + + vault.RetryUntil(t, 15*time.Second, func() error { + // Use standby deliberately to make sure it forwards + client := cluster.Cores[1].Client + resp, err := client.Sys().HAStatus() + if err != nil { + t.Fatal(err) + } + + if len(resp.Nodes) != len(cluster.Cores) { + return fmt.Errorf("expected %d nodes, got %d", len(cluster.Cores), len(resp.Nodes)) + } + return nil + }) +} diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index 06fae5226a79..78ef45a8fbd7 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -476,6 +476,19 @@ func (b *SystemBackend) statusPaths() []*framework.Path { HelpSynopsis: strings.TrimSpace(sysHelp["seal-status"][0]), HelpDescription: strings.TrimSpace(sysHelp["seal-status"][1]), }, + { + Pattern: "ha-status$", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handleHAStatus, + Summary: "Check the HA status of a Vault cluster", + }, + }, + + HelpSynopsis: strings.TrimSpace(sysHelp["ha-status"][0]), + HelpDescription: strings.TrimSpace(sysHelp["ha-status"][1]), + }, } } @@ -927,8 +940,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/ui/namespaces", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: pathInternalUINamespacesRead(b), - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: pathInternalUINamespacesRead(b), + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-ui-namespaces"][0]), @@ -938,8 +951,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/ui/resultant-acl", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalUIResultantACL, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalUIResultantACL, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-ui-resultant-acl"][0]), @@ -949,8 +962,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/requests", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersRequests, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersRequests, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-requests"][0]), @@ -960,8 +973,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/tokens", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersTokens, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersTokens, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-tokens"][0]), @@ -971,8 +984,8 @@ func (b *SystemBackend) internalPaths() []*framework.Path { Pattern: "internal/counters/entities", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ - Callback: b.pathInternalCountersEntities, - Summary: "Backwards compatibility is not guaranteed for this API", + Callback: b.pathInternalCountersEntities, + Summary: "Backwards compatibility is not guaranteed for this API", }, }, HelpSynopsis: strings.TrimSpace(sysHelp["internal-counters-entities"][0]), diff --git a/vault/testing.go b/vault/testing.go index c50eda911787..966fddf9e6ca 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -2271,3 +2271,18 @@ func (n *NoopAudit) Invalidate(ctx context.Context) { defer n.saltMutex.Unlock() n.salt = nil } + +// RetryUntil runs f until it returns a nil result or the timeout is reached. +// If a nil result hasn't been obtained by timeout, calls t.Fatal. +func RetryUntil(t testing.T, timeout time.Duration, f func() error) { + t.Helper() + deadline := time.Now().Add(timeout) + var err error + for time.Now().Before(deadline) { + if err = f(); err == nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("did not complete before deadline, err: %v", err) +} From 696cc15bad725f0d4986f389034b0f94df84c564 Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Tue, 30 Nov 2021 09:22:10 -0500 Subject: [PATCH 8/8] We don't need to call Core.Leader, and we shouldn't because we're now running with a read stateLock held. --- vault/logical_system.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/vault/logical_system.go b/vault/logical_system.go index e09927081fbc..a801a6b5e766 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -4121,19 +4121,13 @@ func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error { } func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { - _, address, clusterAddr, err := b.Core.Leader() - if errwrap.Contains(err, ErrHANotEnabled.Error()) { - err = nil - } - if err != nil { - return nil, err - } + // We're always the leader if we're handling this request. h, _ := host.Info() nodes := []HAStatusNode{ { Hostname: h.Hostname, - APIAddress: address, - ClusterAddress: clusterAddr, + APIAddress: b.Core.redirectAddr, + ClusterAddress: b.Core.ClusterAddr(), ActiveNode: true, }, }