loqrecovery: use captured meta range content for LOQ plans

Previously loss of quorum recovery planner was using local replica
info collected from all nodes to find source of truth for replicas
that lost quorum.
With online approach local info snapshots don't have atomicity.
This could cause planner to fail if available replicas are caught
in different states on different nodes.
This commit adds alternative planning approach when range metadata
is available. Instead of fixing individual replicas that can't make
progress it finds ranges that can't make progress from metadata
using descriptors and updates their replicas to recover from loss
of quorum.
This commit also adds replica collection stage as a part of make-plan
command itself. To invoke collection from a cluster instead of using
files one needs to provide --host and other standard cluster
connection related flags (--cert-dir, --insecure etc.) as appropriate.

Release note: None

pkg/cli/debug.go

@@ -1432,7 +1432,9 @@ func init() {
 	f.StringVarP(&debugRecoverPlanOpts.outputFileName, "plan", "o", "",
 		"filename to write plan to")
 	f.IntSliceVar(&debugRecoverPlanOpts.deadStoreIDs, "dead-store-ids", nil,
-		"list of dead store IDs")
+		"list of dead store IDs (can't be used together with dead-node-ids)")
+	f.IntSliceVar(&debugRecoverPlanOpts.deadNodeIDs, "dead-node-ids", nil,
+		"list of dead node IDs (can't be used together with dead-store-ids)")
 	f.VarP(&debugRecoverPlanOpts.confirmAction, cliflags.ConfirmActions.Name, cliflags.ConfirmActions.Shorthand,
 		cliflags.ConfirmActions.Usage())
 	f.BoolVar(&debugRecoverPlanOpts.force, "force", false,

pkg/cli/debug_recover_loss_of_quorum.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -32,6 +33,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/errors/hintdetail"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 // confirmActionFlag defines a pflag to parse a confirm option.
@@ -291,61 +293,117 @@ var debugRecoverPlanCmd = &cobra.Command{
 	Long: `
 Devise a plan to restore ranges that lost a quorum.
 
-This command will read files with information about replicas collected from all
-surviving nodes of a cluster and make a decision which replicas should be survivors
-for the ranges where quorum was lost.
-Decision is then written into a file or stdout.
+The command analyzes information about replicas from all surviving nodes of a
+cluster, finds ranges that lost quorum and makes decisions about which replicas
+should act as survivors to restore quorum.
+
+Information about replicas could be collected directly by connecting to the
+cluster or from files generated by the collect-info command. In former case,
+cluster connection parameters must be specified. If latter case, file names
+should be provided as arguments. 
+
+After the data is analyzed, a recovery plan is written into a file or stdout.
 
 This command only creates a plan and doesn't change any data.'
 
 See debug recover command help for more details on how to use this command.
 `,
-	Args: cobra.MinimumNArgs(1),
+	Args: cobra.MinimumNArgs(0),
 	RunE: runDebugPlanReplicaRemoval,
 }
 
 var debugRecoverPlanOpts struct {
 	outputFileName string
 	deadStoreIDs   []int
+	deadNodeIDs    []int
 	confirmAction  confirmActionFlag
 	force          bool
 }
 
+var planSpecificFlags = map[string]struct{}{
+	"plan":           {},
+	"dead-store-ids": {},
+	"dead-node-ids":  {},
+	"force":          {},
+	"confirm":        {},
+}
+
 func runDebugPlanReplicaRemoval(cmd *cobra.Command, args []string) error {
-	replicas, err := readReplicaInfoData(args)
-	if err != nil {
-		return err
+	ctx, cancel := context.WithCancel(cmd.Context())
+	defer cancel()
+
+	var replicas loqrecoverypb.ClusterReplicaInfo
+	var err error
+
+	if debugRecoverPlanOpts.deadStoreIDs != nil && debugRecoverPlanOpts.deadNodeIDs != nil {
+		return errors.New("debug recover make-plan command accepts either --dead-node-ids or --dead-store-ids")
+	}
+
+	var stats loqrecovery.CollectionStats
+	if len(args) == 0 {
+		// If no replica info is provided, try to connect to a cluster default or
+		// explicitly provided to retrieve replica info.
+		c, finish, err := getAdminClient(ctx, serverCfg)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get admin connection to cluster")
+		}
+		defer finish()
+		replicas, stats, err = loqrecovery.CollectRemoteReplicaInfo(ctx, c)
+		if err != nil {
+			return errors.Wrapf(err, "failed to retrieve replica info from cluster")
+		}
+	} else {
+		replicas, err = readReplicaInfoData(args)
+		if err != nil {
+			return err
+		}
 	}
 
 	var deadStoreIDs []roachpb.StoreID
 	for _, id := range debugRecoverPlanOpts.deadStoreIDs {
 		deadStoreIDs = append(deadStoreIDs, roachpb.StoreID(id))
 	}
 
-	plan, report, err := loqrecovery.PlanReplicas(cmd.Context(), replicas.LocalInfo, deadStoreIDs)
+	var deadNodeIDs []roachpb.NodeID
+	for _, id := range debugRecoverPlanOpts.deadNodeIDs {
+		deadNodeIDs = append(deadNodeIDs, roachpb.NodeID(id))
+	}
+
+	plan, report, err := loqrecovery.PlanReplicas(
+		ctx,
+		replicas,
+		deadStoreIDs,
+		deadNodeIDs)
 	if err != nil {
 		return err
 	}
 
+	if stats.Nodes > 0 {
+		_, _ = fmt.Fprintf(stderr, `Nodes scanned:           %d
+`, stats.Nodes)
+	}
 	_, _ = fmt.Fprintf(stderr, `Total replicas analyzed: %d
 Ranges without quorum:   %d
 Discarded live replicas: %d
 
 `, report.TotalReplicas, len(report.PlannedUpdates), report.DiscardedNonSurvivors)
+	_, _ = fmt.Fprintf(stderr, "Proposed changes:\n")
 	for _, r := range report.PlannedUpdates {
-		_, _ = fmt.Fprintf(stderr, "Recovering range r%d:%s updating replica %s to %s. "+
+		_, _ = fmt.Fprintf(stderr, "  range r%d:%s updating replica %s to %s. "+
 			"Discarding available replicas: [%s], discarding dead replicas: [%s].\n",
-			r.RangeID, r.StartKey, r.OldReplica, r.Replica,
+			r.RangeID, r.StartKey, r.OldReplica, r.NewReplica,
 			r.DiscardedAvailableReplicas, r.DiscardedDeadReplicas)
 	}
 
-	deadStoreMsg := fmt.Sprintf("\nDiscovered dead stores from provided files: %s",
-		joinStoreIDs(report.MissingStores))
+	argStoresMsg := ""
 	if len(deadStoreIDs) > 0 {
-		_, _ = fmt.Fprintf(stderr, "%s, (matches --dead-store-ids)\n\n", deadStoreMsg)
-	} else {
-		_, _ = fmt.Fprintf(stderr, "%s\n\n", deadStoreMsg)
+		argStoresMsg = ", (matches --dead-store-ids)"
+	}
+	if len(deadNodeIDs) > 0 {
+		argStoresMsg = ", (matches --dead-node-ids)"
 	}
+	_, _ = fmt.Fprintf(stderr, "\nDiscovered dead nodes, will be marked as decommissioned:\n%s\n%s\n\n",
+		formatNodeStores(report.MissingNodes, "  "), argStoresMsg)
 
 	planningErr := report.Error()
 	if planningErr != nil {
@@ -406,6 +464,7 @@ Discarded live replicas: %d
 		return nil
 	}
 
+	planFile := "<plan file>"
 	var writer io.Writer = os.Stdout
 	if len(debugRecoverPlanOpts.outputFileName) > 0 {
 		if _, err = os.Stat(debugRecoverPlanOpts.outputFileName); err == nil {
@@ -417,6 +476,7 @@ Discarded live replicas: %d
 		}
 		defer outFile.Close()
 		writer = outFile
+		planFile = path.Base(debugRecoverPlanOpts.outputFileName)
 	}
 
 	jsonpb := protoutil.JSONPb{Indent: "  "}
@@ -428,10 +488,22 @@ Discarded live replicas: %d
 		return errors.Wrap(err, "failed to write recovery plan")
 	}
 
-	_, _ = fmt.Fprint(stderr, "Plan created\nTo complete recovery, distribute the plan to the"+
-		" below nodes and invoke `debug recover apply-plan` on:\n")
-	for node, stores := range report.UpdatedNodes {
-		_, _ = fmt.Fprintf(stderr, "- node n%d, store(s) %s\n", node, joinStoreIDs(stores))
+	// No args means we collected connection info from cluster and need to
+	// preserve flags for subsequent invocation.
+	remoteArgs := getCLIClusterFlags(len(args) == 0, cmd, func(flag string) bool {
+		_, filter := planSpecificFlags[flag]
+		return filter
+	})
+
+	_, _ = fmt.Fprintf(stderr, `Plan created.
+To stage recovery application in half-online mode invoke:
+
+'cockroach debug recover apply-plan %s %s'
+
+Alternatively distribute plan to below nodes and invoke 'debug recover apply-plan --store=<store-dir> %s' on:
+`, remoteArgs, planFile, planFile)
+	for _, node := range report.UpdatedNodes {
+		_, _ = fmt.Fprintf(stderr, "- node n%d, store(s) %s\n", node.NodeID, joinStoreIDs(node.StoreIDs))
 	}
 
 	return nil
@@ -593,13 +665,50 @@ func joinStoreIDs(storeIDs []roachpb.StoreID) string {
 	return strings.Join(storeNames, ", ")
 }
 
+func formatNodeStores(locations []loqrecovery.NodeStores, indent string) string {
+	hasMultiStore := false
+	for _, v := range locations {
+		hasMultiStore = hasMultiStore || len(v.StoreIDs) > 1
+	}
+	if !hasMultiStore {
+		// we only have a single store per node, no need to list stores.
+		nodeNames := make([]string, 0, len(locations))
+		for _, node := range locations {
+			nodeNames = append(nodeNames, fmt.Sprintf("n%d", node.NodeID))
+		}
+		return indent + strings.Join(nodeNames, ", ")
+	}
+	nodeDetails := make([]string, 0, len(locations))
+	for _, node := range locations {
+		nodeDetails = append(nodeDetails,
+			indent+fmt.Sprintf("n%d: store(s): %s", node.NodeID, joinStoreIDs(node.StoreIDs)))
+	}
+	return strings.Join(nodeDetails, "\n")
+}
+
+// getCLIClusterFlags recreates command line flags from current command
+// discarding any flags that filter returns true for.
+func getCLIClusterFlags(fromCfg bool, cmd *cobra.Command, filter func(flag string) bool) string {
+	if !fromCfg {
+		return " --host <node-hostname>[:<port>] [--certs-dir <certificates-dir>|--insecure]"
+	}
+	var buf strings.Builder
+	cmd.Flags().VisitAll(func(f *pflag.Flag) {
+		if f.Changed && !filter(f.Name) {
+			_, _ = fmt.Fprintf(&buf, " --%s=%v", f.Name, f.Value.String())
+		}
+	})
+	return buf.String()
+}
+
 // setDebugRecoverContextDefaults resets values of command line flags to
 // their default values to ensure tests don't interfere with each other.
 func setDebugRecoverContextDefaults() {
 	debugRecoverCollectInfoOpts.Stores.Specs = nil
 	debugRecoverPlanOpts.outputFileName = ""
 	debugRecoverPlanOpts.confirmAction = prompt
 	debugRecoverPlanOpts.deadStoreIDs = nil
+	debugRecoverPlanOpts.deadStoreIDs = nil
 	debugRecoverExecuteOpts.Stores.Specs = nil
 	debugRecoverExecuteOpts.confirmAction = prompt
 }

pkg/kv/kvserver/loqrecovery/apply.go

@@ -97,7 +97,7 @@ func PrepareUpdateReplicas(
 
 	// Map contains a set of store names that were found in plan for this node,
 	// but were not configured in this command invocation.
-	missing := make(map[roachpb.StoreID]struct{})
+	missing := make(storeIDSet)
 	for _, update := range plan.Updates {
 		if nodeID != update.NodeID() {
 			continue
@@ -132,7 +132,7 @@ func PrepareUpdateReplicas(
 	}
 
 	if len(missing) > 0 {
-		report.MissingStores = storeSliceFromSet(missing)
+		report.MissingStores = missing.storeSliceFromSet()
 	}
 	return report, nil
 }

pkg/kv/kvserver/loqrecovery/loqrecoverypb/recovery.proto

@@ -103,11 +103,14 @@ message ReplicaUpdatePlan {
   // PlanID contains ID generated by cli when generating recovery plan and is subsequently
   // used for status checks and auditing purposes.
   bytes plan_id = 2 [(gogoproto.customname) = "PlanID",
+    (gogoproto.nullable) = false,
     (gogoproto.customtype) = "github.com/cockroachdb/cockroach/pkg/util/uuid.UUID"];
   // DecommissionedNodeIDs is a set of node IDs that need to be marked as decommissioned as a
   // part of loss of quorum recovery process.
   repeated int32 decommissioned_node_ids = 3 [(gogoproto.customname) = "DecommissionedNodeIDs",
     (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/roachpb.NodeID"];
+  // ClusterID contains id of the cluster from which info was collected.
+  string cluster_id = 4 [(gogoproto.customname) = "ClusterID"];
 }
 
 // ReplicaRecoveryRecord is a struct that loss of quorum recovery commands