CLI Enhancements (#3897)

* Use Colored UI if stdout is a tty

* Add format options to operator unseal

* Add format test on operator unseal

* Add -no-color output flag, and use BasicUi if no-color flag is provided

* Move seal status formatting logic to OutputSealStatus

* Apply no-color to warnings from DeprecatedCommands as well

* Add OutputWithFormat to support arbitrary data, add format option to auth list

* Add ability to output arbitrary list data on TableFormatter

* Clear up switch logic on format

* Add format option for list-related commands

* Add format option to rest of commands that returns a client API response

* Remove initOutputYAML and initOutputJSON, and use OutputWithFormat instead

* Remove outputAsYAML and outputAsJSON, and use OutputWithFormat instead

* Remove -no-color flag, use env var exclusively to toggle colored output

* Fix compile

* Remove -no-color flag in main.go

* Add missing FlagSetOutputFormat

* Fix generate-root/decode test

* Migrate init functions to main.go

* Add no-color flag back as hidden

* Handle non-supported data types for TableFormatter.OutputList

* Pull formatting much further up to remove the need to use c.flagFormat (#3950)

* Pull formatting much further up to remove the need to use c.flagFormat

Also remove OutputWithFormat as the logic can cause issues.

* Use const for env var

* Minor updates

* Remove unnecessary check

* Fix SSH output and some tests

* Fix tests

* Make race detector not run on generate root since it kills Travis these days

* Update docs

* Update docs

* Address review feedback

* Handle --format as well as -format

api/sys_generate_root.go

@@ -99,11 +99,11 @@ func (c *Sys) generateRootUpdateCommon(path, shard, nonce string) (*GenerateRoot
 }
 
 type GenerateRootStatusResponse struct {
-	Nonce            string
-	Started          bool
-	Progress         int
-	Required         int
-	Complete         bool
+	Nonce            string `json:"nonce"`
+	Started          bool   `json:"started"`
+	Progress         int    `json:"progress"`
+	Required         int    `json:"required"`
+	Complete         bool   `json:"complete"`
 	EncodedToken     string `json:"encoded_token"`
 	EncodedRootToken string `json:"encoded_root_token"`
 	PGPFingerprint   string `json:"pgp_fingerprint"`

api/sys_rekey.go

@@ -177,27 +177,27 @@ type RekeyInitRequest struct {
 }
 
 type RekeyStatusResponse struct {
-	Nonce           string
-	Started         bool
-	T               int
-	N               int
-	Progress        int
-	Required        int
+	Nonce           string   `json:"nonce"`
+	Started         bool     `json:"started"`
+	T               int      `json:"t"`
+	N               int      `json:"n"`
+	Progress        int      `json:"progress"`
+	Required        int      `json:"required"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyUpdateResponse struct {
-	Nonce           string
-	Complete        bool
-	Keys            []string
+	Nonce           string   `json:"nonce"`
+	Complete        bool     `json:"complete"`
+	Keys            []string `json:"keys"`
 	KeysB64         []string `json:"keys_base64"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyRetrieveResponse struct {
-	Nonce   string
-	Keys    map[string][]string
+	Nonce   string              `json:"nonce"`
+	Keys    map[string][]string `json:"keys"`
 	KeysB64 map[string][]string `json:"keys_base64"`
 }

command/audit_list.go

@@ -44,7 +44,7 @@ Usage: vault audit list [options]
 }
 
 func (c *AuditListCommand) Flags() *FlagSets {
-	set := c.flagSet(FlagSetHTTP)
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
 
 	f := set.NewFlagSet("Command Options")
 
@@ -99,13 +99,17 @@ func (c *AuditListCommand) Run(args []string) int {
 		return 0
 	}
 
-	if c.flagDetailed {
-		c.UI.Output(tableOutput(c.detailedAudits(audits), nil))
+	switch Format(c.UI) {
+	case "table":
+		if c.flagDetailed {
+			c.UI.Output(tableOutput(c.detailedAudits(audits), nil))
+			return 0
+		}
+		c.UI.Output(tableOutput(c.simpleAudits(audits), nil))
 		return 0
+	default:
+		return OutputData(c.UI, audits)
 	}
-
-	c.UI.Output(tableOutput(c.simpleAudits(audits), nil))
-	return 0
 }
 
 func (c *AuditListCommand) simpleAudits(audits map[string]*api.Audit) []string {

command/auth_list.go

@@ -46,7 +46,7 @@ Usage: vault auth list [options]
 }
 
 func (c *AuthListCommand) Flags() *FlagSets {
-	set := c.flagSet(FlagSetHTTP)
+	set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
 
 	f := set.NewFlagSet("Command Options")
 
@@ -55,7 +55,8 @@ func (c *AuthListCommand) Flags() *FlagSets {
 		Target:  &c.flagDetailed,
 		Default: false,
 		Usage: "Print detailed information such as configuration and replication " +
-			"status about each auth method.",
+			"status about each auth method. This option is only applicable to " +
+			"table-formatted output.",
 	})
 
 	return set
@@ -95,13 +96,17 @@ func (c *AuthListCommand) Run(args []string) int {
 		return 2
 	}
 
-	if c.flagDetailed {
-		c.UI.Output(tableOutput(c.detailedMounts(auths), nil))
+	switch Format(c.UI) {
+	case "table":
+		if c.flagDetailed {
+			c.UI.Output(tableOutput(c.detailedMounts(auths), nil))
+			return 0
+		}
+		c.UI.Output(tableOutput(c.simpleMounts(auths), nil))
 		return 0
+	default:
+		return OutputData(c.UI, auths)
 	}
-
-	c.UI.Output(tableOutput(c.simpleMounts(auths), nil))
-	return 0
 }
 
 func (c *AuthListCommand) simpleMounts(auths map[string]*api.AuthMount) []string {

command/base.go

@@ -152,6 +152,11 @@ func (c *BaseCommand) flagSet(bit FlagSetBit) *FlagSets {
 	c.flagsOnce.Do(func() {
 		set := NewFlagSets(c.UI)
 
+		// These flag sets will apply to all leaf subcommands.
+		// TODO: Optional, but FlagSetHTTP can be safely removed from the individual
+		// Flags() subcommands.
+		bit = bit | FlagSetHTTP
+
 		if bit&FlagSetHTTP != 0 {
 			f := set.NewFlagSet("HTTP Options")
 
@@ -260,7 +265,7 @@ func (c *BaseCommand) flagSet(bit FlagSetBit) *FlagSets {
 					Name:       "format",
 					Target:     &c.flagFormat,
 					Default:    "table",
-					EnvVar:     "VAULT_FORMAT",
+					EnvVar:     EnvVaultFormat,
 					Completion: complete.PredictSet("table", "json", "yaml"),
 					Usage: "Print the output in the given format. Valid formats " +
 						"are \"table\", \"json\", or \"yaml\".",
@@ -317,6 +322,11 @@ func (f *FlagSets) Parse(args []string) error {
 	return f.mainSet.Parse(args)
 }
 
+// Parsed reports whether the command-line flags have been parsed.
+func (f *FlagSets) Parsed() bool {
+	return f.mainSet.Parsed()
+}
+
 // Args returns the remaining args after parsing.
 func (f *FlagSets) Args() []string {
 	return f.mainSet.Args()

command/commands.go

@@ -64,6 +64,13 @@ import (
 	physZooKeeper "github.com/hashicorp/vault/physical/zookeeper"
 )
 
+const (
+	// EnvVaultCLINoColor is an env var that toggles colored UI output.
+	EnvVaultCLINoColor = `VAULT_CLI_NO_COLOR`
+	// EnvVaultFormat is the output format
+	EnvVaultFormat = `VAULT_FORMAT`
+)
+
 var (
 	auditBackends = map[string]audit.Factory{
 		"file":   auditFile.Factory,
@@ -149,7 +156,9 @@ func (c *DeprecatedCommand) Help() string {
 
 // Run wraps the embedded Run command and prints a warning about deprecation.
 func (c *DeprecatedCommand) Run(args []string) int {
-	c.warn()
+	if Format(c.UI) == "table" {
+		c.warn()
+	}
 	return c.Command.Run(args)
 }
 
@@ -166,24 +175,7 @@ func (c *DeprecatedCommand) warn() {
 var Commands map[string]cli.CommandFactory
 var DeprecatedCommands map[string]cli.CommandFactory
 
-func init() {
-	ui := &cli.ColoredUi{
-		ErrorColor: cli.UiColorRed,
-		WarnColor:  cli.UiColorYellow,
-		Ui: &cli.BasicUi{
-			Writer:      os.Stdout,
-			ErrorWriter: os.Stderr,
-		},
-	}
-
-	serverCmdUi := &cli.ColoredUi{
-		ErrorColor: cli.UiColorRed,
-		WarnColor:  cli.UiColorYellow,
-		Ui: &cli.BasicUi{
-			Writer: os.Stdout,
-		},
-	}
-
+func initCommands(ui, serverCmdUi cli.Ui) {
 	loginHandlers := map[string]LoginHandler{
 		"aws":      &credAws.CLIHandler{},
 		"centrify": &credCentrify.CLIHandler{},
@@ -572,7 +564,7 @@ func init() {
 
 	// Deprecated commands
 	//
-	// TODO: Remove in 0.9.0
+	// TODO: Remove not before 0.11.0
 	DeprecatedCommands = map[string]cli.CommandFactory{
 		"audit-disable": func() (cli.Command, error) {
 			return &DeprecatedCommand{

command/format.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 
@@ -19,29 +20,38 @@ const (
 	hopeDelim = "♨"
 )
 
-func OutputSecret(ui cli.Ui, format string, secret *api.Secret) int {
-	return outputWithFormat(ui, format, secret, secret)
+type FormatOptions struct {
+	Format string
 }
 
-func OutputList(ui cli.Ui, format string, secret *api.Secret) int {
-	return outputWithFormat(ui, format, secret, secret.Data["keys"])
+func OutputSecret(ui cli.Ui, secret *api.Secret) int {
+	return outputWithFormat(ui, secret, secret)
 }
 
-func outputWithFormat(ui cli.Ui, format string, secret *api.Secret, data interface{}) int {
-	// If we had a colored UI, pull out the nested ui so we don't add escape
-	// sequences for outputting json, etc.
-	colorUI, ok := ui.(*cli.ColoredUi)
-	if ok {
-		ui = colorUI.Ui
+func OutputList(ui cli.Ui, data interface{}) int {
+	switch data.(type) {
+	case *api.Secret:
+		secret := data.(*api.Secret)
+		return outputWithFormat(ui, secret, secret.Data["keys"])
+	default:
+		return outputWithFormat(ui, nil, data)
 	}
+}
+
+func OutputData(ui cli.Ui, data interface{}) int {
+	return outputWithFormat(ui, nil, data)
+}
 
-	formatter, ok := Formatters[strings.ToLower(format)]
+func outputWithFormat(ui cli.Ui, secret *api.Secret, data interface{}) int {
+	format := Format(ui)
+	formatter, ok := Formatters[format]
 	if !ok {
 		ui.Error(fmt.Sprintf("Invalid output format: %s", format))
 		return 1
 	}
+
 	if err := formatter.Output(ui, secret, data); err != nil {
-		ui.Error(fmt.Sprintf("Could not output secret: %s", err.Error()))
+		ui.Error(fmt.Sprintf("Could not parse output: %s", err.Error()))
 		return 1
 	}
 	return 0
@@ -58,6 +68,22 @@ var Formatters = map[string]Formatter{
 	"yml":   YamlFormatter{},
 }
 
+func format() string {
+	format := os.Getenv(EnvVaultFormat)
+	if format == "" {
+		format = "table"
+	}
+	return format
+}
+
+func Format(ui cli.Ui) string {
+	switch ui.(type) {
+	case *VaultUI:
+		return ui.(*VaultUI).format
+	}
+	return format()
+}
+
 // An output formatter for json output of an object
 type JsonFormatter struct{}
 
@@ -87,19 +113,32 @@ type TableFormatter struct {
 }
 
 func (t TableFormatter) Output(ui cli.Ui, secret *api.Secret, data interface{}) error {
-	// TODO: this should really use reflection like the other formatters do
-	if s, ok := data.(*api.Secret); ok {
-		return t.OutputSecret(ui, s)
-	}
-	if s, ok := data.([]interface{}); ok {
-		return t.OutputList(ui, secret, s)
+	switch data.(type) {
+	case *api.Secret:
+		return t.OutputSecret(ui, secret)
+	case []interface{}:
+		return t.OutputList(ui, secret, data)
+	case []string:
+		return t.OutputList(ui, nil, data)
+	default:
+		return errors.New("Cannot use the table formatter for this type")
 	}
-	return errors.New("Cannot use the table formatter for this type")
 }
 
-func (t TableFormatter) OutputList(ui cli.Ui, secret *api.Secret, list []interface{}) error {
+func (t TableFormatter) OutputList(ui cli.Ui, secret *api.Secret, data interface{}) error {
 	t.printWarnings(ui, secret)
 
+	switch data.(type) {
+	case []interface{}:
+	case []string:
+		ui.Output(tableOutput(data.([]string), nil))
+		return nil
+	default:
+		return errors.New("Error: table formatter cannot output list for this data type")
+	}
+
+	list := data.([]interface{})
+
 	if len(list) > 0 {
 		keys := make([]string, len(list))
 		for i, v := range list {
@@ -208,7 +247,14 @@ func (t TableFormatter) OutputSecret(ui cli.Ui, secret *api.Secret) error {
 	return nil
 }
 
+// OutputSealStatus will print *api.SealStatusResponse in the CLI according to the format provided
 func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusResponse) int {
+	switch Format(ui) {
+	case "table":
+	default:
+		return OutputData(ui, status)
+	}
+
 	var sealPrefix string
 	if status.RecoverySeal {
 		sealPrefix = "Recovery "