Merge branch 'master-oss' into raft-seal-migration

CHANGELOG.md

@@ -1,19 +1,27 @@
 ## 1.4 (Unreleased)
 
+FEATURES:
+
+ * **Kerberos Authentication**: We now support Kerberos authentication using
+   a SPNEGO token. Login can be performed using the Vault CLI, API, or agent.
+
 IMPROVEMENTS:
 
+* agent: add option to force the use of the auth-auth token, and ignore the Vault token in the request [GH-8101]
 * auth/jwt: Additional OIDC callback parameters available for CLI logins [JWT-80 & JWT-86]
 * auth/jwt: Bound claims may be optionally configured using globs [JWT-89]
 * core: Separate out service discovery interface from storage interface to allow
   new types of service discovery not coupled to storage [GH-7887]
 * cli: Incorrect TLS configuration will now correctly fail [GH-8025] 
 * secrets/database/mongodb: Switched internal MongoDB driver to mongo-driver [GH-8140]
+* secrets/consul: Add support to specify TLS options per Consul backend [GH-4800]
 * secrets/gcp: Allow specifying the TTL for a service key [GCP-54]
 * secrets/gcp: Add support for rotating root keys [GCP-53]
 * secrets/nomad: Add support to specify TLS options per Nomad backend [GH-8083]
 * storage/raft: Nodes in the raft cluster can all be given possible leader
   addresses for them to continuously try and join one of them, thus automating
   the process of join to a greater extent [GH-7856]
+* storage/etcd: Bumped etcd client API SDK [GH-7931 & GH-4961 & GH-4349 & GH-7582]
 
 BUG FIXES:
 
@@ -24,6 +32,7 @@ BUG FIXES:
 
 BUG FIXES:
 
+* cli: Fix issue where Raft snapshots from standby nodes created an empty backup file [GH-8097]
 * ui: Fix missing License menu item [GH-8230]
 * ui: Fix bug where default auth method on login is defaulted to auth method that is listing-visibility=unauth instead of “other” [GH-8218]
 * ui: Fix bug where KMIP details were not shown in the UI Wizard [GH-8255]

api/go.sum

@@ -72,6 +72,7 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj
 github.com/hashicorp/go-hclog v0.10.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v0.10.1 h1:uyt/l0dWjJ879yiAu+T7FG3/6QX+zwm4bQ8P7XsYt3o=
 github.com/hashicorp/go-hclog v0.10.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-kms-wrapping v0.0.0-20191129225826-634facde9f88/go.mod h1:Pm+Umb/6Gij6ZG534L7QDyvkauaOQWGb+arj9aFjCE0=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=

builtin/logical/consul/client.go

@@ -24,6 +24,9 @@ func (b *backend) client(ctx context.Context, s logical.Storage) (*api.Client, e
 	consulConf.Address = conf.Address
 	consulConf.Scheme = conf.Scheme
 	consulConf.Token = conf.Token
+	consulConf.TLSConfig.CAPem = []byte(conf.CACert)
+	consulConf.TLSConfig.CertPEM = []byte(conf.ClientCert)
+	consulConf.TLSConfig.KeyPEM = []byte(conf.ClientKey)
 
 	client, err := api.NewClient(consulConf)
 	return client, nil, err

builtin/logical/consul/path_config.go

@@ -32,6 +32,24 @@ func pathConfigAccess(b *backend) *framework.Path {
 				Type:        framework.TypeString,
 				Description: "Token for API calls",
 			},
+
+			"ca_cert": &framework.FieldSchema{
+				Type: framework.TypeString,
+				Description: `CA certificate to use when verifying Consul server certificate,
+must be x509 PEM encoded.`,
+			},
+
+			"client_cert": &framework.FieldSchema{
+				Type: framework.TypeString,
+				Description: `Client certificate used for Consul's TLS communication,
+must be x509 PEM encoded and if this is set you need to also set client_key.`,
+			},
+
+			"client_key": &framework.FieldSchema{
+				Type: framework.TypeString,
+				Description: `Client key used for Consul's TLS communication,
+must be x509 PEM encoded and if this is set you need to also set client_cert.`,
+			},
 		},
 
 		Callbacks: map[logical.Operation]framework.OperationFunc{
@@ -80,9 +98,12 @@ func (b *backend) pathConfigAccessRead(ctx context.Context, req *logical.Request
 
 func (b *backend) pathConfigAccessWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 	entry, err := logical.StorageEntryJSON("config/access", accessConfig{
-		Address: data.Get("address").(string),
-		Scheme:  data.Get("scheme").(string),
-		Token:   data.Get("token").(string),
+		Address:    data.Get("address").(string),
+		Scheme:     data.Get("scheme").(string),
+		Token:      data.Get("token").(string),
+		CACert:     data.Get("ca_cert").(string),
+		ClientCert: data.Get("client_cert").(string),
+		ClientKey:  data.Get("client_key").(string),
 	})
 	if err != nil {
 		return nil, err
@@ -96,7 +117,10 @@ func (b *backend) pathConfigAccessWrite(ctx context.Context, req *logical.Reques
 }
 
 type accessConfig struct {
-	Address string `json:"address"`
-	Scheme  string `json:"scheme"`
-	Token   string `json:"token"`
+	Address    string `json:"address"`
+	Scheme     string `json:"scheme"`
+	Token      string `json:"token"`
+	CACert     string `json:"ca_cert"`
+	ClientCert string `json:"client_cert"`
+	ClientKey  string `json:"client_key"`
 }

command/agent.go

@@ -464,8 +464,10 @@ func (c *AgentCommand) Run(args []string) int {
 			})
 		}
 
+		var proxyVaultToken = !config.Cache.UseAutoAuthTokenEnforce
+
 		// Create the request handler
-		cacheHandler := cache.Handler(ctx, cacheLogger, leaseCache, inmemSink)
+		cacheHandler := cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken)
 
 		var listeners []net.Listener
 		for i, lnConfig := range config.Listeners {

command/agent/cache/cache_test.go

@@ -152,7 +152,7 @@ func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *v
 	mux := http.NewServeMux()
 	mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx))
 
-	mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, nil))
+	mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, nil, true))
 	server := &http.Server{
 		Handler:           mux,
 		ReadHeaderTimeout: 10 * time.Second,
@@ -243,7 +243,7 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
 
-	mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink("testid")))
+	mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true))
 	server := &http.Server{
 		Handler:           mux,
 		ReadHeaderTimeout: 10 * time.Second,
@@ -281,6 +281,63 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) {
 	}
 }
 
+func TestCache_AutoAuthClientTokenProxyStripping(t *testing.T) {
+	leaseCache := &mockTokenVerifierProxier{}
+	dummyToken := "DUMMY"
+	realToken := "testid"
+
+	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	cores := cluster.Cores
+	vault.TestWaitActive(t, cores[0].Core)
+	client := cores[0].Client
+
+	cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := namespace.RootContext(nil)
+
+	// Create a muxer and add paths relevant for the lease cache layer
+	mux := http.NewServeMux()
+	//mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
+
+	mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false))
+	server := &http.Server{
+		Handler:           mux,
+		ReadHeaderTimeout: 10 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		IdleTimeout:       5 * time.Minute,
+		ErrorLog:          cacheLogger.StandardLogger(nil),
+	}
+	go server.Serve(listener)
+
+	testClient, err := client.Clone()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := testClient.SetAddress("http://" + listener.Addr().String()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Empty the token in the client. Auto-auth token should be put to use.
+	testClient.SetToken(dummyToken)
+	_, err = testClient.Auth().Token().LookupSelf()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if leaseCache.currentToken != realToken {
+		t.Fatalf("failed to use real token from auto-auth")
+	}
+}
+
 func TestCache_ConcurrentRequests(t *testing.T) {
 	coreConfig := &vault.CoreConfig{
 		DisableMlock: true,

command/agent/cache/handler.go

@@ -20,11 +20,16 @@ import (
 	"github.com/hashicorp/vault/sdk/logical"
 )
 
-func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink) http.Handler {
+func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		logger.Info("received request", "method", r.Method, "path", r.URL.Path)
 
+		if !proxyVaultToken {
+			r.Header.Del(consts.AuthHeaderName)
+		}
+
 		token := r.Header.Get(consts.AuthHeaderName)
+
 		if token == "" && inmemSink != nil {
 			logger.Debug("using auto auth token", "method", r.Method, "path", r.URL.Path)
 			token = inmemSink.(sink.SinkReader).Token()

command/agent/cache/testing.go

@@ -64,3 +64,19 @@ func newTestSendResponse(status int, body string) *SendResponse {
 
 	return resp
 }
+
+type mockTokenVerifierProxier struct {
+	currentToken string
+}
+
+func (p *mockTokenVerifierProxier) Send(ctx context.Context, req *SendRequest) (*SendResponse, error) {
+	p.currentToken = req.Token
+	resp := newTestSendResponse(http.StatusOK,
+		`{"data": {"id": "` + p.currentToken + `"}}`)
+
+	return resp, nil
+}
+
+func (p *mockTokenVerifierProxier) GetCurrentRequestToken() (string) {
+	return p.currentToken
+}

command/agent/cache_end_to_end_test.go

@@ -298,7 +298,7 @@ func TestCache_UsingAutoAuthToken(t *testing.T) {
 	mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx))
 
 	// Passing a non-nil inmemsink tells the agent to use the auto-auth token
-	mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink))
+	mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, true))
 	server := &http.Server{
 		Handler:           mux,
 		ReadHeaderTimeout: 10 * time.Second,

command/agent/config/config.go

@@ -43,7 +43,9 @@ type Vault struct {
 
 // Cache contains any configuration needed for Cache mode
 type Cache struct {
-	UseAutoAuthToken bool `hcl:"use_auto_auth_token"`
+	UseAutoAuthTokenRaw     interface{} `hcl:"use_auto_auth_token"`
+	UseAutoAuthToken        bool `hcl:"-"`
+	UseAutoAuthTokenEnforce bool `hcl:"-"`
 }
 
 // Listener contains configuration for any Vault Agent listeners
@@ -219,6 +221,26 @@ func parseCache(result *Config, list *ast.ObjectList) error {
 		return err
 	}
 
+	if c.UseAutoAuthTokenRaw != nil {
+		c.UseAutoAuthToken, err = parseutil.ParseBool(c.UseAutoAuthTokenRaw)
+		if err != nil {
+			// Could be a value of "force" instead of "true"/"false"
+			switch c.UseAutoAuthTokenRaw.(type) {
+			case string:
+				v := c.UseAutoAuthTokenRaw.(string)
+
+				if !strings.EqualFold(v, "force") {
+					return fmt.Errorf("value of 'use_auto_auth_token' can be either true/false/force, %q is an invalid option", c.UseAutoAuthTokenRaw)
+				}
+				c.UseAutoAuthToken = true
+				c.UseAutoAuthTokenEnforce = true
+
+			default:
+				return err
+			}
+		}
+	}
+
 	result.Cache = &c
 	return nil
 }