vault agent cache

.gitignore

@@ -48,7 +48,9 @@ Vagrantfile
 # Configs
 *.hcl
 !command/agent/config/test-fixtures/config.hcl
+!command/agent/config/test-fixtures/config-cache.hcl
 !command/agent/config/test-fixtures/config-embedded-type.hcl
+!command/agent/config/test-fixtures/config-cache-embedded-type.hcl
 
 .DS_Store
 .idea

api/client.go

@@ -25,6 +25,7 @@ import (
 	"golang.org/x/time/rate"
 )
 
+const EnvVaultAgentAddress = "VAULT_AGENT_ADDR"
 const EnvVaultAddress = "VAULT_ADDR"
 const EnvVaultCACert = "VAULT_CACERT"
 const EnvVaultCAPath = "VAULT_CAPATH"
@@ -237,6 +238,10 @@ func (c *Config) ReadEnvironment() error {
 	if v := os.Getenv(EnvVaultAddress); v != "" {
 		envAddress = v
 	}
+	// Agent's address will take precedence over Vault's address
+	if v := os.Getenv(EnvVaultAgentAddress); v != "" {
+		envAddress = v
+	}
 	if v := os.Getenv(EnvVaultMaxRetries); v != "" {
 		maxRetries, err := strconv.ParseUint(v, 10, 32)
 		if err != nil {
@@ -366,6 +371,21 @@ func NewClient(c *Config) (*Client, error) {
 	c.modifyLock.Lock()
 	defer c.modifyLock.Unlock()
 
+	// If address begins with a `/`, treat it as a socket file path and set
+	// the HttpClient's transport to the corresponding socket dialer.
+	if strings.HasPrefix(c.Address, "/") {
+		socketFilePath := c.Address
+		c.HttpClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext: func(context.Context, string, string) (net.Conn, error) {
+					return net.Dial("unix", socketFilePath)
+				},
+			},
+		}
+		// Set the unix address for URL parsing below
+		c.Address = "http://unix"
+	}
+
 	u, err := url.Parse(c.Address)
 	if err != nil {
 		return nil, err
@@ -707,7 +727,7 @@ func (c *Client) RawRequestWithContext(ctx context.Context, r *Request) (*Respon
 
 	redirectCount := 0
 START:
-	req, err := r.toRetryableHTTP()
+	req, err := r.ToRetryableHTTP()
 	if err != nil {
 		return nil, err
 	}

api/request.go

@@ -62,7 +62,7 @@ func (r *Request) ResetJSONBody() error {
 // DEPRECATED: ToHTTP turns this request into a valid *http.Request for use
 // with the net/http package.
 func (r *Request) ToHTTP() (*http.Request, error) {
-	req, err := r.toRetryableHTTP()
+	req, err := r.ToRetryableHTTP()
 	if err != nil {
 		return nil, err
 	}
@@ -85,7 +85,7 @@ func (r *Request) ToHTTP() (*http.Request, error) {
 	return req.Request, nil
 }
 
-func (r *Request) toRetryableHTTP() (*retryablehttp.Request, error) {
+func (r *Request) ToRetryableHTTP() (*retryablehttp.Request, error) {
 	// Encode the query parameters
 	r.URL.RawQuery = r.Params.Encode()
 

api/secret.go

@@ -292,6 +292,7 @@ type SecretAuth struct {
 	TokenPolicies    []string          `json:"token_policies"`
 	IdentityPolicies []string          `json:"identity_policies"`
 	Metadata         map[string]string `json:"metadata"`
+	Orphan           bool              `json:"orphan"`
 
 	LeaseDuration int  `json:"lease_duration"`
 	Renewable     bool `json:"renewable"`

command/agent.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
+
 	"os"
 	"sort"
 	"strings"
@@ -23,6 +25,7 @@ import (
 	"github.com/hashicorp/vault/command/agent/auth/gcp"
 	"github.com/hashicorp/vault/command/agent/auth/jwt"
 	"github.com/hashicorp/vault/command/agent/auth/kubernetes"
+	"github.com/hashicorp/vault/command/agent/cache"
 	"github.com/hashicorp/vault/command/agent/config"
 	"github.com/hashicorp/vault/command/agent/sink"
 	"github.com/hashicorp/vault/command/agent/sink/file"
@@ -332,10 +335,40 @@ func (c *AgentCommand) Run(args []string) int {
 		EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials,
 	})
 
-	// Start things running
+	// Start auto-auth and sink servers
 	go ah.Run(ctx, method)
 	go ss.Run(ctx, ah.OutputCh, sinks)
 
+	// Parse agent listener configurations
+	var listeners []net.Listener
+	if len(config.Cache.Listeners) != 0 {
+		listeners, err = cache.ServerListeners(config.Cache.Listeners, c.logWriter, c.UI)
+		if err != nil {
+			c.UI.Error(fmt.Sprintf("Error running listeners: %v", err))
+			return 1
+		}
+	}
+
+	// Start listening to requests
+	err = cache.Run(ctx, &cache.Config{
+		Token:            c.client.Token(),
+		UseAutoAuthToken: config.Cache.UseAutoAuthToken,
+		Listeners:        listeners,
+		Logger:           c.logger.Named("cache"),
+	})
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error starting the cache listeners: %v", err))
+		return 1
+	}
+
+	// Ensure that listeners are closed at all the exits
+	listenerCloseFunc := func() {
+		for _, ln := range listeners {
+			ln.Close()
+		}
+	}
+	defer c.cleanupGuard.Do(listenerCloseFunc)
+
 	// Release the log gate.
 	c.logGate.Flush()
 

command/agent/auth/auth.go

@@ -7,6 +7,7 @@ import (
 
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/helper/contextutil"
 	"github.com/hashicorp/vault/helper/jsonutil"
 )
 
@@ -59,13 +60,6 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
 	return ah
 }
 
-func backoffOrQuit(ctx context.Context, backoff time.Duration) {
-	select {
-	case <-time.After(backoff):
-	case <-ctx.Done():
-	}
-}
-
 func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 	if am == nil {
 		panic("nil auth method")
@@ -116,7 +110,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 		path, data, err := am.Authenticate(ctx, ah.client)
 		if err != nil {
 			ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoff.Seconds())
-			backoffOrQuit(ctx, backoff)
+			contextutil.BackoffOrQuit(ctx, backoff)
 			continue
 		}
 
@@ -125,7 +119,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 			wrapClient, err := ah.client.Clone()
 			if err != nil {
 				ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			wrapClient.SetWrappingLookupFunc(func(string, string) string {
@@ -138,26 +132,26 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 		// Check errors/sanity
 		if err != nil {
 			ah.logger.Error("error authenticating", "error", err, "backoff", backoff.Seconds())
-			backoffOrQuit(ctx, backoff)
+			contextutil.BackoffOrQuit(ctx, backoff)
 			continue
 		}
 
 		switch {
 		case ah.wrapTTL > 0:
 			if secret.WrapInfo == nil {
 				ah.logger.Error("authentication returned nil wrap info", "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			if secret.WrapInfo.Token == "" {
 				ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			wrappedResp, err := jsonutil.EncodeJSON(secret.WrapInfo)
 			if err != nil {
 				ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing")
@@ -178,12 +172,12 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 		default:
 			if secret == nil || secret.Auth == nil {
 				ah.logger.Error("authentication returned nil auth info", "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			if secret.Auth.ClientToken == "" {
 				ah.logger.Error("authentication returned empty client token", "backoff", backoff.Seconds())
-				backoffOrQuit(ctx, backoff)
+				contextutil.BackoffOrQuit(ctx, backoff)
 				continue
 			}
 			ah.logger.Info("authentication successful, sending token to sinks")
@@ -201,7 +195,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
 		})
 		if err != nil {
 			ah.logger.Error("error creating renewer, backing off and retrying", "error", err, "backoff", backoff.Seconds())
-			backoffOrQuit(ctx, backoff)
+			contextutil.BackoffOrQuit(ctx, backoff)
 			continue
 		}
 

command/agent/cache/api_proxy.go

@@ -0,0 +1,58 @@
+package cache
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/api"
+)
+
+// APIProxy is an implementation of the proxier interface that is used to
+// forward the request to Vault and get the response.
+type APIProxy struct {
+	logger hclog.Logger
+}
+
+type APIProxyConfig struct {
+	Logger hclog.Logger
+}
+
+func NewAPIProxy(config *APIProxyConfig) Proxier {
+	return &APIProxy{
+		logger: config.Logger,
+	}
+}
+
+func (ap *APIProxy) Send(ctx context.Context, req *SendRequest) (*SendResponse, error) {
+	client, err := api.NewClient(api.DefaultConfig())
+	if err != nil {
+		return nil, err
+	}
+	client.SetToken(req.Token)
+	client.SetHeaders(req.Request.Header)
+
+	fwReq := client.NewRequest(req.Request.Method, req.Request.URL.Path)
+	fwReq.BodyBytes = req.RequestBody
+
+	// Make the request to Vault and get the response
+	ap.logger.Info("forwarding request", "path", req.Request.URL.Path)
+	resp, err := client.RawRequestWithContext(ctx, fwReq)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse and reset response body
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		ap.logger.Error("failed to read request body", "error", err)
+		return nil, err
+	}
+	resp.Body = ioutil.NopCloser(bytes.NewBuffer(respBody))
+
+	return &SendResponse{
+		Response:     resp,
+		ResponseBody: respBody,
+	}, nil
+}

command/agent/cache/api_proxy_test.go

@@ -0,0 +1,43 @@
+package cache
+
+import (
+	"testing"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/helper/jsonutil"
+	"github.com/hashicorp/vault/helper/logging"
+	"github.com/hashicorp/vault/helper/namespace"
+)
+
+func TestCache_APIProxy(t *testing.T) {
+	cleanup, client, _ := setupClusterAndAgent(t, nil)
+	defer cleanup()
+
+	proxier := NewAPIProxy(&APIProxyConfig{
+		Logger: logging.NewVaultLogger(hclog.Trace),
+	})
+
+	r := client.NewRequest("GET", "/v1/sys/health")
+	req, err := r.ToRetryableHTTP()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err := proxier.Send(namespace.RootContext(nil), &SendRequest{
+		Request: req.Request,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var result api.HealthResponse
+	err = jsonutil.DecodeJSONFromReader(resp.Response.Body, &result)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !result.Initialized || result.Sealed || result.Standby {
+		t.Fatalf("bad sys/health response")
+	}
+}