Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TemplateServer: render secrets with Consul Template #7621

Merged
merged 28 commits into from
Oct 14, 2019
Merged

TemplateServer: render secrets with Consul Template #7621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
6b80d5d
add template config parsing, but it's wrong b/c it's not using mapstr…
catsby Sep 24, 2019
0fe3f61
parsing consul templates in agent config
catsby Sep 24, 2019
5d8dcf9
add additional test to configuration parsing, to cover basics
catsby Sep 25, 2019
a4bd223
another test fixture, rework simple test into table
catsby Sep 30, 2019
d024060
refactor into table test
catsby Sep 30, 2019
8933841
rename test
catsby Sep 30, 2019
1213f30
remove flattenKeys and add other test fixture
catsby Oct 1, 2019
651f37a
add template package
catsby Oct 2, 2019
cb57c4b
merge config branch
catsby Oct 3, 2019
2680f68
WIP: add runner
catsby Oct 7, 2019
8e15a5d
fix panic, actually copy templates, etc
catsby Oct 7, 2019
fdbd1dd
rework how the config.Vault is created and enable reading from the en…
catsby Oct 8, 2019
ff66aab
this was supposed to be a part of the prior commit
catsby Oct 8, 2019
c2d06a3
Merge branch 'f-vault-agent-template' into f-vat-template-server
catsby Oct 9, 2019
268d869
move/add methods to testhelpers for converting some values to pointers
catsby Oct 9, 2019
6d165c4
use new methods in testhelpers
catsby Oct 9, 2019
7245fc7
add an unblock channel to block agent until a template has been rendered
catsby Oct 9, 2019
054f394
add note
catsby Oct 9, 2019
370b874
unblock if there are no templates
catsby Oct 10, 2019
459354e
cleanups
catsby Oct 10, 2019
28a6a51
go mod tidy
catsby Oct 10, 2019
a1e3c83
remove dead code
catsby Oct 10, 2019
541a77a
simple test to starT
catsby Oct 10, 2019
fa85039
add simple, empty templates test
catsby Oct 10, 2019
14daed7
Update package doc, error logs, and add missing close() on channel
catsby Oct 11, 2019
c5720ee
update code comment to be clear what I'm referring to
catsby Oct 11, 2019
429285a
have template.NewServer return a (<- chan) type, even though it's a n…
catsby Oct 11, 2019
19d1f74
Update command/agent.go
catsby Oct 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 132 additions & 61 deletions command/agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/hashicorp/vault/command/agent/sink"
"github.com/hashicorp/vault/command/agent/sink/file"
"github.com/hashicorp/vault/command/agent/sink/inmem"
"github.com/hashicorp/vault/command/agent/template"
gatedwriter "github.com/hashicorp/vault/helper/gated-writer"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/helper/logging"
Expand Down Expand Up @@ -192,65 +193,69 @@ func (c *AgentCommand) Run(args []string) int {
}

// Load the configuration
config, err := config.LoadConfig(c.flagConfigs[0])
agentConfig, err := config.LoadConfig(c.flagConfigs[0])
if err != nil {
c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", c.flagConfigs[0], err))
return 1
}

// Ensure at least one config was found.
if config == nil {
if agentConfig == nil {
c.UI.Output(wrapAtLength(
"No configuration read. Please provide the configuration with the " +
"-config flag."))
return 1
}
if config.AutoAuth == nil && config.Cache == nil {
if agentConfig.AutoAuth == nil && agentConfig.Cache == nil {
c.UI.Error("No auto_auth or cache block found in config file")
return 1
}
if config.AutoAuth == nil {
if agentConfig.AutoAuth == nil {
c.UI.Info("No auto_auth block found in config file, not starting automatic authentication feature")
}

if config.Vault != nil {
c.setStringFlag(f, config.Vault.Address, &StringVar{
Name: flagNameAddress,
Target: &c.flagAddress,
Default: "https://127.0.0.1:8200",
EnvVar: api.EnvVaultAddress,
})
c.setStringFlag(f, config.Vault.CACert, &StringVar{
Name: flagNameCACert,
Target: &c.flagCACert,
Default: "",
EnvVar: api.EnvVaultCACert,
})
c.setStringFlag(f, config.Vault.CAPath, &StringVar{
Name: flagNameCAPath,
Target: &c.flagCAPath,
Default: "",
EnvVar: api.EnvVaultCAPath,
})
c.setStringFlag(f, config.Vault.ClientCert, &StringVar{
Name: flagNameClientCert,
Target: &c.flagClientCert,
Default: "",
EnvVar: api.EnvVaultClientCert,
})
c.setStringFlag(f, config.Vault.ClientKey, &StringVar{
Name: flagNameClientKey,
Target: &c.flagClientKey,
Default: "",
EnvVar: api.EnvVaultClientKey,
})
c.setBoolFlag(f, config.Vault.TLSSkipVerify, &BoolVar{
Name: flagNameTLSSkipVerify,
Target: &c.flagTLSSkipVerify,
Default: false,
EnvVar: api.EnvVaultSkipVerify,
})
// create an empty Vault configuration if none was loaded from file. The
// follow-up setStringFlag calls will populate with defaults if otherwise
// omitted
if agentConfig.Vault == nil {
agentConfig.Vault = new(config.Vault)
}
c.setStringFlag(f, agentConfig.Vault.Address, &StringVar{
Name: flagNameAddress,
Target: &c.flagAddress,
Default: "https://127.0.0.1:8200",
EnvVar: api.EnvVaultAddress,
})
c.setStringFlag(f, agentConfig.Vault.CACert, &StringVar{
Name: flagNameCACert,
Target: &c.flagCACert,
Default: "",
EnvVar: api.EnvVaultCACert,
})
c.setStringFlag(f, agentConfig.Vault.CAPath, &StringVar{
Name: flagNameCAPath,
Target: &c.flagCAPath,
Default: "",
EnvVar: api.EnvVaultCAPath,
})
c.setStringFlag(f, agentConfig.Vault.ClientCert, &StringVar{
Name: flagNameClientCert,
Target: &c.flagClientCert,
Default: "",
EnvVar: api.EnvVaultClientCert,
})
c.setStringFlag(f, agentConfig.Vault.ClientKey, &StringVar{
Name: flagNameClientKey,
Target: &c.flagClientKey,
Default: "",
EnvVar: api.EnvVaultClientKey,
})
c.setBoolFlag(f, agentConfig.Vault.TLSSkipVerify, &BoolVar{
Name: flagNameTLSSkipVerify,
Target: &c.flagTLSSkipVerify,
Default: false,
EnvVar: api.EnvVaultSkipVerify,
})

infoKeys := make([]string, 0, 10)
info := make(map[string]string)
Expand All @@ -276,7 +281,7 @@ func (c *AgentCommand) Run(args []string) int {
if os.Getenv("VAULT_TEST_VERIFY_ONLY_DUMP_CONFIG") != "" {
c.UI.Output(fmt.Sprintf(
"\nConfiguration:\n%s\n",
pretty.Sprint(*config)))
pretty.Sprint(*agentConfig)))
}
return 0
}
Expand All @@ -292,12 +297,16 @@ func (c *AgentCommand) Run(args []string) int {
return 1
}

// ctx and cancelFunc are passed to the AuthHandler, SinkServer, and
// TemplateServer that periodically listen for ctx.Done() to fire and shut
// down accordingly.
ctx, cancelFunc := context.WithCancel(context.Background())

var method auth.AuthMethod
var sinks []*sink.SinkConfig
if config.AutoAuth != nil {
for _, sc := range config.AutoAuth.Sinks {
var namespace string
if agentConfig.AutoAuth != nil {
for _, sc := range agentConfig.AutoAuth.Sinks {
switch sc.Type {
case "file":
config := &sink.SinkConfig{
Expand All @@ -323,17 +332,18 @@ func (c *AgentCommand) Run(args []string) int {
}

// Check if a default namespace has been set
mountPath := config.AutoAuth.Method.MountPath
if config.AutoAuth.Method.Namespace != "" {
mountPath = path.Join(config.AutoAuth.Method.Namespace, mountPath)
mountPath := agentConfig.AutoAuth.Method.MountPath
if agentConfig.AutoAuth.Method.Namespace != "" {
namespace = agentConfig.AutoAuth.Method.Namespace
mountPath = path.Join(namespace, mountPath)
}

authConfig := &auth.AuthConfig{
Logger: c.logger.Named(fmt.Sprintf("auth.%s", config.AutoAuth.Method.Type)),
Logger: c.logger.Named(fmt.Sprintf("auth.%s", agentConfig.AutoAuth.Method.Type)),
MountPath: mountPath,
Config: config.AutoAuth.Method.Config,
Config: agentConfig.AutoAuth.Method.Config,
}
switch config.AutoAuth.Method.Type {
switch agentConfig.AutoAuth.Method.Type {
case "alicloud":
method, err = alicloud.NewAliCloudAuthMethod(authConfig)
case "aws":
Expand All @@ -355,11 +365,11 @@ func (c *AgentCommand) Run(args []string) int {
case "pcf": // Deprecated.
method, err = cf.NewCFAuthMethod(authConfig)
default:
c.UI.Error(fmt.Sprintf("Unknown auth method %q", config.AutoAuth.Method.Type))
c.UI.Error(fmt.Sprintf("Unknown auth method %q", agentConfig.AutoAuth.Method.Type))
return 1
}
if err != nil {
c.UI.Error(errwrap.Wrapf(fmt.Sprintf("Error creating %s auth method: {{err}}", config.AutoAuth.Method.Type), err).Error())
c.UI.Error(errwrap.Wrapf(fmt.Sprintf("Error creating %s auth method: {{err}}", agentConfig.AutoAuth.Method.Type), err).Error())
return 1
}
}
Expand All @@ -376,7 +386,7 @@ func (c *AgentCommand) Run(args []string) int {
}

// Parse agent listener configurations
if config.Cache != nil && len(config.Listeners) != 0 {
if agentConfig.Cache != nil && len(agentConfig.Listeners) != 0 {
cacheLogger := c.logger.Named("cache")

// Create the API proxier
Expand All @@ -403,7 +413,7 @@ func (c *AgentCommand) Run(args []string) int {
}

var inmemSink sink.Sink
if config.Cache.UseAutoAuthToken {
if agentConfig.Cache.UseAutoAuthToken {
cacheLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink")
inmemSink, err = inmem.New(&sink.SinkConfig{
Logger: cacheLogger,
Expand All @@ -425,7 +435,7 @@ func (c *AgentCommand) Run(args []string) int {
mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink))

var listeners []net.Listener
for i, lnConfig := range config.Listeners {
for i, lnConfig := range agentConfig.Listeners {
ln, tlsConf, err := cache.StartListener(lnConfig)
if err != nil {
c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
Expand Down Expand Up @@ -468,26 +478,43 @@ func (c *AgentCommand) Run(args []string) int {
defer c.cleanupGuard.Do(listenerCloseFunc)
}

var ssDoneCh, ahDoneCh chan struct{}
// Listen for signals
// TODO: implement support for SIGHUP reloading of configuration
// signal.Notify(c.signalCh)

var ssDoneCh, ahDoneCh, tsDoneCh chan struct{}
var unblockCh <-chan struct{}
var ts *template.Server
// Start auto-auth and sink servers
if method != nil {
ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{
Logger: c.logger.Named("auth.handler"),
Client: c.client,
WrapTTL: config.AutoAuth.Method.WrapTTL,
EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials,
WrapTTL: agentConfig.AutoAuth.Method.WrapTTL,
EnableReauthOnNewCredentials: agentConfig.AutoAuth.EnableReauthOnNewCredentials,
})
ahDoneCh = ah.DoneCh

ss := sink.NewSinkServer(&sink.SinkServerConfig{
Logger: c.logger.Named("sink.server"),
Client: client,
ExitAfterAuth: config.ExitAfterAuth,
ExitAfterAuth: agentConfig.ExitAfterAuth,
})
ssDoneCh = ss.DoneCh

// create an independent vault configuration for Consul Template to use
vaultConfig := c.setupTemplateConfig()
ts, unblockCh = template.NewServer(&template.ServerConfig{
Logger: c.logger.Named("template.server"),
VaultConf: vaultConfig,
Namespace: namespace,
ExitAfterAuth: agentConfig.ExitAfterAuth,
})
tsDoneCh = ts.DoneCh

go ah.Run(ctx, method)
go ss.Run(ctx, ah.OutputCh, sinks)
go ts.Run(ctx, ah.TemplateTokenCh, agentConfig.Templates)
}

// Server configuration output
Expand All @@ -507,21 +534,30 @@ func (c *AgentCommand) Run(args []string) int {
c.logGate.Flush()

// Write out the PID to the file now that server has successfully started
if err := c.storePidFile(config.PidFile); err != nil {
if err := c.storePidFile(agentConfig.PidFile); err != nil {
c.UI.Error(fmt.Sprintf("Error storing PID: %s", err))
return 1
}

defer func() {
if err := c.removePidFile(config.PidFile); err != nil {
if err := c.removePidFile(agentConfig.PidFile); err != nil {
c.UI.Error(fmt.Sprintf("Error deleting the PID file: %s", err))
}
}()

// Wait for the template to render
select {
case <-ctx.Done():
case <-unblockCh:
tyrannosaurus-becks marked this conversation as resolved.
Show resolved Hide resolved
}

select {
case <-ssDoneCh:
// This will happen if we exit-on-auth
c.logger.Info("sinks finished, exiting")
// case <-c.SighupCh:
// TODO: implement reloading
// c.UI.Output("==> Vault Agent reload triggered")
case <-c.ShutdownCh:
c.UI.Output("==> Vault agent shutdown triggered")
cancelFunc()
Expand All @@ -531,6 +567,10 @@ func (c *AgentCommand) Run(args []string) int {
if ssDoneCh != nil {
<-ssDoneCh
}

if tsDoneCh != nil {
<-tsDoneCh
}
}

return 0
Expand Down Expand Up @@ -614,3 +654,34 @@ func (c *AgentCommand) removePidFile(pidPath string) error {
}
return os.Remove(pidPath)
}

// setupTemplateConfig creates a config.Vault struct for use by Consul Template.
// Consul Template does not currently allow us to pass in a configured API
// client, unlike the AuthHandler and SinkServer that reuse the client creted in
// this Run() method. Here we build a config.Vault struct for use by the
// Template Server that matches the configuration used to create the client
// (c.client), but in a struct of type config.Vault so that Consul Template can
// create it's own api client internally.
// TODO test setupTemplateConfig
func (c *AgentCommand) setupTemplateConfig() *config.Vault {
cfg := new(config.Vault)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think since you're just checking for the zero value, all of this can be a single literal:

return &config.Vault {
    Address:   c.flagAddress,
    CACert:    c.flagCACert,
    ...
}


if c.flagAddress != "" {
cfg.Address = c.flagAddress
}
if c.flagCACert != "" {
cfg.CACert = c.flagCACert
}
if c.flagCAPath != "" {
cfg.CAPath = c.flagCAPath
}
if c.flagClientCert != "" {
cfg.ClientCert = c.flagClientCert
}
if c.flagClientKey != "" {
cfg.ClientKey = c.flagClientKey
}
cfg.TLSSkipVerify = c.flagTLSSkipVerify

return cfg
}
5 changes: 5 additions & 0 deletions command/agent/auth/auth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type AuthConfig struct {
type AuthHandler struct {
DoneCh chan struct{}
OutputCh chan string
TemplateTokenCh chan string
logger hclog.Logger
client *api.Client
random *rand.Rand
Expand All @@ -50,6 +51,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
// This is buffered so that if we try to output after the sink server
// has been shut down, during agent shutdown, we won't block
OutputCh: make(chan string, 1),
TemplateTokenCh: make(chan string, 1),
logger: conf.Logger,
client: conf.Client,
random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))),
Expand Down Expand Up @@ -77,6 +79,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
am.Shutdown()
close(ah.OutputCh)
close(ah.DoneCh)
close(ah.TemplateTokenCh)
ah.logger.Info("auth handler stopped")
}()

Expand Down Expand Up @@ -163,6 +166,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
}
ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing")
ah.OutputCh <- string(wrappedResp)
ah.TemplateTokenCh <- string(wrappedResp)

am.CredSuccess()

Expand All @@ -189,6 +193,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) {
}
ah.logger.Info("authentication successful, sending token to sinks")
ah.OutputCh <- secret.Auth.ClientToken
ah.TemplateTokenCh <- secret.Auth.ClientToken

am.CredSuccess()
}
Expand Down
Loading