From 6b80d5dc0ed9dd1812bd33631ab7402df0ff41ca Mon Sep 17 00:00:00 2001 From: catsby Date: Tue, 24 Sep 2019 15:38:17 -0500 Subject: [PATCH 01/26] add template config parsing, but it's wrong b/c it's not using mapstructure --- command/agent/config/config.go | 131 +++++++++++++++++- command/agent/config/config_test.go | 68 ++++++++- .../config/test-fixtures/config-template.hcl | 46 ++++++ 3 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 command/agent/config/test-fixtures/config-template.hcl diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 9f9fafa1a83a..705f7cd055f6 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -14,16 +14,18 @@ import ( "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/helper/parseutil" + "github.com/y0ssar1an/q" ) // Config is the configuration for the vault server. type Config struct { - AutoAuth *AutoAuth `hcl:"auto_auth"` - ExitAfterAuth bool `hcl:"exit_after_auth"` - PidFile string `hcl:"pid_file"` - Listeners []*Listener `hcl:"listeners"` - Cache *Cache `hcl:"cache"` - Vault *Vault `hcl:"vault"` + AutoAuth *AutoAuth `hcl:"auto_auth"` + ExitAfterAuth bool `hcl:"exit_after_auth"` + PidFile string `hcl:"pid_file"` + Listeners []*Listener `hcl:"listeners"` + Cache *Cache `hcl:"cache"` + Vault *Vault `hcl:"vault"` + Templates []*TemplateConfig `hcl:"templates"` } type Vault struct { @@ -74,6 +76,68 @@ type Sink struct { Config map[string]interface{} } +type TemplateConfig struct { + // Backup determines if this template should retain a backup. The default + // value is false. + Backup *bool `hcl:"backup"` + + // Command is the arbitrary command to execute after a template has + // successfully rendered. This is DEPRECATED. Use Exec instead. + Command *string `hcl:"command"` + + // CommandTimeout is the amount of time to wait for the command to finish + // before force-killing it. This is DEPRECATED. Use Exec instead. + CommandTimeoutRaw interface{} `hcl:"command_timeout"` + CommandTimeout *time.Duration `hcl:"-"` + + // Contents are the raw template contents to evaluate. Either this or Source + // must be specified, but not both. + Contents *string `hcl:"contents"` + + // CreateDestDirs tells Consul Template to create the parent directories of + // the destination path if they do not exist. The default value is true. + CreateDestDirs *bool `hcl:"create_dest_dirs"` + + // Destination is the location on disk where the template should be rendered. + // This is required unless running in debug/dry mode. + Destination *string `hcl:"destination"` + + // ErrMissingKey is used to control how the template behaves when attempting + // to index a struct or map key that does not exist. + ErrMissingKey *bool `hcl:"error_on_missing_key"` + + // // Exec is the configuration for the command to run when the template renders + // // successfully. + // Exec *ExecConfig `hcl:"exec"` + + // Perms are the file system permissions to use when creating the file on + // disk. This is useful for when files contain sensitive information, such as + // secrets from Vault. + PermsRaw interface{} `hcl:"perms"` + Perms *os.FileMode `hcl:"-"` + + // Source is the path on disk to the template contents to evaluate. Either + // this or Contents should be specified, but not both. + Source *string `hcl:"source"` + + // // Wait configures per-template quiescence timers. + // Wait *WaitConfig `hcl:"wait"` + + // LeftDelim and RightDelim are optional configurations to control what + // delimiter is utilized when parsing the template. + LeftDelim *string `hcl:"left_delimiter"` + RightDelim *string `hcl:"right_delimiter"` + + // FunctionBlacklist is a list of functions that this template is not + // permitted to run. + FunctionBlacklist []string `hcl:"function_blacklist"` + + // SandboxPath adds a prefix to any path provided to the `file` function + // and causes an error if a relative path tries to traverse outside that + // prefix. + SandboxPath *string `hcl:"sandbox_path"` +} + // LoadConfig loads the configuration at the given path, regardless if // its a file or directory. func LoadConfig(path string) (*Config, error) { @@ -123,6 +187,11 @@ func LoadConfig(path string) (*Config, error) { return nil, errwrap.Wrapf("error parsing 'cache':{{err}}", err) } + err = parseTemplates(&result, list) + if err != nil { + return nil, errwrap.Wrapf("error parsing 'templates':{{err}}", err) + } + if result.Cache != nil { if len(result.Listeners) < 1 { return nil, fmt.Errorf("at least one listener required when cache enabled") @@ -408,3 +477,53 @@ func parseSinks(result *Config, list *ast.ObjectList) error { result.AutoAuth.Sinks = ts return nil } + +func parseTemplates(result *Config, list *ast.ObjectList) error { + name := "template" + + templateList := list.Filter(name) + if len(templateList.Items) < 1 { + return nil + } + + var tcs []*TemplateConfig + + for _, item := range templateList.Items { + var tc TemplateConfig + if err := hcl.DecodeObject(&tc, item.Val); err != nil { + q.Q("error here:", err) + return err + } + + if tc.CommandTimeoutRaw != nil { + timeout, err := parseutil.ParseDurationSecond(tc.CommandTimeoutRaw) + if err != nil { + return err + } + tc.CommandTimeout = &timeout + tc.CommandTimeoutRaw = nil + } + + q.Q(tc.PermsRaw) + perms := os.FileMode(0644) + if tc.PermsRaw != nil { + switch tc.PermsRaw.(type) { + case int: + perms = os.FileMode(tc.PermsRaw.(int)) + default: + return errors.New("error parsing perms") + } + tc.PermsRaw = nil + tc.Perms = &perms + } + q.Q(tc.Perms.String()) + + // check source vs contents + // check command / timeout + + tcs = append(tcs, &tc) + } + + result.Templates = tcs + return nil +} diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index ff133ae576c8..a12506cd4d5f 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/go-test/deep" + "github.com/y0ssar1an/q" ) func TestLoadConfigFile_AgentCache(t *testing.T) { @@ -93,8 +94,14 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { } func TestLoadConfigFile(t *testing.T) { - os.Setenv("TEST_AAD_ENV", "aad") - defer os.Unsetenv("TEST_AAD_ENV") + if err := os.Setenv("TEST_AAD_ENV", "aad"); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Unsetenv("TEST_AAD_ENV"); err != nil { + t.Fatal(err) + } + }() config, err := LoadConfig("./test-fixtures/config.hcl") if err != nil { @@ -278,3 +285,60 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { t.Fatal(diff) } } + +// TestLoadConfigFile_Template_Single tests a single template definition in a +// configuration file, with most entries supplied +func TestLoadConfigFile_Template_Single(t *testing.T) { + q.Q("---------") + q.Q("starting") + q.Q("---------") + defer func() { + q.Q("---------") + q.Q("end") + q.Q("---------") + q.Q("") + }() + config, err := LoadConfig("./test-fixtures/config-template.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + &Sink{ + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + PidFile: "./pidfile", + } + + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } + + config, err = LoadConfig("./test-fixtures/config-embedded-type.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + q.Q(config) + + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} diff --git a/command/agent/config/test-fixtures/config-template.hcl b/command/agent/config/test-fixtures/config-template.hcl new file mode 100644 index 000000000000..1735c8042ca0 --- /dev/null +++ b/command/agent/config/test-fixtures/config-template.hcl @@ -0,0 +1,46 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + namespace = "/my-namespace" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + + config = { + path = "/tmp/file-foo" + } + + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +template { + source = "/path/on/disk/to/template.ctmpl" + destination = "/path/on/disk/where/template/will/render.txt" + create_dest_dirs = true + + #contents = "{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}" + command = "restart service foo" + command_timeout = "60s" + error_on_missing_key = false + perms = 0600 + backup = true + left_delimiter = "{{" + right_delimiter = "}}" + function_blacklist = [] + sandbox_path = "" + + wait { + min = "2s" + max = "10s" + } +} From 0fe3f61702a848fc6f14287d364f870d1812ef9e Mon Sep 17 00:00:00 2001 From: catsby Date: Tue, 24 Sep 2019 16:26:32 -0500 Subject: [PATCH 02/26] parsing consul templates in agent config --- command/agent/config/config.go | 187 +++++++++--------- command/agent/config/config_test.go | 23 ++- .../config/test-fixtures/config-template.hcl | 35 ++-- go.mod | 5 +- go.sum | 18 ++ 5 files changed, 143 insertions(+), 125 deletions(-) diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 705f7cd055f6..e9b8b0c73f80 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -8,24 +8,25 @@ import ( "strings" "time" + ctconfig "github.com/hashicorp/consul-template/config" "github.com/hashicorp/errwrap" "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/helper/parseutil" - "github.com/y0ssar1an/q" + "github.com/mitchellh/mapstructure" ) // Config is the configuration for the vault server. type Config struct { - AutoAuth *AutoAuth `hcl:"auto_auth"` - ExitAfterAuth bool `hcl:"exit_after_auth"` - PidFile string `hcl:"pid_file"` - Listeners []*Listener `hcl:"listeners"` - Cache *Cache `hcl:"cache"` - Vault *Vault `hcl:"vault"` - Templates []*TemplateConfig `hcl:"templates"` + AutoAuth *AutoAuth `hcl:"auto_auth"` + ExitAfterAuth bool `hcl:"exit_after_auth"` + PidFile string `hcl:"pid_file"` + Listeners []*Listener `hcl:"listeners"` + Cache *Cache `hcl:"cache"` + Vault *Vault `hcl:"vault"` + Templates []*ctconfig.TemplateConfig `hcl:"templates"` } type Vault struct { @@ -76,68 +77,6 @@ type Sink struct { Config map[string]interface{} } -type TemplateConfig struct { - // Backup determines if this template should retain a backup. The default - // value is false. - Backup *bool `hcl:"backup"` - - // Command is the arbitrary command to execute after a template has - // successfully rendered. This is DEPRECATED. Use Exec instead. - Command *string `hcl:"command"` - - // CommandTimeout is the amount of time to wait for the command to finish - // before force-killing it. This is DEPRECATED. Use Exec instead. - CommandTimeoutRaw interface{} `hcl:"command_timeout"` - CommandTimeout *time.Duration `hcl:"-"` - - // Contents are the raw template contents to evaluate. Either this or Source - // must be specified, but not both. - Contents *string `hcl:"contents"` - - // CreateDestDirs tells Consul Template to create the parent directories of - // the destination path if they do not exist. The default value is true. - CreateDestDirs *bool `hcl:"create_dest_dirs"` - - // Destination is the location on disk where the template should be rendered. - // This is required unless running in debug/dry mode. - Destination *string `hcl:"destination"` - - // ErrMissingKey is used to control how the template behaves when attempting - // to index a struct or map key that does not exist. - ErrMissingKey *bool `hcl:"error_on_missing_key"` - - // // Exec is the configuration for the command to run when the template renders - // // successfully. - // Exec *ExecConfig `hcl:"exec"` - - // Perms are the file system permissions to use when creating the file on - // disk. This is useful for when files contain sensitive information, such as - // secrets from Vault. - PermsRaw interface{} `hcl:"perms"` - Perms *os.FileMode `hcl:"-"` - - // Source is the path on disk to the template contents to evaluate. Either - // this or Contents should be specified, but not both. - Source *string `hcl:"source"` - - // // Wait configures per-template quiescence timers. - // Wait *WaitConfig `hcl:"wait"` - - // LeftDelim and RightDelim are optional configurations to control what - // delimiter is utilized when parsing the template. - LeftDelim *string `hcl:"left_delimiter"` - RightDelim *string `hcl:"right_delimiter"` - - // FunctionBlacklist is a list of functions that this template is not - // permitted to run. - FunctionBlacklist []string `hcl:"function_blacklist"` - - // SandboxPath adds a prefix to any path provided to the `file` function - // and causes an error if a relative path tries to traverse outside that - // prefix. - SandboxPath *string `hcl:"sandbox_path"` -} - // LoadConfig loads the configuration at the given path, regardless if // its a file or directory. func LoadConfig(path string) (*Config, error) { @@ -486,44 +425,94 @@ func parseTemplates(result *Config, list *ast.ObjectList) error { return nil } - var tcs []*TemplateConfig + var tcs []*ctconfig.TemplateConfig for _, item := range templateList.Items { - var tc TemplateConfig - if err := hcl.DecodeObject(&tc, item.Val); err != nil { - q.Q("error here:", err) - return err + var shadow interface{} + if err := hcl.DecodeObject(&shadow, item.Val); err != nil { + return fmt.Errorf("error decoding config: %s", err) } - if tc.CommandTimeoutRaw != nil { - timeout, err := parseutil.ParseDurationSecond(tc.CommandTimeoutRaw) - if err != nil { - return err - } - tc.CommandTimeout = &timeout - tc.CommandTimeoutRaw = nil + // Convert to a map and flatten the keys we want to flatten + parsed, ok := shadow.(map[string]interface{}) + if !ok { + return errors.New("error converting config") } - q.Q(tc.PermsRaw) - perms := os.FileMode(0644) - if tc.PermsRaw != nil { - switch tc.PermsRaw.(type) { - case int: - perms = os.FileMode(tc.PermsRaw.(int)) - default: - return errors.New("error parsing perms") - } - tc.PermsRaw = nil - tc.Perms = &perms - } - q.Q(tc.Perms.String()) - - // check source vs contents - // check command / timeout + // Flattenkeys belonging to the templates + flattenKeys(parsed, []string{ + "env", + "exec", + "exec.env", + "wait", + }) + var tc ctconfig.TemplateConfig + + // Use mapstructure to populate the basic config fields + var md mapstructure.Metadata + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.ComposeDecodeHookFunc( + ctconfig.StringToFileModeFunc(), + ctconfig.StringToWaitDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + mapstructure.StringToTimeDurationHookFunc(), + ), + ErrorUnused: true, + Metadata: &md, + Result: &tc, + }) + if err != nil { + return errors.New("mapstructure decoder creation failed") + } + if err := decoder.Decode(parsed); err != nil { + return errors.New("mapstructure decode failed") + } tcs = append(tcs, &tc) } - result.Templates = tcs return nil } + +// flattenKeys is a function that takes a map[string]interface{} and recursively +// flattens any keys that are a []map[string]interface{} where the key is in the +// given list of keys. +func flattenKeys(m map[string]interface{}, keys []string) { + keyMap := make(map[string]struct{}) + for _, key := range keys { + keyMap[key] = struct{}{} + } + + var flatten func(map[string]interface{}, string) + flatten = func(m map[string]interface{}, parent string) { + for k, v := range m { + // Calculate the map key, since it could include a parent. + mapKey := k + if parent != "" { + mapKey = parent + "." + k + } + + if _, ok := keyMap[mapKey]; !ok { + continue + } + + switch typed := v.(type) { + case []map[string]interface{}: + if len(typed) > 0 { + last := typed[len(typed)-1] + flatten(last, mapKey) + m[k] = last + } else { + m[k] = nil + } + case map[string]interface{}: + flatten(typed, mapKey) + m[k] = typed + default: + m[k] = v + } + } + } + + flatten(m, "") +} diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index a12506cd4d5f..8d27619baf77 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/go-test/deep" + ctconfig "github.com/hashicorp/consul-template/config" "github.com/y0ssar1an/q" ) @@ -325,20 +326,26 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { }, }, }, + Templates: []*ctconfig.TemplateConfig{ + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render.txt"), + Perms: fileMode(0600), + }, + }, PidFile: "./pidfile", } if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } +} - config, err = LoadConfig("./test-fixtures/config-embedded-type.hcl") - if err != nil { - t.Fatalf("err: %s", err) - } - q.Q(config) +func strPtr(s string) *string { + return &s +} - if diff := deep.Equal(config, expected); diff != nil { - t.Fatal(diff) - } +// FileMode returns a pointer to the given os.FileMode. +func fileMode(o os.FileMode) *os.FileMode { + return &o } diff --git a/command/agent/config/test-fixtures/config-template.hcl b/command/agent/config/test-fixtures/config-template.hcl index 1735c8042ca0..5618d742972c 100644 --- a/command/agent/config/test-fixtures/config-template.hcl +++ b/command/agent/config/test-fixtures/config-template.hcl @@ -24,23 +24,24 @@ auto_auth { } template { - source = "/path/on/disk/to/template.ctmpl" - destination = "/path/on/disk/where/template/will/render.txt" - create_dest_dirs = true + source = "/path/on/disk/to/template.ctmpl" + destination = "/path/on/disk/where/template/will/render.txt" + + #create_dest_dirs = true #contents = "{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}" - command = "restart service foo" - command_timeout = "60s" - error_on_missing_key = false - perms = 0600 - backup = true - left_delimiter = "{{" - right_delimiter = "}}" - function_blacklist = [] - sandbox_path = "" - - wait { - min = "2s" - max = "10s" - } + # command = "restart service foo" + # command_timeout = "60s" + # error_on_missing_key = false + perms = 0600 + + # backup = true + # left_delimiter = "{{" + # right_delimiter = "}}" + # function_blacklist = [] + # sandbox_path = "" + # wait { + # min = "2s" + # max = "10s" + # } } diff --git a/go.mod b/go.mod index befb49285a50..c2170b017a48 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,8 @@ require ( github.com/google/go-github v17.0.0+incompatible github.com/google/go-metrics-stackdriver v0.0.0-20190816035513-b52628e82e2a github.com/google/go-querystring v1.0.0 // indirect - github.com/hashicorp/consul/api v1.0.1 + github.com/hashicorp/consul-template v0.22.0 + github.com/hashicorp/consul/api v1.1.0 github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/go-cleanhttp v0.5.1 github.com/hashicorp/go-gcp-common v0.5.0 @@ -91,6 +92,7 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f github.com/kr/pretty v0.1.0 + github.com/kr/pty v1.1.3 // indirect github.com/kr/text v0.1.0 github.com/lib/pq v1.2.0 github.com/mattn/go-colorable v0.1.2 @@ -118,6 +120,7 @@ require ( github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 // indirect github.com/stretchr/testify v1.3.0 + github.com/y0ssar1an/q v1.0.7 go.etcd.io/bbolt v1.3.2 go.etcd.io/etcd v0.0.0-20190412021913-f29b1ada1971 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 diff --git a/go.sum b/go.sum index 62b14cd48f43..b15fc41e1cf2 100644 --- a/go.sum +++ b/go.sum @@ -145,6 +145,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg= github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -205,6 +206,7 @@ github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= @@ -252,8 +254,12 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/consul-template v0.22.0 h1:ti5cqAekOeMfFYLJCjlPtKGwBcqwVxoZO/Y2vctwuUE= +github.com/hashicorp/consul-template v0.22.0/go.mod h1:lHrykBIcPobCuEcIMLJryKxDyk2lUMnQWmffOEONH0k= github.com/hashicorp/consul/api v1.0.1 h1:LkHu3cLXjya4lgrAyZVe/CUBXgJ7AcDWKSeCjAYN9w0= github.com/hashicorp/consul/api v1.0.1/go.mod h1:LQlewHPiuaRhn1mP2XE4RrjnlRgOeWa/ZM0xWLCen2M= +github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.0 h1:tTfutTNVUTDXpNM4YCImLfiiY3yCDpfgS6tNlUioIUE= github.com/hashicorp/consul/sdk v0.1.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= @@ -263,6 +269,7 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-gatedio v0.5.0/go.mod h1:Lr3t8L6IyxD3DAeaUxGcgl2JnRUpWMCsmBl4Omu/2t4= github.com/hashicorp/go-gcp-common v0.5.0 h1:kkIQTjNTopn4eXQ1+lCiHYZXUtgIZvbc6YtAQkMnTos= github.com/hashicorp/go-gcp-common v0.5.0/go.mod h1:IDGUI2N/OS3PiU4qZcXJeWKPI6O/9Y8hOrbSiMcqyYw= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= @@ -321,6 +328,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf h1:U/40PQvWkaXCDdK9QHKf1pVDVcA+NIDVbzzonFGkgIA= github.com/hashicorp/nomad/api v0.0.0-20190412184103-1c38ced33adf/go.mod h1:BDngVi1f4UA6aJq9WYTgxhfWSE1+42xshvstLU2fRGk= github.com/hashicorp/raft v1.0.1/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= @@ -331,6 +339,8 @@ github.com/hashicorp/raft-snapshot v1.0.2-0.20190827162939-8117efcc5aab h1:WzGMw github.com/hashicorp/raft-snapshot v1.0.2-0.20190827162939-8117efcc5aab/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.8.3 h1:MWYcmct5EtKz0efYooPcL0yNkem+7kWxqXDi/UIh+8k= +github.com/hashicorp/serf v0.8.3/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= github.com/hashicorp/vault-plugin-auth-alicloud v0.5.2-0.20190814210027-93970f08f2ec h1:HXVE8h6RXFsPJgwWpE+5CscsgekqtX4nhDlZGV9jEe4= github.com/hashicorp/vault-plugin-auth-alicloud v0.5.2-0.20190814210027-93970f08f2ec/go.mod h1:TYFfVFgKF9x92T7uXouI9rLPkNnyXo/KkNcj5t+mjdM= github.com/hashicorp/vault-plugin-auth-azure v0.5.2-0.20190814210035-08e00d801115 h1:E57y918o+c+NoI5k7ohbpZu7vRm1XZKZfC5VQVpJvDI= @@ -426,6 +436,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -434,6 +445,7 @@ github.com/michaelklishin/rabbit-hole v1.5.0 h1:Bex27BiFDsijCM9D0ezSHqyy0kehpYHu github.com/michaelklishin/rabbit-hole v1.5.0/go.mod h1:vvI1uOitYZi0O5HEGXhaWC1XT80Gy+HvFheJ+5Krlhk= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= @@ -446,6 +458,7 @@ github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdI github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -499,6 +512,7 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= @@ -594,6 +608,8 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTw github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/y0ssar1an/q v1.0.7 h1:s3ckTY+wjk6Y0sFce4rIS1Ezf8S6d0UFJrKwe40MyiQ= +github.com/y0ssar1an/q v1.0.7/go.mod h1:Q1Rk1StqWjSOfA/CF4zJEW1fLmkl5Cy8EsILdkB+DgE= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190412021913-f29b1ada1971 h1:C+ye4QyWT3rbVj8As5DUc+Dsp067xJxCC6aa9+UnCmU= @@ -652,6 +668,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2eP golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -693,6 +710,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= From 5d8dcf942c95a43d4799cb0aa3c6224f9e6607f1 Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 25 Sep 2019 16:04:34 -0500 Subject: [PATCH 03/26] add additional test to configuration parsing, to cover basics --- command/agent/config/config_test.go | 83 ++++++++++++++++--- ...-template.hcl => config-template-many.hcl} | 37 +++++---- .../test-fixtures/config-template-min.hcl | 29 +++++++ 3 files changed, 120 insertions(+), 29 deletions(-) rename command/agent/config/test-fixtures/{config-template.hcl => config-template-many.hcl} (50%) create mode 100644 command/agent/config/test-fixtures/config-template-min.hcl diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 8d27619baf77..ca902c48eac1 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -288,18 +288,9 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { } // TestLoadConfigFile_Template_Single tests a single template definition in a -// configuration file, with most entries supplied +// configuration file, with minimum entries func TestLoadConfigFile_Template_Single(t *testing.T) { - q.Q("---------") - q.Q("starting") - q.Q("---------") - defer func() { - q.Q("---------") - q.Q("end") - q.Q("---------") - q.Q("") - }() - config, err := LoadConfig("./test-fixtures/config-template.hcl") + config, err := LoadConfig("./test-fixtures/config-template-min.hcl") if err != nil { t.Fatalf("err: %s", err) } @@ -330,7 +321,6 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { &ctconfig.TemplateConfig{ Source: strPtr("/path/on/disk/to/template.ctmpl"), Destination: strPtr("/path/on/disk/where/template/will/render.txt"), - Perms: fileMode(0600), }, }, PidFile: "./pidfile", @@ -341,10 +331,79 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { } } +// TestLoadConfigFile_Template_Multiple tests multiple template definition in a +// configuration file, with most entries supplied +func TestLoadConfigFile_Template_Many(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-template-many.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + &Sink{ + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Templates: []*ctconfig.TemplateConfig{ + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render.txt"), + ErrMissingKey: boolPtr(false), + CreateDestDirs: boolPtr(true), + Command: strPtr("restart service foo"), + Perms: fileMode(0600), + }, + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template2.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render2.txt"), + Backup: boolPtr(true), + Perms: fileMode(0755), + Wait: &ctconfig.WaitConfig{ + Min: timeDurationPtr("2s"), + Max: timeDurationPtr("10s"), + }, + }, + }, + PidFile: "./pidfile", + } + q.Q(config.Templates[1].Wait) + + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + func strPtr(s string) *string { return &s } +func boolPtr(b bool) *bool { + return &b +} + +func timeDurationPtr(duration string) *time.Duration { + d, _ := time.ParseDuration(duration) + + return &d +} + // FileMode returns a pointer to the given os.FileMode. func fileMode(o os.FileMode) *os.FileMode { return &o diff --git a/command/agent/config/test-fixtures/config-template.hcl b/command/agent/config/test-fixtures/config-template-many.hcl similarity index 50% rename from command/agent/config/test-fixtures/config-template.hcl rename to command/agent/config/test-fixtures/config-template-many.hcl index 5618d742972c..2f6fe7b70b6d 100644 --- a/command/agent/config/test-fixtures/config-template.hcl +++ b/command/agent/config/test-fixtures/config-template-many.hcl @@ -27,21 +27,24 @@ template { source = "/path/on/disk/to/template.ctmpl" destination = "/path/on/disk/where/template/will/render.txt" - #create_dest_dirs = true - - #contents = "{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}" - # command = "restart service foo" - # command_timeout = "60s" - # error_on_missing_key = false - perms = 0600 - - # backup = true - # left_delimiter = "{{" - # right_delimiter = "}}" - # function_blacklist = [] - # sandbox_path = "" - # wait { - # min = "2s" - # max = "10s" - # } + create_dest_dirs = true + + command = "restart service foo" + + error_on_missing_key = false + perms = 0600 +} + +template { + source = "/path/on/disk/to/template2.ctmpl" + destination = "/path/on/disk/where/template/will/render2.txt" + + perms = 0755 + + backup = true + + wait { + min = "2s" + max = "10s" + } } diff --git a/command/agent/config/test-fixtures/config-template-min.hcl b/command/agent/config/test-fixtures/config-template-min.hcl new file mode 100644 index 000000000000..5d37dbefbab1 --- /dev/null +++ b/command/agent/config/test-fixtures/config-template-min.hcl @@ -0,0 +1,29 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + namespace = "/my-namespace" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + + config = { + path = "/tmp/file-foo" + } + + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +template { + source = "/path/on/disk/to/template.ctmpl" + destination = "/path/on/disk/where/template/will/render.txt" +} From a4bd2230da19d295ab3fbc0bb250062684c8172e Mon Sep 17 00:00:00 2001 From: catsby Date: Mon, 30 Sep 2019 15:16:45 -0500 Subject: [PATCH 04/26] another test fixture, rework simple test into table --- command/agent/config/config_test.go | 97 +++++++++++++------ .../test-fixtures/config-template-full.hcl | 45 +++++++++ 2 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 command/agent/config/test-fixtures/config-template-full.hcl diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index ca902c48eac1..d0227072a960 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -290,44 +290,83 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { // TestLoadConfigFile_Template_Single tests a single template definition in a // configuration file, with minimum entries func TestLoadConfigFile_Template_Single(t *testing.T) { - config, err := LoadConfig("./test-fixtures/config-template-min.hcl") - if err != nil { - t.Fatalf("err: %s", err) - } - expected := &Config{ - AutoAuth: &AutoAuth{ - Method: &Method{ - Type: "aws", - MountPath: "auth/aws", - Namespace: "my-namespace/", - Config: map[string]interface{}{ - "role": "foobar", + testCases := map[string]struct { + fixturePath string + expectedTemplates []*ctconfig.TemplateConfig + }{ + "min": { + fixturePath: "./test-fixtures/config-template-min.hcl", + expectedTemplates: []*ctconfig.TemplateConfig{ + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render.txt"), }, }, - Sinks: []*Sink{ - &Sink{ - Type: "file", - DHType: "curve25519", - DHPath: "/tmp/file-foo-dhpath", - AAD: "foobar", - Config: map[string]interface{}{ - "path": "/tmp/file-foo", + }, + + "full": { + fixturePath: "./test-fixtures/config-template-full.hcl", + expectedTemplates: []*ctconfig.TemplateConfig{ + &ctconfig.TemplateConfig{ + Backup: boolPtr(true), + Command: strPtr("restart service foo"), + CommandTimeout: timeDurationPtr("60s"), + Contents: strPtr("{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}"), + CreateDestDirs: boolPtr(true), + Destination: strPtr("/path/on/disk/where/template/will/render.txt"), + ErrMissingKey: boolPtr(true), + LeftDelim: strPtr("<<"), + Perms: fileMode(0655), + RightDelim: strPtr(">>"), + SandboxPath: strPtr("/path/on/disk/where"), + + Wait: &ctconfig.WaitConfig{ + Min: timeDurationPtr("5s"), + Max: timeDurationPtr("30s"), }, }, }, }, - Templates: []*ctconfig.TemplateConfig{ - &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render.txt"), - }, - }, - PidFile: "./pidfile", } - if diff := deep.Equal(config, expected); diff != nil { - t.Fatal(diff) + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + config, err := LoadConfig(tc.fixturePath) + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + &Sink{ + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Templates: tc.expectedTemplates, + PidFile: "./pidfile", + } + + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } + }) } } diff --git a/command/agent/config/test-fixtures/config-template-full.hcl b/command/agent/config/test-fixtures/config-template-full.hcl new file mode 100644 index 000000000000..7234689eeccd --- /dev/null +++ b/command/agent/config/test-fixtures/config-template-full.hcl @@ -0,0 +1,45 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + namespace = "/my-namespace" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + + config = { + path = "/tmp/file-foo" + } + + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +template { + destination = "/path/on/disk/where/template/will/render.txt" + create_dest_dirs = true + contents = "{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}" + + command = "restart service foo" + command_timeout = "60s" + + error_on_missing_key = true + perms = 0655 + backup = true + left_delimiter = "<<" + right_delimiter = ">>" + + sandbox_path = "/path/on/disk/where" + wait { + min = "5s" + max = "30s" + } +} \ No newline at end of file From d024060d8d641b8b2ec92e77541f3e46b69ecd6e Mon Sep 17 00:00:00 2001 From: catsby Date: Mon, 30 Sep 2019 15:20:03 -0500 Subject: [PATCH 05/26] refactor into table test --- command/agent/config/config_test.go | 89 ++++++++--------------------- 1 file changed, 25 insertions(+), 64 deletions(-) diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index d0227072a960..37cab38a2852 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -7,7 +7,6 @@ import ( "github.com/go-test/deep" ctconfig "github.com/hashicorp/consul-template/config" - "github.com/y0ssar1an/q" ) func TestLoadConfigFile_AgentCache(t *testing.T) { @@ -287,10 +286,9 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { } } -// TestLoadConfigFile_Template_Single tests a single template definition in a -// configuration file, with minimum entries +// TestLoadConfigFile_Template_Single tests template definitions in Vault Agent +// configuration files func TestLoadConfigFile_Template_Single(t *testing.T) { - testCases := map[string]struct { fixturePath string expectedTemplates []*ctconfig.TemplateConfig @@ -304,7 +302,6 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { }, }, }, - "full": { fixturePath: "./test-fixtures/config-template-full.hcl", expectedTemplates: []*ctconfig.TemplateConfig{ @@ -328,6 +325,29 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { }, }, }, + "many": { + fixturePath: "./test-fixtures/config-template-many.hcl", + expectedTemplates: []*ctconfig.TemplateConfig{ + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render.txt"), + ErrMissingKey: boolPtr(false), + CreateDestDirs: boolPtr(true), + Command: strPtr("restart service foo"), + Perms: fileMode(0600), + }, + &ctconfig.TemplateConfig{ + Source: strPtr("/path/on/disk/to/template2.ctmpl"), + Destination: strPtr("/path/on/disk/where/template/will/render2.txt"), + Backup: boolPtr(true), + Perms: fileMode(0755), + Wait: &ctconfig.WaitConfig{ + Min: timeDurationPtr("2s"), + Max: timeDurationPtr("10s"), + }, + }, + }, + }, } for name, tc := range testCases { @@ -370,65 +390,6 @@ func TestLoadConfigFile_Template_Single(t *testing.T) { } } -// TestLoadConfigFile_Template_Multiple tests multiple template definition in a -// configuration file, with most entries supplied -func TestLoadConfigFile_Template_Many(t *testing.T) { - config, err := LoadConfig("./test-fixtures/config-template-many.hcl") - if err != nil { - t.Fatalf("err: %s", err) - } - - expected := &Config{ - AutoAuth: &AutoAuth{ - Method: &Method{ - Type: "aws", - MountPath: "auth/aws", - Namespace: "my-namespace/", - Config: map[string]interface{}{ - "role": "foobar", - }, - }, - Sinks: []*Sink{ - &Sink{ - Type: "file", - DHType: "curve25519", - DHPath: "/tmp/file-foo-dhpath", - AAD: "foobar", - Config: map[string]interface{}{ - "path": "/tmp/file-foo", - }, - }, - }, - }, - Templates: []*ctconfig.TemplateConfig{ - &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render.txt"), - ErrMissingKey: boolPtr(false), - CreateDestDirs: boolPtr(true), - Command: strPtr("restart service foo"), - Perms: fileMode(0600), - }, - &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template2.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render2.txt"), - Backup: boolPtr(true), - Perms: fileMode(0755), - Wait: &ctconfig.WaitConfig{ - Min: timeDurationPtr("2s"), - Max: timeDurationPtr("10s"), - }, - }, - }, - PidFile: "./pidfile", - } - q.Q(config.Templates[1].Wait) - - if diff := deep.Equal(config, expected); diff != nil { - t.Fatal(diff) - } -} - func strPtr(s string) *string { return &s } From 89338418082ccf8a0ebf9523cccce319aaec9878 Mon Sep 17 00:00:00 2001 From: catsby Date: Mon, 30 Sep 2019 16:53:36 -0500 Subject: [PATCH 06/26] rename test --- command/agent/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 37cab38a2852..cce9d0d949cf 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -288,7 +288,7 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { // TestLoadConfigFile_Template_Single tests template definitions in Vault Agent // configuration files -func TestLoadConfigFile_Template_Single(t *testing.T) { +func TestLoadConfigFile_Template(t *testing.T) { testCases := map[string]struct { fixturePath string expectedTemplates []*ctconfig.TemplateConfig From 1213f3014fd9960f65a2f53be84aa84dd89367f6 Mon Sep 17 00:00:00 2001 From: catsby Date: Tue, 1 Oct 2019 10:30:30 -0500 Subject: [PATCH 07/26] remove flattenKeys and add other test fixture --- command/agent/config/config.go | 62 ++++--------------- command/agent/config/config_test.go | 4 +- .../test-fixtures/config-template-full.hcl | 4 ++ 3 files changed, 18 insertions(+), 52 deletions(-) diff --git a/command/agent/config/config.go b/command/agent/config/config.go index e9b8b0c73f80..84b78992194b 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -439,13 +439,18 @@ func parseTemplates(result *Config, list *ast.ObjectList) error { return errors.New("error converting config") } - // Flattenkeys belonging to the templates - flattenKeys(parsed, []string{ - "env", - "exec", - "exec.env", - "wait", - }) + // flatten the wait field. The initial "wait" value, if given, is a + // []map[string]interface{}, but we need it to be map[string]interface{}. + // Consul Template has a method flattenKeys that walks all of parsed and + // flattens every key. For Vault Agent, we only care about the wait input. + // Only one wait stanza is supported, however Consul Template does not error + // with multiple instead it flattens them down, with last value winning. + // Here we take the last element of the parsed["wait"] slice to keep + // consistency with Consul Template behavior. + wait, ok := parsed["wait"].([]map[string]interface{}) + if ok { + parsed["wait"] = wait[len(wait)-1] + } var tc ctconfig.TemplateConfig @@ -473,46 +478,3 @@ func parseTemplates(result *Config, list *ast.ObjectList) error { result.Templates = tcs return nil } - -// flattenKeys is a function that takes a map[string]interface{} and recursively -// flattens any keys that are a []map[string]interface{} where the key is in the -// given list of keys. -func flattenKeys(m map[string]interface{}, keys []string) { - keyMap := make(map[string]struct{}) - for _, key := range keys { - keyMap[key] = struct{}{} - } - - var flatten func(map[string]interface{}, string) - flatten = func(m map[string]interface{}, parent string) { - for k, v := range m { - // Calculate the map key, since it could include a parent. - mapKey := k - if parent != "" { - mapKey = parent + "." + k - } - - if _, ok := keyMap[mapKey]; !ok { - continue - } - - switch typed := v.(type) { - case []map[string]interface{}: - if len(typed) > 0 { - last := typed[len(typed)-1] - flatten(last, mapKey) - m[k] = last - } else { - m[k] = nil - } - case map[string]interface{}: - flatten(typed, mapKey) - m[k] = typed - default: - m[k] = v - } - } - } - - flatten(m, "") -} diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index cce9d0d949cf..5a5bf3073777 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -319,8 +319,8 @@ func TestLoadConfigFile_Template(t *testing.T) { SandboxPath: strPtr("/path/on/disk/where"), Wait: &ctconfig.WaitConfig{ - Min: timeDurationPtr("5s"), - Max: timeDurationPtr("30s"), + Min: timeDurationPtr("10s"), + Max: timeDurationPtr("40s"), }, }, }, diff --git a/command/agent/config/test-fixtures/config-template-full.hcl b/command/agent/config/test-fixtures/config-template-full.hcl index 7234689eeccd..ee05c23098cb 100644 --- a/command/agent/config/test-fixtures/config-template-full.hcl +++ b/command/agent/config/test-fixtures/config-template-full.hcl @@ -42,4 +42,8 @@ template { min = "5s" max = "30s" } + wait { + min = "10s" + max = "40s" + } } \ No newline at end of file From 651f37a391e2e57ccb9c98754c2d219ff2984fea Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 2 Oct 2019 15:00:18 -0500 Subject: [PATCH 08/26] add template package --- command/agent.go | 3 +++ command/agent/template/template.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 command/agent/template/template.go diff --git a/command/agent.go b/command/agent.go index e338ad86ed1a..9bb2802f3a9c 100644 --- a/command/agent.go +++ b/command/agent.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/vault/command/agent/sink" "github.com/hashicorp/vault/command/agent/sink/file" "github.com/hashicorp/vault/command/agent/sink/inmem" + "github.com/hashicorp/vault/command/agent/template" gatedwriter "github.com/hashicorp/vault/helper/gated-writer" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/logging" @@ -490,6 +491,8 @@ func (c *AgentCommand) Run(args []string) int { go ss.Run(ctx, ah.OutputCh, sinks) } + _ := template.TemplateServer{} + // Server configuration output padding := 24 sort.Strings(infoKeys) diff --git a/command/agent/template/template.go b/command/agent/template/template.go new file mode 100644 index 000000000000..d55b4fd1596a --- /dev/null +++ b/command/agent/template/template.go @@ -0,0 +1,28 @@ +package template + +import ( + "sync" +) + +// TemplateServer +type TemplateServer struct { + // config holds the template managers configuration + // config *TaskTemplateManagerConfig + + // // lookup allows looking up the set of Nomad templates by their consul-template ID + // lookup map[string][]*structs.Template + + // runner is the consul-template runner + // runner *manager.Runner + + // // signals is a lookup map from the string representation of a signal to its + // // actual signal + // signals map[string]os.Signal + + // shutdownCh is used to signal and started goroutine to shutdown + shutdownCh chan struct{} + + // shutdown marks whether the manager has been shutdown + shutdown bool + shutdownLock sync.Mutex +} From 2680f689a28f1849d529c48a8553438344709b85 Mon Sep 17 00:00:00 2001 From: catsby Date: Mon, 7 Oct 2019 16:08:49 -0500 Subject: [PATCH 09/26] WIP: add runner --- command/agent/auth/auth.go | 5 + command/agent/config/config.go | 6 + command/agent/template/template.go | 179 ++++++++++++++++++++++++++--- go.sum | 3 + 4 files changed, 179 insertions(+), 14 deletions(-) diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index 7111bab93efd..b7fb0fb557f2 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -30,6 +30,7 @@ type AuthConfig struct { type AuthHandler struct { DoneCh chan struct{} OutputCh chan string + TemplateTokenCh chan string logger hclog.Logger client *api.Client random *rand.Rand @@ -50,6 +51,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { // This is buffered so that if we try to output after the sink server // has been shut down, during agent shutdown, we won't block OutputCh: make(chan string, 1), + TemplateTokenCh: make(chan string, 1), logger: conf.Logger, client: conf.Client, random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))), @@ -77,6 +79,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) { am.Shutdown() close(ah.OutputCh) close(ah.DoneCh) + close(ah.TemplateTokenCh) ah.logger.Info("auth handler stopped") }() @@ -163,6 +166,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) { } ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing") ah.OutputCh <- string(wrappedResp) + ah.TemplateTokenCh <- string(wrappedResp) am.CredSuccess() @@ -189,6 +193,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) { } ah.logger.Info("authentication successful, sending token to sinks") ah.OutputCh <- secret.Auth.ClientToken + ah.TemplateTokenCh <- secret.Auth.ClientToken am.CredSuccess() } diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 30e38374564b..0d3db58d71a5 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -29,6 +29,7 @@ type Config struct { Templates []*ctconfig.TemplateConfig `hcl:"templates"` } +// Vault contains configuration for connnecting to Vault servers type Vault struct { Address string `hcl:"address"` CACert string `hcl:"ca_cert"` @@ -39,15 +40,18 @@ type Vault struct { ClientKey string `hcl:"client_key"` } +// Cache contains any configuration needed for Cache mode type Cache struct { UseAutoAuthToken bool `hcl:"use_auto_auth_token"` } +// Listener contains configuration for any Vault Agent listeners type Listener struct { Type string Config map[string]interface{} } +// AutoAuth is the configured authentication method and sinks type AutoAuth struct { Method *Method `hcl:"-"` Sinks []*Sink `hcl:"sinks"` @@ -57,6 +61,7 @@ type AutoAuth struct { EnableReauthOnNewCredentials bool `hcl:"enable_reauth_on_new_credentials"` } +// Method represents the configuration for the authentication backend type Method struct { Type string MountPath string `hcl:"mount_path"` @@ -66,6 +71,7 @@ type Method struct { Config map[string]interface{} } +// Sink defines a location to write the authenticated token type Sink struct { Type string WrapTTLRaw interface{} `hcl:"wrap_ttl"` diff --git a/command/agent/template/template.go b/command/agent/template/template.go index d55b4fd1596a..e5b76ea4657c 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -1,28 +1,179 @@ +// Package template is responsible for rendering user supplied templates to +// disk. The Server type managing the lifecycle of an internal Consul Template +// Runner package template import ( - "sync" + "context" + "strings" + + ctconfig "github.com/hashicorp/consul-template/config" + "github.com/hashicorp/consul-template/manager" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/command/agent/config" ) -// TemplateServer -type TemplateServer struct { - // config holds the template managers configuration - // config *TaskTemplateManagerConfig +// ServerConfig is a config struct for setting up the basic parts of the +// Server +type ServerConfig struct { + Logger hclog.Logger + // Client *api.Client + VaultConf *config.Vault + ExitAfterAuth bool + + Namespace string +} + +// Server manages the Consul Template Runner which renders templates +type Server struct { + // config holds the ServerConfig used to create it. It's passed along in other + // methods + config *ServerConfig // // lookup allows looking up the set of Nomad templates by their consul-template ID // lookup map[string][]*structs.Template // runner is the consul-template runner - // runner *manager.Runner - - // // signals is a lookup map from the string representation of a signal to its - // // actual signal - // signals map[string]os.Signal + runner *manager.Runner - // shutdownCh is used to signal and started goroutine to shutdown - shutdownCh chan struct{} + // shutdownCh is used to signal the started goroutine to shutdown + // shutdownCh chan struct{} // shutdown marks whether the manager has been shutdown - shutdown bool - shutdownLock sync.Mutex + // shutdown bool + // shutdownLock sync.Mutex + Templates []*ctconfig.TemplateConfig + + DoneCh chan struct{} + logger hclog.Logger + // client *api.Client + exitAfterAuth bool +} + +// NewServer returns a new configured server +func NewServer(conf *ServerConfig) *Server { + ts := Server{ + DoneCh: make(chan struct{}), + logger: conf.Logger, + config: conf, + exitAfterAuth: conf.ExitAfterAuth, + } + return &ts +} + +// Run kicks off the internal Consul Template runner, and listens for changes to +// the token from the AuthHandler. If Done() is called on the context, shut down +// the Runner and return +func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ctconfig.TemplateConfig) { + latestToken := new(string) + ts.logger.Info("starting sink server") + defer func() { + ts.logger.Info("template server stopped") + close(ts.DoneCh) + }() + + if incoming == nil { + panic("incoming channel is nil") + } + + // construct a consul template vault config based the agents vault + // configuration + var runnerConfig *ctconfig.Config + if runnerConfig = newRunnerConfig(ts.config, ts.Templates); runnerConfig == nil { + ts.logger.Info("template server failed to generate runner config") + close(ts.DoneCh) + return + } + + runner, err := manager.NewRunner(runnerConfig, false) + if err != nil { + ts.logger.Info("template server failed to create") + close(ts.DoneCh) + return + } + + for { + select { + case <-ctx.Done(): + ts.runner.StopImmediately() + close(ts.DoneCh) + return + + case token := <-incoming: + // q.Q(">> incoming token") + if token != *latestToken { + // q.Q(">>:: new token") + ts.runner.Stop() + *latestToken = token + ctv := ctconfig.Config{ + Vault: &ctconfig.VaultConfig{ + Token: latestToken, + }, + } + runnerConfig.Merge(&ctv) + var runnerErr error + runner, runnerErr = manager.NewRunner(runnerConfig, false) + if runnerErr != nil { + ts.logger.Info("template server failed with new Vault token") + close(ts.DoneCh) + return + } + go ts.runner.Start() + } + case err := <-runner.ErrCh: + ts.logger.Info("template server error:", err) + close(ts.DoneCh) + return + } + } +} + +// newRunnerConfig returns a consul-template runner configuration, setting the +// Vault and Consul configurations based on the clients configs. +func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctconfig.Config { + conf := ctconfig.DefaultConfig() + conf.Templates = templates.Copy() + + // Setup the Vault config + // Always set these to ensure nothing is picked up from the environment + emptyStr := "" + conf.Vault.RenewToken = boolPtr(false) + // TODO: need token here + conf.Vault.Token = &emptyStr + conf.Vault.Address = &sc.VaultConf.Address + // conf.Vault.Token = &config.VaultToken + if sc.Namespace != "" { + conf.Vault.Namespace = &sc.Namespace + } + + if strings.HasPrefix(sc.VaultConf.Address, "https") || sc.VaultConf.CACert != "" { + skipVerify := sc.VaultConf.TLSSkipVerify + verify := !skipVerify + conf.Vault.SSL = &ctconfig.SSLConfig{ + Enabled: boolPtr(true), + Verify: &verify, + Cert: &sc.VaultConf.ClientCert, + Key: &sc.VaultConf.ClientKey, + CaCert: &sc.VaultConf.CACert, + CaPath: &sc.VaultConf.CAPath, + // ServerName: &sc.VaultConf.TLSServerName, + } + } else { + conf.Vault.SSL = &ctconfig.SSLConfig{ + Enabled: boolPtr(false), + Verify: boolPtr(false), + Cert: &emptyStr, + Key: &emptyStr, + CaCert: &emptyStr, + CaPath: &emptyStr, + ServerName: &emptyStr, + } + } + + conf.Finalize() + return conf +} + +func boolPtr(b bool) *bool { + return &b } diff --git a/go.sum b/go.sum index 86c220188168..d394f146e4ac 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7O github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v11.7.1+incompatible h1:M2YZIajBBVekV86x0rr1443Lc1F/Ylxb9w+5EtSyX3Q= github.com/Azure/go-autorest v11.7.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -433,6 +434,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.5 h1:JhhFTIOslh5ZsPrpa3Wdg8bF0WI3b44EMblmU9wIsXc= github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -455,6 +457,7 @@ github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdI github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= From 8e15a5dfbb82d62059bd86e54dd2dd41f34ceab1 Mon Sep 17 00:00:00 2001 From: catsby Date: Mon, 7 Oct 2019 17:47:52 -0500 Subject: [PATCH 10/26] fix panic, actually copy templates, etc --- command/agent.go | 30 ++++++++++++++++++++++++++---- command/agent/template/template.go | 13 +++++++------ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/command/agent.go b/command/agent.go index 9bb2802f3a9c..e7d3594a790e 100644 --- a/command/agent.go +++ b/command/agent.go @@ -293,10 +293,14 @@ func (c *AgentCommand) Run(args []string) int { return 1 } + // ctx and cancelFunc are passed to the AuthHandler, SinkServer, and + // Server that periodically listen for ctx.Done() to fire and shut + // down accordingly. ctx, cancelFunc := context.WithCancel(context.Background()) var method auth.AuthMethod var sinks []*sink.SinkConfig + var namespace string if config.AutoAuth != nil { for _, sc := range config.AutoAuth.Sinks { switch sc.Type { @@ -326,7 +330,8 @@ func (c *AgentCommand) Run(args []string) int { // Check if a default namespace has been set mountPath := config.AutoAuth.Method.MountPath if config.AutoAuth.Method.Namespace != "" { - mountPath = path.Join(config.AutoAuth.Method.Namespace, mountPath) + namespace = config.AutoAuth.Method.Namespace + mountPath = path.Join(namespace, mountPath) } authConfig := &auth.AuthConfig{ @@ -469,7 +474,7 @@ func (c *AgentCommand) Run(args []string) int { defer c.cleanupGuard.Do(listenerCloseFunc) } - var ssDoneCh, ahDoneCh chan struct{} + var ssDoneCh, ahDoneCh, tsDoneCh chan struct{} // Start auto-auth and sink servers if method != nil { ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{ @@ -487,11 +492,20 @@ func (c *AgentCommand) Run(args []string) int { }) ssDoneCh = ss.DoneCh + ts := template.NewServer(&template.ServerConfig{ + Logger: c.logger.Named("template.server"), + VaultConf: config.Vault, + Namespace: namespace, + ExitAfterAuth: config.ExitAfterAuth, + }) + tsDoneCh = ts.DoneCh + go ah.Run(ctx, method) go ss.Run(ctx, ah.OutputCh, sinks) - } - _ := template.TemplateServer{} + // TODO: should this be conditional? + go ts.Run(ctx, ah.TemplateTokenCh, config.Templates) + } // Server configuration output padding := 24 @@ -525,6 +539,10 @@ func (c *AgentCommand) Run(args []string) int { case <-ssDoneCh: // This will happen if we exit-on-auth c.logger.Info("sinks finished, exiting") + case <-tsDoneCh: + // TODO: wait for any templates to render(?) + c.logger.Info("template finished, exiting") + // TODO do we actually finish here? case <-c.ShutdownCh: c.UI.Output("==> Vault agent shutdown triggered") cancelFunc() @@ -534,6 +552,10 @@ func (c *AgentCommand) Run(args []string) int { if ssDoneCh != nil { <-ssDoneCh } + + if tsDoneCh != nil { + <-tsDoneCh + } } return 0 diff --git a/command/agent/template/template.go b/command/agent/template/template.go index e5b76ea4657c..1c074ca3a8c2 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -79,13 +79,14 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct // construct a consul template vault config based the agents vault // configuration var runnerConfig *ctconfig.Config - if runnerConfig = newRunnerConfig(ts.config, ts.Templates); runnerConfig == nil { + if runnerConfig = newRunnerConfig(ts.config, templates); runnerConfig == nil { ts.logger.Info("template server failed to generate runner config") close(ts.DoneCh) return } - runner, err := manager.NewRunner(runnerConfig, false) + var err error + ts.runner, err = manager.NewRunner(runnerConfig, false) if err != nil { ts.logger.Info("template server failed to create") close(ts.DoneCh) @@ -100,9 +101,8 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct return case token := <-incoming: - // q.Q(">> incoming token") if token != *latestToken { - // q.Q(">>:: new token") + ts.logger.Info("template server recieved new token") ts.runner.Stop() *latestToken = token ctv := ctconfig.Config{ @@ -111,8 +111,9 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct }, } runnerConfig.Merge(&ctv) + runnerConfig.Finalize() var runnerErr error - runner, runnerErr = manager.NewRunner(runnerConfig, false) + ts.runner, runnerErr = manager.NewRunner(runnerConfig, false) if runnerErr != nil { ts.logger.Info("template server failed with new Vault token") close(ts.DoneCh) @@ -120,7 +121,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct } go ts.runner.Start() } - case err := <-runner.ErrCh: + case err := <-ts.runner.ErrCh: ts.logger.Info("template server error:", err) close(ts.DoneCh) return From fdbd1dd118062e926355406a3dfe8934dc58bf20 Mon Sep 17 00:00:00 2001 From: catsby Date: Tue, 8 Oct 2019 16:06:49 -0500 Subject: [PATCH 11/26] rework how the config.Vault is created and enable reading from the environment --- command/agent/template/template.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 1c074ca3a8c2..d24f8f14aeaf 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -102,7 +102,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct case token := <-incoming: if token != *latestToken { - ts.logger.Info("template server recieved new token") + ts.logger.Info("template server received new token") ts.runner.Stop() *latestToken = token ctv := ctconfig.Config{ @@ -122,7 +122,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct go ts.runner.Start() } case err := <-ts.runner.ErrCh: - ts.logger.Info("template server error:", err) + ts.logger.Info("template server error:", err.Error()) close(ts.DoneCh) return } @@ -132,6 +132,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct // newRunnerConfig returns a consul-template runner configuration, setting the // Vault and Consul configurations based on the clients configs. func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctconfig.Config { + // TODO only use default Vault config conf := ctconfig.DefaultConfig() conf.Templates = templates.Copy() From ff66aab4419fb87ebdb83187d45e28646317564d Mon Sep 17 00:00:00 2001 From: catsby Date: Tue, 8 Oct 2019 16:13:03 -0500 Subject: [PATCH 12/26] this was supposed to be a part of the prior commit --- command/agent.go | 162 +++++++++++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 63 deletions(-) diff --git a/command/agent.go b/command/agent.go index e7d3594a790e..8a8db9956e9d 100644 --- a/command/agent.go +++ b/command/agent.go @@ -193,65 +193,69 @@ func (c *AgentCommand) Run(args []string) int { } // Load the configuration - config, err := config.LoadConfig(c.flagConfigs[0]) + agentConfig, err := config.LoadConfig(c.flagConfigs[0]) if err != nil { c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", c.flagConfigs[0], err)) return 1 } // Ensure at least one config was found. - if config == nil { + if agentConfig == nil { c.UI.Output(wrapAtLength( "No configuration read. Please provide the configuration with the " + "-config flag.")) return 1 } - if config.AutoAuth == nil && config.Cache == nil { + if agentConfig.AutoAuth == nil && agentConfig.Cache == nil { c.UI.Error("No auto_auth or cache block found in config file") return 1 } - if config.AutoAuth == nil { + if agentConfig.AutoAuth == nil { c.UI.Info("No auto_auth block found in config file, not starting automatic authentication feature") } - if config.Vault != nil { - c.setStringFlag(f, config.Vault.Address, &StringVar{ - Name: flagNameAddress, - Target: &c.flagAddress, - Default: "https://127.0.0.1:8200", - EnvVar: api.EnvVaultAddress, - }) - c.setStringFlag(f, config.Vault.CACert, &StringVar{ - Name: flagNameCACert, - Target: &c.flagCACert, - Default: "", - EnvVar: api.EnvVaultCACert, - }) - c.setStringFlag(f, config.Vault.CAPath, &StringVar{ - Name: flagNameCAPath, - Target: &c.flagCAPath, - Default: "", - EnvVar: api.EnvVaultCAPath, - }) - c.setStringFlag(f, config.Vault.ClientCert, &StringVar{ - Name: flagNameClientCert, - Target: &c.flagClientCert, - Default: "", - EnvVar: api.EnvVaultClientCert, - }) - c.setStringFlag(f, config.Vault.ClientKey, &StringVar{ - Name: flagNameClientKey, - Target: &c.flagClientKey, - Default: "", - EnvVar: api.EnvVaultClientKey, - }) - c.setBoolFlag(f, config.Vault.TLSSkipVerify, &BoolVar{ - Name: flagNameTLSSkipVerify, - Target: &c.flagTLSSkipVerify, - Default: false, - EnvVar: api.EnvVaultSkipVerify, - }) + // create an empty Vault configuration if none was loaded from file. The + // follow-up setStringFlag calls will populate with defaults if otherwise + // omitted + if agentConfig.Vault == nil { + agentConfig.Vault = new(config.Vault) } + c.setStringFlag(f, agentConfig.Vault.Address, &StringVar{ + Name: flagNameAddress, + Target: &c.flagAddress, + Default: "https://127.0.0.1:8200", + EnvVar: api.EnvVaultAddress, + }) + c.setStringFlag(f, agentConfig.Vault.CACert, &StringVar{ + Name: flagNameCACert, + Target: &c.flagCACert, + Default: "", + EnvVar: api.EnvVaultCACert, + }) + c.setStringFlag(f, agentConfig.Vault.CAPath, &StringVar{ + Name: flagNameCAPath, + Target: &c.flagCAPath, + Default: "", + EnvVar: api.EnvVaultCAPath, + }) + c.setStringFlag(f, agentConfig.Vault.ClientCert, &StringVar{ + Name: flagNameClientCert, + Target: &c.flagClientCert, + Default: "", + EnvVar: api.EnvVaultClientCert, + }) + c.setStringFlag(f, agentConfig.Vault.ClientKey, &StringVar{ + Name: flagNameClientKey, + Target: &c.flagClientKey, + Default: "", + EnvVar: api.EnvVaultClientKey, + }) + c.setBoolFlag(f, agentConfig.Vault.TLSSkipVerify, &BoolVar{ + Name: flagNameTLSSkipVerify, + Target: &c.flagTLSSkipVerify, + Default: false, + EnvVar: api.EnvVaultSkipVerify, + }) infoKeys := make([]string, 0, 10) info := make(map[string]string) @@ -277,7 +281,7 @@ func (c *AgentCommand) Run(args []string) int { if os.Getenv("VAULT_TEST_VERIFY_ONLY_DUMP_CONFIG") != "" { c.UI.Output(fmt.Sprintf( "\nConfiguration:\n%s\n", - pretty.Sprint(*config))) + pretty.Sprint(*agentConfig))) } return 0 } @@ -301,8 +305,8 @@ func (c *AgentCommand) Run(args []string) int { var method auth.AuthMethod var sinks []*sink.SinkConfig var namespace string - if config.AutoAuth != nil { - for _, sc := range config.AutoAuth.Sinks { + if agentConfig.AutoAuth != nil { + for _, sc := range agentConfig.AutoAuth.Sinks { switch sc.Type { case "file": config := &sink.SinkConfig{ @@ -328,18 +332,18 @@ func (c *AgentCommand) Run(args []string) int { } // Check if a default namespace has been set - mountPath := config.AutoAuth.Method.MountPath - if config.AutoAuth.Method.Namespace != "" { - namespace = config.AutoAuth.Method.Namespace + mountPath := agentConfig.AutoAuth.Method.MountPath + if agentConfig.AutoAuth.Method.Namespace != "" { + namespace = agentConfig.AutoAuth.Method.Namespace mountPath = path.Join(namespace, mountPath) } authConfig := &auth.AuthConfig{ - Logger: c.logger.Named(fmt.Sprintf("auth.%s", config.AutoAuth.Method.Type)), + Logger: c.logger.Named(fmt.Sprintf("auth.%s", agentConfig.AutoAuth.Method.Type)), MountPath: mountPath, - Config: config.AutoAuth.Method.Config, + Config: agentConfig.AutoAuth.Method.Config, } - switch config.AutoAuth.Method.Type { + switch agentConfig.AutoAuth.Method.Type { case "alicloud": method, err = alicloud.NewAliCloudAuthMethod(authConfig) case "aws": @@ -361,11 +365,11 @@ func (c *AgentCommand) Run(args []string) int { case "pcf": // Deprecated. method, err = cf.NewCFAuthMethod(authConfig) default: - c.UI.Error(fmt.Sprintf("Unknown auth method %q", config.AutoAuth.Method.Type)) + c.UI.Error(fmt.Sprintf("Unknown auth method %q", agentConfig.AutoAuth.Method.Type)) return 1 } if err != nil { - c.UI.Error(errwrap.Wrapf(fmt.Sprintf("Error creating %s auth method: {{err}}", config.AutoAuth.Method.Type), err).Error()) + c.UI.Error(errwrap.Wrapf(fmt.Sprintf("Error creating %s auth method: {{err}}", agentConfig.AutoAuth.Method.Type), err).Error()) return 1 } } @@ -382,7 +386,7 @@ func (c *AgentCommand) Run(args []string) int { } // Parse agent listener configurations - if config.Cache != nil && len(config.Listeners) != 0 { + if agentConfig.Cache != nil && len(agentConfig.Listeners) != 0 { cacheLogger := c.logger.Named("cache") // Create the API proxier @@ -409,7 +413,7 @@ func (c *AgentCommand) Run(args []string) int { } var inmemSink sink.Sink - if config.Cache.UseAutoAuthToken { + if agentConfig.Cache.UseAutoAuthToken { cacheLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink") inmemSink, err = inmem.New(&sink.SinkConfig{ Logger: cacheLogger, @@ -431,7 +435,7 @@ func (c *AgentCommand) Run(args []string) int { mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink)) var listeners []net.Listener - for i, lnConfig := range config.Listeners { + for i, lnConfig := range agentConfig.Listeners { ln, tlsConf, err := cache.StartListener(lnConfig) if err != nil { c.UI.Error(fmt.Sprintf("Error starting listener: %v", err)) @@ -480,23 +484,25 @@ func (c *AgentCommand) Run(args []string) int { ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{ Logger: c.logger.Named("auth.handler"), Client: c.client, - WrapTTL: config.AutoAuth.Method.WrapTTL, - EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials, + WrapTTL: agentConfig.AutoAuth.Method.WrapTTL, + EnableReauthOnNewCredentials: agentConfig.AutoAuth.EnableReauthOnNewCredentials, }) ahDoneCh = ah.DoneCh ss := sink.NewSinkServer(&sink.SinkServerConfig{ Logger: c.logger.Named("sink.server"), Client: client, - ExitAfterAuth: config.ExitAfterAuth, + ExitAfterAuth: agentConfig.ExitAfterAuth, }) ssDoneCh = ss.DoneCh + // create an independant vault configuration for Consul Template to use + vaultConfig := c.setupTemplateConfig() ts := template.NewServer(&template.ServerConfig{ Logger: c.logger.Named("template.server"), - VaultConf: config.Vault, + VaultConf: vaultConfig, Namespace: namespace, - ExitAfterAuth: config.ExitAfterAuth, + ExitAfterAuth: agentConfig.ExitAfterAuth, }) tsDoneCh = ts.DoneCh @@ -504,7 +510,7 @@ func (c *AgentCommand) Run(args []string) int { go ss.Run(ctx, ah.OutputCh, sinks) // TODO: should this be conditional? - go ts.Run(ctx, ah.TemplateTokenCh, config.Templates) + go ts.Run(ctx, ah.TemplateTokenCh, agentConfig.Templates) } // Server configuration output @@ -524,13 +530,13 @@ func (c *AgentCommand) Run(args []string) int { c.logGate.Flush() // Write out the PID to the file now that server has successfully started - if err := c.storePidFile(config.PidFile); err != nil { + if err := c.storePidFile(agentConfig.PidFile); err != nil { c.UI.Error(fmt.Sprintf("Error storing PID: %s", err)) return 1 } defer func() { - if err := c.removePidFile(config.PidFile); err != nil { + if err := c.removePidFile(agentConfig.PidFile); err != nil { c.UI.Error(fmt.Sprintf("Error deleting the PID file: %s", err)) } }() @@ -639,3 +645,33 @@ func (c *AgentCommand) removePidFile(pidPath string) error { } return os.Remove(pidPath) } + +// setupTemplateConfig creates a config.Vault struct for use by Consul Template. +// Consul Template does not currently allow us to pass in a configured API +// client, unlike the AuthHandler and SinkServer that reuse the client creted in +// this Run() method. Here we build a config.Vault struct for use by the +// Template Server that matches the configuration used to create the client +// (c.client), but in a struct of type config.Vault so that Consul Template can +// create it's own api client internally. +func (c *AgentCommand) setupTemplateConfig() *config.Vault { + cfg := new(config.Vault) + + if c.flagAddress != "" { + cfg.Address = c.flagAddress + } + if c.flagCACert != "" { + cfg.CACert = c.flagCACert + } + if c.flagCAPath != "" { + cfg.CAPath = c.flagCAPath + } + if c.flagClientCert != "" { + cfg.ClientCert = c.flagClientCert + } + if c.flagClientKey != "" { + cfg.ClientKey = c.flagClientKey + } + cfg.TLSSkipVerify = c.flagTLSSkipVerify + + return cfg +} From 268d869637a63356d8ad390f4982cdccae25d278 Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 9 Oct 2019 13:08:53 -0500 Subject: [PATCH 13/26] move/add methods to testhelpers for converting some values to pointers --- helper/testhelpers/testhelpers.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/helper/testhelpers/testhelpers.go b/helper/testhelpers/testhelpers.go index 6fde258df6b0..ac28aba77884 100644 --- a/helper/testhelpers/testhelpers.go +++ b/helper/testhelpers/testhelpers.go @@ -7,6 +7,7 @@ import ( "fmt" "math/rand" "net/url" + "os" "sync/atomic" "time" @@ -350,3 +351,25 @@ func RaftClusterJoinNodes(t testing.T, cluster *vault.TestCluster) { WaitForNCoresUnsealed(t, cluster, 3) } + +// StrPtr returns a pointer to a string value +func StrPtr(s string) *string { + return &s +} + +// BoolPtr returns a pointer to a boolean value +func BoolPtr(b bool) *bool { + return &b +} + +// TimeDurationPtr returns a pointer to a time duration value +func TimeDurationPtr(duration string) *time.Duration { + d, _ := time.ParseDuration(duration) + + return &d +} + +// FileMode returns a pointer to the given os.FileMode +func FileModePtr(o os.FileMode) *os.FileMode { + return &o +} From 6d165c4cc1c1fcdb1c8d0139ccd17b6865e4489b Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 9 Oct 2019 13:09:04 -0500 Subject: [PATCH 14/26] use new methods in testhelpers --- command/agent/config/config_test.go | 74 +++++++++++------------------ 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index a4fc2c6e7b94..52f996c9f8ca 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-test/deep" ctconfig "github.com/hashicorp/consul-template/config" + "github.com/hashicorp/vault/helper/testhelpers" ) func TestLoadConfigFile_AgentCache(t *testing.T) { @@ -296,8 +297,8 @@ func TestLoadConfigFile_Template(t *testing.T) { fixturePath: "./test-fixtures/config-template-min.hcl", expectedTemplates: []*ctconfig.TemplateConfig{ &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render.txt"), + Source: testhelpers.StrPtr("/path/on/disk/to/template.ctmpl"), + Destination: testhelpers.StrPtr("/path/on/disk/where/template/will/render.txt"), }, }, }, @@ -305,21 +306,21 @@ func TestLoadConfigFile_Template(t *testing.T) { fixturePath: "./test-fixtures/config-template-full.hcl", expectedTemplates: []*ctconfig.TemplateConfig{ &ctconfig.TemplateConfig{ - Backup: boolPtr(true), - Command: strPtr("restart service foo"), - CommandTimeout: timeDurationPtr("60s"), - Contents: strPtr("{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}"), - CreateDestDirs: boolPtr(true), - Destination: strPtr("/path/on/disk/where/template/will/render.txt"), - ErrMissingKey: boolPtr(true), - LeftDelim: strPtr("<<"), - Perms: fileMode(0655), - RightDelim: strPtr(">>"), - SandboxPath: strPtr("/path/on/disk/where"), + Backup: testhelpers.BoolPtr(true), + Command: testhelpers.StrPtr("restart service foo"), + CommandTimeout: testhelpers.TimeDurationPtr("60s"), + Contents: testhelpers.StrPtr("{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}"), + CreateDestDirs: testhelpers.BoolPtr(true), + Destination: testhelpers.StrPtr("/path/on/disk/where/template/will/render.txt"), + ErrMissingKey: testhelpers.BoolPtr(true), + LeftDelim: testhelpers.StrPtr("<<"), + Perms: testhelpers.FileModePtr(0655), + RightDelim: testhelpers.StrPtr(">>"), + SandboxPath: testhelpers.StrPtr("/path/on/disk/where"), Wait: &ctconfig.WaitConfig{ - Min: timeDurationPtr("10s"), - Max: timeDurationPtr("40s"), + Min: testhelpers.TimeDurationPtr("10s"), + Max: testhelpers.TimeDurationPtr("40s"), }, }, }, @@ -328,21 +329,21 @@ func TestLoadConfigFile_Template(t *testing.T) { fixturePath: "./test-fixtures/config-template-many.hcl", expectedTemplates: []*ctconfig.TemplateConfig{ &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render.txt"), - ErrMissingKey: boolPtr(false), - CreateDestDirs: boolPtr(true), - Command: strPtr("restart service foo"), - Perms: fileMode(0600), + Source: testhelpers.StrPtr("/path/on/disk/to/template.ctmpl"), + Destination: testhelpers.StrPtr("/path/on/disk/where/template/will/render.txt"), + ErrMissingKey: testhelpers.BoolPtr(false), + CreateDestDirs: testhelpers.BoolPtr(true), + Command: testhelpers.StrPtr("restart service foo"), + Perms: testhelpers.FileModePtr(0600), }, &ctconfig.TemplateConfig{ - Source: strPtr("/path/on/disk/to/template2.ctmpl"), - Destination: strPtr("/path/on/disk/where/template/will/render2.txt"), - Backup: boolPtr(true), - Perms: fileMode(0755), + Source: testhelpers.StrPtr("/path/on/disk/to/template2.ctmpl"), + Destination: testhelpers.StrPtr("/path/on/disk/where/template/will/render2.txt"), + Backup: testhelpers.BoolPtr(true), + Perms: testhelpers.FileModePtr(0755), Wait: &ctconfig.WaitConfig{ - Min: timeDurationPtr("2s"), - Max: timeDurationPtr("10s"), + Min: testhelpers.TimeDurationPtr("2s"), + Max: testhelpers.TimeDurationPtr("10s"), }, }, }, @@ -388,22 +389,3 @@ func TestLoadConfigFile_Template(t *testing.T) { }) } } - -func strPtr(s string) *string { - return &s -} - -func boolPtr(b bool) *bool { - return &b -} - -func timeDurationPtr(duration string) *time.Duration { - d, _ := time.ParseDuration(duration) - - return &d -} - -// FileMode returns a pointer to the given os.FileMode. -func fileMode(o os.FileMode) *os.FileMode { - return &o -} From 7245fc762b040634ec5a7d7a33592c3633a0923f Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 9 Oct 2019 17:07:25 -0500 Subject: [PATCH 15/26] add an unblock channel to block agent until a template has been rendered --- command/agent.go | 21 +++++++++++++++++---- command/agent/template/template.go | 30 ++++++++++++++++++------------ command/commands.go | 1 + 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/command/agent.go b/command/agent.go index 8a8db9956e9d..4829acabffc5 100644 --- a/command/agent.go +++ b/command/agent.go @@ -478,7 +478,12 @@ func (c *AgentCommand) Run(args []string) int { defer c.cleanupGuard.Do(listenerCloseFunc) } - var ssDoneCh, ahDoneCh, tsDoneCh chan struct{} + // TODO: listen for SIGHUP + // Listen for signals + // signal.Notify(c.signalCh) + + var ssDoneCh, ahDoneCh, tsDoneCh, unblockCh chan struct{} + var ts *template.Server // Start auto-auth and sink servers if method != nil { ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{ @@ -498,7 +503,7 @@ func (c *AgentCommand) Run(args []string) int { // create an independant vault configuration for Consul Template to use vaultConfig := c.setupTemplateConfig() - ts := template.NewServer(&template.ServerConfig{ + ts, unblockCh = template.NewServer(&template.ServerConfig{ Logger: c.logger.Named("template.server"), VaultConf: vaultConfig, Namespace: namespace, @@ -541,14 +546,21 @@ func (c *AgentCommand) Run(args []string) int { } }() + // Wait for the template to render + select { + case <-ctx.Done(): + case <-unblockCh: + } + select { case <-ssDoneCh: // This will happen if we exit-on-auth c.logger.Info("sinks finished, exiting") case <-tsDoneCh: - // TODO: wait for any templates to render(?) c.logger.Info("template finished, exiting") - // TODO do we actually finish here? + // TODO: implement reloading + // case <-c.SighupCh: + // c.UI.Output("==> Vault Agent reload triggered") case <-c.ShutdownCh: c.UI.Output("==> Vault agent shutdown triggered") cancelFunc() @@ -653,6 +665,7 @@ func (c *AgentCommand) removePidFile(pidPath string) error { // Template Server that matches the configuration used to create the client // (c.client), but in a struct of type config.Vault so that Consul Template can // create it's own api client internally. +// TODO test setupTemplateConfig func (c *AgentCommand) setupTemplateConfig() *config.Vault { cfg := new(config.Vault) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index d24f8f14aeaf..4a20e1b1ebaa 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -39,26 +39,30 @@ type Server struct { // shutdownCh is used to signal the started goroutine to shutdown // shutdownCh chan struct{} + // unblockCh is used to block until a template is rendered + unblockCh chan struct{} + // shutdown marks whether the manager has been shutdown // shutdown bool // shutdownLock sync.Mutex Templates []*ctconfig.TemplateConfig - DoneCh chan struct{} - logger hclog.Logger - // client *api.Client + DoneCh chan struct{} + logger hclog.Logger exitAfterAuth bool } // NewServer returns a new configured server -func NewServer(conf *ServerConfig) *Server { +func NewServer(conf *ServerConfig) (*Server, chan struct{}) { + unblock := make(chan struct{}) ts := Server{ DoneCh: make(chan struct{}), logger: conf.Logger, + unblockCh: unblock, config: conf, exitAfterAuth: conf.ExitAfterAuth, } - return &ts + return &ts, unblock } // Run kicks off the internal Consul Template runner, and listens for changes to @@ -81,7 +85,6 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct var runnerConfig *ctconfig.Config if runnerConfig = newRunnerConfig(ts.config, templates); runnerConfig == nil { ts.logger.Info("template server failed to generate runner config") - close(ts.DoneCh) return } @@ -89,7 +92,6 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct ts.runner, err = manager.NewRunner(runnerConfig, false) if err != nil { ts.logger.Info("template server failed to create") - close(ts.DoneCh) return } @@ -97,7 +99,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct select { case <-ctx.Done(): ts.runner.StopImmediately() - close(ts.DoneCh) + // TODO close ts.DoneCh return case token := <-incoming: @@ -116,15 +118,19 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct ts.runner, runnerErr = manager.NewRunner(runnerConfig, false) if runnerErr != nil { ts.logger.Info("template server failed with new Vault token") - close(ts.DoneCh) - return + // TODO: continue or fail here? Right now we just continue + continue + } else { + go ts.runner.Start() } - go ts.runner.Start() } case err := <-ts.runner.ErrCh: ts.logger.Info("template server error:", err.Error()) - close(ts.DoneCh) + // TODO close ts.DoneCh return + case <-ts.runner.TemplateRenderedCh(): + // A template has been rendered, unblock + close(ts.unblockCh) } } } diff --git a/command/commands.go b/command/commands.go index 68946bb682b6..73d29b850202 100644 --- a/command/commands.go +++ b/command/commands.go @@ -196,6 +196,7 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { UI: serverCmdUi, }, ShutdownCh: MakeShutdownCh(), + // SighupCh: MakeSighupCh(), }, nil }, "audit": func() (cli.Command, error) { From 054f394e00b01100d75efa455dd3b29dc882fa7f Mon Sep 17 00:00:00 2001 From: catsby Date: Wed, 9 Oct 2019 17:10:36 -0500 Subject: [PATCH 16/26] add note --- command/agent/template/template.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 4a20e1b1ebaa..5886d519f0bf 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -80,6 +80,8 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct panic("incoming channel is nil") } + // TODO: if there are no templates, close the unblock ch + // construct a consul template vault config based the agents vault // configuration var runnerConfig *ctconfig.Config From 370b874f9d5fbeebd5b815523fd2e8f65aad7d11 Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 10:56:40 -0500 Subject: [PATCH 17/26] unblock if there are no templates --- command/agent/template/template.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 5886d519f0bf..6383955b38e7 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -80,7 +80,12 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct panic("incoming channel is nil") } - // TODO: if there are no templates, close the unblock ch + // If there are no templates, close the unblockCh + if len(templates) == 0 { + // nothing to do + close(ts.unblockCh) + return + } // construct a consul template vault config based the agents vault // configuration From 459354e9076df3368a95c83f02003b87f5ff523f Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 15:18:26 -0500 Subject: [PATCH 18/26] cleanups --- command/agent.go | 8 ++----- command/agent/template/template.go | 37 ++++++++++++------------------ command/commands.go | 1 - 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/command/agent.go b/command/agent.go index 4829acabffc5..24c086516de9 100644 --- a/command/agent.go +++ b/command/agent.go @@ -478,8 +478,8 @@ func (c *AgentCommand) Run(args []string) int { defer c.cleanupGuard.Do(listenerCloseFunc) } - // TODO: listen for SIGHUP // Listen for signals + // TODO: implement support for SIGHUP reloading of configuration // signal.Notify(c.signalCh) var ssDoneCh, ahDoneCh, tsDoneCh, unblockCh chan struct{} @@ -513,8 +513,6 @@ func (c *AgentCommand) Run(args []string) int { go ah.Run(ctx, method) go ss.Run(ctx, ah.OutputCh, sinks) - - // TODO: should this be conditional? go ts.Run(ctx, ah.TemplateTokenCh, agentConfig.Templates) } @@ -556,10 +554,8 @@ func (c *AgentCommand) Run(args []string) int { case <-ssDoneCh: // This will happen if we exit-on-auth c.logger.Info("sinks finished, exiting") - case <-tsDoneCh: - c.logger.Info("template finished, exiting") - // TODO: implement reloading // case <-c.SighupCh: + // TODO: implement reloading // c.UI.Output("==> Vault Agent reload triggered") case <-c.ShutdownCh: c.UI.Output("==> Vault agent shutdown triggered") diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 6383955b38e7..ad6100f07320 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -30,21 +30,13 @@ type Server struct { // methods config *ServerConfig - // // lookup allows looking up the set of Nomad templates by their consul-template ID - // lookup map[string][]*structs.Template - // runner is the consul-template runner runner *manager.Runner - // shutdownCh is used to signal the started goroutine to shutdown - // shutdownCh chan struct{} - // unblockCh is used to block until a template is rendered unblockCh chan struct{} - // shutdown marks whether the manager has been shutdown - // shutdown bool - // shutdownLock sync.Mutex + // Templates holds the parsed Consul Templates Templates []*ctconfig.TemplateConfig DoneCh chan struct{} @@ -106,7 +98,6 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct select { case <-ctx.Done(): ts.runner.StopImmediately() - // TODO close ts.DoneCh return case token := <-incoming: @@ -133,10 +124,13 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct } case err := <-ts.runner.ErrCh: ts.logger.Info("template server error:", err.Error()) - // TODO close ts.DoneCh return case <-ts.runner.TemplateRenderedCh(): // A template has been rendered, unblock + if ts.exitAfterAuth { + // if we want to exit after auth, go ahead and shut down the runner + ts.runner.Stop() + } close(ts.unblockCh) } } @@ -153,7 +147,6 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctco // Always set these to ensure nothing is picked up from the environment emptyStr := "" conf.Vault.RenewToken = boolPtr(false) - // TODO: need token here conf.Vault.Token = &emptyStr conf.Vault.Address = &sc.VaultConf.Address // conf.Vault.Token = &config.VaultToken @@ -161,6 +154,16 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctco conf.Vault.Namespace = &sc.Namespace } + conf.Vault.SSL = &ctconfig.SSLConfig{ + Enabled: boolPtr(false), + Verify: boolPtr(false), + Cert: &emptyStr, + Key: &emptyStr, + CaCert: &emptyStr, + CaPath: &emptyStr, + ServerName: &emptyStr, + } + if strings.HasPrefix(sc.VaultConf.Address, "https") || sc.VaultConf.CACert != "" { skipVerify := sc.VaultConf.TLSSkipVerify verify := !skipVerify @@ -173,16 +176,6 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctco CaPath: &sc.VaultConf.CAPath, // ServerName: &sc.VaultConf.TLSServerName, } - } else { - conf.Vault.SSL = &ctconfig.SSLConfig{ - Enabled: boolPtr(false), - Verify: boolPtr(false), - Cert: &emptyStr, - Key: &emptyStr, - CaCert: &emptyStr, - CaPath: &emptyStr, - ServerName: &emptyStr, - } } conf.Finalize() diff --git a/command/commands.go b/command/commands.go index 73d29b850202..68946bb682b6 100644 --- a/command/commands.go +++ b/command/commands.go @@ -196,7 +196,6 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { UI: serverCmdUi, }, ShutdownCh: MakeShutdownCh(), - // SighupCh: MakeSighupCh(), }, nil }, "audit": func() (cli.Command, error) { From 28a6a517fb96f1e0618001000d31dd961d8b45d1 Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 15:18:48 -0500 Subject: [PATCH 19/26] go mod tidy --- go.mod | 1 - go.sum | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9332e715e34b..4ad9cbea61ef 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,6 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f github.com/kr/pretty v0.1.0 - github.com/kr/pty v1.1.3 // indirect github.com/kr/text v0.1.0 github.com/lib/pq v1.2.0 github.com/mattn/go-colorable v0.1.2 diff --git a/go.sum b/go.sum index d394f146e4ac..8254f3dea9b9 100644 --- a/go.sum +++ b/go.sum @@ -267,6 +267,7 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-gatedio v0.5.0 h1:Jm1X5yP4yCqqWj5L1TgW7iZwCVPGtVc+mro5r/XX7Tg= github.com/hashicorp/go-gatedio v0.5.0/go.mod h1:Lr3t8L6IyxD3DAeaUxGcgl2JnRUpWMCsmBl4Omu/2t4= github.com/hashicorp/go-gcp-common v0.5.0 h1:kkIQTjNTopn4eXQ1+lCiHYZXUtgIZvbc6YtAQkMnTos= github.com/hashicorp/go-gcp-common v0.5.0/go.mod h1:IDGUI2N/OS3PiU4qZcXJeWKPI6O/9Y8hOrbSiMcqyYw= From a1e3c83368e8ae2fe6d54759c24cff53a74ee901 Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 15:55:49 -0500 Subject: [PATCH 20/26] remove dead code --- command/agent/template/template.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index ad6100f07320..a50c81ade666 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -149,7 +149,7 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctco conf.Vault.RenewToken = boolPtr(false) conf.Vault.Token = &emptyStr conf.Vault.Address = &sc.VaultConf.Address - // conf.Vault.Token = &config.VaultToken + if sc.Namespace != "" { conf.Vault.Namespace = &sc.Namespace } @@ -174,7 +174,6 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) *ctco Key: &sc.VaultConf.ClientKey, CaCert: &sc.VaultConf.CACert, CaPath: &sc.VaultConf.CAPath, - // ServerName: &sc.VaultConf.TLSServerName, } } From 541a77a0f43c2cf6a15d02a5b5c8ab2012f3b4a5 Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 16:25:22 -0500 Subject: [PATCH 21/26] simple test to starT --- command/agent/template/template_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 command/agent/template/template_test.go diff --git a/command/agent/template/template_test.go b/command/agent/template/template_test.go new file mode 100644 index 000000000000..4ab7f4b9adeb --- /dev/null +++ b/command/agent/template/template_test.go @@ -0,0 +1,15 @@ +package template + +import "testing" + +// TestNewServer is a simple test to make sure NewServer returns a Server and +// channel +func TestNewServer(t *testing.T) { + ts, ch := NewServer(&ServerConfig{}) + if ts == nil { + t.Fatal("nil server returned") + } + if ch == nil { + t.Fatal("nil blocking channel returned") + } +} From fa8503909f9d99e1626713b912969231fec1e0b9 Mon Sep 17 00:00:00 2001 From: catsby Date: Thu, 10 Oct 2019 16:40:37 -0500 Subject: [PATCH 22/26] add simple, empty templates test --- command/agent/template/template.go | 1 + command/agent/template/template_test.go | 40 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index a50c81ade666..cd3a5b8cea1f 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -39,6 +39,7 @@ type Server struct { // Templates holds the parsed Consul Templates Templates []*ctconfig.TemplateConfig + // TODO: remove donech? DoneCh chan struct{} logger hclog.Logger exitAfterAuth bool diff --git a/command/agent/template/template_test.go b/command/agent/template/template_test.go index 4ab7f4b9adeb..5d1b8342e35e 100644 --- a/command/agent/template/template_test.go +++ b/command/agent/template/template_test.go @@ -1,6 +1,14 @@ package template -import "testing" +import ( + "context" + "testing" + + ctconfig "github.com/hashicorp/consul-template/config" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/command/agent/config" + "github.com/hashicorp/vault/sdk/helper/logging" +) // TestNewServer is a simple test to make sure NewServer returns a Server and // channel @@ -13,3 +21,33 @@ func TestNewServer(t *testing.T) { t.Fatal("nil blocking channel returned") } } + +func TestServerRun(t *testing.T) { + // create http test server + + templateTokenCh := make(chan string, 1) + ctx, _ := context.WithCancel(context.Background()) + sc := ServerConfig{ + Logger: logging.NewVaultLogger(hclog.Trace), + VaultConf: &config.Vault{ + Address: "http://127.0.0.1:8200", // replace with test server address + }, + } + // var tsDoneCh, unblockCh chan struct{} + var unblockCh chan struct{} + ts, unblockCh := NewServer(&sc) + if ts == nil { + t.Fatal("nil server returned") + } + if unblockCh == nil { + t.Fatal("nil blocking channel returned") + } + var tcs []*ctconfig.TemplateConfig + ts.Run(ctx, templateTokenCh, tcs) + + // Unblock should close immediately b/c there are no templates to render + select { + case <-ctx.Done(): + case <-unblockCh: + } +} From 14daed773700e5f3978f5b74b4a19a4c40144075 Mon Sep 17 00:00:00 2001 From: catsby Date: Fri, 11 Oct 2019 11:28:48 -0500 Subject: [PATCH 23/26] Update package doc, error logs, and add missing close() on channel --- command/agent/template/template.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index cd3a5b8cea1f..a8693e9d7881 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -1,6 +1,8 @@ // Package template is responsible for rendering user supplied templates to -// disk. The Server type managing the lifecycle of an internal Consul Template -// Runner +// disk. The Server type accepts configuration to communicate to a Vault server +// and a Vault token for authentication. Internally, the Server creates a Consul +// Template Runner which manages reading secrets from Vault and rendering +// templates to disk at configured locations package template import ( @@ -84,14 +86,16 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct // configuration var runnerConfig *ctconfig.Config if runnerConfig = newRunnerConfig(ts.config, templates); runnerConfig == nil { - ts.logger.Info("template server failed to generate runner config") + ts.logger.Error("template server failed to generate runner config") + close(ts.unblockCh) return } var err error ts.runner, err = manager.NewRunner(runnerConfig, false) if err != nil { - ts.logger.Info("template server failed to create") + ts.logger.Error("template server failed to create", "error", err) + close(ts.unblockCh) return } @@ -99,6 +103,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct select { case <-ctx.Done(): ts.runner.StopImmediately() + close(ts.unblockCh) return case token := <-incoming: @@ -116,7 +121,7 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct var runnerErr error ts.runner, runnerErr = manager.NewRunner(runnerConfig, false) if runnerErr != nil { - ts.logger.Info("template server failed with new Vault token") + ts.logger.Error("template server failed with new Vault token", "error", runnerErr) // TODO: continue or fail here? Right now we just continue continue } else { @@ -124,7 +129,8 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct } } case err := <-ts.runner.ErrCh: - ts.logger.Info("template server error:", err.Error()) + ts.logger.Error("template server error", "error", err.Error()) + close(ts.unblockCh) return case <-ts.runner.TemplateRenderedCh(): // A template has been rendered, unblock From c5720ee87b9fa73571f5f39016111ac80bc2bd05 Mon Sep 17 00:00:00 2001 From: catsby Date: Fri, 11 Oct 2019 12:08:25 -0500 Subject: [PATCH 24/26] update code comment to be clear what I'm referring to --- command/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent.go b/command/agent.go index 24c086516de9..4809dd341631 100644 --- a/command/agent.go +++ b/command/agent.go @@ -298,7 +298,7 @@ func (c *AgentCommand) Run(args []string) int { } // ctx and cancelFunc are passed to the AuthHandler, SinkServer, and - // Server that periodically listen for ctx.Done() to fire and shut + // TemplateServer that periodically listen for ctx.Done() to fire and shut // down accordingly. ctx, cancelFunc := context.WithCancel(context.Background()) From 429285a928ac8b93688ad8faacc6d5148f9df5b5 Mon Sep 17 00:00:00 2001 From: catsby Date: Fri, 11 Oct 2019 12:30:04 -0500 Subject: [PATCH 25/26] have template.NewServer return a (<- chan) type, even though it's a normal chan, as a better practice to enforce reading only --- command/agent.go | 3 ++- command/agent/template/template.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/command/agent.go b/command/agent.go index 4809dd341631..3f7c104e1fe3 100644 --- a/command/agent.go +++ b/command/agent.go @@ -482,7 +482,8 @@ func (c *AgentCommand) Run(args []string) int { // TODO: implement support for SIGHUP reloading of configuration // signal.Notify(c.signalCh) - var ssDoneCh, ahDoneCh, tsDoneCh, unblockCh chan struct{} + var ssDoneCh, ahDoneCh, tsDoneCh chan struct{} + var unblockCh <-chan struct{} var ts *template.Server // Start auto-auth and sink servers if method != nil { diff --git a/command/agent/template/template.go b/command/agent/template/template.go index a8693e9d7881..dbcc54bec01e 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -48,7 +48,7 @@ type Server struct { } // NewServer returns a new configured server -func NewServer(conf *ServerConfig) (*Server, chan struct{}) { +func NewServer(conf *ServerConfig) (*Server, <-chan struct{}) { unblock := make(chan struct{}) ts := Server{ DoneCh: make(chan struct{}), From 19d1f74afc84b6082cf40821a40c5b50ef7872f6 Mon Sep 17 00:00:00 2001 From: Clint Date: Fri, 11 Oct 2019 12:38:20 -0500 Subject: [PATCH 26/26] Update command/agent.go Co-Authored-By: Jim Kalafut --- command/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent.go b/command/agent.go index 3f7c104e1fe3..65d093a68ba5 100644 --- a/command/agent.go +++ b/command/agent.go @@ -502,7 +502,7 @@ func (c *AgentCommand) Run(args []string) int { }) ssDoneCh = ss.DoneCh - // create an independant vault configuration for Consul Template to use + // create an independent vault configuration for Consul Template to use vaultConfig := c.setupTemplateConfig() ts, unblockCh = template.NewServer(&template.ServerConfig{ Logger: c.logger.Named("template.server"),