diff --git a/cli.go b/cli.go index d4ba12e2c..7a6bd3302 100644 --- a/cli.go +++ b/cli.go @@ -585,6 +585,11 @@ func (cli *CLI) ParseFlags(args []string) ( return nil }), "vault-transport-max-idle-conns-per-host", "") + flags.Var((funcIntVar)(func(i int) error { + c.Vault.Transport.MaxConnsPerHost = config.Int(i) + return nil + }), "vault-transport-max-conns-per-host", "") + flags.Var((funcDurationVar)(func(d time.Duration) error { c.Vault.Transport.TLSHandshakeTimeout = config.TimeDuration(d) return nil @@ -920,6 +925,9 @@ Options: -vault-transport-max-idle-conns-per-host= Sets the maximum number of idle connections to permit per host + -vault-transport-max-conns-per-host= + Sets the maximum number of total connections to permit per host + -vault-transport-tls-handshake-timeout= Sets the handshake timeout diff --git a/cli_test.go b/cli_test.go index af002241d..f0365785a 100644 --- a/cli_test.go +++ b/cli_test.go @@ -805,6 +805,18 @@ func TestCLI_ParseFlags(t *testing.T) { }, false, }, + { + "vault-transport-max-conns-per-host", + []string{"-vault-transport-max-conns-per-host", "25"}, + &config.Config{ + Vault: &config.VaultConfig{ + Transport: &config.TransportConfig{ + MaxConnsPerHost: config.Int(25), + }, + }, + }, + false, + }, { "vault-transport-tls-handshake-timeout", []string{"-vault-transport-tls-handshake-timeout", "30s"}, diff --git a/config/config_test.go b/config/config_test.go index 556c3c208..948909749 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1443,6 +1443,22 @@ func TestParse(t *testing.T) { }, false, }, + { + "vault_transport_max_conns_per_host", + `vault { + transport { + max_conns_per_host = 25 + } + }`, + &Config{ + Vault: &VaultConfig{ + Transport: &TransportConfig{ + MaxConnsPerHost: Int(25), + }, + }, + }, + false, + }, { "vault_transport_tls_handshake_timeout", `vault { diff --git a/config/consul_test.go b/config/consul_test.go index 97c5a0031..353ae9c93 100644 --- a/config/consul_test.go +++ b/config/consul_test.go @@ -327,6 +327,7 @@ func TestConsulConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, }, diff --git a/config/convert_test.go b/config/convert_test.go index aec730dd8..c19f9c972 100644 --- a/config/convert_test.go +++ b/config/convert_test.go @@ -216,7 +216,7 @@ func TestFileModePresent(t *testing.T) { }, { "present", - FileMode(0644), + FileMode(0o644), true, }, { diff --git a/config/nomad_test.go b/config/nomad_test.go index 4413150d8..d7ffdadc0 100644 --- a/config/nomad_test.go +++ b/config/nomad_test.go @@ -247,6 +247,7 @@ func TestNomadConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, Retry: &RetryConfig{ @@ -286,6 +287,7 @@ func TestNomadConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, Retry: &RetryConfig{ @@ -332,6 +334,7 @@ func TestNomadConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, Retry: &RetryConfig{ @@ -374,6 +377,7 @@ func TestNomadConfig_Finalize(t *testing.T) { MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), CustomDialer: mockDialer{}, }, Retry: &RetryConfig{ @@ -418,6 +422,7 @@ func TestNomadConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, Retry: &RetryConfig{ diff --git a/config/transport.go b/config/transport.go index de1cf7b62..c1352fe00 100644 --- a/config/transport.go +++ b/config/transport.go @@ -30,6 +30,14 @@ const ( // per host. DefaultMaxIdleConnsPerHost = 100 + // DefaultMaxConnsPerHost is the default number of maximum connections to use + // per host. The associated HTTP Transport MaxConnsPerHost is used to limit + // the total number of connections per host, including connections in the + // dialing, active, and idle states. On limit violation, dials will block. + // + // Zero means no limit. + DefaultMaxConnsPerHost = 0 + // DefaultTLSHandshakeTimeout is the amount of time to negotiate the TLS // handshake. DefaultTLSHandshakeTimeout = 10 * time.Second @@ -63,6 +71,9 @@ type TransportConfig struct { // host. MaxIdleConnsPerHost *int `mapstructure:"max_idle_conns_per_host"` + // MaxConns is the maximum number of total connections. + MaxConnsPerHost *int `mapstructure:"max_conns_per_host"` + // TLSHandshakeTimeout is the amount of time to wait to complete the TLS // handshake. TLSHandshakeTimeout *time.Duration `mapstructure:"tls_handshake_timeout"` @@ -88,6 +99,7 @@ func (c *TransportConfig) Copy() *TransportConfig { o.DisableKeepAlives = c.DisableKeepAlives o.IdleConnTimeout = c.IdleConnTimeout o.MaxIdleConns = c.MaxIdleConns + o.MaxConnsPerHost = c.MaxConnsPerHost o.MaxIdleConnsPerHost = c.MaxIdleConnsPerHost o.TLSHandshakeTimeout = c.TLSHandshakeTimeout @@ -140,6 +152,10 @@ func (c *TransportConfig) Merge(o *TransportConfig) *TransportConfig { r.MaxIdleConnsPerHost = o.MaxIdleConnsPerHost } + if o.MaxConnsPerHost != nil { + r.MaxConnsPerHost = o.MaxConnsPerHost + } + if o.TLSHandshakeTimeout != nil { r.TLSHandshakeTimeout = o.TLSHandshakeTimeout } @@ -173,6 +189,10 @@ func (c *TransportConfig) Finalize() { c.MaxIdleConnsPerHost = Int(DefaultMaxIdleConnsPerHost) } + if c.MaxConnsPerHost == nil { + c.MaxConnsPerHost = Int(DefaultMaxConnsPerHost) + } + if c.TLSHandshakeTimeout == nil { c.TLSHandshakeTimeout = TimeDuration(DefaultTLSHandshakeTimeout) } @@ -189,12 +209,14 @@ func (c *TransportConfig) GoString() string { "DialTimeout:%s, "+ "DisableKeepAlives:%t, "+ "MaxIdleConnsPerHost:%d, "+ - "TLSHandshakeTimeout:%s"+ + "TLSHandshakeTimeout:%s,"+ + "MaxConnsPerHost:%d"+ "}", TimeDurationVal(c.DialKeepAlive), TimeDurationVal(c.DialTimeout), BoolVal(c.DisableKeepAlives), IntVal(c.MaxIdleConnsPerHost), TimeDurationVal(c.TLSHandshakeTimeout), + IntVal(c.MaxConnsPerHost), ) } diff --git a/config/transport_test.go b/config/transport_test.go index dd2ba04be..043a088ab 100644 --- a/config/transport_test.go +++ b/config/transport_test.go @@ -33,6 +33,7 @@ func TestTransportConfig_Copy(t *testing.T) { IdleConnTimeout: TimeDuration(40 * time.Second), MaxIdleConns: Int(150), MaxIdleConnsPerHost: Int(15), + MaxConnsPerHost: Int(10), TLSHandshakeTimeout: TimeDuration(30 * time.Second), }, }, @@ -44,6 +45,7 @@ func TestTransportConfig_Copy(t *testing.T) { IdleConnTimeout: TimeDuration(40 * time.Second), MaxIdleConns: Int(150), MaxIdleConnsPerHost: Int(15), + MaxConnsPerHost: Int(10), TLSHandshakeTimeout: TimeDuration(30 * time.Second), }, }, @@ -234,6 +236,30 @@ func TestTransportConfig_Merge(t *testing.T) { &TransportConfig{MaxIdleConnsPerHost: Int(10)}, &TransportConfig{MaxIdleConnsPerHost: Int(10)}, }, + { + "max_conns_overrides", + &TransportConfig{MaxConnsPerHost: Int(10)}, + &TransportConfig{MaxConnsPerHost: Int(20)}, + &TransportConfig{MaxConnsPerHost: Int(20)}, + }, + { + "max_conns_empty_one", + &TransportConfig{MaxConnsPerHost: Int(10)}, + &TransportConfig{}, + &TransportConfig{MaxConnsPerHost: Int(10)}, + }, + { + "max_conns_empty_two", + &TransportConfig{}, + &TransportConfig{MaxConnsPerHost: Int(10)}, + &TransportConfig{MaxConnsPerHost: Int(10)}, + }, + { + "max_conns_same", + &TransportConfig{MaxConnsPerHost: Int(10)}, + &TransportConfig{MaxConnsPerHost: Int(10)}, + &TransportConfig{MaxConnsPerHost: Int(10)}, + }, { "tls_handshake_timeout_overrides", &TransportConfig{TLSHandshakeTimeout: TimeDuration(10 * time.Second)}, @@ -309,6 +335,7 @@ func TestTransportConfig_Finalize(t *testing.T) { DisableKeepAlives: Bool(false), IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, diff --git a/config/vault_test.go b/config/vault_test.go index bdc2830ce..0ac232113 100644 --- a/config/vault_test.go +++ b/config/vault_test.go @@ -510,6 +510,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -556,6 +557,59 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), + TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), + }, + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold), + K8SAuthRoleName: String(""), + K8SServiceAccountTokenPath: String(DefaultK8SServiceAccountTokenPath), + K8SServiceAccountToken: String(""), + K8SServiceMountPath: String(DefaultK8SServiceMountPath), + }, + }, + { + "with_max_conns", + nil, + &VaultConfig{ + Address: String("address"), + Transport: &TransportConfig{ + MaxIdleConns: Int(20), + MaxIdleConnsPerHost: Int(5), + MaxConnsPerHost: Int(100), + }, + }, + &VaultConfig{ + Address: String("address"), + Enabled: Bool(true), + Namespace: String(""), + RenewToken: Bool(false), + Retry: &RetryConfig{ + Backoff: TimeDuration(DefaultRetryBackoff), + MaxBackoff: TimeDuration(DefaultRetryMaxBackoff), + Enabled: Bool(true), + Attempts: Int(DefaultRetryAttempts), + }, + SSL: &SSLConfig{ + CaCert: String(""), + CaCertBytes: String(""), + CaPath: String(""), + Cert: String(""), + Enabled: Bool(true), + Key: String(""), + ServerName: String(""), + Verify: Bool(true), + }, + Token: String(""), + Transport: &TransportConfig{ + DialKeepAlive: TimeDuration(DefaultDialKeepAlive), + DialTimeout: TimeDuration(DefaultDialTimeout), + DisableKeepAlives: Bool(false), + IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), + MaxIdleConns: Int(20), + MaxIdleConnsPerHost: Int(5), + MaxConnsPerHost: Int(100), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -612,6 +666,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -666,6 +721,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -713,6 +769,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -760,6 +817,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), @@ -809,6 +867,7 @@ func TestVaultConfig_Finalize(t *testing.T) { IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), MaxIdleConns: Int(DefaultMaxIdleConns), MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + MaxConnsPerHost: Int(DefaultMaxConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, UnwrapToken: Bool(DefaultVaultUnwrapToken), diff --git a/dependency/client_set.go b/dependency/client_set.go index b19cccd99..298a61831 100644 --- a/dependency/client_set.go +++ b/dependency/client_set.go @@ -112,6 +112,7 @@ type CreateVaultClientInput struct { TransportIdleConnTimeout time.Duration TransportMaxIdleConns int TransportMaxIdleConnsPerHost int + TransportMaxConnsPerHost int TransportTLSHandshakeTimeout time.Duration } @@ -280,6 +281,7 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error { MaxIdleConns: i.TransportMaxIdleConns, IdleConnTimeout: i.TransportIdleConnTimeout, MaxIdleConnsPerHost: i.TransportMaxIdleConnsPerHost, + MaxConnsPerHost: i.TransportMaxConnsPerHost, TLSHandshakeTimeout: i.TransportTLSHandshakeTimeout, } diff --git a/dependency/nomad_var_get_test.go b/dependency/nomad_var_get_test.go index e173070af..cc80473e3 100644 --- a/dependency/nomad_var_get_test.go +++ b/dependency/nomad_var_get_test.go @@ -14,7 +14,6 @@ import ( ) func TestNewNVGetQuery(t *testing.T) { - cases := []struct { name string i string @@ -126,7 +125,6 @@ func TestNewNVGetQuery(t *testing.T) { } func TestNVGetQuery_Fetch(t *testing.T) { - type nvmap map[string]string _ = testNomad.CreateVariable("test-kv-get/path", nvmap{"bar": "barp"}, nil) _ = testNomad.CreateNamespace("test", nil) @@ -274,7 +272,6 @@ func TestNVGetQuery_Fetch(t *testing.T) { } func TestNVGetQuery_String(t *testing.T) { - cases := []struct { name string i string diff --git a/dependency/nomad_var_list_test.go b/dependency/nomad_var_list_test.go index b1534ef05..16bad174d 100644 --- a/dependency/nomad_var_list_test.go +++ b/dependency/nomad_var_list_test.go @@ -13,7 +13,6 @@ import ( ) func TestNewNVListQuery(t *testing.T) { - cases := []struct { name string i string @@ -177,7 +176,6 @@ func TestNewNVListQuery(t *testing.T) { } func TestNVListQuery_Fetch(t *testing.T) { - type nvmap map[string]string _ = testNomad.CreateVariable("test-kv-list/prefix/foo", nvmap{"bar": "barp"}, nil) _ = testNomad.CreateVariable("test-kv-list/prefix/zip", nvmap{"zap": "zapp"}, nil) @@ -346,7 +344,6 @@ func TestNVListQuery_Fetch(t *testing.T) { } func TestNVListQuery_String(t *testing.T) { - cases := []struct { name string i string diff --git a/manager/example_extfuncmap_test.go b/manager/example_extfuncmap_test.go index 168dda3af..e8f025573 100644 --- a/manager/example_extfuncmap_test.go +++ b/manager/example_extfuncmap_test.go @@ -22,7 +22,6 @@ import ( // It is not comprehensive and does not demonstrate the dependencies, polling, // and rerendering features available in the manager func Example_customFuncMap() { - // Consul-template uses the standard logger, which needs to be silenced // in this example log.SetOutput(io.Discard) diff --git a/manager/example_manager_test.go b/manager/example_manager_test.go index 75a1e701d..7a847b48c 100644 --- a/manager/example_manager_test.go +++ b/manager/example_manager_test.go @@ -19,7 +19,6 @@ import ( // demonstrate the dependencies, polling, and rerendering features available in // the manager func Example() { - // Consul-template uses the standard logger, which needs to be silenced // in this example log.SetOutput(io.Discard) diff --git a/manager/runner.go b/manager/runner.go index cef90504f..72128d979 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -1363,6 +1363,7 @@ func NewClientSet(c *config.Config) (*dep.ClientSet, error) { TransportIdleConnTimeout: config.TimeDurationVal(c.Vault.Transport.IdleConnTimeout), TransportMaxIdleConns: config.IntVal(c.Vault.Transport.MaxIdleConns), TransportMaxIdleConnsPerHost: config.IntVal(c.Vault.Transport.MaxIdleConnsPerHost), + TransportMaxConnsPerHost: config.IntVal(c.Vault.Transport.MaxConnsPerHost), TransportTLSHandshakeTimeout: config.TimeDurationVal(c.Vault.Transport.TLSHandshakeTimeout), K8SAuthRoleName: config.StringVal(c.Vault.K8SAuthRoleName), K8SServiceAccountTokenPath: config.StringVal(c.Vault.K8SServiceAccountTokenPath),