Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Transport config parameter for MaxConnsPerHost #1858

Merged
merged 8 commits into from
Jan 2, 2024
+155 −10
Merged

Add Transport config parameter for MaxConnsPerHost #1858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
91cb33a
add MaxConnsPerHost transport config parameter
ccapurso Dec 14, 2023
4ced69a
add support for MaxConnsPerHost to Vault client
ccapurso Dec 14, 2023
116eb1b
fix config finalize tests
ccapurso Dec 15, 2023
b0b8180
gofumpt
peteski22 Dec 15, 2023
56c6a96
update DefaultMaxConnsPerHost godoc comment
ccapurso Dec 15, 2023
16ab278
add MaxConnsPerHost to transport GoString
ccapurso Jan 2, 2024
6c70796
add MaxConnsPerHost case to config TestParse
ccapurso Jan 2, 2024
c2d9cef
add support for vault-transport-max-conns-per-host
ccapurso Jan 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cli.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,11 @@ func (cli *CLI) ParseFlags(args []string) (
return nil
}), "vault-transport-max-idle-conns-per-host", "")

flags.Var((funcIntVar)(func(i int) error {
c.Vault.Transport.MaxConnsPerHost = config.Int(i)
return nil
}), "vault-transport-max-conns-per-host", "")

flags.Var((funcDurationVar)(func(d time.Duration) error {
c.Vault.Transport.TLSHandshakeTimeout = config.TimeDuration(d)
return nil
Expand Down Expand Up @@ -920,6 +925,9 @@ Options:
-vault-transport-max-idle-conns-per-host=<int>
Sets the maximum number of idle connections to permit per host

-vault-transport-max-conns-per-host=<int>
Sets the maximum number of total connections to permit per host

-vault-transport-tls-handshake-timeout=<duration>
Sets the handshake timeout

Expand Down
12 changes: 12 additions & 0 deletions cli_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,18 @@ func TestCLI_ParseFlags(t *testing.T) {
},
false,
},
{
"vault-transport-max-conns-per-host",
[]string{"-vault-transport-max-conns-per-host", "25"},
&config.Config{
Vault: &config.VaultConfig{
Transport: &config.TransportConfig{
MaxConnsPerHost: config.Int(25),
},
},
},
false,
},
{
"vault-transport-tls-handshake-timeout",
[]string{"-vault-transport-tls-handshake-timeout", "30s"},
Expand Down
16 changes: 16 additions & 0 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1443,6 +1443,22 @@ func TestParse(t *testing.T) {
},
false,
},
{
"vault_transport_max_conns_per_host",
`vault {
transport {
max_conns_per_host = 25
}
}`,
&Config{
Vault: &VaultConfig{
Transport: &TransportConfig{
MaxConnsPerHost: Int(25),
},
},
},
false,
},
{
"vault_transport_tls_handshake_timeout",
`vault {
Expand Down
1 change: 1 addition & 0 deletions config/consul_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,7 @@ func TestConsulConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
},
Expand Down
2 changes: 1 addition & 1 deletion config/convert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ func TestFileModePresent(t *testing.T) {
},
{
"present",
FileMode(0644),
FileMode(0o644),
lornasong marked this conversation as resolved.
Show resolved Hide resolved
true,
},
{
Expand Down
5 changes: 5 additions & 0 deletions config/nomad_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
Retry: &RetryConfig{
Expand Down Expand Up @@ -286,6 +287,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
Retry: &RetryConfig{
Expand Down Expand Up @@ -332,6 +334,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
Retry: &RetryConfig{
Expand Down Expand Up @@ -374,6 +377,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
CustomDialer: mockDialer{},
},
Retry: &RetryConfig{
Expand Down Expand Up @@ -418,6 +422,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
Retry: &RetryConfig{
Expand Down
24 changes: 23 additions & 1 deletion config/transport.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ const (
// per host.
DefaultMaxIdleConnsPerHost = 100

// DefaultMaxConnsPerHost is the default number of maximum connections to use
// per host. The associated HTTP Transport MaxConnsPerHost is used to limit
// the total number of connections per host, including connections in the
// dialing, active, and idle states. On limit violation, dials will block.
//
// Zero means no limit.
DefaultMaxConnsPerHost = 0

// DefaultTLSHandshakeTimeout is the amount of time to negotiate the TLS
// handshake.
DefaultTLSHandshakeTimeout = 10 * time.Second
Expand Down Expand Up @@ -63,6 +71,9 @@ type TransportConfig struct {
// host.
MaxIdleConnsPerHost *int `mapstructure:"max_idle_conns_per_host"`

// MaxConns is the maximum number of total connections.
MaxConnsPerHost *int `mapstructure:"max_conns_per_host"`

// TLSHandshakeTimeout is the amount of time to wait to complete the TLS
// handshake.
TLSHandshakeTimeout *time.Duration `mapstructure:"tls_handshake_timeout"`
Expand All @@ -88,6 +99,7 @@ func (c *TransportConfig) Copy() *TransportConfig {
o.DisableKeepAlives = c.DisableKeepAlives
o.IdleConnTimeout = c.IdleConnTimeout
o.MaxIdleConns = c.MaxIdleConns
o.MaxConnsPerHost = c.MaxConnsPerHost
o.MaxIdleConnsPerHost = c.MaxIdleConnsPerHost
o.TLSHandshakeTimeout = c.TLSHandshakeTimeout

Expand Down Expand Up @@ -140,6 +152,10 @@ func (c *TransportConfig) Merge(o *TransportConfig) *TransportConfig {
r.MaxIdleConnsPerHost = o.MaxIdleConnsPerHost
}

if o.MaxConnsPerHost != nil {
r.MaxConnsPerHost = o.MaxConnsPerHost
}

if o.TLSHandshakeTimeout != nil {
r.TLSHandshakeTimeout = o.TLSHandshakeTimeout
}
Expand Down Expand Up @@ -173,6 +189,10 @@ func (c *TransportConfig) Finalize() {
c.MaxIdleConnsPerHost = Int(DefaultMaxIdleConnsPerHost)
}

if c.MaxConnsPerHost == nil {
c.MaxConnsPerHost = Int(DefaultMaxConnsPerHost)
}

if c.TLSHandshakeTimeout == nil {
c.TLSHandshakeTimeout = TimeDuration(DefaultTLSHandshakeTimeout)
}
Expand All @@ -189,12 +209,14 @@ func (c *TransportConfig) GoString() string {
"DialTimeout:%s, "+
"DisableKeepAlives:%t, "+
"MaxIdleConnsPerHost:%d, "+
"TLSHandshakeTimeout:%s"+
"TLSHandshakeTimeout:%s,"+
"MaxConnsPerHost:%d"+
"}",
TimeDurationVal(c.DialKeepAlive),
TimeDurationVal(c.DialTimeout),
BoolVal(c.DisableKeepAlives),
IntVal(c.MaxIdleConnsPerHost),
TimeDurationVal(c.TLSHandshakeTimeout),
IntVal(c.MaxConnsPerHost),
)
}
27 changes: 27 additions & 0 deletions config/transport_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestTransportConfig_Copy(t *testing.T) {
IdleConnTimeout: TimeDuration(40 * time.Second),
MaxIdleConns: Int(150),
MaxIdleConnsPerHost: Int(15),
MaxConnsPerHost: Int(10),
TLSHandshakeTimeout: TimeDuration(30 * time.Second),
},
},
Expand All @@ -44,6 +45,7 @@ func TestTransportConfig_Copy(t *testing.T) {
IdleConnTimeout: TimeDuration(40 * time.Second),
MaxIdleConns: Int(150),
MaxIdleConnsPerHost: Int(15),
MaxConnsPerHost: Int(10),
TLSHandshakeTimeout: TimeDuration(30 * time.Second),
},
},
Expand Down Expand Up @@ -234,6 +236,30 @@ func TestTransportConfig_Merge(t *testing.T) {
&TransportConfig{MaxIdleConnsPerHost: Int(10)},
&TransportConfig{MaxIdleConnsPerHost: Int(10)},
},
{
"max_conns_overrides",
&TransportConfig{MaxConnsPerHost: Int(10)},
&TransportConfig{MaxConnsPerHost: Int(20)},
&TransportConfig{MaxConnsPerHost: Int(20)},
},
{
"max_conns_empty_one",
&TransportConfig{MaxConnsPerHost: Int(10)},
&TransportConfig{},
&TransportConfig{MaxConnsPerHost: Int(10)},
},
{
"max_conns_empty_two",
&TransportConfig{},
&TransportConfig{MaxConnsPerHost: Int(10)},
&TransportConfig{MaxConnsPerHost: Int(10)},
},
{
"max_conns_same",
&TransportConfig{MaxConnsPerHost: Int(10)},
&TransportConfig{MaxConnsPerHost: Int(10)},
&TransportConfig{MaxConnsPerHost: Int(10)},
},
{
"tls_handshake_timeout_overrides",
&TransportConfig{TLSHandshakeTimeout: TimeDuration(10 * time.Second)},
Expand Down Expand Up @@ -309,6 +335,7 @@ func TestTransportConfig_Finalize(t *testing.T) {
DisableKeepAlives: Bool(false),
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
Expand Down
59 changes: 59 additions & 0 deletions config/vault_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -556,6 +557,59 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration),
LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold),
K8SAuthRoleName: String(""),
K8SServiceAccountTokenPath: String(DefaultK8SServiceAccountTokenPath),
K8SServiceAccountToken: String(""),
K8SServiceMountPath: String(DefaultK8SServiceMountPath),
},
},
{
"with_max_conns",
nil,
&VaultConfig{
Address: String("address"),
Transport: &TransportConfig{
MaxIdleConns: Int(20),
MaxIdleConnsPerHost: Int(5),
MaxConnsPerHost: Int(100),
},
},
&VaultConfig{
Address: String("address"),
Enabled: Bool(true),
Namespace: String(""),
RenewToken: Bool(false),
Retry: &RetryConfig{
Backoff: TimeDuration(DefaultRetryBackoff),
MaxBackoff: TimeDuration(DefaultRetryMaxBackoff),
Enabled: Bool(true),
Attempts: Int(DefaultRetryAttempts),
},
SSL: &SSLConfig{
CaCert: String(""),
CaCertBytes: String(""),
CaPath: String(""),
Cert: String(""),
Enabled: Bool(true),
Key: String(""),
ServerName: String(""),
Verify: Bool(true),
},
Token: String(""),
Transport: &TransportConfig{
DialKeepAlive: TimeDuration(DefaultDialKeepAlive),
DialTimeout: TimeDuration(DefaultDialTimeout),
DisableKeepAlives: Bool(false),
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(20),
MaxIdleConnsPerHost: Int(5),
MaxConnsPerHost: Int(100),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -612,6 +666,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -666,6 +721,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -713,6 +769,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -760,6 +817,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down Expand Up @@ -809,6 +867,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout),
MaxIdleConns: Int(DefaultMaxIdleConns),
MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
MaxConnsPerHost: Int(DefaultMaxConnsPerHost),
TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
},
UnwrapToken: Bool(DefaultVaultUnwrapToken),
Expand Down
2 changes: 2 additions & 0 deletions dependency/client_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ type CreateVaultClientInput struct {
TransportIdleConnTimeout time.Duration
TransportMaxIdleConns int
TransportMaxIdleConnsPerHost int
TransportMaxConnsPerHost int
TransportTLSHandshakeTimeout time.Duration
}

Expand Down Expand Up @@ -280,6 +281,7 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
MaxIdleConns: i.TransportMaxIdleConns,
IdleConnTimeout: i.TransportIdleConnTimeout,
MaxIdleConnsPerHost: i.TransportMaxIdleConnsPerHost,
MaxConnsPerHost: i.TransportMaxConnsPerHost,
TLSHandshakeTimeout: i.TransportTLSHandshakeTimeout,
}

Expand Down
3 changes: 0 additions & 3 deletions dependency/nomad_var_get_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
)

func TestNewNVGetQuery(t *testing.T) {

cases := []struct {
name string
i string
Expand Down Expand Up @@ -126,7 +125,6 @@ func TestNewNVGetQuery(t *testing.T) {
}

func TestNVGetQuery_Fetch(t *testing.T) {

type nvmap map[string]string
_ = testNomad.CreateVariable("test-kv-get/path", nvmap{"bar": "barp"}, nil)
_ = testNomad.CreateNamespace("test", nil)
Expand Down Expand Up @@ -274,7 +272,6 @@ func TestNVGetQuery_Fetch(t *testing.T) {
}

func TestNVGetQuery_String(t *testing.T) {

cases := []struct {
name string
i string
Expand Down
Loading