From 9f526338bdf4dd89d6b45fa35881ea9a44071357 Mon Sep 17 00:00:00 2001 From: Mohsen Sarmadi Date: Wed, 14 Feb 2018 11:49:04 +0000 Subject: [PATCH 01/10] Add MaxRetries setting for aws sdk to be used by AWS AUTH credential --- builtin/credential/aws/client.go | 3 +++ builtin/credential/aws/path_config_client.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/builtin/credential/aws/client.go b/builtin/credential/aws/client.go index 2b4c4aba4ec8..3cc774b2f8b7 100644 --- a/builtin/credential/aws/client.go +++ b/builtin/credential/aws/client.go @@ -34,6 +34,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg } endpoint := aws.String("") + var MaxRetries int if config != nil { // Override the default endpoint with the configured endpoint. switch { @@ -47,6 +48,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg credsConfig.AccessKey = config.AccessKey credsConfig.SecretKey = config.SecretKey + MaxRetries = config.MaxRetries } credsConfig.HTTPClient = cleanhttp.DefaultClient() @@ -65,6 +67,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg Region: aws.String(region), HTTPClient: cleanhttp.DefaultClient(), Endpoint: endpoint, + MaxRetries: aws.Int(MaxRetries), }, nil } diff --git a/builtin/credential/aws/path_config_client.go b/builtin/credential/aws/path_config_client.go index 0d7532ce4b00..aace5725c371 100644 --- a/builtin/credential/aws/path_config_client.go +++ b/builtin/credential/aws/path_config_client.go @@ -47,6 +47,10 @@ func pathConfigClient(b *backend) *framework.Path { Default: "", Description: "Value to require in the X-Vault-AWS-IAM-Server-ID request header", }, + "max_retries": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Maximum number of retries for recoverable exceptions of AWS APIs", + }, }, ExistenceCheck: b.pathConfigClientExistenceCheck, @@ -254,6 +258,7 @@ type clientConfig struct { IAMEndpoint string `json:"iam_endpoint" structs:"iam_endpoint" mapstructure:"iam_endpoint"` STSEndpoint string `json:"sts_endpoint" structs:"sts_endpoint" mapstructure:"sts_endpoint"` IAMServerIdHeaderValue string `json:"iam_server_id_header_value" structs:"iam_server_id_header_value" mapstructure:"iam_server_id_header_value"` + MaxRetries int `json:"max_retries"` } const pathConfigClientHelpSyn = ` From 82c442dbbd2dc160c299d0ea434b9c61456c1559 Mon Sep 17 00:00:00 2001 From: Mohsen Sarmadi Date: Wed, 14 Feb 2018 11:40:14 +0000 Subject: [PATCH 02/10] Add MaxRetries setting for aws sdk to be used by AWS secret backend --- builtin/logical/aws/client.go | 3 +++ builtin/logical/aws/path_config_root.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/builtin/logical/aws/client.go b/builtin/logical/aws/client.go index 91309ad28aae..feab31ef62df 100644 --- a/builtin/logical/aws/client.go +++ b/builtin/logical/aws/client.go @@ -17,6 +17,7 @@ import ( func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) { credsConfig := &awsutil.CredentialsConfig{} var endpoint string + var MaxRetries int entry, err := s.Get(ctx, "config/root") if err != nil { @@ -31,6 +32,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (* credsConfig.AccessKey = config.AccessKey credsConfig.SecretKey = config.SecretKey credsConfig.Region = config.Region + MaxRetries = config.MaxRetries switch { case clientType == "iam" && config.IAMEndpoint != "": endpoint = *aws.String(config.IAMEndpoint) @@ -61,6 +63,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (* Region: aws.String(credsConfig.Region), Endpoint: &endpoint, HTTPClient: cleanhttp.DefaultClient(), + MaxRetries: aws.Int(MaxRetries), }, nil } diff --git a/builtin/logical/aws/path_config_root.go b/builtin/logical/aws/path_config_root.go index d420c1ef10f0..3ee636804b32 100644 --- a/builtin/logical/aws/path_config_root.go +++ b/builtin/logical/aws/path_config_root.go @@ -33,6 +33,10 @@ func pathConfigRoot() *framework.Path { Type: framework.TypeString, Description: "Endpoint to custom STS server URL", }, + "max_retries": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Maximum number of retries for recoverable exceptions of AWS APIs", + }, }, Callbacks: map[logical.Operation]framework.OperationFunc{ @@ -48,6 +52,7 @@ func pathConfigRootWrite(ctx context.Context, req *logical.Request, data *framew region := data.Get("region").(string) iamendpoint := data.Get("iam_endpoint").(string) stsendpoint := data.Get("sts_endpoint").(string) + maxretries := data.Get("max_retries").(int) entry, err := logical.StorageEntryJSON("config/root", rootConfig{ AccessKey: data.Get("access_key").(string), @@ -55,6 +60,7 @@ func pathConfigRootWrite(ctx context.Context, req *logical.Request, data *framew IAMEndpoint: iamendpoint, STSEndpoint: stsendpoint, Region: region, + MaxRetries: maxretries, }) if err != nil { return nil, err @@ -73,6 +79,7 @@ type rootConfig struct { IAMEndpoint string `json:"iam_endpoint"` STSEndpoint string `json:"sts_endpoint"` Region string `json:"region"` + MaxRetries int `json:"max_retries"` } const pathConfigRootHelpSyn = ` From fc92db15dcc5f9f3d9e3df0809764d3c31912616 Mon Sep 17 00:00:00 2001 From: Mohsen Sarmadi Date: Wed, 14 Feb 2018 17:16:47 +0000 Subject: [PATCH 03/10] Set the default aws default values for aws auth and aws secret backend --- builtin/credential/aws/path_config_client.go | 2 ++ builtin/logical/aws/path_config_root.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/builtin/credential/aws/path_config_client.go b/builtin/credential/aws/path_config_client.go index aace5725c371..59be31245f03 100644 --- a/builtin/credential/aws/path_config_client.go +++ b/builtin/credential/aws/path_config_client.go @@ -3,6 +3,7 @@ package awsauth import ( "context" + "github.com/aws/aws-sdk-go/aws" "github.com/fatih/structs" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/logical/framework" @@ -49,6 +50,7 @@ func pathConfigClient(b *backend) *framework.Path { }, "max_retries": &framework.FieldSchema{ Type: framework.TypeInt, + Default: aws.UseServiceDefaultRetries, Description: "Maximum number of retries for recoverable exceptions of AWS APIs", }, }, diff --git a/builtin/logical/aws/path_config_root.go b/builtin/logical/aws/path_config_root.go index 3ee636804b32..12d8142928ca 100644 --- a/builtin/logical/aws/path_config_root.go +++ b/builtin/logical/aws/path_config_root.go @@ -3,6 +3,7 @@ package aws import ( "context" + "github.com/aws/aws-sdk-go/aws" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/logical/framework" ) @@ -35,6 +36,7 @@ func pathConfigRoot() *framework.Path { }, "max_retries": &framework.FieldSchema{ Type: framework.TypeInt, + Default: aws.UseServiceDefaultRetries, Description: "Maximum number of retries for recoverable exceptions of AWS APIs", }, }, From 4702c1da7be80011202b905f17e2026a9116b9f3 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:19:15 -0500 Subject: [PATCH 04/10] Update client.go --- builtin/credential/aws/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builtin/credential/aws/client.go b/builtin/credential/aws/client.go index a384b175a0f4..88c4be06c113 100644 --- a/builtin/credential/aws/client.go +++ b/builtin/credential/aws/client.go @@ -34,7 +34,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg } endpoint := aws.String("") - var MaxRetries int + var maxRetries int if config != nil { // Override the default endpoint with the configured endpoint. switch { @@ -48,7 +48,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg credsConfig.AccessKey = config.AccessKey credsConfig.SecretKey = config.SecretKey - MaxRetries = config.MaxRetries + maxRetries = config.MaxRetries } credsConfig.HTTPClient = cleanhttp.DefaultClient() @@ -67,7 +67,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg Region: aws.String(region), HTTPClient: cleanhttp.DefaultClient(), Endpoint: endpoint, - MaxRetries: aws.Int(MaxRetries), + MaxRetries: aws.Int(maxRetries), }, nil } From c443c81d2391af65b892e7f6fcaa152b0c0eb9a9 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:20:07 -0500 Subject: [PATCH 05/10] Update client.go --- builtin/logical/aws/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builtin/logical/aws/client.go b/builtin/logical/aws/client.go index feab31ef62df..e853c3ac81fd 100644 --- a/builtin/logical/aws/client.go +++ b/builtin/logical/aws/client.go @@ -17,7 +17,7 @@ import ( func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) { credsConfig := &awsutil.CredentialsConfig{} var endpoint string - var MaxRetries int + var maxRetries int entry, err := s.Get(ctx, "config/root") if err != nil { @@ -32,7 +32,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (* credsConfig.AccessKey = config.AccessKey credsConfig.SecretKey = config.SecretKey credsConfig.Region = config.Region - MaxRetries = config.MaxRetries + maxRetries = config.MaxRetries switch { case clientType == "iam" && config.IAMEndpoint != "": endpoint = *aws.String(config.IAMEndpoint) @@ -63,7 +63,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (* Region: aws.String(credsConfig.Region), Endpoint: &endpoint, HTTPClient: cleanhttp.DefaultClient(), - MaxRetries: aws.Int(MaxRetries), + MaxRetries: aws.Int(maxRetries), }, nil } From 9e0c6e2944107ba875a2f8b132c05afa78342542 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:22:39 -0500 Subject: [PATCH 06/10] Update path_config_client.go --- builtin/credential/aws/path_config_client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builtin/credential/aws/path_config_client.go b/builtin/credential/aws/path_config_client.go index 59be31245f03..845ee24bda15 100644 --- a/builtin/credential/aws/path_config_client.go +++ b/builtin/credential/aws/path_config_client.go @@ -226,6 +226,13 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical configEntry.IAMServerIdHeaderValue = data.Get("iam_server_id_header_value").(string) } + maxRetriesInt, ok := data.GetOk("max_retries") + if ok { + configEntry.MaxRetries = maxRetriesInt.(int) + } else if req.Operation == logical.CreateOperation { + configEntry.MaxRetries = data.Get("max_retries").(string) + } + // Since this endpoint supports both create operation and update operation, // the error checks for access_key and secret_key not being set are not present. // This allows calling this endpoint multiple times to provide the values. From 8647cd4d5d8975f04b59ea6e6966214fb4fedb0a Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:23:57 -0500 Subject: [PATCH 07/10] Update client.go --- builtin/credential/aws/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/credential/aws/client.go b/builtin/credential/aws/client.go index 88c4be06c113..13bc17ba7b24 100644 --- a/builtin/credential/aws/client.go +++ b/builtin/credential/aws/client.go @@ -34,7 +34,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg } endpoint := aws.String("") - var maxRetries int + var maxRetries int = aws.UseServiceDefaultRetries if config != nil { // Override the default endpoint with the configured endpoint. switch { From 5ef5c2a3f25e68778627388cd35aa5dcd08dd603 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:24:08 -0500 Subject: [PATCH 08/10] Update client.go --- builtin/logical/aws/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/logical/aws/client.go b/builtin/logical/aws/client.go index e853c3ac81fd..7787cfa679d2 100644 --- a/builtin/logical/aws/client.go +++ b/builtin/logical/aws/client.go @@ -17,7 +17,7 @@ import ( func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) { credsConfig := &awsutil.CredentialsConfig{} var endpoint string - var maxRetries int + var maxRetries int = aws.UseServiceDefaultRetries entry, err := s.Get(ctx, "config/root") if err != nil { From d62adf926935fcb533edf0c42d53f6a701f2bb91 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 15 Feb 2018 15:26:11 -0500 Subject: [PATCH 09/10] Update path_config_client.go --- builtin/credential/aws/path_config_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/credential/aws/path_config_client.go b/builtin/credential/aws/path_config_client.go index 845ee24bda15..90aef49f2317 100644 --- a/builtin/credential/aws/path_config_client.go +++ b/builtin/credential/aws/path_config_client.go @@ -230,7 +230,7 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical if ok { configEntry.MaxRetries = maxRetriesInt.(int) } else if req.Operation == logical.CreateOperation { - configEntry.MaxRetries = data.Get("max_retries").(string) + configEntry.MaxRetries = data.Get("max_retries").(int) } // Since this endpoint supports both create operation and update operation, From c4f35b752b4d0f0461455c8b71c093335689191d Mon Sep 17 00:00:00 2001 From: Mohsen Date: Thu, 15 Feb 2018 22:40:33 +0000 Subject: [PATCH 10/10] Running make fmt --- builtin/credential/aws/path_config_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/credential/aws/path_config_client.go b/builtin/credential/aws/path_config_client.go index 90aef49f2317..05e080af5b78 100644 --- a/builtin/credential/aws/path_config_client.go +++ b/builtin/credential/aws/path_config_client.go @@ -232,7 +232,7 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical } else if req.Operation == logical.CreateOperation { configEntry.MaxRetries = data.Get("max_retries").(int) } - + // Since this endpoint supports both create operation and update operation, // the error checks for access_key and secret_key not being set are not present. // This allows calling this endpoint multiple times to provide the values.