diff --git a/helper/strutil/strutil.go b/helper/strutil/strutil.go index eaab5007bb13..b5e69c4f25a3 100644 --- a/helper/strutil/strutil.go +++ b/helper/strutil/strutil.go @@ -303,3 +303,11 @@ func GlobbedStringsMatch(item, val string) bool { return val == item } + +// AppendIfMissing adds a string to a slice if the given string is not present +func AppendIfMissing(slice []string, i string) []string { + if StrListContains(slice, i) { + return slice + } + return append(slice, i) +} diff --git a/helper/strutil/strutil_test.go b/helper/strutil/strutil_test.go index 5a76cd2ddab5..ce02719d1f5c 100644 --- a/helper/strutil/strutil_test.go +++ b/helper/strutil/strutil_test.go @@ -330,3 +330,40 @@ func TestTrimStrings(t *testing.T) { t.Fatalf("Bad TrimStrings: expected:%#v, got:%#v", expected, actual) } } + +func TestStrutil_AppendIfMissing(t *testing.T) { + keys := []string{} + + keys = AppendIfMissing(keys, "foo") + + if len(keys) != 1 { + t.Fatalf("expected slice to be length of 1: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to contain key 'foo': %v", keys) + } + + keys = AppendIfMissing(keys, "bar") + + if len(keys) != 2 { + t.Fatalf("expected slice to be length of 2: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to contain key 'foo': %v", keys) + } + if keys[1] != "bar" { + t.Fatalf("expected slice to contain key 'bar': %v", keys) + } + + keys = AppendIfMissing(keys, "foo") + + if len(keys) != 2 { + t.Fatalf("expected slice to still be length of 2: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to still contain key 'foo': %v", keys) + } + if keys[1] != "bar" { + t.Fatalf("expected slice to still contain key 'bar': %v", keys) + } +} diff --git a/physical/azure.go b/physical/azure.go index 4d5083e71411..94a0024bf1ab 100644 --- a/physical/azure.go +++ b/physical/azure.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/azure-storage-go" "github.com/armon/go-metrics" "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/helper/strutil" ) // MaxBlobSize at this time @@ -181,7 +182,7 @@ func (a *AzureBackend) List(prefix string) ([]string, error) { if i := strings.Index(key, "/"); i == -1 { keys = append(keys, key) } else { - keys = appendIfMissing(keys, key[:i+1]) + keys = strutil.AppendIfMissing(keys, key[:i+1]) } } diff --git a/physical/etcd3.go b/physical/etcd3.go index 6fecc7372061..7f86e4eed17e 100644 --- a/physical/etcd3.go +++ b/physical/etcd3.go @@ -15,6 +15,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/coreos/etcd/pkg/transport" + "github.com/hashicorp/vault/helper/strutil" log "github.com/mgutz/logxi/v1" "golang.org/x/net/context" ) @@ -204,7 +205,7 @@ func (c *EtcdBackend) List(prefix string) ([]string, error) { if i := strings.Index(key, "/"); i == -1 { keys = append(keys, key) } else if i != -1 { - keys = appendIfMissing(keys, key[:i+1]) + keys = strutil.AppendIfMissing(keys, key[:i+1]) } } return keys, nil diff --git a/physical/mssql.go b/physical/mssql.go index 25709a22b159..9764574b0e21 100644 --- a/physical/mssql.go +++ b/physical/mssql.go @@ -9,6 +9,7 @@ import ( "github.com/armon/go-metrics" _ "github.com/denisenkom/go-mssqldb" + "github.com/hashicorp/vault/helper/strutil" log "github.com/mgutz/logxi/v1" ) @@ -206,7 +207,7 @@ func (m *MsSQLBackend) List(prefix string) ([]string, error) { if i := strings.Index(key, "/"); i == -1 { keys = append(keys, key) } else if i != -1 { - keys = appendIfMissing(keys, string(key[:i+1])) + keys = strutil.AppendIfMissing(keys, string(key[:i+1])) } } diff --git a/physical/mysql.go b/physical/mysql.go index d063df4223aa..73aed84e54fc 100644 --- a/physical/mysql.go +++ b/physical/mysql.go @@ -17,6 +17,7 @@ import ( "github.com/armon/go-metrics" mysql "github.com/go-sql-driver/mysql" "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/helper/strutil" ) // Unreserved tls key @@ -222,7 +223,7 @@ func (m *MySQLBackend) List(prefix string) ([]string, error) { keys = append(keys, key) } else if i != -1 { // Add truncated 'folder' paths - keys = appendIfMissing(keys, string(key[:i+1])) + keys = strutil.AppendIfMissing(keys, string(key[:i+1])) } } diff --git a/physical/s3.go b/physical/s3.go index 16571cabd299..ce2b91a8740d 100644 --- a/physical/s3.go +++ b/physical/s3.go @@ -205,8 +205,9 @@ func (s *S3Backend) List(prefix string) ([]string, error) { defer s.permitPool.Release() params := &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(prefix), + Bucket: aws.String(s.bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), } keys := []string{} @@ -214,6 +215,17 @@ func (s *S3Backend) List(prefix string) ([]string, error) { err := s.client.ListObjectsV2Pages(params, func(page *s3.ListObjectsV2Output, lastPage bool) bool { if page != nil { + // Add truncated 'folder' paths + for _, commonPrefix := range page.CommonPrefixes { + // Avoid panic + if commonPrefix == nil { + continue + } + + commonPrefix := strings.TrimPrefix(*commonPrefix.Prefix, prefix) + keys = append(keys, commonPrefix) + } + // Add objects only from the current 'folder' for _, key := range page.Contents { // Avoid panic if key == nil { @@ -221,14 +233,7 @@ func (s *S3Backend) List(prefix string) ([]string, error) { } key := strings.TrimPrefix(*key.Key, prefix) - - if i := strings.Index(key, "/"); i == -1 { - // Add objects only from the current 'folder' - keys = append(keys, key) - } else if i != -1 { - // Add truncated 'folder' paths - keys = appendIfMissing(keys, key[:i+1]) - } + keys = append(keys, key) } } return true @@ -242,12 +247,3 @@ func (s *S3Backend) List(prefix string) ([]string, error) { return keys, nil } - -func appendIfMissing(slice []string, i string) []string { - for _, ele := range slice { - if ele == i { - return slice - } - } - return append(slice, i) -} diff --git a/physical/s3_test.go b/physical/s3_test.go index 8fdb882fb5a2..7191cfec44ba 100644 --- a/physical/s3_test.go +++ b/physical/s3_test.go @@ -7,23 +7,26 @@ import ( "testing" "time" + "github.com/hashicorp/vault/helper/awsutil" "github.com/hashicorp/vault/helper/logformat" log "github.com/mgutz/logxi/v1" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) func TestS3Backend(t *testing.T) { - if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { + credsConfig := &awsutil.CredentialsConfig{} + + credsChain, err := credsConfig.GenerateCredentialChain() + if err != nil { t.SkipNow() } - creds, err := credentials.NewEnvCredentials().Get() + _, err = credsChain.Get() if err != nil { - t.Fatalf("err: %v", err) + t.SkipNow() } // If the variable is empty or doesn't exist, the default @@ -36,7 +39,7 @@ func TestS3Backend(t *testing.T) { } s3conn := s3.New(session.New(&aws.Config{ - Credentials: credentials.NewEnvCredentials(), + Credentials: credsChain, Endpoint: aws.String(endpoint), Region: aws.String(region), })) @@ -77,11 +80,9 @@ func TestS3Backend(t *testing.T) { logger := logformat.NewVaultLogger(log.LevelTrace) + // This uses the same logic to find the AWS credentials as we did at the beginning of the test b, err := NewBackend("s3", logger, map[string]string{ - "access_key": creds.AccessKeyID, - "secret_key": creds.SecretAccessKey, - "session_token": creds.SessionToken, - "bucket": bucket, + "bucket": bucket, }) if err != nil { t.Fatalf("err: %s", err) diff --git a/physical/swift.go b/physical/swift.go index eab8dc98cbc8..cff664e6fd4a 100644 --- a/physical/swift.go +++ b/physical/swift.go @@ -13,6 +13,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/errwrap" "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/vault/helper/strutil" "github.com/ncw/swift" ) @@ -207,7 +208,7 @@ func (s *SwiftBackend) List(prefix string) ([]string, error) { keys = append(keys, key) } else if i != -1 { // Add truncated 'folder' paths - keys = appendIfMissing(keys, key[:i+1]) + keys = strutil.AppendIfMissing(keys, key[:i+1]) } }