From 9ab3542d5eb2b5546e5fdf2065e9ee1d0293cb2c Mon Sep 17 00:00:00 2001 From: Michel Vocks Date: Mon, 22 Jul 2019 13:36:09 +0200 Subject: [PATCH 1/3] Added s3 storage path parameter --- physical/s3/s3.go | 34 +++++++++++++++++++++++++++++----- physical/s3/s3_test.go | 1 + 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index 89db6cdc6dd8..a1564f699832 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -7,20 +7,20 @@ import ( "io" "net/http" "os" + "path" "sort" "strconv" "strings" "time" - log "github.com/hashicorp/go-hclog" - - metrics "github.com/armon/go-metrics" + "github.com/armon/go-metrics" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/hashicorp/errwrap" - cleanhttp "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/go-cleanhttp" + log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/helper/awsutil" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/parseutil" @@ -34,6 +34,7 @@ var _ physical.Backend = (*S3Backend)(nil) // within an S3 bucket. type S3Backend struct { bucket string + path string kmsKeyId string client *s3.S3 logger log.Logger @@ -52,6 +53,11 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, } } + path, ok := conf["path"] + if !ok { + path = "" + } + accessKey, ok := conf["access_key"] if !ok { accessKey = "" @@ -144,6 +150,7 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, s := &S3Backend{ client: s3conn, bucket: bucket, + path: path, kmsKeyId: kmsKeyId, logger: logger, permitPool: physical.NewPermitPool(maxParInt), @@ -158,9 +165,12 @@ func (s *S3Backend) Put(ctx context.Context, entry *physical.Entry) error { s.permitPool.Acquire() defer s.permitPool.Release() + // Setup key + key := path.Join(s.path, entry.Key) + putObjectInput := &s3.PutObjectInput{ Bucket: aws.String(s.bucket), - Key: aws.String(entry.Key), + Key: aws.String(key), Body: bytes.NewReader(entry.Value), } @@ -185,6 +195,9 @@ func (s *S3Backend) Get(ctx context.Context, key string) (*physical.Entry, error s.permitPool.Acquire() defer s.permitPool.Release() + // Setup key + key = path.Join(s.path, key) + resp, err := s.client.GetObject(&s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), @@ -230,6 +243,9 @@ func (s *S3Backend) Delete(ctx context.Context, key string) error { s.permitPool.Acquire() defer s.permitPool.Release() + // Setup key + key = path.Join(s.path, key) + _, err := s.client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), @@ -250,6 +266,14 @@ func (s *S3Backend) List(ctx context.Context, prefix string) ([]string, error) { s.permitPool.Acquire() defer s.permitPool.Release() + // Setup prefix + prefix = path.Join(s.path, prefix) + + // Validate prefix is ending with a "/" + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + params := &s3.ListObjectsV2Input{ Bucket: aws.String(s.bucket), Prefix: aws.String(prefix), diff --git a/physical/s3/s3_test.go b/physical/s3/s3_test.go index 54b87e7b9c25..849ab6b27857 100644 --- a/physical/s3/s3_test.go +++ b/physical/s3/s3_test.go @@ -97,6 +97,7 @@ func DoS3BackendTest(t *testing.T, kmsKeyId string) { b, err := NewS3Backend(map[string]string{ "bucket": bucket, "kmsKeyId": kmsKeyId, + "path": "test/vault", }, logger) if err != nil { t.Fatalf("err: %s", err) From f1d7c8c26cf355e1f15152d297a793d9b77a06b2 Mon Sep 17 00:00:00 2001 From: Michel Vocks Date: Mon, 22 Jul 2019 15:25:51 +0200 Subject: [PATCH 2/3] Added documentation --- website/source/docs/configuration/storage/s3.html.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index a4d56e9b7425..17d54589a33d 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -73,6 +73,9 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. permissions for this key. You can use `alias/aws/s3` to specify the default key for the account. +- `path` `(string: "")` - Specifies the path in the S3 Bucket where Vault + data will be stored. + ## `s3` Examples ### Default Example From b3a1fd90303dbba983e7ee160ad131efee6c1653 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 24 Jul 2019 12:48:08 -0400 Subject: [PATCH 3/3] Simplify logic slightly --- physical/s3/s3.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index a1564f699832..315b6d326965 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -53,10 +53,7 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, } } - path, ok := conf["path"] - if !ok { - path = "" - } + path := conf["path"] accessKey, ok := conf["access_key"] if !ok {