From 377b7230af69025df464adddcbcd5f82ecb1b963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Nov=C3=A1k?= Date: Tue, 5 Oct 2021 07:19:45 +0000 Subject: [PATCH 1/5] Add configurable AWS STS Endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Novák --- pkg/objstore/s3/s3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 8d50156d18..fe91a4b887 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -86,6 +86,7 @@ type Config struct { // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` SSEConfig SSEConfig `yaml:"sse_config"` + STSEndpoint string `yaml:"sts_endpoint"` } // SSEConfig deals with the configuration of SSE for Minio. The following options are valid: @@ -228,6 +229,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B Client: &http.Client{ Transport: http.DefaultTransport, }, + Endpoint: config.STSEndpoint, }), } } From d155761bf38542c3281abc0c8ce6913a7b07f3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Nov=C3=A1k?= Date: Tue, 5 Oct 2021 07:27:53 +0000 Subject: [PATCH 2/5] Edit Docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Novák --- docs/storage.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/storage.md b/docs/storage.md index bd5558f76d..f40ed976b1 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -88,6 +88,7 @@ config: kms_key_id: "" kms_encryption_context: {} encryption_key: "" + sts_endpoint: "" ``` At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. @@ -226,6 +227,12 @@ With this policy you should be able to run set `THANOS_TEST_OBJSTORE_SKIP=GCS,AZ Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html +##### STS Endpoint + +If you want to use IAM credential retrieved from an instance profile, Thanos needs to authenticate through AWS STS. For this purposes you can specify your own STS Endpoint. + +By default Thanos will use endpoint: https://sts.amazonaws.com and AWS region coresponding endpoints. + #### GCS To configure Google Cloud Storage bucket as an object store you need to set `bucket` with GCS bucket name and configure Google Application credentials. From 311b3088cfce3b76705c942584fb86d00d05f14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Nov=C3=A1k?= Date: Tue, 5 Oct 2021 08:55:33 +0000 Subject: [PATCH 3/5] Edit Changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Novák --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29fbfe848f..bd2b1edccc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response. - [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus. - [#4696](https://github.com/thanos-io/thanos/pull/4696) Query: add cache name to tracing spans. +- [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint ### Fixed From 0373184254b374eb61bfa93da6047fb81d8e63dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Nov=C3=A1k?= Date: Tue, 19 Oct 2021 07:07:07 +0000 Subject: [PATCH 4/5] Fix formating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Novák --- pkg/objstore/s3/s3.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index fe91a4b887..321d8fe57d 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -84,9 +84,9 @@ type Config struct { ListObjectsVersion string `yaml:"list_objects_version"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. // NOTE we need to make sure this number does not produce more parts than 10 000. - PartSize uint64 `yaml:"part_size"` - SSEConfig SSEConfig `yaml:"sse_config"` - STSEndpoint string `yaml:"sts_endpoint"` + PartSize uint64 `yaml:"part_size"` + SSEConfig SSEConfig `yaml:"sse_config"` + STSEndpoint string `yaml:"sts_endpoint"` } // SSEConfig deals with the configuration of SSE for Minio. The following options are valid: From c5b678c54a543c2ce5cd3acb01e0da0fd3946b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Nov=C3=A1k?= Date: Tue, 19 Oct 2021 10:45:47 +0200 Subject: [PATCH 5/5] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bartlomiej Plotka Signed-off-by: Tomáš Novák --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5e72064d..5ceefc193a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response. - [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus. - [#4696](https://github.com/thanos-io/thanos/pull/4696) Query: add cache name to tracing spans. -- [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint +- [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint. - [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas. ### Fixed