config: Add possibility to inline ServiceAccount into GCS config (#963)

* Add possibility to inline ServiceAccount into GCS config

* Update comment in pkg/objstore/gcs/gcs.go

Co-Authored-By: metalmatze <[email protected]>

* Generate docs for GCS config

docs/storage.md

@@ -155,9 +155,13 @@ For example:
 type: GCS
 config:
   bucket: ""
+  service_account: ""
 ```
 
-Application credentials are configured via JSON file, the client looks for:
+### Using GOOGLE_APPLICATION_CREDENTIALS
+
+Application credentials are configured via JSON file and only the bucket needs to be specified,
+the client looks for:
 
 1. A JSON file whose path is specified by the
    `GOOGLE_APPLICATION_CREDENTIALS` environment variable.
@@ -171,6 +175,30 @@ Application credentials are configured via JSON file, the client looks for:
 
 You can read more on how to get application credential json file in [https://cloud.google.com/docs/authentication/production](https://cloud.google.com/docs/authentication/production)
 
+### Using inline a Service Account
+
+Another possibility is to inline the ServiceAccount into the Thanos configuration and only maintain one file.
+This feature was added, so that the Prometheus Operator only needs to take care of one secret file.
+
+```yaml
+type: GCS
+config:
+  bucket: "thanos"
+  service_account: |-
+    {
+      "type": "service_account",
+      "project_id": "project",
+      "private_key_id": "abcdefghijklmnopqrstuvwxyz12345678906666",
+      "private_key": "-----BEGIN PRIVATE KEY-----\...\n-----END PRIVATE KEY-----\n",
+      "client_email": "[email protected]",
+      "client_id": "123456789012345678901",
+      "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+      "token_uri": "https://oauth2.googleapis.com/token",
+      "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+      "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/thanos%40gitpods.iam.gserviceaccount.com"
+    }
+```
+
 ### GCS Policies
 
 For deployment:

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/prometheus/tsdb v0.4.0
 	go.opencensus.io v0.19.0 // indirect
 	golang.org/x/net v0.0.0-20190213061140-3a22650c66bd
+	golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890
 	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4
 	golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 // indirect
 	golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2

pkg/objstore/gcs/gcs.go

@@ -16,6 +16,7 @@ import (
 	"github.com/improbable-eng/thanos/pkg/objstore"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/version"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	yaml "gopkg.in/yaml.v2"
@@ -26,7 +27,8 @@ const DirDelim = "/"
 
 // Config stores the configuration for gcs bucket.
 type Config struct {
-	Bucket string `yaml:"bucket"`
+	Bucket         string `yaml:"bucket"`
+	ServiceAccount string `yaml:"service_account"`
 }
 
 // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS.
@@ -47,8 +49,23 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component st
 	if gc.Bucket == "" {
 		return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks")
 	}
-	gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version()))
-	gcsClient, err := storage.NewClient(ctx, gcsOptions)
+
+	var opts []option.ClientOption
+
+	// If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic.
+	if gc.ServiceAccount != "" {
+		credentials, err := google.CredentialsFromJSON(ctx, []byte(gc.ServiceAccount))
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create credentials from JSON")
+		}
+		opts = append(opts, option.WithCredentials(credentials))
+	}
+
+	opts = append(opts,
+		option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())),
+	)
+
+	gcsClient, err := storage.NewClient(ctx, opts...)
 	if err != nil {
 		return nil, err
 	}