From 6e403c7a6ea50a365d8863e4011fe7fe3c364874 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 14:59:32 +0800 Subject: [PATCH 01/43] Add unified flags for bucket configuration Signed-off-by: jojohappy --- pkg/objstore/config.go | 152 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 pkg/objstore/config.go diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go new file mode 100644 index 0000000000..11a6417674 --- /dev/null +++ b/pkg/objstore/config.go @@ -0,0 +1,152 @@ +package objstore + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "gopkg.in/alecthomas/kingpin.v2" +) + +type ObjProvider string + +const ( + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + Unsupported ObjProvider = "UNSUPPORTED" +) + +var ErrUnsupported = errors.New("unsupported provider") +var ErrMissingGCSBucket = errors.New("missing Google Cloud Storage bucket name for stored blocks") +var ErrInsufficientS3Info = errors.New("insufficient s3 configuration information") + +// BucketConfig encapsulates the necessary config values to instantiate an object store client. +// Use Provider to represent the Object Stores +type BucketConfig struct { + Provider ObjProvider + Bucket string + Endpoint string + AccessKey string + secretKey string + Insecure bool + SignatureV2 bool + SSEEncryption bool +} + +// NewBucketConfig return the configuration of object store +// TODO(jojohappy) should it support multiple bucket? +func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { + var bucketConfig BucketConfig + provider := cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported provider are GCS / S3."). + PlaceHolder("").String() + + bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) + cmd.Flag("provider.bucket", "The bucket name for stored blocks."). + PlaceHolder("").Envar("PROVIDER_BUCKET").StringVar(&bucketConfig.Bucket) + + switch bucketConfig.Provider { + case GCS: + case S3: + cmd.Flag("provider.endpoint", "The object store API endpoint for stored blocks. Support S3-Compatible API"). + PlaceHolder("").Envar("PROVIDER_ENDPOINT").StringVar(&bucketConfig.Endpoint) + + cmd.Flag("provider.access-key", "Access key for an object store API. Support S3-Compatible API"). + PlaceHolder("").Envar("PROVIDER_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) + + // TODO(jojohappy): should it be encrypted? + bucketConfig.secretKey = os.Getenv("PROVIDER_SECRET_KEY") + + cmd.Flag("provider.insecure", "Whether to use an insecure connection with an object store API. Support S3-Compatible API"). + Default("false").Envar("PROVIDER_INSECURE").BoolVar(&bucketConfig.Insecure) + + cmd.Flag("provider.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). + Default("false").Envar("PROVIDER_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) + + cmd.Flag("provider.encrypt-sse", "Whether to use Server Side Encryption"). + Default("false").Envar("PROVIDER_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) + default: + bucketConfig.Provider = Unsupported + } + + return &bucketConfig +} + +// NewBackupBucketConfig return the configuration of backup object store +func NewBackupBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { + var bucketConfig BucketConfig + provider := cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported provider are GCS / S3."). + PlaceHolder("").String() + + bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) + cmd.Flag("provider-backup.bucket", "The bucket name for backup stored blocks."). + PlaceHolder("").StringVar(&bucketConfig.Bucket) + + switch bucketConfig.Provider { + case GCS: + case S3: + default: + bucketConfig.Provider = Unsupported + } + + return &bucketConfig +} + +// Provider return the provider of Object Store +func (conf *BucketConfig) BucketProvider() string { + return string(conf.Provider) +} + +// String returns the Provider information +func (conf *BucketConfig) String() string { + return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", conf.BucketProvider(), conf.Bucket, conf.Endpoint) +} + +// BucketSecretKey returns the Provider Secret Key +// TODO(jojohappy): it should return the key decrypted. +func (conf *BucketConfig) BucketSecretKey() string { + return conf.secretKey +} + +// SetSecretKey to setup the Secret Key for the Provider +func (conf *BucketConfig) SetSecretKey(secretKey string) { + conf.secretKey = secretKey +} + +// Validate checks to see the config options are set. +func (conf *BucketConfig) Validate() error { + switch conf.Provider { + case GCS: + if conf.Bucket == "" { + return ErrMissingGCSBucket + } + case S3: + if conf.Endpoint == "" || + (conf.AccessKey == "" && conf.secretKey != "") || + (conf.AccessKey != "" && conf.secretKey == "") { + return ErrInsufficientS3Info + } + default: + return ErrUnsupported + } + return nil +} + +// ValidateForTests checks to see the config options for tests are set. +func (conf *BucketConfig) ValidateForTests() error { + switch conf.Provider { + case GCS: + if conf.Bucket == "" { + return ErrMissingGCSBucket + } + case S3: + if conf.Endpoint == "" || + conf.AccessKey == "" || + conf.secretKey == "" { + return ErrInsufficientS3Info + } + default: + return ErrUnsupported + } + return nil +} From 5bfbd287fed78acb4a13584901216f6f8d7fc3f2 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 15:01:43 +0800 Subject: [PATCH 02/43] Use the unified bucket config for components Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 25 ++++++++++--------------- cmd/thanos/compact.go | 15 +++++---------- cmd/thanos/downsample.go | 13 ++++--------- cmd/thanos/rule.go | 21 ++++++++------------- cmd/thanos/sidecar.go | 23 +++++++++-------------- cmd/thanos/store.go | 17 ++++++----------- pkg/objstore/client/factory.go | 24 ++++++++++++------------ pkg/objstore/s3/s3.go | 10 +++++----- 8 files changed, 59 insertions(+), 89 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index fa8c75797e..dd21fd82c1 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -11,8 +11,8 @@ import ( "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/block" + "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/verifier" "github.com/oklog/run" @@ -42,35 +42,30 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - gcsBucket := cmd.Flag("gcs-bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConf := objstore.NewBucketConfig(cmd) // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. - verifyBackupGCSBucket := cmd.Flag("gcs-backup-bucket", "Google Cloud Storage bucket name to backup blocks on repair operations."). - PlaceHolder("").String() - verifyBackupS3Bucket := cmd.Flag("s3-backup-bucket", "S3 bucket name to backup blocks on repair operations."). - PlaceHolder("").String() + backupBucketConf := objstore.NewBackupBucketConfig(verify) verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ "If none is specified, all blocks will be verified. Repeated field").Strings() m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, gcsBucket, *s3Config, reg, name) + bkt, err := client.NewBucket(logger, bucketConf, reg, name) if err != nil { return err } defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - backupS3Config := *s3Config - backupS3Config.Bucket = *verifyBackupS3Bucket - backupBkt, err := client.NewBucket(logger, verifyBackupGCSBucket, backupS3Config, reg, name) - if err == client.ErrNotFound { + backupBucketConfig := *bucketConf + backupBucketConfig.Provider = backupBucketConf.Provider + backupBucketConfig.Bucket = backupBucketConf.Bucket + backupBkt, err := client.NewBucket(logger, &backupBucketConfig, reg, name) + if err == objstore.ErrUnsupported { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") } @@ -129,7 +124,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin lsOutput := ls.Flag("output", "Format in which to print each block's information. May be 'json' or custom template."). Short('o').Default("").String() m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, gcsBucket, *s3Config, reg, name) + bkt, err := client.NewBucket(logger, bucketConf, reg, name) if err != nil { return err } diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 0781285523..14b941ca26 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -10,8 +10,8 @@ import ( "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/compact" "github.com/improbable-eng/thanos/pkg/compact/downsample" + "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/oklog/run" "github.com/opentracing/opentracing-go" @@ -32,10 +32,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").String() - - s3config := s3.RegisterS3Params(cmd) + bucketConf := objstore.NewBucketConfig(cmd) syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) @@ -56,8 +53,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri return runCompact(g, logger, reg, *httpAddr, *dataDir, - *gcsBucket, - s3config, + bucketConf, time.Duration(*syncDelay), *haltOnError, *wait, @@ -78,8 +74,7 @@ func runCompact( reg *prometheus.Registry, httpBindAddr string, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConf *objstore.BucketConfig, syncDelay time.Duration, haltOnError bool, wait bool, @@ -99,7 +94,7 @@ func runCompact( reg.MustRegister(halted) - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 6ea50f8a07..130b98db12 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -16,7 +16,6 @@ import ( "github.com/improbable-eng/thanos/pkg/compact/downsample" "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/oklog/run" "github.com/oklog/ulid" @@ -33,13 +32,10 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks."). - PlaceHolder("").Required().String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConf := objstore.NewBucketConfig(cmd) m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - return runDownsample(g, logger, reg, *dataDir, *gcsBucket, s3Config, name) + return runDownsample(g, logger, reg, *dataDir, bucketConf, name) } } @@ -48,12 +44,11 @@ func runDownsample( logger log.Logger, reg *prometheus.Registry, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConf *objstore.BucketConfig, component string, ) error { - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 2bb8e16944..616fdd8ace 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -24,8 +24,8 @@ import ( "github.com/improbable-eng/thanos/pkg/alert" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" + "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/shipper" "github.com/improbable-eng/thanos/pkg/store" @@ -72,12 +72,9 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager URLs to push firing alerts to. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path."). Strings() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty, ruler won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - s3Config := s3.RegisterS3Params(cmd) + bucketConf := objstore.NewBucketConfig(cmd) m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) @@ -112,8 +109,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) *dataDir, *ruleFiles, peer, - *gcsBucket, - s3Config, + bucketConf, tsdbOpts, name, alertQueryURL, @@ -136,8 +132,7 @@ func runRule( dataDir string, ruleFiles []string, peer *cluster.Peer, - gcsBucket string, - s3Config *s3.Config, + bucketConf *objstore.BucketConfig, tsdbOpts *tsdb.Options, component string, alertQueryURL *url.URL, @@ -415,13 +410,13 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) - if err != nil && err != client.ErrNotFound { + bkt, err := client.NewBucket(logger, bucketConf, reg, component) + if err != nil && err != objstore.ErrUnsupported { return err } - if err == client.ErrNotFound { - level.Info(logger).Log("msg", "No GCS or S3 bucket was configured, uploads will be disabled") + if err == objstore.ErrUnsupported { + level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 284c2fc651..8247ce848e 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -15,8 +15,8 @@ import ( "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" + "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/reloader" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/shipper" @@ -43,11 +43,6 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty, sidecar won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) - reloaderCfgFile := cmd.Flag("reloader.config-file", "Config file watched by the reloader."). Default("").String() @@ -56,6 +51,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() + bucketConf := objstore.NewBucketConfig(cmd) + m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( log.With(logger, "component", "reloader"), @@ -77,8 +74,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri *httpBindAddr, *promURL, *dataDir, - *gcsBucket, - s3Config, + bucketConf, peer, rl, name, @@ -95,8 +91,7 @@ func runSidecar( httpBindAddr string, promURL *url.URL, dataDir string, - gcsBucket string, - s3Config *s3.Config, + bucketConf *objstore.BucketConfig, peer *cluster.Peer, reloader *reloader.Reloader, component string, @@ -224,13 +219,13 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) - if err != nil && err != client.ErrNotFound { + bkt, err := client.NewBucket(logger, bucketConf, reg, component) + if err != nil && err != objstore.ErrUnsupported { return err } - if err == client.ErrNotFound { - level.Info(logger).Log("msg", "No GCS or S3 bucket was configured, uploads will be disabled") + if err == objstore.ErrUnsupported { + level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0e92c8c774..c5886d4a92 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -9,8 +9,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/cluster" + "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" - "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/store" "github.com/improbable-eng/thanos/pkg/store/storepb" @@ -24,17 +24,14 @@ import ( // registerStore registers a store command. func registerStore(m map[string]setupFunc, app *kingpin.Application, name string) { - cmd := app.Command(name, "store node giving access to blocks in a GCS bucket") + cmd := app.Command(name, "store node giving access to blocks in a supported bucket provider") grpcBindAddr, httpBindAddr, newPeerFn := regCommonServerFlags(cmd) dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - gcsBucket := cmd.Flag("gcs.bucket", "Google Cloud Storage bucket name for stored blocks. If empty sidecar won't store any block inside Google Cloud Storage."). - PlaceHolder("").String() - - s3Config := s3.RegisterS3Params(cmd) + bucketConf := objstore.NewBucketConfig(cmd) indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() @@ -51,8 +48,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string logger, reg, tracer, - *gcsBucket, - s3Config, + bucketConf, *dataDir, *grpcBindAddr, *httpBindAddr, @@ -71,8 +67,7 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - gcsBucket string, - s3Config *s3.Config, + bucketConf *objstore.BucketConfig, dataDir string, grpcBindAddr string, httpBindAddr string, @@ -83,7 +78,7 @@ func runStore( verbose bool, ) error { { - bkt, err := client.NewBucket(logger, &gcsBucket, *s3Config, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 7324bfc5e3..13f828fce0 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -16,26 +16,26 @@ import ( "google.golang.org/api/option" ) -var ErrNotFound = errors.New("no valid GCS or S3 configuration supplied") - // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, gcsBucket *string, s3Config s3.Config, reg *prometheus.Registry, component string) (objstore.Bucket, error) { - if *gcsBucket != "" { +func NewBucket(logger log.Logger, bucketConf *objstore.BucketConfig, reg *prometheus.Registry, component string) (objstore.Bucket, error) { + err := bucketConf.Validate() + if err != nil { + return nil, err + } + switch bucketConf.Provider { + case objstore.GCS: gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) gcsClient, err := storage.NewClient(context.Background(), gcsOptions) if err != nil { return nil, errors.Wrap(err, "create GCS client") } - return objstore.BucketWithMetrics(*gcsBucket, gcs.NewBucket(*gcsBucket, gcsClient, reg), reg), nil - } - - if s3Config.Validate() == nil { - b, err := s3.NewBucket(logger, &s3Config, reg, component) + return objstore.BucketWithMetrics(bucketConf.Bucket, gcs.NewBucket(bucketConf.Bucket, gcsClient, reg), reg), nil + case objstore.S3: + b, err := s3.NewBucket(logger, bucketConf, reg, component) if err != nil { return nil, errors.Wrap(err, "create s3 client") } - return objstore.BucketWithMetrics(s3Config.Bucket, b, reg), nil + return objstore.BucketWithMetrics(bucketConf.Bucket, b, reg), nil } - - return nil, ErrNotFound + return nil, objstore.ErrUnsupported } diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 9678cd288b..ca5838f646 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -107,7 +107,7 @@ func (conf *Config) ValidateForTests() error { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf *objstore.BucketConfig, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider if conf.AccessKey != "" { signature := credentials.SignatureV4 @@ -118,7 +118,7 @@ func NewBucket(logger log.Logger, conf *Config, reg prometheus.Registerer, compo chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ AccessKeyID: conf.AccessKey, - SecretAccessKey: conf.secretKey, + SecretAccessKey: conf.BucketSecretKey(), SignerType: signature, }, }} @@ -285,14 +285,14 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { func (b *Bucket) Close() error { return nil } -func configFromEnv() *Config { - c := &Config{ +func configFromEnv() *objstore.BucketConfig { + c := &objstore.BucketConfig{ Bucket: os.Getenv("S3_BUCKET"), Endpoint: os.Getenv("S3_ENDPOINT"), AccessKey: os.Getenv("S3_ACCESS_KEY"), - secretKey: os.Getenv("S3_SECRET_KEY"), } + c.SetSecretKey(os.Getenv("S3_SECRET_KEY")) insecure, err := strconv.ParseBool(os.Getenv("S3_INSECURE")) if err != nil { c.Insecure = insecure From 51801e6b2727d5a42cc959eceaac474fe7e45edb Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 15:03:06 +0800 Subject: [PATCH 03/43] Use double quotes instead of single quotes Signed-off-by: jojohappy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 23beaab07b..fe98a7b6d6 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ tarballs-release: $(PROMU) # test runs all Thanos golang tests against each supported version of Prometheus. .PHONY: test test: test-deps - @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS="true" or/and export THANOS_SKIP_S3_AWS_TESTS="true" if you want to skip e2e tests against real store buckets" + @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and export THANOS_SKIP_S3_AWS_TESTS='true' if you want to skip e2e tests against real store buckets" @for ver in $(SUPPORTED_PROM_VERSIONS); do \ THANOS_TEST_PROMETHEUS_PATH="prometheus-$$ver" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/ | grep -v /benchmark/); \ done From c8e311b99c021668f57c18a516defda68c3436aa Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 15:28:09 +0800 Subject: [PATCH 04/43] Fixed missing flags in man page Signed-off-by: jojohappy --- pkg/objstore/config.go | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go index 11a6417674..2a1d00a88d 100644 --- a/pkg/objstore/config.go +++ b/pkg/objstore/config.go @@ -38,36 +38,30 @@ type BucketConfig struct { // TODO(jojohappy) should it support multiple bucket? func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - provider := cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported provider are GCS / S3."). + provider := cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). PlaceHolder("").String() bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) cmd.Flag("provider.bucket", "The bucket name for stored blocks."). PlaceHolder("").Envar("PROVIDER_BUCKET").StringVar(&bucketConfig.Bucket) - switch bucketConfig.Provider { - case GCS: - case S3: - cmd.Flag("provider.endpoint", "The object store API endpoint for stored blocks. Support S3-Compatible API"). - PlaceHolder("").Envar("PROVIDER_ENDPOINT").StringVar(&bucketConfig.Endpoint) + cmd.Flag("provider.endpoint", "The object store API endpoint for stored blocks. Support S3-Compatible API"). + PlaceHolder("").Envar("PROVIDER_ENDPOINT").StringVar(&bucketConfig.Endpoint) - cmd.Flag("provider.access-key", "Access key for an object store API. Support S3-Compatible API"). - PlaceHolder("").Envar("PROVIDER_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) + cmd.Flag("provider.access-key", "Access key for an object store API. Support S3-Compatible API"). + PlaceHolder("").Envar("PROVIDER_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) - // TODO(jojohappy): should it be encrypted? - bucketConfig.secretKey = os.Getenv("PROVIDER_SECRET_KEY") + // TODO(jojohappy): should it be encrypted? + bucketConfig.secretKey = os.Getenv("PROVIDER_SECRET_KEY") - cmd.Flag("provider.insecure", "Whether to use an insecure connection with an object store API. Support S3-Compatible API"). - Default("false").Envar("PROVIDER_INSECURE").BoolVar(&bucketConfig.Insecure) + cmd.Flag("provider.insecure", "Whether to use an insecure connection with an object store API. Support S3-Compatible API"). + Default("false").Envar("PROVIDER_INSECURE").BoolVar(&bucketConfig.Insecure) - cmd.Flag("provider.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). - Default("false").Envar("PROVIDER_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) + cmd.Flag("provider.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). + Default("false").Envar("PROVIDER_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) - cmd.Flag("provider.encrypt-sse", "Whether to use Server Side Encryption"). - Default("false").Envar("PROVIDER_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) - default: - bucketConfig.Provider = Unsupported - } + cmd.Flag("provider.encrypt-sse", "Whether to use Server Side Encryption"). + Default("false").Envar("PROVIDER_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) return &bucketConfig } @@ -75,20 +69,13 @@ func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { // NewBackupBucketConfig return the configuration of backup object store func NewBackupBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - provider := cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported provider are GCS / S3."). + provider := cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). PlaceHolder("").String() bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) cmd.Flag("provider-backup.bucket", "The bucket name for backup stored blocks."). PlaceHolder("").StringVar(&bucketConfig.Bucket) - switch bucketConfig.Provider { - case GCS: - case S3: - default: - bucketConfig.Provider = Unsupported - } - return &bucketConfig } From f5fda0c1f9e0ff7c9bf20ebad59a53c1699bd862 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 17:34:06 +0800 Subject: [PATCH 05/43] To use value of bucket config instead of pointer to get value from command-line arguments Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 8 +++---- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 6 ++--- cmd/thanos/sidecar.go | 6 ++--- cmd/thanos/store.go | 2 +- pkg/objstore/client/factory.go | 6 ++--- pkg/objstore/config.go | 40 ++++++++++++++++------------------ pkg/objstore/s3/s3.go | 4 ++-- 9 files changed, 37 insertions(+), 39 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index dd21fd82c1..e0ce42e0ab 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -55,7 +55,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ "If none is specified, all blocks will be verified. Repeated field").Strings() m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, bucketConf, reg, name) + bkt, err := client.NewBucket(logger, *bucketConf, reg, name) if err != nil { return err } @@ -64,8 +64,8 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin backupBucketConfig := *bucketConf backupBucketConfig.Provider = backupBucketConf.Provider backupBucketConfig.Bucket = backupBucketConf.Bucket - backupBkt, err := client.NewBucket(logger, &backupBucketConfig, reg, name) - if err == objstore.ErrUnsupported { + backupBkt, err := client.NewBucket(logger, backupBucketConfig, reg, name) + if err == objstore.ErrNotFound { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") } @@ -124,7 +124,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin lsOutput := ls.Flag("output", "Format in which to print each block's information. May be 'json' or custom template."). Short('o').Default("").String() m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, bucketConf, reg, name) + bkt, err := client.NewBucket(logger, *bucketConf, reg, name) if err != nil { return err } diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 14b941ca26..be85e6a3a2 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -94,7 +94,7 @@ func runCompact( reg.MustRegister(halted) - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, *bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 130b98db12..56172ef350 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -48,7 +48,7 @@ func runDownsample( component string, ) error { - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, *bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 616fdd8ace..c5348aec12 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -410,12 +410,12 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, bucketConf, reg, component) - if err != nil && err != objstore.ErrUnsupported { + bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + if err != nil && err != objstore.ErrNotFound { return err } - if err == objstore.ErrUnsupported { + if err == objstore.ErrNotFound { level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 8247ce848e..370da80637 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -219,12 +219,12 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, bucketConf, reg, component) - if err != nil && err != objstore.ErrUnsupported { + bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + if err != nil && err != objstore.ErrNotFound { return err } - if err == objstore.ErrUnsupported { + if err == objstore.ErrNotFound { level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index c5886d4a92..8bf80b6d00 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -78,7 +78,7 @@ func runStore( verbose bool, ) error { { - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, *bucketConf, reg, component) if err != nil { return err } diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 13f828fce0..6461eace32 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -17,12 +17,12 @@ import ( ) // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, bucketConf *objstore.BucketConfig, reg *prometheus.Registry, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, bucketConf objstore.BucketConfig, reg *prometheus.Registry, component string) (objstore.Bucket, error) { err := bucketConf.Validate() if err != nil { return nil, err } - switch bucketConf.Provider { + switch bucketConf.GetProvider() { case objstore.GCS: gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) gcsClient, err := storage.NewClient(context.Background(), gcsOptions) @@ -37,5 +37,5 @@ func NewBucket(logger log.Logger, bucketConf *objstore.BucketConfig, reg *promet } return objstore.BucketWithMetrics(bucketConf.Bucket, b, reg), nil } - return nil, objstore.ErrUnsupported + return nil, objstore.ErrNotFound } diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go index 2a1d00a88d..8b95f83391 100644 --- a/pkg/objstore/config.go +++ b/pkg/objstore/config.go @@ -3,28 +3,27 @@ package objstore import ( "fmt" "os" - "strings" "github.com/pkg/errors" "gopkg.in/alecthomas/kingpin.v2" ) -type ObjProvider string +type objProvider string const ( - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - Unsupported ObjProvider = "UNSUPPORTED" + GCS objProvider = "GCS" + S3 objProvider = "S3" + Unsupported objProvider = "UNSUPPORTED" ) -var ErrUnsupported = errors.New("unsupported provider") +var ErrNotFound = errors.New("no valid provider configuration supplied") var ErrMissingGCSBucket = errors.New("missing Google Cloud Storage bucket name for stored blocks") var ErrInsufficientS3Info = errors.New("insufficient s3 configuration information") // BucketConfig encapsulates the necessary config values to instantiate an object store client. // Use Provider to represent the Object Stores type BucketConfig struct { - Provider ObjProvider + Provider string Bucket string Endpoint string AccessKey string @@ -38,10 +37,10 @@ type BucketConfig struct { // TODO(jojohappy) should it support multiple bucket? func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - provider := cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). - PlaceHolder("").String() - bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) + cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). + PlaceHolder("").StringVar(&bucketConfig.Provider) + cmd.Flag("provider.bucket", "The bucket name for stored blocks."). PlaceHolder("").Envar("PROVIDER_BUCKET").StringVar(&bucketConfig.Bucket) @@ -69,24 +68,23 @@ func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { // NewBackupBucketConfig return the configuration of backup object store func NewBackupBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - provider := cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). - PlaceHolder("").String() - bucketConfig.Provider = ObjProvider(strings.ToUpper(strings.Trim(*provider, " "))) + cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). + PlaceHolder("").StringVar(&bucketConfig.Provider) cmd.Flag("provider-backup.bucket", "The bucket name for backup stored blocks."). PlaceHolder("").StringVar(&bucketConfig.Bucket) return &bucketConfig } -// Provider return the provider of Object Store -func (conf *BucketConfig) BucketProvider() string { - return string(conf.Provider) +// GetProvider return the provider type +func (conf *BucketConfig) GetProvider() objProvider { + return objProvider(conf.Provider) } // String returns the Provider information func (conf *BucketConfig) String() string { - return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", conf.BucketProvider(), conf.Bucket, conf.Endpoint) + return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", conf.Provider, conf.Bucket, conf.Endpoint) } // BucketSecretKey returns the Provider Secret Key @@ -102,7 +100,7 @@ func (conf *BucketConfig) SetSecretKey(secretKey string) { // Validate checks to see the config options are set. func (conf *BucketConfig) Validate() error { - switch conf.Provider { + switch objProvider(conf.Provider) { case GCS: if conf.Bucket == "" { return ErrMissingGCSBucket @@ -114,14 +112,14 @@ func (conf *BucketConfig) Validate() error { return ErrInsufficientS3Info } default: - return ErrUnsupported + return ErrNotFound } return nil } // ValidateForTests checks to see the config options for tests are set. func (conf *BucketConfig) ValidateForTests() error { - switch conf.Provider { + switch objProvider(conf.Provider) { case GCS: if conf.Bucket == "" { return ErrMissingGCSBucket @@ -133,7 +131,7 @@ func (conf *BucketConfig) ValidateForTests() error { return ErrInsufficientS3Info } default: - return ErrUnsupported + return ErrNotFound } return nil } diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index ca5838f646..b54b803c48 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -107,7 +107,7 @@ func (conf *Config) ValidateForTests() error { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf *objstore.BucketConfig, reg prometheus.Registerer, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider if conf.AccessKey != "" { signature := credentials.SignatureV4 @@ -312,7 +312,7 @@ func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), erro return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), c, nil, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), *c, nil, "thanos-e2e-test") if err != nil { return nil, nil, err } From af99b66d9a51a9447346069a550f5352a04ba49c Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 17:41:34 +0800 Subject: [PATCH 06/43] Remove useless code Signed-off-by: jojohappy --- pkg/objstore/s3/s3.go | 60 ------------------------------------------- 1 file changed, 60 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index b54b803c48..b10a39f20f 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" - "gopkg.in/alecthomas/kingpin.v2" ) const ( @@ -47,65 +46,6 @@ type Bucket struct { opsTotal *prometheus.CounterVec } -// Config encapsulates the necessary config values to instantiate an s3 client. -type Config struct { - Bucket string - Endpoint string - AccessKey string - secretKey string - Insecure bool - SignatureV2 bool - SSEEncryption bool -} - -// RegisterS3Params registers the s3 flags and returns an initialized Config struct. -func RegisterS3Params(cmd *kingpin.CmdClause) *Config { - var s3config Config - - cmd.Flag("s3.bucket", "S3-Compatible API bucket name for stored blocks."). - PlaceHolder("").Envar("S3_BUCKET").StringVar(&s3config.Bucket) - - cmd.Flag("s3.endpoint", "S3-Compatible API endpoint for stored blocks."). - PlaceHolder("").Envar("S3_ENDPOINT").StringVar(&s3config.Endpoint) - - cmd.Flag("s3.access-key", "Access key for an S3-Compatible API."). - PlaceHolder("").Envar("S3_ACCESS_KEY").StringVar(&s3config.AccessKey) - - s3config.secretKey = os.Getenv("S3_SECRET_KEY") - - cmd.Flag("s3.insecure", "Whether to use an insecure connection with an S3-Compatible API."). - Default("false").Envar("S3_INSECURE").BoolVar(&s3config.Insecure) - - cmd.Flag("s3.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used."). - Default("false").Envar("S3_SIGNATURE_VERSION2").BoolVar(&s3config.SignatureV2) - - cmd.Flag("s3.encrypt-sse", "Whether to use Server Side Encryption"). - Default("false").Envar("S3_SSE_ENCRYPTION").BoolVar(&s3config.SSEEncryption) - - return &s3config -} - -// Validate checks to see if mandatory s3 config options are set. -func (conf *Config) Validate() error { - if conf.Bucket == "" || - conf.Endpoint == "" || - (conf.AccessKey == "" && conf.secretKey != "") || - (conf.AccessKey != "" && conf.secretKey == "") { - return errors.New("insufficient s3 configuration information") - } - return nil -} - -// ValidateForTests checks to see if mandatory s3 config options for tests are set. -func (conf *Config) ValidateForTests() error { - if conf.Endpoint == "" || - conf.AccessKey == "" || - conf.secretKey == "" { - return errors.New("insufficient s3 test configuration information") - } - return nil -} - // NewBucket returns a new Bucket using the provided s3 config values. func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider From 75111fc0e7627906382192a60cd311c1a29181e8 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 31 Aug 2018 18:26:51 +0800 Subject: [PATCH 07/43] Update documents Signed-off-by: jojohappy --- docs/components/bucket.md | 192 ++++++++++++++++++++----------------- docs/components/compact.md | 63 ++++++------ docs/components/rule.md | 30 +++--- docs/components/sidecar.md | 31 +++--- docs/components/store.md | 32 ++++--- 5 files changed, 198 insertions(+), 150 deletions(-) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 5a37dafaa2..5d964fc6ef 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -23,34 +23,37 @@ usage: thanos bucket [] [ ...] inspect metric data in an object storage bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings + to. If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. + How often we send traces (1/). If 0 + no trace will be sent periodically, unless forced + by baggage item. See `pkg/tracing/tracing.go` for + details. + --provider.type= + Specify the provider for object store. If empty or + unsupport provider, Thanos won't read and store + any block to the object store. Now supported GCS / + S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption Subcommands: bucket verify [] @@ -79,44 +82,54 @@ usage: thanos bucket verify [] verify all blocks in the bucket against specified issues Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings + to. If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. - -r, --repair attempt to repair blocks for which issues were - detected + How often we send traces (1/). If 0 + no trace will be sent periodically, unless forced + by baggage item. See `pkg/tracing/tracing.go` for + details. + --provider.type= + Specify the provider for object store. If empty or + unsupport provider, Thanos won't read and store + any block to the object store. Now supported GCS / + S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption + -r, --repair attempt to repair blocks for which issues were + detected + --provider-backup.type= + Specify the provider for backup object store. If + empty or unsupport provider, Thanos won't backup + any block to the object store. Now supported GCS / + S3. + --provider-backup.bucket= + The bucket name for backup stored blocks. -i, --issues=index_issue... ... - Issues to verify (and optionally repair). - Possible values: [duplicated_compaction - index_issue overlapped_blocks] + Issues to verify (and optionally repair). Possible + values: [duplicated_compaction index_issue + overlapped_blocks] --id-whitelist=ID-WHITELIST ... - Block IDs to verify (and optionally repair) only. - If none is specified, all blocks will be - verified. Repeated field + Block IDs to verify (and optionally repair) only. + If none is specified, all blocks will be verified. + Repeated field ``` @@ -137,36 +150,39 @@ usage: thanos bucket ls [] list all blocks in the bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings + to. If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. - --gcs-bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --gcs-backup-bucket= - Google Cloud Storage bucket name to backup blocks - on repair operations. - --s3-backup-bucket= - S3 bucket name to backup blocks on repair - operations. - -o, --output="" Format in which to print each block's - information. May be 'json' or custom template. + How often we send traces (1/). If 0 + no trace will be sent periodically, unless forced + by baggage item. See `pkg/tracing/tracing.go` for + details. + --provider.type= + Specify the provider for object store. If empty or + unsupport provider, Thanos won't read and store + any block to the object store. Now supported GCS / + S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption + -o, --output="" Format in which to print each block's information. + May be 'json' or custom template. ``` diff --git a/docs/components/compact.md b/docs/components/compact.md index 45e730b87c..73e8e1e08d 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -23,34 +23,43 @@ usage: thanos compact [] continuously compacts blocks in an object store bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings + to. If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If - 0 no trace will be sent periodically, unless - forced by baggage item. See - `pkg/tracing/tracing.go` for details. + How often we send traces (1/). If 0 + no trace will be sent periodically, unless forced + by baggage item. See `pkg/tracing/tracing.go` for + details. --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --data-dir="./data" Data directory in which to cache blocks and - process compactions. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption - --sync-delay=30m Minimum age of fresh (non-compacted) blocks - before they are being processed. + Listen host:port for HTTP endpoints. + --data-dir="./data" Data directory in which to cache blocks and + process compactions. + --provider.type= + Specify the provider for object store. If empty or + unsupport provider, Thanos won't read and store + any block to the object store. Now supported GCS / + S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption + --sync-delay=30m Minimum age of fresh (non-compacted) blocks before + they are being processed. --retention.resolution-raw=0d How long to retain raw samples in bucket. 0d - disables this retention @@ -60,7 +69,7 @@ Flags: --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 hour) in bucket. 0d - disables this retention - -w, --wait Do not exit after all compactions have been - processed and wait for new work. + -w, --wait Do not exit after all compactions have been + processed and wait for new work. ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index 4f82600266..b9efbe9258 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -108,19 +108,27 @@ Flags: DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty, ruler won't store any block - inside Google Cloud Storage. --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption + --provider.type= + Specify the provider for object store. If empty + or unsupport provider, Thanos won't read and + store any block to the object store. Now + supported GCS / S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 2fd00b147e..d67059eb11 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -92,18 +92,6 @@ Flags: URL at which to reach Prometheus's API. For better performance use local network. --tsdb.path="./data" Data directory of TSDB. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty, sidecar won't store any block - inside Google Cloud Storage. - --s3.bucket= S3-Compatible API bucket name for stored - blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; - otherwise Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption --reloader.config-file="" Config file watched by the reloader. --reloader.config-envsubst-file="" Output file for environment variable @@ -111,6 +99,25 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). + --provider.type= + Specify the provider for object store. If empty + or unsupport provider, Thanos won't read and + store any block to the object store. Now + supported GCS / S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored + blocks. Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; + otherwise Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption ``` diff --git a/docs/components/store.md b/docs/components/store.md index 55d417304b..a2e3c07bc9 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -19,7 +19,7 @@ In general about 1MB of local disk space is required per TSDB block stored in th ```$ usage: thanos store [] -store node giving access to blocks in a GCS bucket +store node giving access to blocks in a supported bucket provider Flags: -h, --help Show context-sensitive help (also try @@ -81,17 +81,25 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --gcs.bucket= Google Cloud Storage bucket name for stored - blocks. If empty sidecar won't store any block - inside Google Cloud Storage. - --s3.bucket= S3-Compatible API bucket name for stored blocks. - --s3.endpoint= S3-Compatible API endpoint for stored blocks. - --s3.access-key= Access key for an S3-Compatible API. - --s3.insecure Whether to use an insecure connection with an - S3-Compatible API. - --s3.signature-version2 Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used. - --s3.encrypt-sse Whether to use Server Side Encryption + --provider.type= + Specify the provider for object store. If empty + or unsupport provider, Thanos won't read and + store any block to the object store. Now + supported GCS / S3. + --provider.bucket= + The bucket name for stored blocks. + --provider.endpoint= + The object store API endpoint for stored blocks. + Support S3-Compatible API + --provider.access-key= + Access key for an object store API. Support + S3-Compatible API + --provider.insecure Whether to use an insecure connection with an + object store API. Support S3-Compatible API + --provider.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --provider.encrypt-sse Whether to use Server Side Encryption --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. From 04c94bd5bdc0283d6b3a5086c9c7b202874f6c97 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 3 Sep 2018 12:58:49 +0800 Subject: [PATCH 08/43] Update - Rename provider to objstore for flags - To use objProvider instead of string for Provider Type - Get rid of bucket configuration errors Signed-off-by: jojohappy --- pkg/objstore/config.go | 110 ++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 73 deletions(-) diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go index 8b95f83391..152d56e0e7 100644 --- a/pkg/objstore/config.go +++ b/pkg/objstore/config.go @@ -4,26 +4,34 @@ import ( "fmt" "os" - "github.com/pkg/errors" "gopkg.in/alecthomas/kingpin.v2" ) type objProvider string const ( - GCS objProvider = "GCS" - S3 objProvider = "S3" - Unsupported objProvider = "UNSUPPORTED" + GCS objProvider = "GCS" + S3 objProvider = "S3" ) -var ErrNotFound = errors.New("no valid provider configuration supplied") -var ErrMissingGCSBucket = errors.New("missing Google Cloud Storage bucket name for stored blocks") -var ErrInsufficientS3Info = errors.New("insufficient s3 configuration information") +func newObjProvider(s kingpin.Settings, op *objProvider) { + s.SetValue(op) + return +} + +func (p *objProvider) Set(value string) error { + *p = objProvider(value) + return nil +} + +func (p *objProvider) String() string { + return string(*p) +} // BucketConfig encapsulates the necessary config values to instantiate an object store client. // Use Provider to represent the Object Stores type BucketConfig struct { - Provider string + Provider objProvider Bucket string Endpoint string AccessKey string @@ -38,29 +46,28 @@ type BucketConfig struct { func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - cmd.Flag("provider.type", "Specify the provider for object store. If empty or unsupport provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). - PlaceHolder("").StringVar(&bucketConfig.Provider) + newObjProvider(cmd.Flag("objstore.type", "Specify the provider for object store. If empty or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). + PlaceHolder(""), &bucketConfig.Provider) - cmd.Flag("provider.bucket", "The bucket name for stored blocks."). - PlaceHolder("").Envar("PROVIDER_BUCKET").StringVar(&bucketConfig.Bucket) + cmd.Flag("objstore.bucket", "The bucket name for stored blocks."). + PlaceHolder("").Envar("OBJSTORE_BUCKET").StringVar(&bucketConfig.Bucket) - cmd.Flag("provider.endpoint", "The object store API endpoint for stored blocks. Support S3-Compatible API"). - PlaceHolder("").Envar("PROVIDER_ENDPOINT").StringVar(&bucketConfig.Endpoint) + cmd.Flag("objstore.endpoint", "The object store API endpoint for stored blocks. Supported S3-Compatible API"). + PlaceHolder("").Envar("OBJSTORE_ENDPOINT").StringVar(&bucketConfig.Endpoint) - cmd.Flag("provider.access-key", "Access key for an object store API. Support S3-Compatible API"). - PlaceHolder("").Envar("PROVIDER_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) + cmd.Flag("objstore.access-key", "Access key for an object store API. Supported S3-Compatible API"). + PlaceHolder("").Envar("OBJSTORE_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) - // TODO(jojohappy): should it be encrypted? bucketConfig.secretKey = os.Getenv("PROVIDER_SECRET_KEY") - cmd.Flag("provider.insecure", "Whether to use an insecure connection with an object store API. Support S3-Compatible API"). - Default("false").Envar("PROVIDER_INSECURE").BoolVar(&bucketConfig.Insecure) + cmd.Flag("objstore.insecure", "Whether to use an insecure connection with an object store API. Supported S3-Compatible API"). + Default("false").Envar("OBJSTORE_INSECURE").BoolVar(&bucketConfig.Insecure) - cmd.Flag("provider.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). - Default("false").Envar("PROVIDER_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) + cmd.Flag("objstore.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). + Default("false").Envar("OBJSTORE_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) - cmd.Flag("provider.encrypt-sse", "Whether to use Server Side Encryption"). - Default("false").Envar("PROVIDER_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) + cmd.Flag("objstore.encrypt-sse", "Whether to use Server Side Encryption"). + Default("false").Envar("OBJSTORE_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) return &bucketConfig } @@ -69,27 +76,22 @@ func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { func NewBackupBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { var bucketConfig BucketConfig - cmd.Flag("provider-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). - PlaceHolder("").StringVar(&bucketConfig.Provider) - cmd.Flag("provider-backup.bucket", "The bucket name for backup stored blocks."). + newObjProvider(cmd.Flag("objstore-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). + PlaceHolder(""), &bucketConfig.Provider) + + cmd.Flag("objstore-backup.bucket", "The bucket name for backup stored blocks."). PlaceHolder("").StringVar(&bucketConfig.Bucket) return &bucketConfig } -// GetProvider return the provider type -func (conf *BucketConfig) GetProvider() objProvider { - return objProvider(conf.Provider) -} - // String returns the Provider information func (conf *BucketConfig) String() string { - return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", conf.Provider, conf.Bucket, conf.Endpoint) + return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", string(conf.Provider), conf.Bucket, conf.Endpoint) } -// BucketSecretKey returns the Provider Secret Key -// TODO(jojohappy): it should return the key decrypted. -func (conf *BucketConfig) BucketSecretKey() string { +// GetSecretKey returns the Provider Secret Key +func (conf *BucketConfig) GetSecretKey() string { return conf.secretKey } @@ -97,41 +99,3 @@ func (conf *BucketConfig) BucketSecretKey() string { func (conf *BucketConfig) SetSecretKey(secretKey string) { conf.secretKey = secretKey } - -// Validate checks to see the config options are set. -func (conf *BucketConfig) Validate() error { - switch objProvider(conf.Provider) { - case GCS: - if conf.Bucket == "" { - return ErrMissingGCSBucket - } - case S3: - if conf.Endpoint == "" || - (conf.AccessKey == "" && conf.secretKey != "") || - (conf.AccessKey != "" && conf.secretKey == "") { - return ErrInsufficientS3Info - } - default: - return ErrNotFound - } - return nil -} - -// ValidateForTests checks to see the config options for tests are set. -func (conf *BucketConfig) ValidateForTests() error { - switch objProvider(conf.Provider) { - case GCS: - if conf.Bucket == "" { - return ErrMissingGCSBucket - } - case S3: - if conf.Endpoint == "" || - conf.AccessKey == "" || - conf.secretKey == "" { - return ErrInsufficientS3Info - } - default: - return ErrNotFound - } - return nil -} From 1e40dab77839b02bbdaa5591df71069154823feb Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 3 Sep 2018 12:59:53 +0800 Subject: [PATCH 09/43] Change errors package Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 2 +- cmd/thanos/rule.go | 4 ++-- cmd/thanos/sidecar.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index e0ce42e0ab..d8a602113e 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -65,7 +65,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin backupBucketConfig.Provider = backupBucketConf.Provider backupBucketConfig.Bucket = backupBucketConf.Bucket backupBkt, err := client.NewBucket(logger, backupBucketConfig, reg, name) - if err == objstore.ErrNotFound { + if err == client.ErrNotFound { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index c5348aec12..8afe5210df 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -411,11 +411,11 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. bkt, err := client.NewBucket(logger, *bucketConf, reg, component) - if err != nil && err != objstore.ErrNotFound { + if err != nil && err != client.ErrNotFound { return err } - if err == objstore.ErrNotFound { + if err == client.ErrNotFound { level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 370da80637..e118aa4690 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -220,11 +220,11 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. bkt, err := client.NewBucket(logger, *bucketConf, reg, component) - if err != nil && err != objstore.ErrNotFound { + if err != nil && err != client.ErrNotFound { return err } - if err == objstore.ErrNotFound { + if err == client.ErrNotFound { level.Info(logger).Log("msg", "No supported bucket was configured, uploads will be disabled") uploads = false } From d340fba77040896771e2db240a57d0cc8a3a2c95 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 3 Sep 2018 13:01:09 +0800 Subject: [PATCH 10/43] To validate the configuration in each provider client Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 16 ++++++++++------ pkg/objstore/s3/s3.go | 24 ++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 6461eace32..6e5b65454a 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -16,14 +16,15 @@ import ( "google.golang.org/api/option" ) +var ErrNotFound = errors.New("not found bucket") + // NewBucket initializes and returns new object storage clients. func NewBucket(logger log.Logger, bucketConf objstore.BucketConfig, reg *prometheus.Registry, component string) (objstore.Bucket, error) { - err := bucketConf.Validate() - if err != nil { - return nil, err - } - switch bucketConf.GetProvider() { + switch bucketConf.Provider { case objstore.GCS: + if bucketConf.Bucket == "" { + return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") + } gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) gcsClient, err := storage.NewClient(context.Background(), gcsOptions) if err != nil { @@ -31,11 +32,14 @@ func NewBucket(logger log.Logger, bucketConf objstore.BucketConfig, reg *prometh } return objstore.BucketWithMetrics(bucketConf.Bucket, gcs.NewBucket(bucketConf.Bucket, gcsClient, reg), reg), nil case objstore.S3: + if err := s3.Validate(&bucketConf); err != nil { + return nil, err + } b, err := s3.NewBucket(logger, bucketConf, reg, component) if err != nil { return nil, errors.Wrap(err, "create s3 client") } return objstore.BucketWithMetrics(bucketConf.Bucket, b, reg), nil } - return nil, objstore.ErrNotFound + return nil, ErrNotFound } diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index b10a39f20f..e3f13ac86d 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -58,7 +58,7 @@ func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Reg chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ AccessKeyID: conf.AccessKey, - SecretAccessKey: conf.BucketSecretKey(), + SecretAccessKey: conf.GetSecretKey(), SignerType: signature, }, }} @@ -125,6 +125,26 @@ func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Reg return bkt, nil } +// Validate checks to see the config options are set. +func Validate(conf *objstore.BucketConfig) error { + if conf.Endpoint == "" || + (conf.AccessKey == "" && conf.GetSecretKey() != "") || + (conf.AccessKey != "" && conf.GetSecretKey() == "") { + return errors.New("insufficient s3 test configuration information") + } + return nil +} + +// ValidateForTests checks to see the config options for tests are set. +func ValidateForTests(conf *objstore.BucketConfig) error { + if conf.Endpoint == "" || + conf.AccessKey == "" || + conf.GetSecretKey() == "" { + return errors.New("insufficient s3 test configuration information") + } + return nil +} + // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { @@ -248,7 +268,7 @@ func configFromEnv() *objstore.BucketConfig { // In a close function it empties and deletes the bucket. func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error) { c := configFromEnv() - if err := c.ValidateForTests(); err != nil { + if err := ValidateForTests(c); err != nil { return nil, nil, err } From c1326913235851578bd12ccd2eb4c29d3704734a Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 3 Sep 2018 13:55:46 +0800 Subject: [PATCH 11/43] To support to make bucket configuration flag using suffix Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 4 +-- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- pkg/objstore/config.go | 53 +++++++++++++++++----------------------- 7 files changed, 30 insertions(+), 37 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index d8a602113e..a9f3630de5 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -42,14 +42,14 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. - backupBucketConf := objstore.NewBackupBucketConfig(verify) + backupBucketConf := objstore.NewBucketConfig(verify, "backup") verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index be85e6a3a2..c5451a2d5e 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -32,7 +32,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 56172ef350..6cd6e35d2d 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,7 +32,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { return runDownsample(g, logger, reg, *dataDir, bucketConf, name) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 8afe5210df..474570d55c 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -74,7 +74,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index e118aa4690..b7ebe8516d 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -51,7 +51,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 8bf80b6d00..ba42eb351a 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -31,7 +31,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd) + bucketConf := objstore.NewBucketConfig(cmd, "") indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go index 152d56e0e7..dda74383e8 100644 --- a/pkg/objstore/config.go +++ b/pkg/objstore/config.go @@ -3,6 +3,7 @@ package objstore import ( "fmt" "os" + "strings" "gopkg.in/alecthomas/kingpin.v2" ) @@ -42,45 +43,37 @@ type BucketConfig struct { } // NewBucketConfig return the configuration of object store -// TODO(jojohappy) should it support multiple bucket? -func NewBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { +func NewBucketConfig(cmd *kingpin.CmdClause, suffix string) *BucketConfig { var bucketConfig BucketConfig - - newObjProvider(cmd.Flag("objstore.type", "Specify the provider for object store. If empty or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). + flagSuffix := "" + envSuffix := "" + if strings.Trim(suffix, " ") != "" { + flagSuffix = "-" + suffix + envSuffix = "_" + strings.ToUpper(suffix) + } + + newObjProvider(cmd.Flag(fmt.Sprintf("objstore%s.type", flagSuffix), "Specify the provider for object store. If empty or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). PlaceHolder(""), &bucketConfig.Provider) - cmd.Flag("objstore.bucket", "The bucket name for stored blocks."). - PlaceHolder("").Envar("OBJSTORE_BUCKET").StringVar(&bucketConfig.Bucket) - - cmd.Flag("objstore.endpoint", "The object store API endpoint for stored blocks. Supported S3-Compatible API"). - PlaceHolder("").Envar("OBJSTORE_ENDPOINT").StringVar(&bucketConfig.Endpoint) - - cmd.Flag("objstore.access-key", "Access key for an object store API. Supported S3-Compatible API"). - PlaceHolder("").Envar("OBJSTORE_ACCESS_KEY").StringVar(&bucketConfig.AccessKey) + cmd.Flag(fmt.Sprintf("objstore%s.bucket", flagSuffix), "The bucket name for stored blocks."). + PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_BUCKET", envSuffix)).StringVar(&bucketConfig.Bucket) - bucketConfig.secretKey = os.Getenv("PROVIDER_SECRET_KEY") + cmd.Flag(fmt.Sprintf("objstore%s.endpoint", flagSuffix), "The object store API endpoint for stored blocks. Supported S3-Compatible API"). + PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_ENDPOINT", envSuffix)).StringVar(&bucketConfig.Endpoint) - cmd.Flag("objstore.insecure", "Whether to use an insecure connection with an object store API. Supported S3-Compatible API"). - Default("false").Envar("OBJSTORE_INSECURE").BoolVar(&bucketConfig.Insecure) + cmd.Flag(fmt.Sprintf("objstore%s.access-key", flagSuffix), "Access key for an object store API. Supported S3-Compatible API"). + PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_ACCESS_KEY", envSuffix)).StringVar(&bucketConfig.AccessKey) - cmd.Flag("objstore.signature-version2", "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). - Default("false").Envar("OBJSTORE_SIGNATURE_VERSION2").BoolVar(&bucketConfig.SignatureV2) + bucketConfig.secretKey = os.Getenv(fmt.Sprintf("OBJSTORE%s_SECRET_KEY", envSuffix)) - cmd.Flag("objstore.encrypt-sse", "Whether to use Server Side Encryption"). - Default("false").Envar("OBJSTORE_SSE_ENCRYPTION").BoolVar(&bucketConfig.SSEEncryption) + cmd.Flag(fmt.Sprintf("objstore%s.insecure", flagSuffix), "Whether to use an insecure connection with an object store API. Supported S3-Compatible API"). + Default("false").Envar(fmt.Sprintf("OBJSTORE%s_INSECURE", envSuffix)).BoolVar(&bucketConfig.Insecure) - return &bucketConfig -} - -// NewBackupBucketConfig return the configuration of backup object store -func NewBackupBucketConfig(cmd *kingpin.CmdClause) *BucketConfig { - var bucketConfig BucketConfig - - newObjProvider(cmd.Flag("objstore-backup.type", "Specify the provider for backup object store. If empty or unsupport provider, Thanos won't backup any block to the object store. Now supported GCS / S3."). - PlaceHolder(""), &bucketConfig.Provider) + cmd.Flag(fmt.Sprintf("objstore%s.signature-version2", flagSuffix), "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). + Default("false").Envar(fmt.Sprintf("OBJSTORE%s_SIGNATURE_VERSION2", envSuffix)).BoolVar(&bucketConfig.SignatureV2) - cmd.Flag("objstore-backup.bucket", "The bucket name for backup stored blocks."). - PlaceHolder("").StringVar(&bucketConfig.Bucket) + cmd.Flag(fmt.Sprintf("objstore%s.encrypt-sse", flagSuffix), "Whether to use Server Side Encryption"). + Default("false").Envar(fmt.Sprintf("OBJSTORE%s_SSE_ENCRYPTION", envSuffix)).BoolVar(&bucketConfig.SSEEncryption) return &bucketConfig } From 2e999c2969a3b666a616f341e71135ca11c3d83e Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 3 Sep 2018 13:59:06 +0800 Subject: [PATCH 12/43] Update documents Signed-off-by: jojohappy --- docs/components/bucket.md | 90 ++++++++++++++++++++++---------------- docs/components/compact.md | 22 +++++----- docs/components/rule.md | 22 +++++----- docs/components/sidecar.md | 22 +++++----- docs/components/store.md | 22 +++++----- 5 files changed, 96 insertions(+), 82 deletions(-) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 5d964fc6ef..ce202f8678 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -35,25 +35,25 @@ Flags: no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --provider.type= + --objstore.type= Specify the provider for object store. If empty or - unsupport provider, Thanos won't read and store + unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption Subcommands: bucket verify [] @@ -94,34 +94,48 @@ Flags: no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --provider.type= + --objstore.type= Specify the provider for object store. If empty or - unsupport provider, Thanos won't read and store + unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption -r, --repair attempt to repair blocks for which issues were detected - --provider-backup.type= - Specify the provider for backup object store. If - empty or unsupport provider, Thanos won't backup + --objstore-backup.type= + Specify the provider for object store. If empty or + unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider-backup.bucket= - The bucket name for backup stored blocks. + --objstore-backup.bucket= + The bucket name for stored blocks. + --objstore-backup.endpoint= + The object store API endpoint for stored blocks. + Supported S3-Compatible API + --objstore-backup.access-key= + Access key for an object store API. Supported + S3-Compatible API + --objstore-backup.insecure + Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore-backup.signature-version2 + Whether to use S3 Signature Version 2; otherwise + Signature Version 4 will be used + --objstore-backup.encrypt-sse + Whether to use Server Side Encryption -i, --issues=index_issue... ... Issues to verify (and optionally repair). Possible values: [duplicated_compaction index_issue @@ -162,25 +176,25 @@ Flags: no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --provider.type= + --objstore.type= Specify the provider for object store. If empty or - unsupport provider, Thanos won't read and store + unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption -o, --output="" Format in which to print each block's information. May be 'json' or custom template. diff --git a/docs/components/compact.md b/docs/components/compact.md index 73e8e1e08d..b6eabf08e7 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -39,25 +39,25 @@ Flags: Listen host:port for HTTP endpoints. --data-dir="./data" Data directory in which to cache blocks and process compactions. - --provider.type= + --objstore.type= Specify the provider for object store. If empty or - unsupport provider, Thanos won't read and store + unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption --sync-delay=30m Minimum age of fresh (non-compacted) blocks before they are being processed. --retention.resolution-raw=0d diff --git a/docs/components/rule.md b/docs/components/rule.md index b9efbe9258..18c1aa1f35 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -111,24 +111,24 @@ Flags: --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --provider.type= + --objstore.type= Specify the provider for object store. If empty - or unsupport provider, Thanos won't read and + or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index d67059eb11..759642e9f5 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -99,25 +99,25 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). - --provider.type= + --objstore.type= Specify the provider for object store. If empty - or unsupport provider, Thanos won't read and + or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored - blocks. Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + blocks. Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption ``` diff --git a/docs/components/store.md b/docs/components/store.md index a2e3c07bc9..50f3d06f50 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -81,25 +81,25 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --provider.type= + --objstore.type= Specify the provider for object store. If empty - or unsupport provider, Thanos won't read and + or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3. - --provider.bucket= + --objstore.bucket= The bucket name for stored blocks. - --provider.endpoint= + --objstore.endpoint= The object store API endpoint for stored blocks. - Support S3-Compatible API - --provider.access-key= - Access key for an object store API. Support + Supported S3-Compatible API + --objstore.access-key= + Access key for an object store API. Supported S3-Compatible API - --provider.insecure Whether to use an insecure connection with an - object store API. Support S3-Compatible API - --provider.signature-version2 + --objstore.insecure Whether to use an insecure connection with an + object store API. Supported S3-Compatible API + --objstore.signature-version2 Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used - --provider.encrypt-sse Whether to use Server Side Encryption + --objstore.encrypt-sse Whether to use Server Side Encryption --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. From c8694e93216654b2823350f8f41ed32a01b349d7 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 7 Sep 2018 16:01:10 +0800 Subject: [PATCH 13/43] Refactor: - Remove all flags of objstore - Add flag as objstore.config to pass the configuration for bucket with yaml - To define the configuration for each provider - Add new method to get the name of bucket Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 12 ++--- cmd/thanos/compact.go | 10 ++-- cmd/thanos/downsample.go | 9 ++-- cmd/thanos/rule.go | 10 ++-- cmd/thanos/sidecar.go | 10 ++-- cmd/thanos/store.go | 10 ++-- pkg/objstore/client/factory.go | 67 ++++++++++++++---------- pkg/objstore/config.go | 94 ---------------------------------- pkg/objstore/gcs/gcs.go | 73 +++++++++++++++++++------- pkg/objstore/objstore.go | 7 +++ pkg/objstore/s3/s3.go | 67 +++++++++++++++++------- 11 files changed, 181 insertions(+), 188 deletions(-) delete mode 100644 pkg/objstore/config.go diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index a9f3630de5..088fa4abf0 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -11,7 +11,6 @@ import ( "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/block" - "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/verifier" @@ -42,14 +41,16 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. - backupBucketConf := objstore.NewBucketConfig(verify, "backup") + backupBucketConf := verify.Flag("objstore-backup.config", "The configuration of backup bucket for stored blocks."). + PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ @@ -61,10 +62,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin } defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - backupBucketConfig := *bucketConf - backupBucketConfig.Provider = backupBucketConf.Provider - backupBucketConfig.Bucket = backupBucketConf.Bucket - backupBkt, err := client.NewBucket(logger, backupBucketConfig, reg, name) + backupBkt, err := client.NewBucket(logger, *backupBucketConf, reg, name) if err == client.ErrNotFound { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c5451a2d5e..f1932c1f72 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -10,7 +10,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/compact" "github.com/improbable-eng/thanos/pkg/compact/downsample" - "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/oklog/run" @@ -32,7 +31,8 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) @@ -53,7 +53,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri return runCompact(g, logger, reg, *httpAddr, *dataDir, - bucketConf, + *bucketConf, time.Duration(*syncDelay), *haltOnError, *wait, @@ -74,7 +74,7 @@ func runCompact( reg *prometheus.Registry, httpBindAddr string, dataDir string, - bucketConf *objstore.BucketConfig, + bucketConf string, syncDelay time.Duration, haltOnError bool, wait bool, @@ -94,7 +94,7 @@ func runCompact( reg.MustRegister(halted) - bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 6cd6e35d2d..c013cbc453 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,10 +32,11 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - return runDownsample(g, logger, reg, *dataDir, bucketConf, name) + return runDownsample(g, logger, reg, *dataDir, *bucketConf, name) } } @@ -44,11 +45,11 @@ func runDownsample( logger log.Logger, reg *prometheus.Registry, dataDir string, - bucketConf *objstore.BucketConfig, + bucketConf string, component string, ) error { - bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 474570d55c..c85929868a 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -24,7 +24,6 @@ import ( "github.com/improbable-eng/thanos/pkg/alert" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" - "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/shipper" @@ -74,7 +73,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) @@ -109,7 +109,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) *dataDir, *ruleFiles, peer, - bucketConf, + *bucketConf, tsdbOpts, name, alertQueryURL, @@ -132,7 +132,7 @@ func runRule( dataDir string, ruleFiles []string, peer *cluster.Peer, - bucketConf *objstore.BucketConfig, + bucketConf string, tsdbOpts *tsdb.Options, component string, alertQueryURL *url.URL, @@ -410,7 +410,7 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil && err != client.ErrNotFound { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index b7ebe8516d..eca363dd2e 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -15,7 +15,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/cluster" - "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/reloader" "github.com/improbable-eng/thanos/pkg/runutil" @@ -51,7 +50,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( @@ -74,7 +74,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri *httpBindAddr, *promURL, *dataDir, - bucketConf, + *bucketConf, peer, rl, name, @@ -91,7 +91,7 @@ func runSidecar( httpBindAddr string, promURL *url.URL, dataDir string, - bucketConf *objstore.BucketConfig, + bucketConf string, peer *cluster.Peer, reloader *reloader.Reloader, component string, @@ -219,7 +219,7 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil && err != client.ErrNotFound { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index ba42eb351a..1d0674ef51 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -9,7 +9,6 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/cluster" - "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/store" @@ -31,7 +30,8 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConf := objstore.NewBucketConfig(cmd, "") + bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + PlaceHolder("").String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() @@ -48,7 +48,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string logger, reg, tracer, - bucketConf, + *bucketConf, *dataDir, *grpcBindAddr, *httpBindAddr, @@ -67,7 +67,7 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - bucketConf *objstore.BucketConfig, + bucketConf string, dataDir string, grpcBindAddr string, httpBindAddr string, @@ -78,7 +78,7 @@ func runStore( verbose bool, ) error { { - bkt, err := client.NewBucket(logger, *bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { return err } diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 6e5b65454a..342ac93c31 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -3,43 +3,58 @@ package client import ( "context" "fmt" - "runtime" - "cloud.google.com/go/storage" "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/gcs" "github.com/improbable-eng/thanos/pkg/objstore/s3" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/version" - "google.golang.org/api/option" + yaml "gopkg.in/yaml.v2" ) +type objProvider string + +const ( + GCS objProvider = "GCS" + S3 objProvider = "S3" +) + +type BucketConfig struct { + Type objProvider `yaml:"type"` + Content interface{} `yaml:"content"` +} + var ErrNotFound = errors.New("not found bucket") // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, bucketConf objstore.BucketConfig, reg *prometheus.Registry, component string) (objstore.Bucket, error) { - switch bucketConf.Provider { - case objstore.GCS: - if bucketConf.Bucket == "" { - return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") - } - gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) - gcsClient, err := storage.NewClient(context.Background(), gcsOptions) - if err != nil { - return nil, errors.Wrap(err, "create GCS client") - } - return objstore.BucketWithMetrics(bucketConf.Bucket, gcs.NewBucket(bucketConf.Bucket, gcsClient, reg), reg), nil - case objstore.S3: - if err := s3.Validate(&bucketConf); err != nil { - return nil, err - } - b, err := s3.NewBucket(logger, bucketConf, reg, component) - if err != nil { - return nil, errors.Wrap(err, "create s3 client") - } - return objstore.BucketWithMetrics(bucketConf.Bucket, b, reg), nil +func NewBucket(logger log.Logger, config string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { + var err error + var bucketConf BucketConfig + if config == "" { + return nil, ErrNotFound + } + err = yaml.Unmarshal([]byte(config), &bucketConf) + if err != nil { + return nil, errors.Wrap(err, "unmarshal objstore.config") + } + + content, err := yaml.Marshal(bucketConf.Content) + if err != nil { + return nil, err + } + + var bucket objstore.Bucket + switch bucketConf.Type { + case GCS: + bucket, err = gcs.NewBucket(logger, context.Background(), content, reg, component) + case S3: + bucket, err = s3.NewBucket(logger, content, reg, component) + default: + return nil, ErrNotFound + } + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("create %s client", bucketConf.Type)) } - return nil, ErrNotFound + return objstore.BucketWithMetrics(bucket.GetBucket(), bucket, reg), nil } diff --git a/pkg/objstore/config.go b/pkg/objstore/config.go deleted file mode 100644 index dda74383e8..0000000000 --- a/pkg/objstore/config.go +++ /dev/null @@ -1,94 +0,0 @@ -package objstore - -import ( - "fmt" - "os" - "strings" - - "gopkg.in/alecthomas/kingpin.v2" -) - -type objProvider string - -const ( - GCS objProvider = "GCS" - S3 objProvider = "S3" -) - -func newObjProvider(s kingpin.Settings, op *objProvider) { - s.SetValue(op) - return -} - -func (p *objProvider) Set(value string) error { - *p = objProvider(value) - return nil -} - -func (p *objProvider) String() string { - return string(*p) -} - -// BucketConfig encapsulates the necessary config values to instantiate an object store client. -// Use Provider to represent the Object Stores -type BucketConfig struct { - Provider objProvider - Bucket string - Endpoint string - AccessKey string - secretKey string - Insecure bool - SignatureV2 bool - SSEEncryption bool -} - -// NewBucketConfig return the configuration of object store -func NewBucketConfig(cmd *kingpin.CmdClause, suffix string) *BucketConfig { - var bucketConfig BucketConfig - flagSuffix := "" - envSuffix := "" - if strings.Trim(suffix, " ") != "" { - flagSuffix = "-" + suffix - envSuffix = "_" + strings.ToUpper(suffix) - } - - newObjProvider(cmd.Flag(fmt.Sprintf("objstore%s.type", flagSuffix), "Specify the provider for object store. If empty or unsupported provider, Thanos won't read and store any block to the object store. Now supported GCS / S3."). - PlaceHolder(""), &bucketConfig.Provider) - - cmd.Flag(fmt.Sprintf("objstore%s.bucket", flagSuffix), "The bucket name for stored blocks."). - PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_BUCKET", envSuffix)).StringVar(&bucketConfig.Bucket) - - cmd.Flag(fmt.Sprintf("objstore%s.endpoint", flagSuffix), "The object store API endpoint for stored blocks. Supported S3-Compatible API"). - PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_ENDPOINT", envSuffix)).StringVar(&bucketConfig.Endpoint) - - cmd.Flag(fmt.Sprintf("objstore%s.access-key", flagSuffix), "Access key for an object store API. Supported S3-Compatible API"). - PlaceHolder("").Envar(fmt.Sprintf("OBJSTORE%s_ACCESS_KEY", envSuffix)).StringVar(&bucketConfig.AccessKey) - - bucketConfig.secretKey = os.Getenv(fmt.Sprintf("OBJSTORE%s_SECRET_KEY", envSuffix)) - - cmd.Flag(fmt.Sprintf("objstore%s.insecure", flagSuffix), "Whether to use an insecure connection with an object store API. Supported S3-Compatible API"). - Default("false").Envar(fmt.Sprintf("OBJSTORE%s_INSECURE", envSuffix)).BoolVar(&bucketConfig.Insecure) - - cmd.Flag(fmt.Sprintf("objstore%s.signature-version2", flagSuffix), "Whether to use S3 Signature Version 2; otherwise Signature Version 4 will be used"). - Default("false").Envar(fmt.Sprintf("OBJSTORE%s_SIGNATURE_VERSION2", envSuffix)).BoolVar(&bucketConfig.SignatureV2) - - cmd.Flag(fmt.Sprintf("objstore%s.encrypt-sse", flagSuffix), "Whether to use Server Side Encryption"). - Default("false").Envar(fmt.Sprintf("OBJSTORE%s_SSE_ENCRYPTION", envSuffix)).BoolVar(&bucketConfig.SSEEncryption) - - return &bucketConfig -} - -// String returns the Provider information -func (conf *BucketConfig) String() string { - return fmt.Sprintf("Provider: %s, Bucket: %s, Endpoint: %s", string(conf.Provider), conf.Bucket, conf.Endpoint) -} - -// GetSecretKey returns the Provider Secret Key -func (conf *BucketConfig) GetSecretKey() string { - return conf.secretKey -} - -// SetSecretKey to setup the Secret Key for the Provider -func (conf *BucketConfig) SetSecretKey(secretKey string) { - conf.secretKey = secretKey -} diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 227c47c051..786cc45dc9 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -3,18 +3,23 @@ package gcs import ( "context" - "io" - "strings" - "fmt" + "io" "math/rand" + "runtime" + "strings" "testing" "time" "cloud.google.com/go/storage" + "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/objstore" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" "google.golang.org/api/iterator" + "google.golang.org/api/option" + yaml "gopkg.in/yaml.v2" ) const ( @@ -32,29 +37,56 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +// gcsConfig stores the configuration for gcs bucket +type gcsConfig struct { + Bucket string `yaml:"bucket"` +} + // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. type Bucket struct { + logger log.Logger bkt *storage.BucketHandle opsTotal *prometheus.CounterVec + bucket string closer io.Closer } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(name string, cl *storage.Client, reg prometheus.Registerer) *Bucket { +func NewBucket(logger log.Logger, ctx context.Context, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { + var gc gcsConfig + err := yaml.Unmarshal(conf, &gc) + if err != nil { + return nil, err + } + if gc.Bucket == "" { + return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") + } + gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) + gcsClient, err := storage.NewClient(ctx, gcsOptions) + if err != nil { + return nil, err + } bkt := &Bucket{ - bkt: cl.Bucket(name), + logger: logger, + bkt: gcsClient.Bucket(gc.Bucket), opsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_objstore_gcs_bucket_operations_total", Help: "Total number of operations that were executed against a Google Compute Storage bucket.", - ConstLabels: prometheus.Labels{"bucket": name}, + ConstLabels: prometheus.Labels{"bucket": gc.Bucket}, }, []string{"operation"}), - closer: cl, + closer: gcsClient, + bucket: gc.Bucket, } if reg != nil { reg.MustRegister() } - return bkt + return bkt, nil +} + +// GetBucket return the bucket name for gcs +func (b *Bucket) GetBucket() string { + return b.bucket } // Iter calls f for each entry in the given directory. The argument to f is the full @@ -151,27 +183,32 @@ func (b *Bucket) Close() error { // In a close function it empties and deletes the bucket. func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) { ctx, cancel := context.WithCancel(context.Background()) - gcsClient, err := storage.NewClient(ctx) + src := rand.NewSource(time.Now().UnixNano()) + gTestConfig := gcsConfig{ + Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), + } + + bc, err := yaml.Marshal(gTestConfig) if err != nil { - cancel() return nil, nil, err } - src := rand.NewSource(time.Now().UnixNano()) - name := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()) - bkt := gcsClient.Bucket(name) - if err = bkt.Create(ctx, project, nil); err != nil { + b, err := NewBucket(log.NewNopLogger(), ctx, bc, nil, "thanos-e2e-test") + if err != nil { cancel() - _ = gcsClient.Close() return nil, nil, err } - b := NewBucket(name, gcsClient, nil) + if err = b.bkt.Create(ctx, project, nil); err != nil { + cancel() + _ = b.Close() + return nil, nil, err + } - t.Log("created temporary GCS bucket for GCS tests with name", name, "in project", project) + t.Log("created temporary GCS bucket for GCS tests with name", b.bucket, "in project", project) return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := bkt.Delete(ctx); err != nil { + if err := b.bkt.Delete(ctx); err != nil { t.Logf("deleting bucket failed: %s", err) } cancel() diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 379de95e33..40b250f770 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -26,6 +26,9 @@ type Bucket interface { // Delete removes the object with the given name. Delete(ctx context.Context, name string) error + + // GetBucket return the bucket name for the provider + GetBucket() string } // BucketReader provides read access to an object storage bucket. @@ -313,6 +316,10 @@ func (b *metricBucket) Close() error { return b.bkt.Close() } +func (b *metricBucket) GetBucket() string { + return b.bkt.GetBucket() +} + type timingReadCloser struct { io.ReadCloser diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index e3f13ac86d..54508bde2a 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" + yaml "gopkg.in/yaml.v2" ) const ( @@ -37,6 +38,17 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +// s3Config stores the configuration for gcs bucket +type s3Config struct { + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"access-key"` + SecretKey string `yaml:"secret-key"` + Insecure bool `yaml:"insecure"` + SignatureV2 bool `yaml:"signature-version2"` + SSEEncryption bool `yaml:"encrypt-sse"` +} + // Bucket implements the store.Bucket interface against s3-compatible APIs. type Bucket struct { logger log.Logger @@ -47,18 +59,27 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Registerer, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider - if conf.AccessKey != "" { + var config s3Config + err := yaml.Unmarshal(conf, &config) + if err != nil { + return nil, err + } + err = Validate(config) + if err != nil { + return nil, err + } + if config.AccessKey != "" { signature := credentials.SignatureV4 - if conf.SignatureV2 { + if config.SignatureV2 { signature = credentials.SignatureV2 } chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ - AccessKeyID: conf.AccessKey, - SecretAccessKey: conf.GetSecretKey(), + AccessKeyID: config.AccessKey, + SecretAccessKey: config.SecretKey, SignerType: signature, }, }} @@ -74,7 +95,7 @@ func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Reg } } - client, err := minio.NewWithCredentials(conf.Endpoint, credentials.NewChainCredentials(chain), !conf.Insecure, "") + client, err := minio.NewWithCredentials(config.Endpoint, credentials.NewChainCredentials(chain), !config.Insecure, "") if err != nil { return nil, errors.Wrap(err, "initialize s3 client") } @@ -104,19 +125,19 @@ func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Reg }) var sse encrypt.ServerSide - if conf.SSEEncryption { + if config.SSEEncryption { sse = encrypt.NewSSE() } bkt := &Bucket{ logger: logger, - bucket: conf.Bucket, + bucket: config.Bucket, client: client, sse: sse, opsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "thanos_objstore_s3_bucket_operations_total", Help: "Total number of operations that were executed against an s3 bucket.", - ConstLabels: prometheus.Labels{"bucket": conf.Bucket}, + ConstLabels: prometheus.Labels{"bucket": config.Bucket}, }, []string{"operation"}), } if reg != nil { @@ -125,21 +146,26 @@ func NewBucket(logger log.Logger, conf objstore.BucketConfig, reg prometheus.Reg return bkt, nil } +// GetBucket return the bucket name for s3 +func (b *Bucket) GetBucket() string { + return b.bucket +} + // Validate checks to see the config options are set. -func Validate(conf *objstore.BucketConfig) error { +func Validate(conf s3Config) error { if conf.Endpoint == "" || - (conf.AccessKey == "" && conf.GetSecretKey() != "") || - (conf.AccessKey != "" && conf.GetSecretKey() == "") { + (conf.AccessKey == "" && conf.SecretKey != "") || + (conf.AccessKey != "" && conf.SecretKey == "") { return errors.New("insufficient s3 test configuration information") } return nil } // ValidateForTests checks to see the config options for tests are set. -func ValidateForTests(conf *objstore.BucketConfig) error { +func ValidateForTests(conf s3Config) error { if conf.Endpoint == "" || conf.AccessKey == "" || - conf.GetSecretKey() == "" { + conf.SecretKey == "" { return errors.New("insufficient s3 test configuration information") } return nil @@ -245,14 +271,14 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { func (b *Bucket) Close() error { return nil } -func configFromEnv() *objstore.BucketConfig { - c := &objstore.BucketConfig{ +func configFromEnv() s3Config { + c := s3Config{ Bucket: os.Getenv("S3_BUCKET"), Endpoint: os.Getenv("S3_ENDPOINT"), AccessKey: os.Getenv("S3_ACCESS_KEY"), + SecretKey: os.Getenv("S3_SECRET_KEY"), } - c.SetSecretKey(os.Getenv("S3_SECRET_KEY")) insecure, err := strconv.ParseBool(os.Getenv("S3_INSECURE")) if err != nil { c.Insecure = insecure @@ -271,8 +297,11 @@ func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), erro if err := ValidateForTests(c); err != nil { return nil, nil, err } - - b, err := NewBucket(log.NewNopLogger(), *c, nil, "thanos-e2e-test") + bc, err := yaml.Marshal(c) + if err != nil { + return nil, nil, err + } + b, err := NewBucket(log.NewNopLogger(), bc, nil, "thanos-e2e-test") if err != nil { return nil, nil, err } From 173a2f194c681eab3052131872c45ef8e82ea2a4 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 7 Sep 2018 16:18:23 +0800 Subject: [PATCH 14/43] Update documents --- cmd/thanos/bucket.go | 4 +- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- docs/components/bucket.md | 166 +++++++++++-------------------------- docs/components/compact.md | 56 +++++-------- docs/components/rule.md | 22 +---- docs/components/sidecar.md | 22 +---- docs/components/store.md | 22 +---- 11 files changed, 84 insertions(+), 218 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 088fa4abf0..1a728747bc 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -41,7 +41,7 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() // Verify command. @@ -49,7 +49,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. - backupBucketConf := verify.Flag("objstore-backup.config", "The configuration of backup bucket for stored blocks."). + backupBucketConf := verify.Flag("objstore-backup.config", "The yaml format configuration of backup bucket for stored blocks."). PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index f1932c1f72..dd39e5e475 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -31,7 +31,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index c013cbc453..b341f1e8c5 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,7 +32,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index c85929868a..89197c71d4 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -73,7 +73,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index eca363dd2e..0e8c66acd7 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -50,7 +50,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 1d0674ef51..0ffb6f7516 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -30,7 +30,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). PlaceHolder("").String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). diff --git a/docs/components/bucket.md b/docs/components/bucket.md index ce202f8678..0b2b16904c 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -23,37 +23,20 @@ usage: thanos bucket [] [ ...] inspect metric data in an object storage bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 - no trace will be sent periodically, unless forced - by baggage item. See `pkg/tracing/tracing.go` for - details. - --objstore.type= - Specify the provider for object store. If empty or - unsupported provider, Thanos won't read and store - any block to the object store. Now supported GCS / - S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config= + The yaml format configuration of bucket for stored + blocks. Subcommands: bucket verify [] @@ -82,68 +65,32 @@ usage: thanos bucket verify [] verify all blocks in the bucket against specified issues Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 - no trace will be sent periodically, unless forced - by baggage item. See `pkg/tracing/tracing.go` for - details. - --objstore.type= - Specify the provider for object store. If empty or - unsupported provider, Thanos won't read and store - any block to the object store. Now supported GCS / - S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption - -r, --repair attempt to repair blocks for which issues were - detected - --objstore-backup.type= - Specify the provider for object store. If empty or - unsupported provider, Thanos won't read and store - any block to the object store. Now supported GCS / - S3. - --objstore-backup.bucket= - The bucket name for stored blocks. - --objstore-backup.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore-backup.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore-backup.insecure - Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore-backup.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore-backup.encrypt-sse - Whether to use Server Side Encryption + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config= + The yaml format configuration of bucket for stored + blocks. + -r, --repair attempt to repair blocks for which issues were detected + --objstore-backup.config= + The yaml format configuration of backup bucket for + stored blocks. -i, --issues=index_issue... ... - Issues to verify (and optionally repair). Possible - values: [duplicated_compaction index_issue - overlapped_blocks] + Issues to verify (and optionally repair). Possible + values: [duplicated_compaction index_issue + overlapped_blocks] --id-whitelist=ID-WHITELIST ... - Block IDs to verify (and optionally repair) only. - If none is specified, all blocks will be verified. - Repeated field + Block IDs to verify (and optionally repair) only. If + none is specified, all blocks will be verified. Repeated + field ``` @@ -164,39 +111,22 @@ usage: thanos bucket ls [] list all blocks in the bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. If + empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 - no trace will be sent periodically, unless forced - by baggage item. See `pkg/tracing/tracing.go` for - details. - --objstore.type= - Specify the provider for object store. If empty or - unsupported provider, Thanos won't read and store - any block to the object store. Now supported GCS / - S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption - -o, --output="" Format in which to print each block's information. - May be 'json' or custom template. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for details. + --objstore.config= + The yaml format configuration of bucket for stored + blocks. + -o, --output="" Format in which to print each block's information. May + be 'json' or custom template. ``` diff --git a/docs/components/compact.md b/docs/components/compact.md index b6eabf08e7..55dfec846b 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -23,43 +23,27 @@ usage: thanos compact [] continuously compacts blocks in an object store bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. + If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 - no trace will be sent periodically, unless forced - by baggage item. See `pkg/tracing/tracing.go` for - details. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for + details. --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --data-dir="./data" Data directory in which to cache blocks and - process compactions. - --objstore.type= - Specify the provider for object store. If empty or - unsupported provider, Thanos won't read and store - any block to the object store. Now supported GCS / - S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption - --sync-delay=30m Minimum age of fresh (non-compacted) blocks before - they are being processed. + Listen host:port for HTTP endpoints. + --data-dir="./data" Data directory in which to cache blocks and process + compactions. + --objstore.config= + The yaml format configuration of bucket for stored + blocks. + --sync-delay=30m Minimum age of fresh (non-compacted) blocks before + they are being processed. --retention.resolution-raw=0d How long to retain raw samples in bucket. 0d - disables this retention @@ -69,7 +53,7 @@ Flags: --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 hour) in bucket. 0d - disables this retention - -w, --wait Do not exit after all compactions have been - processed and wait for new work. + -w, --wait Do not exit after all compactions have been processed + and wait for new work. ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index 18c1aa1f35..1413af8fea 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -111,24 +111,8 @@ Flags: --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --objstore.type= - Specify the provider for object store. If empty - or unsupported provider, Thanos won't read and - store any block to the object store. Now - supported GCS / S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption + --objstore.config= + The yaml format configuration of bucket for + stored blocks. ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 759642e9f5..4d94ae59e4 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -99,25 +99,9 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). - --objstore.type= - Specify the provider for object store. If empty - or unsupported provider, Thanos won't read and - store any block to the object store. Now - supported GCS / S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored - blocks. Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; - otherwise Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption + --objstore.config= + The yaml format configuration of bucket for + stored blocks. ``` diff --git a/docs/components/store.md b/docs/components/store.md index 50f3d06f50..d624977104 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -81,25 +81,9 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --objstore.type= - Specify the provider for object store. If empty - or unsupported provider, Thanos won't read and - store any block to the object store. Now - supported GCS / S3. - --objstore.bucket= - The bucket name for stored blocks. - --objstore.endpoint= - The object store API endpoint for stored blocks. - Supported S3-Compatible API - --objstore.access-key= - Access key for an object store API. Supported - S3-Compatible API - --objstore.insecure Whether to use an insecure connection with an - object store API. Supported S3-Compatible API - --objstore.signature-version2 - Whether to use S3 Signature Version 2; otherwise - Signature Version 4 will be used - --objstore.encrypt-sse Whether to use Server Side Encryption + --objstore.config= + The yaml format configuration of bucket for + stored blocks. --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. From 49e66f9d3a691aebc71ef602a1eb7fc2ff6ba261 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Fri, 7 Sep 2018 16:31:07 +0800 Subject: [PATCH 15/43] Fixed missing method for inmem bucket --- pkg/objstore/inmem/inmem.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 57e9e0562b..2849e9bbc5 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -145,3 +145,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { } func (b *Bucket) Close() error { return nil } + +// GetBucket return the bucket name +func (b *Bucket) GetBucket() string { + return "inmem" +} From 8758b9ce8a7d19bec80a5011973837dd1da98359 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:07:07 +0800 Subject: [PATCH 16/43] Update the describe for objstore.config --- cmd/thanos/bucket.go | 4 ++-- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 1a728747bc..438a3cf87d 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -41,7 +41,7 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() // Verify command. @@ -49,7 +49,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. - backupBucketConf := verify.Flag("objstore-backup.config", "The yaml format configuration of backup bucket for stored blocks."). + backupBucketConf := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index dd39e5e475..630876bb17 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -31,7 +31,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index b341f1e8c5..4f987d3c20 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,7 +32,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 89197c71d4..2b29cd160a 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -73,7 +73,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 0e8c66acd7..7a358edb6a 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -50,7 +50,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0ffb6f7516..6c6f422b4b 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -30,7 +30,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The yaml format configuration of bucket for stored blocks."). + bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). PlaceHolder("").String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). From 9f1666861ea4c2098c27bc302464f9f709e8665e Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:10:24 +0800 Subject: [PATCH 17/43] To setup bucket flags required for component bucket and downsample --- cmd/thanos/bucket.go | 2 +- cmd/thanos/downsample.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 438a3cf87d..a9770d05dc 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -42,7 +42,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin cmd := app.Command(name, "inspect metric data in an object storage bucket") bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").Required().String() // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 4f987d3c20..0ed7224e25 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -33,7 +33,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").Required().String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { return runDownsample(g, logger, reg, *dataDir, *bucketConf, name) From a8cf8d61da870a471008f5a206bfd4f24f90e387 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:15:22 +0800 Subject: [PATCH 18/43] Rename Content to Config for Bucket.Config Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 342ac93c31..3372ed3bc2 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -21,25 +21,25 @@ const ( ) type BucketConfig struct { - Type objProvider `yaml:"type"` - Content interface{} `yaml:"content"` + Type objProvider `yaml:"type"` + Config interface{} `yaml:"config"` } var ErrNotFound = errors.New("not found bucket") // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, config string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { var err error var bucketConf BucketConfig - if config == "" { + if conf == "" { return nil, ErrNotFound } - err = yaml.Unmarshal([]byte(config), &bucketConf) + err = yaml.Unmarshal([]byte(conf), &bucketConf) if err != nil { return nil, errors.Wrap(err, "unmarshal objstore.config") } - content, err := yaml.Marshal(bucketConf.Content) + config, err := yaml.Marshal(bucketConf.Config) if err != nil { return nil, err } @@ -47,9 +47,9 @@ func NewBucket(logger log.Logger, config string, reg *prometheus.Registry, compo var bucket objstore.Bucket switch bucketConf.Type { case GCS: - bucket, err = gcs.NewBucket(logger, context.Background(), content, reg, component) + bucket, err = gcs.NewBucket(logger, context.Background(), config, reg, component) case S3: - bucket, err = s3.NewBucket(logger, content, reg, component) + bucket, err = s3.NewBucket(logger, config, reg, component) default: return nil, ErrNotFound } From e787a313fa7f282006de58726b3483025dc3c2d5 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:17:39 +0800 Subject: [PATCH 19/43] To change error handler idiom Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 3372ed3bc2..a49b1a0df9 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -34,8 +34,7 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone if conf == "" { return nil, ErrNotFound } - err = yaml.Unmarshal([]byte(conf), &bucketConf) - if err != nil { + if err := yaml.Unmarshal([]byte(conf), &bucketConf); err != nil { return nil, errors.Wrap(err, "unmarshal objstore.config") } From ff2cbf5c8173931ca37890cba26c26b8c05f5b47 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:29:52 +0800 Subject: [PATCH 20/43] Update describe for component store Signed-off-by: jojohappy --- cmd/thanos/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 6c6f422b4b..018415b73e 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -23,7 +23,7 @@ import ( // registerStore registers a store command. func registerStore(m map[string]setupFunc, app *kingpin.Application, name string) { - cmd := app.Command(name, "store node giving access to blocks in a supported bucket provider") + cmd := app.Command(name, "store node giving access to blocks in a bucket provider. Now supported GCS / S3.") grpcBindAddr, httpBindAddr, newPeerFn := regCommonServerFlags(cmd) From 4b51cec3de6f565d087388296335b951b1ecc227 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:37:50 +0800 Subject: [PATCH 21/43] To setup secret-key just use envvar Signed-off-by: jojohappy --- pkg/objstore/s3/s3.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 54508bde2a..213f970be3 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -43,10 +43,10 @@ type s3Config struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` AccessKey string `yaml:"access-key"` - SecretKey string `yaml:"secret-key"` Insecure bool `yaml:"insecure"` SignatureV2 bool `yaml:"signature-version2"` SSEEncryption bool `yaml:"encrypt-sse"` + secretKey string } // Bucket implements the store.Bucket interface against s3-compatible APIs. @@ -66,6 +66,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon if err != nil { return nil, err } + config.secretKey = os.Getenv("S3_SECRET_KEY") err = Validate(config) if err != nil { return nil, err @@ -79,7 +80,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ AccessKeyID: config.AccessKey, - SecretAccessKey: config.SecretKey, + SecretAccessKey: config.secretKey, SignerType: signature, }, }} @@ -154,8 +155,8 @@ func (b *Bucket) GetBucket() string { // Validate checks to see the config options are set. func Validate(conf s3Config) error { if conf.Endpoint == "" || - (conf.AccessKey == "" && conf.SecretKey != "") || - (conf.AccessKey != "" && conf.SecretKey == "") { + (conf.AccessKey == "" && conf.secretKey != "") || + (conf.AccessKey != "" && conf.secretKey == "") { return errors.New("insufficient s3 test configuration information") } return nil @@ -165,7 +166,7 @@ func Validate(conf s3Config) error { func ValidateForTests(conf s3Config) error { if conf.Endpoint == "" || conf.AccessKey == "" || - conf.SecretKey == "" { + conf.secretKey == "" { return errors.New("insufficient s3 test configuration information") } return nil @@ -276,7 +277,7 @@ func configFromEnv() s3Config { Bucket: os.Getenv("S3_BUCKET"), Endpoint: os.Getenv("S3_ENDPOINT"), AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), + secretKey: os.Getenv("S3_SECRET_KEY"), } insecure, err := strconv.ParseBool(os.Getenv("S3_INSECURE")) From 62e0c890cdafa4646a73138ac6c1728631c80c95 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:43:57 +0800 Subject: [PATCH 22/43] Update the placeholder of flags and documents Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 4 ++-- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- docs/components/bucket.md | 22 +++++++++------------ docs/components/compact.md | 39 +++++++++++++++++++------------------- docs/components/rule.md | 5 ++--- docs/components/sidecar.md | 5 ++--- docs/components/store.md | 7 +++---- 11 files changed, 42 insertions(+), 50 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index a9770d05dc..5e5124b09d 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -42,7 +42,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin cmd := app.Command(name, "inspect metric data in an object storage bucket") bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + PlaceHolder("").Required().String() // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") @@ -50,7 +50,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin Short('r').Default("false").Bool() // NOTE(bplotka): Currently we support backup buckets only in the same project. backupBucketConf := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 630876bb17..24c72eca5e 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -32,7 +32,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 0ed7224e25..4f1b63cf8c 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -33,7 +33,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + PlaceHolder("").Required().String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { return runDownsample(g, logger, reg, *dataDir, *bucketConf, name) diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 2b29cd160a..eb9f72b998 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -74,7 +74,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 7a358edb6a..424965e01c 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -51,7 +51,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 018415b73e..8985dc2fa5 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -31,7 +31,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 0b2b16904c..9c454457f6 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -18,7 +18,7 @@ by adding a new command within `/cmd/thanos/bucket.go` [embedmd]:# (flags/bucket.txt $) ```$ -usage: thanos bucket [] [ ...] +usage: thanos bucket --objstore.config= [ ...] inspect metric data in an object storage bucket @@ -34,9 +34,8 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The yaml format configuration of bucket for stored - blocks. + --objstore.config= + The object store configuration in yaml format. Subcommands: bucket verify [] @@ -76,13 +75,11 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The yaml format configuration of bucket for stored - blocks. + --objstore.config= + The object store configuration in yaml format. -r, --repair attempt to repair blocks for which issues were detected - --objstore-backup.config= - The yaml format configuration of backup bucket for - stored blocks. + --objstore-backup.config= + The backup object store configuration in yaml format. -i, --issues=index_issue... ... Issues to verify (and optionally repair). Possible values: [duplicated_compaction index_issue @@ -122,9 +119,8 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The yaml format configuration of bucket for stored - blocks. + --objstore.config= + The object store configuration in yaml format. -o, --output="" Format in which to print each block's information. May be 'json' or custom template. diff --git a/docs/components/compact.md b/docs/components/compact.md index 55dfec846b..d3a4a4fb72 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -23,27 +23,26 @@ usage: thanos compact [] continuously compacts blocks in an object store bucket Flags: - -h, --help Show context-sensitive help (also try --help-long and - --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings to. - If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings + to. If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 no - trace will be sent periodically, unless forced by - baggage item. See `pkg/tracing/tracing.go` for - details. + How often we send traces (1/). If 0 + no trace will be sent periodically, unless forced + by baggage item. See `pkg/tracing/tracing.go` for + details. --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --data-dir="./data" Data directory in which to cache blocks and process - compactions. - --objstore.config= - The yaml format configuration of bucket for stored - blocks. - --sync-delay=30m Minimum age of fresh (non-compacted) blocks before - they are being processed. + Listen host:port for HTTP endpoints. + --data-dir="./data" Data directory in which to cache blocks and + process compactions. + --objstore.config= + The object store configuration in yaml format. + --sync-delay=30m Minimum age of fresh (non-compacted) blocks before + they are being processed. --retention.resolution-raw=0d How long to retain raw samples in bucket. 0d - disables this retention @@ -53,7 +52,7 @@ Flags: --retention.resolution-1h=0d How long to retain samples of resolution 2 (1 hour) in bucket. 0d - disables this retention - -w, --wait Do not exit after all compactions have been processed - and wait for new work. + -w, --wait Do not exit after all compactions have been + processed and wait for new work. ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index 1413af8fea..a0de317eb9 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -111,8 +111,7 @@ Flags: --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --objstore.config= - The yaml format configuration of bucket for - stored blocks. + --objstore.config= + The object store configuration in yaml format. ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 4d94ae59e4..02311146f9 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -99,9 +99,8 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). - --objstore.config= - The yaml format configuration of bucket for - stored blocks. + --objstore.config= + The object store configuration in yaml format. ``` diff --git a/docs/components/store.md b/docs/components/store.md index d624977104..96abacab27 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -19,7 +19,7 @@ In general about 1MB of local disk space is required per TSDB block stored in th ```$ usage: thanos store [] -store node giving access to blocks in a supported bucket provider +store node giving access to blocks in a bucket provider. Now supported GCS / S3. Flags: -h, --help Show context-sensitive help (also try @@ -81,9 +81,8 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --objstore.config= - The yaml format configuration of bucket for - stored blocks. + --objstore.config= + The object store configuration in yaml format. --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. From 7666ac61cdf124cffef07964b602253e647a3bf9 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 21:53:18 +0800 Subject: [PATCH 23/43] Update example for bucket Signed-off-by: jojohappy --- docs/components/bucket.md | 4 +++- docs/components/compact.md | 4 +++- docs/components/rule.md | 8 +++++--- docs/components/sidecar.md | 4 +++- docs/components/store.md | 8 +++++--- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 9c454457f6..b68a36045c 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -6,7 +6,9 @@ It is normally run as a stand alone command to aid with troubleshooting. Example: ``` -$ thanos bucket verify --gcs.bucket example-bucket +$ thanos bucket verify --objstore.config="type: GCS +config: + bucket: example-bucket" ``` Bucket can be extended to add more subcommands that will be helpful when working with object storage buckets diff --git a/docs/components/compact.md b/docs/components/compact.md index d3a4a4fb72..9dccb988f9 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -6,7 +6,9 @@ It is generally not semantically concurrency safe and must be deployed as a sing Example: ``` -$ thanos compact --gcs.bucket example-bucket --data-dir /tmp/thanos-compact +$ thanos compact --data-dir /tmp/thanos-compact --objstore.config="type: GCS +config: + bucket: example-bucket" ``` The compactor needs local disk space to store intermediate data for its processing. Generally, about 100GB are recommended for it to keep working as the compacted time ranges grow over time. diff --git a/docs/components/rule.md b/docs/components/rule.md index a0de317eb9..e74a24a7e0 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -14,9 +14,11 @@ $ thanos rule \ --eval-interval "30s" \ --rule-file "/path/to/rules/*.rules.yaml" \ --alert.query-url "http://0.0.0.0:9090" \ - --alertmanagers.url "alert.thanos.io" - --gcs.bucket "example-bucket" \ - --cluster.peers "thanos-cluster.example.org" + --alertmanagers.url "alert.thanos.io" \ + --cluster.peers "thanos-cluster.example.org" \ + --objstore.config="type: GCS +config: + bucket: example-bucket" ``` As rule nodes outsource query processing to query nodes, they should generally experience little load. If necessary, functional sharding can be applied by splitting up the sets of rules between HA pairs. diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 02311146f9..5b92dc8dec 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -15,8 +15,10 @@ The retention is recommended to not be lower than three times the block duration $ thanos sidecar \ --tsdb.path "/path/to/prometheus/data/dir" \ --prometheus.url "http://localhost:9090" \ - --gcs.bucket "example-bucket" \ --cluster.peers "thanos-cluster.example.org" \ + --objstore.config="type: GCS +config: + bucket: example-bucket" ``` ## Deployment diff --git a/docs/components/store.md b/docs/components/store.md index 96abacab27..e35addf947 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -5,9 +5,11 @@ It keeps a small amount of information about all remote blocks on local disk and ``` $ thanos store \ - --data-dir "/local/state/data/dir" \ - --gcs.bucket "example-bucket" \ - --cluster.peers "thanos-cluster.example.org" + --data-dir "/local/state/data/dir" \ + --cluster.peers "thanos-cluster.example.org" \ + --objstore.config="type: GCS +config: + bucket: example-bucket" ``` In general about 1MB of local disk space is required per TSDB block stored in the object storage bucket. From 61c25aaabab3a7cdb34d0125cba0940fa9253096 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 22:24:23 +0800 Subject: [PATCH 24/43] Update documents Signed-off-by: jojohappy --- docs/storage.md | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index 4a4ff7e1bd..822a42f905 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -26,23 +26,26 @@ At that point, anyone can use your provider! Thanos uses minio client to upload Prometheus data into AWS s3. -To configure S3 bucket as an object store you need to set these mandatory S3 flags: -- --s3.endpoint -- --s3.bucket - -Instead of using flags you can pass all the configuration via environment variables: -- `S3_BUCKET` -- `S3_ENDPOINT` -- `S3_ACCESS_KEY` +To configure S3 bucket as an object store you need to set these mandatory S3 variables in yaml format: +``` +--objstore.config="type: S3 +config: + bucket: + endpoint: + access-key: + insecure: + signature-version2: + encrypt-sse: " +``` + +Meanwhile you also should set the `secret-key` via environment variables: - `S3_SECRET_KEY` -- `S3_INSECURE` -- `S3_SIGNATURE_VERSION2` AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) -Make sure you use a correct signature version with `--s3.signature-version2`, otherwise, you will get Access Denied error. +Make sure you use a correct signature version to set `signature-version2: true`, otherwise, you will get Access Denied error. -For debug purposes you can `--s3.insecure` to switch to plain insecure HTTP instead of HTTPS +For debug purposes you can set `insecure: true` to switch to plain insecure HTTP instead of HTTPS ### Credentials Credentials will by default try to retrieve from the following sources: @@ -122,7 +125,14 @@ Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/usin ## GCP Configuration -To configure Google Cloud Storage bucket as an object store you need to set `--gcs.bucket` with GCS bucket name and configure Google Application credentials. +To configure Google Cloud Storage bucket as an object store you need to set `bucket` with GCS bucket name and configure Google Application credentials. + +For example: +``` +--objstore.config="type: GCS +config: + bucket: " +``` Application credentials are configured via JSON file, the client looks for: From 27f9fc2c5e11e4394d95660cbe3ca24c60970ab8 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Sat, 8 Sep 2018 22:24:35 +0800 Subject: [PATCH 25/43] Update CHANGELOG Signed-off-by: jojohappy --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8787cddb..97adb08067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan ## Unreleased +- Add Thanos Rule UI +- Add Thanos Rule reload via HTTP POST to /-/reload +- Add Thanos compact --retention.default flag, for configuring storage bucket retention period. +- Removes support for multiple units in duration. For example: 1m0s won't work, while 1m will work. +- Adds support for y,w,d time units +- Add Thanos bucket ls -o wide, which provides more detailed information about blocks stored in the bucket. +- Use the configuration of bucket in yaml format. + ## [v0.1.0](https://github.com/improbable-eng/thanos/releases/tag/v0.1.0) - 2018.09.14 Initial version to have a stable reference before [gossip protocol removal](https://github.com/improbable-eng/thanos/blob/master/docs/proposals/gossip-removal.md). From b5943ba23e79699560c9c88c96c1c0d1088db33e Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 10 Sep 2018 21:57:03 +0800 Subject: [PATCH 26/43] Fixed something nits --- cmd/thanos/bucket.go | 1 - pkg/objstore/gcs/gcs.go | 7 +++---- pkg/objstore/inmem/inmem.go | 2 +- pkg/objstore/objstore.go | 2 +- pkg/objstore/s3/s3.go | 10 ++++------ 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 5e5124b09d..c1329ba358 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -48,7 +48,6 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() - // NOTE(bplotka): Currently we support backup buckets only in the same project. backupBucketConf := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 786cc45dc9..0b2acc2f77 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -37,7 +37,7 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -// gcsConfig stores the configuration for gcs bucket +// gcsConfig stores the configuration for gcs bucket. type gcsConfig struct { Bucket string `yaml:"bucket"` } @@ -55,8 +55,7 @@ type Bucket struct { // NewBucket returns a new Bucket against the given bucket handle. func NewBucket(logger log.Logger, ctx context.Context, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { var gc gcsConfig - err := yaml.Unmarshal(conf, &gc) - if err != nil { + if err := yaml.Unmarshal(conf, &gc); err != nil { return nil, err } if gc.Bucket == "" { @@ -84,7 +83,7 @@ func NewBucket(logger log.Logger, ctx context.Context, conf []byte, reg promethe return bkt, nil } -// GetBucket return the bucket name for gcs +// GetBucket returns the bucket name for gcs. func (b *Bucket) GetBucket() string { return b.bucket } diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 2849e9bbc5..1362eb1cb0 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -146,7 +146,7 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { func (b *Bucket) Close() error { return nil } -// GetBucket return the bucket name +// GetBucket returns the bucket name. func (b *Bucket) GetBucket() string { return "inmem" } diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 40b250f770..4b8cc0b08d 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -27,7 +27,7 @@ type Bucket interface { // Delete removes the object with the given name. Delete(ctx context.Context, name string) error - // GetBucket return the bucket name for the provider + // GetBucket returns the bucket name for the provider. GetBucket() string } diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 213f970be3..e6459e826a 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -38,7 +38,7 @@ const ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -// s3Config stores the configuration for gcs bucket +// s3Config stores the configuration for s3 bucket. type s3Config struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` @@ -62,13 +62,11 @@ type Bucket struct { func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { var chain []credentials.Provider var config s3Config - err := yaml.Unmarshal(conf, &config) - if err != nil { + if err := yaml.Unmarshal(conf, &config); err != nil { return nil, err } config.secretKey = os.Getenv("S3_SECRET_KEY") - err = Validate(config) - if err != nil { + if err := Validate(config); err != nil { return nil, err } if config.AccessKey != "" { @@ -147,7 +145,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon return bkt, nil } -// GetBucket return the bucket name for s3 +// GetBucket returns the bucket name for s3. func (b *Bucket) GetBucket() string { return b.bucket } From 845148fd4b108332bf90eb5782d2ed629768a0b7 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 10 Sep 2018 22:49:59 +0800 Subject: [PATCH 27/43] To distinguish no bucket is configured or not supported Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 4 +++- pkg/objstore/client/factory_test.go | 36 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 pkg/objstore/client/factory_test.go diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index a49b1a0df9..380fd555b9 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -49,8 +49,10 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone bucket, err = gcs.NewBucket(logger, context.Background(), config, reg, component) case S3: bucket, err = s3.NewBucket(logger, config, reg, component) - default: + case "": return nil, ErrNotFound + default: + return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("create %s client", bucketConf.Type)) diff --git a/pkg/objstore/client/factory_test.go b/pkg/objstore/client/factory_test.go new file mode 100644 index 0000000000..b39fdcf814 --- /dev/null +++ b/pkg/objstore/client/factory_test.go @@ -0,0 +1,36 @@ +package client + +import ( + "testing" + + "github.com/improbable-eng/thanos/pkg/testutil" + + "github.com/go-kit/kit/log" +) + +func TestErrorBucketConfig(t *testing.T) { + conf := ` + type: FAKE-GCS + config: + bucket: test-bucket + ` + _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") + testutil.NotOk(t, err) + testutil.Assert(t, err != ErrNotFound, "it should not error with not found") +} + +func TestBlankBucketConfig(t *testing.T) { + conf := "" + _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") + testutil.NotOk(t, err) + testutil.Assert(t, err == ErrNotFound, "it should error with not found") +} + +func TestBlankBucketConfigContent(t *testing.T) { + conf := ` + type: GCS + ` + _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") + testutil.NotOk(t, err) + testutil.Assert(t, err != ErrNotFound, "it should not error with not found") +} From 0e2f7ae1508e47f33af29462b2ae15e56a80f948 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 10 Sep 2018 23:14:06 +0800 Subject: [PATCH 28/43] Update CHANGELOG Signed-off-by: jojohappy --- CHANGELOG.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97adb08067..e46bf1a080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,23 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan - Removes support for multiple units in duration. For example: 1m0s won't work, while 1m will work. - Adds support for y,w,d time units - Add Thanos bucket ls -o wide, which provides more detailed information about blocks stored in the bucket. -- Use the configuration of bucket in yaml format. +- Remove support of those flags for bucket + - --gcs-bucket=\ + - --s3.bucket=\ + - --s3.endpoint=\ + - --s3.access-key=\ + - --s3.insecure + - --s3.signature-version2 + - --s3.encrypt-sse + - --gcs-backup-bucket=\ + - --s3-backup-bucket=\ +- Remove support of those environment variables for bucket + * S3_BUCKET + * S3_ENDPOINT + * S3_ACCESS_KEY + * S3_INSECURE + * S3_SIGNATURE_VERSION2 +- Add flag `--objstore.config` for the configuration of bucket in yaml format. Note that detailed information in document [storage](docs/storage.md). ## [v0.1.0](https://github.com/improbable-eng/thanos/releases/tag/v0.1.0) - 2018.09.14 From 7411fa7710fe9f2a25e8ba3b59dcae07449e9c7a Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 10 Sep 2018 23:19:33 +0800 Subject: [PATCH 29/43] Remove unrequired unit test Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 2 -- pkg/objstore/client/factory_test.go | 7 ------- 2 files changed, 9 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 380fd555b9..87be4155d1 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -49,8 +49,6 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone bucket, err = gcs.NewBucket(logger, context.Background(), config, reg, component) case S3: bucket, err = s3.NewBucket(logger, config, reg, component) - case "": - return nil, ErrNotFound default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/pkg/objstore/client/factory_test.go b/pkg/objstore/client/factory_test.go index b39fdcf814..efe0d80f0c 100644 --- a/pkg/objstore/client/factory_test.go +++ b/pkg/objstore/client/factory_test.go @@ -19,13 +19,6 @@ func TestErrorBucketConfig(t *testing.T) { testutil.Assert(t, err != ErrNotFound, "it should not error with not found") } -func TestBlankBucketConfig(t *testing.T) { - conf := "" - _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") - testutil.NotOk(t, err) - testutil.Assert(t, err == ErrNotFound, "it should error with not found") -} - func TestBlankBucketConfigContent(t *testing.T) { conf := ` type: GCS From 8580d0cd8b41c292b0a13bccf7e295ec813b7238 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 11 Sep 2018 23:05:09 +0800 Subject: [PATCH 30/43] To set bucket flag required for component store and compact Signed-off-by: jojohappy --- cmd/thanos/compact.go | 2 +- cmd/thanos/store.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 24c72eca5e..f2f1dc1ab0 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -32,7 +32,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").Required().String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 8985dc2fa5..5a784f6919 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -31,7 +31,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string Default("./data").String() bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + PlaceHolder("").Required().String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() From c61b82ef2808edc3675e5a2371a6f12afd5e7af5 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 11 Sep 2018 23:06:16 +0800 Subject: [PATCH 31/43] Rename GetBucket to Name & Set context as first argument Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 4 ++-- pkg/objstore/gcs/gcs.go | 8 ++++---- pkg/objstore/inmem/inmem.go | 4 ++-- pkg/objstore/objstore.go | 8 ++++---- pkg/objstore/s3/s3.go | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 87be4155d1..e001453f52 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -46,7 +46,7 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone var bucket objstore.Bucket switch bucketConf.Type { case GCS: - bucket, err = gcs.NewBucket(logger, context.Background(), config, reg, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, reg, component) case S3: bucket, err = s3.NewBucket(logger, config, reg, component) default: @@ -55,5 +55,5 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("create %s client", bucketConf.Type)) } - return objstore.BucketWithMetrics(bucket.GetBucket(), bucket, reg), nil + return objstore.BucketWithMetrics(bucket.Name(), bucket, reg), nil } diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 0b2acc2f77..c20298a66b 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -53,7 +53,7 @@ type Bucket struct { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(logger log.Logger, ctx context.Context, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, reg prometheus.Registerer, component string) (*Bucket, error) { var gc gcsConfig if err := yaml.Unmarshal(conf, &gc); err != nil { return nil, err @@ -83,8 +83,8 @@ func NewBucket(logger log.Logger, ctx context.Context, conf []byte, reg promethe return bkt, nil } -// GetBucket returns the bucket name for gcs. -func (b *Bucket) GetBucket() string { +// Name returns the bucket name for gcs. +func (b *Bucket) Name() string { return b.bucket } @@ -192,7 +192,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), ctx, bc, nil, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, nil, "thanos-e2e-test") if err != nil { cancel() return nil, nil, err diff --git a/pkg/objstore/inmem/inmem.go b/pkg/objstore/inmem/inmem.go index 1362eb1cb0..29f95d7b8e 100644 --- a/pkg/objstore/inmem/inmem.go +++ b/pkg/objstore/inmem/inmem.go @@ -146,7 +146,7 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { func (b *Bucket) Close() error { return nil } -// GetBucket returns the bucket name. -func (b *Bucket) GetBucket() string { +// Name returns the bucket name. +func (b *Bucket) Name() string { return "inmem" } diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 4b8cc0b08d..81ae6fbb8f 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -27,8 +27,8 @@ type Bucket interface { // Delete removes the object with the given name. Delete(ctx context.Context, name string) error - // GetBucket returns the bucket name for the provider. - GetBucket() string + // Name returns the bucket name for the provider. + Name() string } // BucketReader provides read access to an object storage bucket. @@ -316,8 +316,8 @@ func (b *metricBucket) Close() error { return b.bkt.Close() } -func (b *metricBucket) GetBucket() string { - return b.bkt.GetBucket() +func (b *metricBucket) Name() string { + return b.bkt.Name() } type timingReadCloser struct { diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index e6459e826a..2c4b651df2 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -145,8 +145,8 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon return bkt, nil } -// GetBucket returns the bucket name for s3. -func (b *Bucket) GetBucket() string { +// Name returns the bucket name for s3. +func (b *Bucket) Name() string { return b.bucket } From c7aa4a90d9851f795dd6982a00a80cf91bc7cc36 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 11 Sep 2018 23:15:35 +0800 Subject: [PATCH 32/43] Wrap error to give more information Signed-off-by: jojohappy --- cmd/thanos/store.go | 2 +- pkg/objstore/client/factory.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 5a784f6919..0ee7f22708 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -80,7 +80,7 @@ func runStore( { bkt, err := client.NewBucket(logger, bucketConf, reg, component) if err != nil { - return err + return errors.Wrap(err, "create bucket client") } // Ensure we close up everything properly. diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index e001453f52..81696712e5 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -40,7 +40,7 @@ func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, compone config, err := yaml.Marshal(bucketConf.Config) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshal content of bucket configuration") } var bucket objstore.Bucket From 31828d74ca8e7a742ac18932701251670435967c Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 11 Sep 2018 23:20:47 +0800 Subject: [PATCH 33/43] Rename field bucket to name for the struct of Bucket Signed-off-by: jojohappy --- pkg/objstore/gcs/gcs.go | 8 ++++---- pkg/objstore/s3/s3.go | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index c20298a66b..e06be4f374 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -47,7 +47,7 @@ type Bucket struct { logger log.Logger bkt *storage.BucketHandle opsTotal *prometheus.CounterVec - bucket string + name string closer io.Closer } @@ -75,7 +75,7 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, reg promethe ConstLabels: prometheus.Labels{"bucket": gc.Bucket}, }, []string{"operation"}), closer: gcsClient, - bucket: gc.Bucket, + name: gc.Bucket, } if reg != nil { reg.MustRegister() @@ -85,7 +85,7 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, reg promethe // Name returns the bucket name for gcs. func (b *Bucket) Name() string { - return b.bucket + return b.name } // Iter calls f for each entry in the given directory. The argument to f is the full @@ -204,7 +204,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - t.Log("created temporary GCS bucket for GCS tests with name", b.bucket, "in project", project) + t.Log("created temporary GCS bucket for GCS tests with name", b.name, "in project", project) return b, func() { objstore.EmptyBucket(t, ctx, b) if err := b.bkt.Delete(ctx); err != nil { diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 2c4b651df2..7acde93bfa 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -52,7 +52,7 @@ type s3Config struct { // Bucket implements the store.Bucket interface against s3-compatible APIs. type Bucket struct { logger log.Logger - bucket string + name string client *minio.Client sse encrypt.ServerSide opsTotal *prometheus.CounterVec @@ -130,7 +130,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon bkt := &Bucket{ logger: logger, - bucket: config.Bucket, + name: config.Bucket, client: client, sse: sse, opsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -147,7 +147,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon // Name returns the bucket name for s3. func (b *Bucket) Name() string { - return b.bucket + return b.name } // Validate checks to see the config options are set. @@ -180,7 +180,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) err dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } - for object := range b.client.ListObjects(b.bucket, dir, false, ctx.Done()) { + for object := range b.client.ListObjects(b.name, dir, false, ctx.Done()) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -205,7 +205,7 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } - r, err := b.client.GetObjectWithContext(ctx, b.bucket, name, *opts) + r, err := b.client.GetObjectWithContext(ctx, b.name, name, *opts) if err != nil { return nil, err } @@ -235,7 +235,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { b.opsTotal.WithLabelValues(opObjectHead).Inc() - _, err := b.client.StatObject(b.bucket, name, minio.StatObjectOptions{}) + _, err := b.client.StatObject(b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -250,7 +250,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { b.opsTotal.WithLabelValues(opObjectInsert).Inc() - _, err := b.client.PutObjectWithContext(ctx, b.bucket, name, r, -1, + _, err := b.client.PutObjectWithContext(ctx, b.name, name, r, -1, minio.PutObjectOptions{ServerSideEncryption: b.sse}, ) @@ -260,7 +260,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { b.opsTotal.WithLabelValues(opObjectDelete).Inc() - return b.client.RemoveObject(b.bucket, name) + return b.client.RemoveObject(b.name, name) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -334,7 +334,7 @@ func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), erro if err := b.client.MakeBucket(tmpBucketName, location); err != nil { return nil, nil, err } - b.bucket = tmpBucketName + b.name = tmpBucketName t.Log("created temporary AWS bucket for AWS tests with name", tmpBucketName, "in", location) return b, func() { From 6a817552ed2ee163f7833a1c76f6639464789c7a Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 11 Sep 2018 23:27:27 +0800 Subject: [PATCH 34/43] Update documents Signed-off-by: jojohappy --- docs/components/compact.md | 2 +- docs/components/store.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/components/compact.md b/docs/components/compact.md index 9dccb988f9..d3cb800afc 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -20,7 +20,7 @@ On-disk data is safe to delete between restarts and should be the first attempt [embedmd]:# (flags/compact.txt $) ```$ -usage: thanos compact [] +usage: thanos compact --objstore.config= [] continuously compacts blocks in an object store bucket diff --git a/docs/components/store.md b/docs/components/store.md index e35addf947..28d3c71c9e 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -19,7 +19,7 @@ In general about 1MB of local disk space is required per TSDB block stored in th [embedmd]:# (flags/store.txt $) ```$ -usage: thanos store [] +usage: thanos store --objstore.config= [] store node giving access to blocks in a bucket provider. Now supported GCS / S3. From b3f8eb1884300b65554cfd19f9d324508d273f98 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 17 Sep 2018 23:29:31 +0800 Subject: [PATCH 35/43] Change the bucket configuration flags to reference to YAML file --- cmd/thanos/bucket.go | 12 ++++++------ cmd/thanos/compact.go | 10 +++++----- cmd/thanos/downsample.go | 10 +++++----- cmd/thanos/rule.go | 10 +++++----- cmd/thanos/sidecar.go | 10 +++++----- cmd/thanos/store.go | 10 +++++----- pkg/objstore/client/factory.go | 28 +++++++++++++++++++++++----- 7 files changed, 54 insertions(+), 36 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index c1329ba358..b67cc90aae 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -41,27 +41,27 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").Required().String() // Verify command. verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() - backupBucketConf := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). + backupBucketConfFile := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ "If none is specified, all blocks will be verified. Repeated field").Strings() m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, *bucketConf, reg, name) + bkt, err := client.NewBucket(logger, *bucketConfFile, reg, name) if err != nil { return err } defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client") - backupBkt, err := client.NewBucket(logger, *backupBucketConf, reg, name) + backupBkt, err := client.NewBucket(logger, *backupBucketConfFile, reg, name) if err == client.ErrNotFound { if *verifyRepair { return errors.Wrap(err, "repair is specified, so backup client is required") @@ -121,7 +121,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin lsOutput := ls.Flag("output", "Format in which to print each block's information. May be 'json' or custom template."). Short('o').Default("").String() m[name+" ls"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ bool) error { - bkt, err := client.NewBucket(logger, *bucketConf, reg, name) + bkt, err := client.NewBucket(logger, *bucketConfFile, reg, name) if err != nil { return err } diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index f2f1dc1ab0..6db3be4846 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -31,8 +31,8 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").Required().String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). Default("30m")) @@ -53,7 +53,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri return runCompact(g, logger, reg, *httpAddr, *dataDir, - *bucketConf, + *bucketConfFile, time.Duration(*syncDelay), *haltOnError, *wait, @@ -74,7 +74,7 @@ func runCompact( reg *prometheus.Registry, httpBindAddr string, dataDir string, - bucketConf string, + bucketConfFile string, syncDelay time.Duration, haltOnError bool, wait bool, @@ -94,7 +94,7 @@ func runCompact( reg.MustRegister(halted) - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 4f1b63cf8c..bb67a491e1 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,11 +32,11 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").Required().String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - return runDownsample(g, logger, reg, *dataDir, *bucketConf, name) + return runDownsample(g, logger, reg, *dataDir, *bucketConfFile, name) } } @@ -45,11 +45,11 @@ func runDownsample( logger log.Logger, reg *prometheus.Registry, dataDir string, - bucketConf string, + bucketConfFile string, component string, ) error { - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index eb9f72b998..78db7217ec 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -73,8 +73,8 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) @@ -109,7 +109,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) *dataDir, *ruleFiles, peer, - *bucketConf, + *bucketConfFile, tsdbOpts, name, alertQueryURL, @@ -132,7 +132,7 @@ func runRule( dataDir string, ruleFiles []string, peer *cluster.Peer, - bucketConf string, + bucketConfFile string, tsdbOpts *tsdb.Options, component string, alertQueryURL *url.URL, @@ -410,7 +410,7 @@ func runRule( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil && err != client.ErrNotFound { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 424965e01c..08e0ab16a2 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -50,8 +50,8 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { rl := reloader.New( @@ -74,7 +74,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri *httpBindAddr, *promURL, *dataDir, - *bucketConf, + *bucketConfFile, peer, rl, name, @@ -91,7 +91,7 @@ func runSidecar( httpBindAddr string, promURL *url.URL, dataDir string, - bucketConf string, + bucketConfFile string, peer *cluster.Peer, reloader *reloader.Reloader, component string, @@ -219,7 +219,7 @@ func runSidecar( // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil && err != client.ErrNotFound { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0ee7f22708..bfd6dc07c3 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -30,8 +30,8 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConf := cmd.Flag("objstore.config", "The object store configuration in yaml format."). - PlaceHolder("").Required().String() + bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + PlaceHolder("").Required().String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). Default("250MB").Bytes() @@ -48,7 +48,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string logger, reg, tracer, - *bucketConf, + *bucketConfFile, *dataDir, *grpcBindAddr, *httpBindAddr, @@ -67,7 +67,7 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - bucketConf string, + bucketConfFile string, dataDir string, grpcBindAddr string, httpBindAddr string, @@ -78,7 +78,7 @@ func runStore( verbose bool, ) error { { - bkt, err := client.NewBucket(logger, bucketConf, reg, component) + bkt, err := client.NewBucket(logger, bucketConfFile, reg, component) if err != nil { return errors.Wrap(err, "create bucket client") } diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 81696712e5..d886a37b44 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -3,8 +3,10 @@ package client import ( "context" "fmt" + "io/ioutil" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/gcs" "github.com/improbable-eng/thanos/pkg/objstore/s3" @@ -27,15 +29,31 @@ type BucketConfig struct { var ErrNotFound = errors.New("not found bucket") +func loadFile(confFile string) (*BucketConfig, error) { + content, err := ioutil.ReadFile(confFile) + if err != nil { + return nil, err + } + + bucketConf := &BucketConfig{} + if err := yaml.UnmarshalStrict(content, bucketConf); err != nil { + return nil, fmt.Errorf("parsing YAML file %s: %v", confFile, err) + } + return bucketConf, nil +} + // NewBucket initializes and returns new object storage clients. -func NewBucket(logger log.Logger, conf string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confFile string, reg *prometheus.Registry, component string) (objstore.Bucket, error) { + level.Info(logger).Log("msg", "loading bucket configuration file", "filename", confFile) + var err error - var bucketConf BucketConfig - if conf == "" { + if confFile == "" { return nil, ErrNotFound } - if err := yaml.Unmarshal([]byte(conf), &bucketConf); err != nil { - return nil, errors.Wrap(err, "unmarshal objstore.config") + + bucketConf, err := loadFile(confFile) + if err != nil { + return nil, errors.Wrap(err, "parsing objstore.config.file") } config, err := yaml.Marshal(bucketConf.Config) From cb5f75d7f57b42392478fd9e807204aba57227ec Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 17 Sep 2018 23:30:19 +0800 Subject: [PATCH 36/43] Do not get the secret-key from envvar --- pkg/objstore/s3/s3.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 7acde93bfa..3794f23607 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -46,7 +46,7 @@ type s3Config struct { Insecure bool `yaml:"insecure"` SignatureV2 bool `yaml:"signature-version2"` SSEEncryption bool `yaml:"encrypt-sse"` - secretKey string + SecretKey string `yaml:"secret-key"` } // Bucket implements the store.Bucket interface against s3-compatible APIs. @@ -65,7 +65,6 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon if err := yaml.Unmarshal(conf, &config); err != nil { return nil, err } - config.secretKey = os.Getenv("S3_SECRET_KEY") if err := Validate(config); err != nil { return nil, err } @@ -78,7 +77,7 @@ func NewBucket(logger log.Logger, conf []byte, reg prometheus.Registerer, compon chain = []credentials.Provider{&credentials.Static{ Value: credentials.Value{ AccessKeyID: config.AccessKey, - SecretAccessKey: config.secretKey, + SecretAccessKey: config.SecretKey, SignerType: signature, }, }} @@ -153,8 +152,8 @@ func (b *Bucket) Name() string { // Validate checks to see the config options are set. func Validate(conf s3Config) error { if conf.Endpoint == "" || - (conf.AccessKey == "" && conf.secretKey != "") || - (conf.AccessKey != "" && conf.secretKey == "") { + (conf.AccessKey == "" && conf.SecretKey != "") || + (conf.AccessKey != "" && conf.SecretKey == "") { return errors.New("insufficient s3 test configuration information") } return nil @@ -164,7 +163,7 @@ func Validate(conf s3Config) error { func ValidateForTests(conf s3Config) error { if conf.Endpoint == "" || conf.AccessKey == "" || - conf.secretKey == "" { + conf.SecretKey == "" { return errors.New("insufficient s3 test configuration information") } return nil @@ -275,7 +274,7 @@ func configFromEnv() s3Config { Bucket: os.Getenv("S3_BUCKET"), Endpoint: os.Getenv("S3_ENDPOINT"), AccessKey: os.Getenv("S3_ACCESS_KEY"), - secretKey: os.Getenv("S3_SECRET_KEY"), + SecretKey: os.Getenv("S3_SECRET_KEY"), } insecure, err := strconv.ParseBool(os.Getenv("S3_INSECURE")) From 8c2dedd5f22203007c4415f26861b9a5c033ff3d Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 17 Sep 2018 23:40:19 +0800 Subject: [PATCH 37/43] Update documents --- docs/components/bucket.md | 24 +++++++++------ docs/components/compact.md | 62 +++++++++++++++++++++----------------- docs/components/rule.md | 14 ++++++--- docs/components/sidecar.md | 14 ++++++--- docs/components/store.md | 16 +++++++--- docs/storage.md | 16 +++++----- 6 files changed, 89 insertions(+), 57 deletions(-) diff --git a/docs/components/bucket.md b/docs/components/bucket.md index b68a36045c..1d1f3ca311 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -6,9 +6,15 @@ It is normally run as a stand alone command to aid with troubleshooting. Example: ``` -$ thanos bucket verify --objstore.config="type: GCS +$ thanos bucket verify --objstore.config.file=bucket.yml +``` + +The content of `bucket.yml`: + +```yaml +type: GCS config: - bucket: example-bucket" + bucket: example-bucket ``` Bucket can be extended to add more subcommands that will be helpful when working with object storage buckets @@ -20,7 +26,7 @@ by adding a new command within `/cmd/thanos/bucket.go` [embedmd]:# (flags/bucket.txt $) ```$ -usage: thanos bucket --objstore.config= [ ...] +usage: thanos bucket --objstore.config.file= [ ...] inspect metric data in an object storage bucket @@ -36,8 +42,8 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. Subcommands: bucket verify [] @@ -77,8 +83,8 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. -r, --repair attempt to repair blocks for which issues were detected --objstore-backup.config= The backup object store configuration in yaml format. @@ -121,8 +127,8 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. -o, --output="" Format in which to print each block's information. May be 'json' or custom template. diff --git a/docs/components/compact.md b/docs/components/compact.md index d3cb800afc..d13ef0d847 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -6,9 +6,15 @@ It is generally not semantically concurrency safe and must be deployed as a sing Example: ``` -$ thanos compact --data-dir /tmp/thanos-compact --objstore.config="type: GCS +$ thanos compact --data-dir /tmp/thanos-compact --objstore.config.file=bucket.yml +``` + +The content of `bucket.yml`: + +```yaml +type: GCS config: - bucket: example-bucket" + bucket: example-bucket ``` The compactor needs local disk space to store intermediate data for its processing. Generally, about 100GB are recommended for it to keep working as the compacted time ranges grow over time. @@ -20,41 +26,41 @@ On-disk data is safe to delete between restarts and should be the first attempt [embedmd]:# (flags/compact.txt $) ```$ -usage: thanos compact --objstore.config= [] +usage: thanos compact --objstore.config.file= [] continuously compacts blocks in an object store bucket Flags: - -h, --help Show context-sensitive help (also try --help-long - and --help-man). - --version Show application version. - --log.level=info Log filtering level. + -h, --help Show context-sensitive help (also try --help-long and + --help-man). + --version Show application version. + --log.level=info Log filtering level. --gcloudtrace.project=GCLOUDTRACE.PROJECT - GCP project to send Google Cloud Trace tracings - to. If empty, tracing will be disabled. + GCP project to send Google Cloud Trace tracings to. + If empty, tracing will be disabled. --gcloudtrace.sample-factor=1 - How often we send traces (1/). If 0 - no trace will be sent periodically, unless forced - by baggage item. See `pkg/tracing/tracing.go` for - details. + How often we send traces (1/). If 0 no + trace will be sent periodically, unless forced by + baggage item. See `pkg/tracing/tracing.go` for + details. --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --data-dir="./data" Data directory in which to cache blocks and - process compactions. - --objstore.config= - The object store configuration in yaml format. - --sync-delay=30m Minimum age of fresh (non-compacted) blocks before - they are being processed. + Listen host:port for HTTP endpoints. + --data-dir="./data" Data directory in which to cache blocks and process + compactions. + --objstore.config.file= + The object store configuration file path. + --sync-delay=30m Minimum age of fresh (non-compacted) blocks before + they are being processed. --retention.resolution-raw=0d - How long to retain raw samples in bucket. 0d - - disables this retention + How long to retain raw samples in bucket. 0d - + disables this retention --retention.resolution-5m=0d - How long to retain samples of resolution 1 (5 - minutes) in bucket. 0d - disables this retention + How long to retain samples of resolution 1 (5 + minutes) in bucket. 0d - disables this retention --retention.resolution-1h=0d - How long to retain samples of resolution 2 (1 - hour) in bucket. 0d - disables this retention - -w, --wait Do not exit after all compactions have been - processed and wait for new work. + How long to retain samples of resolution 2 (1 hour) + in bucket. 0d - disables this retention + -w, --wait Do not exit after all compactions have been processed + and wait for new work. ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index e74a24a7e0..a65d346d36 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -16,9 +16,15 @@ $ thanos rule \ --alert.query-url "http://0.0.0.0:9090" \ --alertmanagers.url "alert.thanos.io" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config="type: GCS + --objstore.config.file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS config: - bucket: example-bucket" + bucket: example-bucket ``` As rule nodes outsource query processing to query nodes, they should generally experience little load. If necessary, functional sharding can be applied by splitting up the sets of rules between HA pairs. @@ -113,7 +119,7 @@ Flags: --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 5b92dc8dec..ceac03e3ee 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -16,9 +16,15 @@ $ thanos sidecar \ --tsdb.path "/path/to/prometheus/data/dir" \ --prometheus.url "http://localhost:9090" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config="type: GCS + --objstore.config.file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS config: - bucket: example-bucket" + bucket: example-bucket ``` ## Deployment @@ -101,8 +107,8 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. ``` diff --git a/docs/components/store.md b/docs/components/store.md index 28d3c71c9e..f9fe517cf8 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -7,9 +7,15 @@ It keeps a small amount of information about all remote blocks on local disk and $ thanos store \ --data-dir "/local/state/data/dir" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config="type: GCS + --objstore.config.file "bucket.yml" +``` + +The content of `bucket.yml`: + +```yaml +type: GCS config: - bucket: example-bucket" + bucket: example-bucket ``` In general about 1MB of local disk space is required per TSDB block stored in the object storage bucket. @@ -19,7 +25,7 @@ In general about 1MB of local disk space is required per TSDB block stored in th [embedmd]:# (flags/store.txt $) ```$ -usage: thanos store --objstore.config= [] +usage: thanos store --objstore.config.file= [] store node giving access to blocks in a bucket provider. Now supported GCS / S3. @@ -83,8 +89,8 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --objstore.config= - The object store configuration in yaml format. + --objstore.config.file= + The object store configuration file path. --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes for chunks. diff --git a/docs/storage.md b/docs/storage.md index 822a42f905..70e863cf2d 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -26,20 +26,20 @@ At that point, anyone can use your provider! Thanos uses minio client to upload Prometheus data into AWS s3. -To configure S3 bucket as an object store you need to set these mandatory S3 variables in yaml format: +To configure S3 bucket as an object store you need to set these mandatory S3 variables in yaml format stored in a file: ``` ---objstore.config="type: S3 +type: S3 config: bucket: endpoint: access-key: insecure: signature-version2: - encrypt-sse: " + encrypt-sse: + secret-key: ``` -Meanwhile you also should set the `secret-key` via environment variables: -- `S3_SECRET_KEY` +Set the flags `--objstore.config.file` to reference to the configuration file. AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) @@ -129,11 +129,13 @@ To configure Google Cloud Storage bucket as an object store you need to set `buc For example: ``` ---objstore.config="type: GCS +type: GCS config: - bucket: " + bucket: ``` +Set the flags `--objstore.config.file` to reference to the configuration file. + Application credentials are configured via JSON file, the client looks for: 1. A JSON file whose path is specified by the From 9ecc7f9ce99bfae93bf6bb6de401b3b57dc2b5ee Mon Sep 17 00:00:00 2001 From: jojohappy Date: Mon, 17 Sep 2018 23:43:56 +0800 Subject: [PATCH 38/43] Update CHANGELOG Signed-off-by: jojohappy --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e46bf1a080..9d80dd86aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,8 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan * S3_ACCESS_KEY * S3_INSECURE * S3_SIGNATURE_VERSION2 -- Add flag `--objstore.config` for the configuration of bucket in yaml format. Note that detailed information in document [storage](docs/storage.md). + * S3_SECRET_KEY +- Add flag `--objstore.config.file` to reference to the bucket configuration file in yaml format. Note that detailed information in document [storage](docs/storage.md). ## [v0.1.0](https://github.com/improbable-eng/thanos/releases/tag/v0.1.0) - 2018.09.14 From 2b64ce08b28141438f1f37f02cba76d10efd9cb6 Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 18 Sep 2018 00:01:59 +0800 Subject: [PATCH 39/43] Update test case for package objstore client Signed-off-by: jojohappy --- pkg/objstore/client/factory_test.go | 10 ++-------- pkg/objstore/client/testconf/blank-gcs.conf.yml | 1 + pkg/objstore/client/testconf/fake-gcs.conf.yml | 3 +++ 3 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 pkg/objstore/client/testconf/blank-gcs.conf.yml create mode 100644 pkg/objstore/client/testconf/fake-gcs.conf.yml diff --git a/pkg/objstore/client/factory_test.go b/pkg/objstore/client/factory_test.go index efe0d80f0c..319c1e20ac 100644 --- a/pkg/objstore/client/factory_test.go +++ b/pkg/objstore/client/factory_test.go @@ -9,20 +9,14 @@ import ( ) func TestErrorBucketConfig(t *testing.T) { - conf := ` - type: FAKE-GCS - config: - bucket: test-bucket - ` + conf := "testconf/fake-gcs.conf.yml" _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") testutil.NotOk(t, err) testutil.Assert(t, err != ErrNotFound, "it should not error with not found") } func TestBlankBucketConfigContent(t *testing.T) { - conf := ` - type: GCS - ` + conf := "testconf/blank-gcs.conf.yml" _, err := NewBucket(log.NewNopLogger(), conf, nil, "bkt-client-test") testutil.NotOk(t, err) testutil.Assert(t, err != ErrNotFound, "it should not error with not found") diff --git a/pkg/objstore/client/testconf/blank-gcs.conf.yml b/pkg/objstore/client/testconf/blank-gcs.conf.yml new file mode 100644 index 0000000000..cb5ef588cc --- /dev/null +++ b/pkg/objstore/client/testconf/blank-gcs.conf.yml @@ -0,0 +1 @@ +type: GCS \ No newline at end of file diff --git a/pkg/objstore/client/testconf/fake-gcs.conf.yml b/pkg/objstore/client/testconf/fake-gcs.conf.yml new file mode 100644 index 0000000000..538c832788 --- /dev/null +++ b/pkg/objstore/client/testconf/fake-gcs.conf.yml @@ -0,0 +1,3 @@ +type: FAKE-GCS +config: + bucket: test-bucket \ No newline at end of file From 248034e85ef44dd1d06126c457f0a7600ca68cba Mon Sep 17 00:00:00 2001 From: jojohappy Date: Tue, 18 Sep 2018 08:38:01 +0800 Subject: [PATCH 40/43] Rename flag objstore.config.file to objstore.config-file Signed-off-by: jojohappy --- CHANGELOG.md | 2 +- cmd/thanos/bucket.go | 2 +- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- docs/components/bucket.md | 10 +++++----- docs/components/compact.md | 6 +++--- docs/components/rule.md | 4 ++-- docs/components/sidecar.md | 4 ++-- docs/components/store.md | 6 +++--- docs/storage.md | 4 ++-- pkg/objstore/client/factory.go | 2 +- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d80dd86aa..f881d753b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan * S3_INSECURE * S3_SIGNATURE_VERSION2 * S3_SECRET_KEY -- Add flag `--objstore.config.file` to reference to the bucket configuration file in yaml format. Note that detailed information in document [storage](docs/storage.md). +- Add flag `--objstore.config-file` to reference to the bucket configuration file in yaml format. Note that detailed information in document [storage](docs/storage.md). ## [v0.1.0](https://github.com/improbable-eng/thanos/releases/tag/v0.1.0) - 2018.09.14 diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index b67cc90aae..0b3f9331b8 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -41,7 +41,7 @@ var ( func registerBucket(m map[string]setupFunc, app *kingpin.Application, name string) { cmd := app.Command(name, "inspect metric data in an object storage bucket") - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").Required().String() // Verify command. diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 6db3be4846..42f5a1a344 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -31,7 +31,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions."). Default("./data").String() - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").Required().String() syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index bb67a491e1..a6e92a696f 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -32,7 +32,7 @@ func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name s dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings."). Default("./data").String() - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").Required().String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 78db7217ec..dd85b0630c 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -73,7 +73,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) alertQueryURL := cmd.Flag("alert.query-url", "The external Thanos Query URL that would be set in all alerts 'Source' field").String() - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 08e0ab16a2..3c78f03597 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -50,7 +50,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri reloaderRuleDirs := cmd.Flag("reloader.rule-dir", "Rule directories for the reloader to refresh (repeated field).").Strings() - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").String() m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index bfd6dc07c3..4110736e78 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -30,7 +30,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application, name string dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks."). Default("./data").String() - bucketConfFile := cmd.Flag("objstore.config.file", "The object store configuration file path."). + bucketConfFile := cmd.Flag("objstore.config-file", "The object store configuration file path."). PlaceHolder("").Required().String() indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache."). diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 1d1f3ca311..3ae92b0770 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -6,7 +6,7 @@ It is normally run as a stand alone command to aid with troubleshooting. Example: ``` -$ thanos bucket verify --objstore.config.file=bucket.yml +$ thanos bucket verify --objstore.config-file=bucket.yml ``` The content of `bucket.yml`: @@ -26,7 +26,7 @@ by adding a new command within `/cmd/thanos/bucket.go` [embedmd]:# (flags/bucket.txt $) ```$ -usage: thanos bucket --objstore.config.file= [ ...] +usage: thanos bucket --objstore.config-file= [ ...] inspect metric data in an object storage bucket @@ -42,7 +42,7 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config.file= + --objstore.config-file= The object store configuration file path. Subcommands: @@ -83,7 +83,7 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config.file= + --objstore.config-file= The object store configuration file path. -r, --repair attempt to repair blocks for which issues were detected --objstore-backup.config= @@ -127,7 +127,7 @@ Flags: How often we send traces (1/). If 0 no trace will be sent periodically, unless forced by baggage item. See `pkg/tracing/tracing.go` for details. - --objstore.config.file= + --objstore.config-file= The object store configuration file path. -o, --output="" Format in which to print each block's information. May be 'json' or custom template. diff --git a/docs/components/compact.md b/docs/components/compact.md index d13ef0d847..df738f747f 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -6,7 +6,7 @@ It is generally not semantically concurrency safe and must be deployed as a sing Example: ``` -$ thanos compact --data-dir /tmp/thanos-compact --objstore.config.file=bucket.yml +$ thanos compact --data-dir /tmp/thanos-compact --objstore.config-file=bucket.yml ``` The content of `bucket.yml`: @@ -26,7 +26,7 @@ On-disk data is safe to delete between restarts and should be the first attempt [embedmd]:# (flags/compact.txt $) ```$ -usage: thanos compact --objstore.config.file= [] +usage: thanos compact --objstore.config-file= [] continuously compacts blocks in an object store bucket @@ -47,7 +47,7 @@ Flags: Listen host:port for HTTP endpoints. --data-dir="./data" Data directory in which to cache blocks and process compactions. - --objstore.config.file= + --objstore.config-file= The object store configuration file path. --sync-delay=30m Minimum age of fresh (non-compacted) blocks before they are being processed. diff --git a/docs/components/rule.md b/docs/components/rule.md index a65d346d36..0237812397 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -16,7 +16,7 @@ $ thanos rule \ --alert.query-url "http://0.0.0.0:9090" \ --alertmanagers.url "alert.thanos.io" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config.file "bucket.yml" + --objstore.config-file "bucket.yml" ``` The content of `bucket.yml`: @@ -119,7 +119,7 @@ Flags: --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field - --objstore.config.file= + --objstore.config-file= The object store configuration file path. ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index ceac03e3ee..dbd92bc088 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -16,7 +16,7 @@ $ thanos sidecar \ --tsdb.path "/path/to/prometheus/data/dir" \ --prometheus.url "http://localhost:9090" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config.file "bucket.yml" + --objstore.config-file "bucket.yml" ``` The content of `bucket.yml`: @@ -107,7 +107,7 @@ Flags: --reloader.rule-dir=RELOADER.RULE-DIR ... Rule directories for the reloader to refresh (repeated field). - --objstore.config.file= + --objstore.config-file= The object store configuration file path. ``` diff --git a/docs/components/store.md b/docs/components/store.md index f9fe517cf8..eae9af5437 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -7,7 +7,7 @@ It keeps a small amount of information about all remote blocks on local disk and $ thanos store \ --data-dir "/local/state/data/dir" \ --cluster.peers "thanos-cluster.example.org" \ - --objstore.config.file "bucket.yml" + --objstore.config-file "bucket.yml" ``` The content of `bucket.yml`: @@ -25,7 +25,7 @@ In general about 1MB of local disk space is required per TSDB block stored in th [embedmd]:# (flags/store.txt $) ```$ -usage: thanos store --objstore.config.file= [] +usage: thanos store --objstore.config-file= [] store node giving access to blocks in a bucket provider. Now supported GCS / S3. @@ -89,7 +89,7 @@ Flags: accounting the latency differences between network types: local, lan, wan. --data-dir="./data" Data directory in which to cache remote blocks. - --objstore.config.file= + --objstore.config-file= The object store configuration file path. --index-cache-size=250MB Maximum size of items held in the index cache. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes diff --git a/docs/storage.md b/docs/storage.md index 70e863cf2d..2d8b4ac209 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -39,7 +39,7 @@ config: secret-key: ``` -Set the flags `--objstore.config.file` to reference to the configuration file. +Set the flags `--objstore.config-file` to reference to the configuration file. AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) @@ -134,7 +134,7 @@ config: bucket: ``` -Set the flags `--objstore.config.file` to reference to the configuration file. +Set the flags `--objstore.config-file` to reference to the configuration file. Application credentials are configured via JSON file, the client looks for: diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index d886a37b44..4d5202429d 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -53,7 +53,7 @@ func NewBucket(logger log.Logger, confFile string, reg *prometheus.Registry, com bucketConf, err := loadFile(confFile) if err != nil { - return nil, errors.Wrap(err, "parsing objstore.config.file") + return nil, errors.Wrap(err, "parsing objstore.config-file") } config, err := yaml.Marshal(bucketConf.Config) From 4d665f15f81b6c5855c9c7ac30dada5e681ca3db Mon Sep 17 00:00:00 2001 From: jojohappy Date: Wed, 19 Sep 2018 07:33:01 +0800 Subject: [PATCH 41/43] Update CHANGELOG --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f881d753b8..bccdcf874e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,6 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan ## Unreleased -- Add Thanos Rule UI -- Add Thanos Rule reload via HTTP POST to /-/reload -- Add Thanos compact --retention.default flag, for configuring storage bucket retention period. -- Removes support for multiple units in duration. For example: 1m0s won't work, while 1m will work. -- Adds support for y,w,d time units -- Add Thanos bucket ls -o wide, which provides more detailed information about blocks stored in the bucket. - Remove support of those flags for bucket - --gcs-bucket=\ - --s3.bucket=\ From 33d1559be22d901c31182062fdb5327db1c4f2af Mon Sep 17 00:00:00 2001 From: jojohappy Date: Wed, 19 Sep 2018 14:16:45 +0800 Subject: [PATCH 42/43] To wrap errors for loading and parsing bucket configuration file Signed-off-by: jojohappy --- pkg/objstore/client/factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 4d5202429d..3a50a38d92 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -32,12 +32,12 @@ var ErrNotFound = errors.New("not found bucket") func loadFile(confFile string) (*BucketConfig, error) { content, err := ioutil.ReadFile(confFile) if err != nil { - return nil, err + return nil, errors.Wrap(err, fmt.Sprintf("loading YAML file %s", confFile)) } bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(content, bucketConf); err != nil { - return nil, fmt.Errorf("parsing YAML file %s: %v", confFile, err) + return nil, errors.Wrap(err, fmt.Sprintf("parsing YAML file %s", confFile)) } return bucketConf, nil } From 5bbb65269897a6773b1b676273f002de844ac48d Mon Sep 17 00:00:00 2001 From: jojohappy Date: Wed, 19 Sep 2018 14:43:50 +0800 Subject: [PATCH 43/43] Rename objstore flag for consistency Signed-off-by: jojohappy --- cmd/thanos/bucket.go | 4 ++-- docs/components/bucket.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 0b3f9331b8..a51a4d14dc 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -48,8 +48,8 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin verify := cmd.Command("verify", "verify all blocks in the bucket against specified issues") verifyRepair := verify.Flag("repair", "attempt to repair blocks for which issues were detected"). Short('r').Default("false").Bool() - backupBucketConfFile := verify.Flag("objstore-backup.config", "The backup object store configuration in yaml format."). - PlaceHolder("").String() + backupBucketConfFile := verify.Flag("objstore-backup.config-file", "The backup object store configuration file path."). + PlaceHolder("").String() verifyIssues := verify.Flag("issues", fmt.Sprintf("Issues to verify (and optionally repair). Possible values: %v", allIssues())). Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings() verifyIDWhitelist := verify.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+ diff --git a/docs/components/bucket.md b/docs/components/bucket.md index 3ae92b0770..5a7c87be78 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -86,8 +86,8 @@ Flags: --objstore.config-file= The object store configuration file path. -r, --repair attempt to repair blocks for which issues were detected - --objstore-backup.config= - The backup object store configuration in yaml format. + --objstore-backup.config-file= + The backup object store configuration file path. -i, --issues=index_issue... ... Issues to verify (and optionally repair). Possible values: [duplicated_compaction index_issue