From dd5e7a09c9b5205e5653b1617b963766fe2fbea7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Tue, 7 Sep 2021 12:19:33 +0200 Subject: [PATCH 1/6] Add runtime config handler --- CHANGELOG.md | 1 + cmd/tempo/app/modules.go | 2 + modules/overrides/overrides.go | 69 +++++++++++++++++++++++++++++++--- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f0ce897f70..2f9eacb94b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ * [CHANGE] Renamed CLI flag from `--storage.trace.maintenance-cycle` to `--storage.trace.blocklist_poll`. This is a **breaking change** [#897](https://github.com/grafana/tempo/pull/897) (@mritunjaysharma394) * [CHANGE] update jsonnet alerts and recording rules to use `job_selectors` and `cluster_selectors` for configurable unique identifier labels [#935](https://github.com/grafana/tempo/pull/935) (@kevinschoonover) * [CHANGE] Modify generated tag keys in Vulture for easier filtering [#934](https://github.com/grafana/tempo/pull/934) (@zalegrala) +* [FEATURE] Add runtime config handler [#936](https://github.com/grafana/tempo/pull/936) (@mapno) ## v1.1.0 / 2021-08-26 * [CHANGE] Upgrade Cortex from v1.9.0 to v1.9.0-131-ga4bf10354 [#841](https://github.com/grafana/tempo/pull/841) (@aknuds1) diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 659dc58506e..693e68e678b 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -102,6 +102,8 @@ func (t *App) initOverrides() (services.Service, error) { } t.overrides = overrides + t.Server.HTTP.Handle("/runtime_config", overrides.Handler()) + return t.overrides, nil } diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index a9a93724b57..e0141582d10 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "io" + "net/http" "time" + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/log" "github.com/grafana/dskit/runtimeconfig" "github.com/grafana/dskit/services" @@ -37,13 +39,18 @@ func loadPerTenantOverrides(r io.Reader) (interface{}, error) { return overrides, nil } +type Config struct { + Defaults *Limits `yaml:"defaults"` + PerTenantOverrides *perTenantOverrides `yaml:"overrides,omitempty"` +} + // Overrides periodically fetch a set of per-user overrides, and provides convenience // functions for fetching the correct value. type Overrides struct { services.Service - defaultLimits *Limits - tenantLimits TenantLimits + config *Config + tenantLimits TenantLimits // Manager for subservices subservices *services.Manager @@ -55,7 +62,13 @@ type Overrides struct { // are defaulted to those values. As such, the last call to NewOverrides will // become the new global defaults. func NewOverrides(defaults Limits) (*Overrides, error) { - var tenantLimits TenantLimits + var ( + tenantLimits TenantLimits + config = &Config{ + Defaults: &defaults, + PerTenantOverrides: &perTenantOverrides{TenantLimits: make(map[string]*Limits)}, + } + ) subservices := []services.Service(nil) if defaults.PerTenantOverrideConfig != "" { @@ -68,13 +81,17 @@ func NewOverrides(defaults Limits) (*Overrides, error) { if err != nil { return nil, fmt.Errorf("failed to create runtime config manager %w", err) } + perTenantOverrides, ok := runtimeCfgMgr.GetConfig().(*perTenantOverrides) + if ok && perTenantOverrides != nil { + config.PerTenantOverrides = perTenantOverrides + } tenantLimits = tenantLimitsFromRuntimeConfig(runtimeCfgMgr) subservices = append(subservices, runtimeCfgMgr) } o := &Overrides{ - tenantLimits: tenantLimits, - defaultLimits: &defaults, + tenantLimits: tenantLimits, + config: config, } if len(subservices) > 0 { @@ -123,6 +140,46 @@ func (o *Overrides) stopping(_ error) error { return nil } +func (o *Overrides) Handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var output interface{} + cfg := o.config + switch r.URL.Query().Get("mode") { + case "diff": + // Default runtime config is just empty struct, but to make diff work, + // we set defaultLimits for every tenant that exists in runtime config. + defaultCfg := perTenantOverrides{} + defaultCfg.TenantLimits = map[string]*Limits{} + for k, v := range o.config.PerTenantOverrides.TenantLimits { + if v != nil { + defaultCfg.TenantLimits[k] = o.config.Defaults + } + } + + cfgYaml, err := util.YAMLMarshalUnmarshal(cfg) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + defaultCfgYaml, err := util.YAMLMarshalUnmarshal(cfg) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + output, err = util.DiffConfig(defaultCfgYaml, cfgYaml) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + default: + output = cfg + } + util.WriteYAMLResponse(w, output) + } +} + // IngestionRateStrategy returns whether the ingestion rate limit should be individually applied // to each distributor instance (local) or evenly shared across the cluster (global). func (o *Overrides) IngestionRateStrategy() string { @@ -179,7 +236,7 @@ func (o *Overrides) getOverridesForUser(userID string) *Limits { return l } } - return o.defaultLimits + return o.config.Defaults } func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) TenantLimits { From 42f33466f69ac9a5509114d7e0830e6e5d945778 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Thu, 9 Sep 2021 18:15:10 +0200 Subject: [PATCH 2/6] Improve code --- cmd/tempo/app/modules.go | 2 +- modules/overrides/overrides.go | 83 ++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 693e68e678b..c9b990d9014 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -284,8 +284,8 @@ func (t *App) setupModuleManager() error { deps := map[string][]string{ // Server: nil, - // Overrides: nil, // Store: nil, + Overrides: {Server}, MemberlistKV: {Server}, QueryFrontend: {Server}, Ring: {Server, MemberlistKV}, diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index e0141582d10..481deeff49a 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -15,9 +15,9 @@ import ( "gopkg.in/yaml.v2" ) -// TenantLimits is a function that returns limits for given tenant, or -// nil, if there are no tenant-specific limits. -type TenantLimits func(userID string) *Limits +// TenantOverrides is a function that returns limits per tenant, or +// nil, if there are no overrides. +type TenantOverrides func() *perTenantOverrides const wildcardTenant = "*" @@ -26,6 +26,15 @@ type perTenantOverrides struct { TenantLimits map[string]*Limits `yaml:"overrides"` } +// ForUser returns limits for a given tenant, or nil if there are no tenant-specific limits. +func (o *perTenantOverrides) ForUser(userID string) *Limits { + l, ok := o.TenantLimits[userID] + if !ok || l == nil { + return nil + } + return l +} + // loadPerTenantOverrides is of type runtimeconfig.Loader func loadPerTenantOverrides(r io.Reader) (interface{}, error) { var overrides = &perTenantOverrides{} @@ -41,7 +50,7 @@ func loadPerTenantOverrides(r io.Reader) (interface{}, error) { type Config struct { Defaults *Limits `yaml:"defaults"` - PerTenantOverrides *perTenantOverrides `yaml:"overrides,omitempty"` + PerTenantOverrides *perTenantOverrides `yaml:"overrides"` } // Overrides periodically fetch a set of per-user overrides, and provides convenience @@ -49,8 +58,8 @@ type Config struct { type Overrides struct { services.Service - config *Config - tenantLimits TenantLimits + defaultLimits *Limits + tenantOverrides TenantOverrides // Manager for subservices subservices *services.Manager @@ -62,13 +71,7 @@ type Overrides struct { // are defaulted to those values. As such, the last call to NewOverrides will // become the new global defaults. func NewOverrides(defaults Limits) (*Overrides, error) { - var ( - tenantLimits TenantLimits - config = &Config{ - Defaults: &defaults, - PerTenantOverrides: &perTenantOverrides{TenantLimits: make(map[string]*Limits)}, - } - ) + var perTenantOverrides TenantOverrides subservices := []services.Service(nil) if defaults.PerTenantOverrideConfig != "" { @@ -81,17 +84,13 @@ func NewOverrides(defaults Limits) (*Overrides, error) { if err != nil { return nil, fmt.Errorf("failed to create runtime config manager %w", err) } - perTenantOverrides, ok := runtimeCfgMgr.GetConfig().(*perTenantOverrides) - if ok && perTenantOverrides != nil { - config.PerTenantOverrides = perTenantOverrides - } - tenantLimits = tenantLimitsFromRuntimeConfig(runtimeCfgMgr) + perTenantOverrides = tenantOverridesFromRuntimeConfig(runtimeCfgMgr) subservices = append(subservices, runtimeCfgMgr) } o := &Overrides{ - tenantLimits: tenantLimits, - config: config, + tenantOverrides: perTenantOverrides, + defaultLimits: &defaults, } if len(subservices) > 0 { @@ -143,26 +142,29 @@ func (o *Overrides) stopping(_ error) error { func (o *Overrides) Handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var output interface{} - cfg := o.config + cfg := Config{ + Defaults: o.defaultLimits, + PerTenantOverrides: o.tenantOverrides(), + } switch r.URL.Query().Get("mode") { case "diff": // Default runtime config is just empty struct, but to make diff work, // we set defaultLimits for every tenant that exists in runtime config. - defaultCfg := perTenantOverrides{} + defaultCfg := perTenantOverrides{TenantLimits: map[string]*Limits{}} defaultCfg.TenantLimits = map[string]*Limits{} - for k, v := range o.config.PerTenantOverrides.TenantLimits { + for k, v := range o.tenantOverrides().TenantLimits { if v != nil { - defaultCfg.TenantLimits[k] = o.config.Defaults + defaultCfg.TenantLimits[k] = o.defaultLimits } } - cfgYaml, err := util.YAMLMarshalUnmarshal(cfg) + cfgYaml, err := util.YAMLMarshalUnmarshal(cfg.PerTenantOverrides) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - defaultCfgYaml, err := util.YAMLMarshalUnmarshal(cfg) + defaultCfgYaml, err := util.YAMLMarshalUnmarshal(defaultCfg) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -225,30 +227,33 @@ func (o *Overrides) BlockRetention(userID string) time.Duration { } func (o *Overrides) getOverridesForUser(userID string) *Limits { - if o.tenantLimits != nil { - l := o.tenantLimits(userID) - if l != nil { - return l - } + if o.tenantOverrides == nil || o.tenantOverrides() == nil { + return o.defaultLimits + } + l := o.tenantOverrides().ForUser(userID) + if l != nil { + return l + } - l = o.tenantLimits(wildcardTenant) - if l != nil { - return l - } + l = o.tenantOverrides().ForUser(wildcardTenant) + if l != nil { + return l } - return o.config.Defaults + + return o.defaultLimits } -func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) TenantLimits { +func tenantOverridesFromRuntimeConfig(c *runtimeconfig.Manager) TenantOverrides { if c == nil { return nil } - return func(userID string) *Limits { + return func() *perTenantOverrides { cfg, ok := c.GetConfig().(*perTenantOverrides) if !ok || cfg == nil { return nil } - return cfg.TenantLimits[userID] + return cfg } + } From d5629a3ee37025eb855eaa433455e2e5b78409ca Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Thu, 9 Sep 2021 20:28:24 +0200 Subject: [PATCH 3/6] Inline overrides to not duplicate the field --- modules/overrides/limits.go | 2 +- modules/overrides/overrides.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/overrides/limits.go b/modules/overrides/limits.go index dc294f6e779..7c4d49999e3 100644 --- a/modules/overrides/limits.go +++ b/modules/overrides/limits.go @@ -37,7 +37,7 @@ type Limits struct { // Compactor enforced limits. BlockRetention model.Duration `yaml:"block_retention" json:"block_retention"` - // Config for overrides, convenient if it goes here. + // Configuration for overrides, convenient if it goes here. PerTenantOverrideConfig string `yaml:"per_tenant_override_config" json:"per_tenant_override_config"` PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"` } diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 481deeff49a..df52fb878fe 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -48,9 +48,10 @@ func loadPerTenantOverrides(r io.Reader) (interface{}, error) { return overrides, nil } +// Config type Config struct { - Defaults *Limits `yaml:"defaults"` - PerTenantOverrides *perTenantOverrides `yaml:"overrides"` + Defaults *Limits `yaml:"defaults"` + PerTenantOverrides perTenantOverrides `yaml:",inline"` } // Overrides periodically fetch a set of per-user overrides, and provides convenience @@ -141,10 +142,14 @@ func (o *Overrides) stopping(_ error) error { func (o *Overrides) Handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var tenantOverrides perTenantOverrides + if o.tenantOverrides != nil && o.tenantOverrides() != nil { + tenantOverrides = *o.tenantOverrides() + } var output interface{} cfg := Config{ Defaults: o.defaultLimits, - PerTenantOverrides: o.tenantOverrides(), + PerTenantOverrides: tenantOverrides, } switch r.URL.Query().Get("mode") { case "diff": From 64be701bc8d7749d44382d0f90351b27406b391e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Thu, 9 Sep 2021 20:32:54 +0200 Subject: [PATCH 4/6] Finish doc comment --- modules/overrides/overrides.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index df52fb878fe..6052f74a2f7 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -48,7 +48,7 @@ func loadPerTenantOverrides(r io.Reader) (interface{}, error) { return overrides, nil } -// Config +// Config is a struct used to print the complete runtime config (defaults + overrides) type Config struct { Defaults *Limits `yaml:"defaults"` PerTenantOverrides perTenantOverrides `yaml:",inline"` From 32250520eda0bfc938f053c9bced7b8ea6de0c13 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Fri, 10 Sep 2021 13:24:09 +0200 Subject: [PATCH 5/6] Use runtime config manager instead of limits struct --- modules/overrides/overrides.go | 76 ++++++++++++++++------------------ 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 6052f74a2f7..45a80746639 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -15,10 +15,6 @@ import ( "gopkg.in/yaml.v2" ) -// TenantOverrides is a function that returns limits per tenant, or -// nil, if there are no overrides. -type TenantOverrides func() *perTenantOverrides - const wildcardTenant = "*" // perTenantOverrides represents the overrides config file @@ -26,8 +22,8 @@ type perTenantOverrides struct { TenantLimits map[string]*Limits `yaml:"overrides"` } -// ForUser returns limits for a given tenant, or nil if there are no tenant-specific limits. -func (o *perTenantOverrides) ForUser(userID string) *Limits { +// forUser returns limits for a given tenant, or nil if there are no tenant-specific limits. +func (o *perTenantOverrides) forUser(userID string) *Limits { l, ok := o.TenantLimits[userID] if !ok || l == nil { return nil @@ -59,8 +55,8 @@ type Config struct { type Overrides struct { services.Service - defaultLimits *Limits - tenantOverrides TenantOverrides + defaultLimits *Limits + runtimeConfigMgr *runtimeconfig.Manager // Manager for subservices subservices *services.Manager @@ -72,7 +68,7 @@ type Overrides struct { // are defaulted to those values. As such, the last call to NewOverrides will // become the new global defaults. func NewOverrides(defaults Limits) (*Overrides, error) { - var perTenantOverrides TenantOverrides + var manager *runtimeconfig.Manager subservices := []services.Service(nil) if defaults.PerTenantOverrideConfig != "" { @@ -85,13 +81,13 @@ func NewOverrides(defaults Limits) (*Overrides, error) { if err != nil { return nil, fmt.Errorf("failed to create runtime config manager %w", err) } - perTenantOverrides = tenantOverridesFromRuntimeConfig(runtimeCfgMgr) + manager = runtimeCfgMgr subservices = append(subservices, runtimeCfgMgr) } o := &Overrides{ - tenantOverrides: perTenantOverrides, - defaultLimits: &defaults, + runtimeConfigMgr: manager, + defaultLimits: &defaults, } if len(subservices) > 0 { @@ -140,11 +136,23 @@ func (o *Overrides) stopping(_ error) error { return nil } +func (o *Overrides) TenantOverrides() *perTenantOverrides { + if o.runtimeConfigMgr == nil { + return nil + } + cfg, ok := o.runtimeConfigMgr.GetConfig().(*perTenantOverrides) + if !ok || cfg == nil { + return nil + } + + return cfg +} + func (o *Overrides) Handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var tenantOverrides perTenantOverrides - if o.tenantOverrides != nil && o.tenantOverrides() != nil { - tenantOverrides = *o.tenantOverrides() + if o.TenantOverrides() != nil { + tenantOverrides = *o.TenantOverrides() } var output interface{} cfg := Config{ @@ -157,7 +165,7 @@ func (o *Overrides) Handler() http.HandlerFunc { // we set defaultLimits for every tenant that exists in runtime config. defaultCfg := perTenantOverrides{TenantLimits: map[string]*Limits{}} defaultCfg.TenantLimits = map[string]*Limits{} - for k, v := range o.tenantOverrides().TenantLimits { + for k, v := range tenantOverrides.TenantLimits { if v != nil { defaultCfg.TenantLimits[k] = o.defaultLimits } @@ -217,48 +225,34 @@ func (o *Overrides) MaxSearchBytesPerTrace(userID string) int { return o.getOverridesForUser(userID).MaxSearchBytesPerTrace } -// IngestionRateSpans is the number of spans per second allowed for this tenant +// IngestionRateLimitBytes is the number of spans per second allowed for this tenant func (o *Overrides) IngestionRateLimitBytes(userID string) float64 { return float64(o.getOverridesForUser(userID).IngestionRateLimitBytes) } -// IngestionBurstSize is the burst size in spans allowed for this tenant +// IngestionBurstSizeBytes is the burst size in spans allowed for this tenant func (o *Overrides) IngestionBurstSizeBytes(userID string) int { return o.getOverridesForUser(userID).IngestionBurstSizeBytes } +// BlockRetention is the duration of the block retention for this tenant func (o *Overrides) BlockRetention(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).BlockRetention) } func (o *Overrides) getOverridesForUser(userID string) *Limits { - if o.tenantOverrides == nil || o.tenantOverrides() == nil { - return o.defaultLimits - } - l := o.tenantOverrides().ForUser(userID) - if l != nil { - return l - } - - l = o.tenantOverrides().ForUser(wildcardTenant) - if l != nil { - return l - } - - return o.defaultLimits -} + if tenantOverrides := o.TenantOverrides(); tenantOverrides != nil { + l := tenantOverrides.forUser(userID) + if l != nil { + return l + } -func tenantOverridesFromRuntimeConfig(c *runtimeconfig.Manager) TenantOverrides { - if c == nil { - return nil - } - return func() *perTenantOverrides { - cfg, ok := c.GetConfig().(*perTenantOverrides) - if !ok || cfg == nil { - return nil + l = tenantOverrides.forUser(wildcardTenant) + if l != nil { + return l } - return cfg } + return o.defaultLimits } From 50e244204066aef92450dc9086721ee53142f021 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Date: Fri, 10 Sep 2021 13:57:01 +0200 Subject: [PATCH 6/6] Unexport tenantOverrides --- modules/overrides/overrides.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 45a80746639..2063863dfb2 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -136,7 +136,7 @@ func (o *Overrides) stopping(_ error) error { return nil } -func (o *Overrides) TenantOverrides() *perTenantOverrides { +func (o *Overrides) tenantOverrides() *perTenantOverrides { if o.runtimeConfigMgr == nil { return nil } @@ -151,8 +151,8 @@ func (o *Overrides) TenantOverrides() *perTenantOverrides { func (o *Overrides) Handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var tenantOverrides perTenantOverrides - if o.TenantOverrides() != nil { - tenantOverrides = *o.TenantOverrides() + if o.tenantOverrides() != nil { + tenantOverrides = *o.tenantOverrides() } var output interface{} cfg := Config{ @@ -241,7 +241,7 @@ func (o *Overrides) BlockRetention(userID string) time.Duration { } func (o *Overrides) getOverridesForUser(userID string) *Limits { - if tenantOverrides := o.TenantOverrides(); tenantOverrides != nil { + if tenantOverrides := o.tenantOverrides(); tenantOverrides != nil { l := tenantOverrides.forUser(userID) if l != nil { return l