diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f0ce897f70..2f9eacb94b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@
* [CHANGE] Renamed CLI flag from `--storage.trace.maintenance-cycle` to `--storage.trace.blocklist_poll`. This is a **breaking change** [#897](https://github.com/grafana/tempo/pull/897) (@mritunjaysharma394)
* [CHANGE] update jsonnet alerts and recording rules to use `job_selectors` and `cluster_selectors` for configurable unique identifier labels [#935](https://github.com/grafana/tempo/pull/935) (@kevinschoonover)
* [CHANGE] Modify generated tag keys in Vulture for easier filtering [#934](https://github.com/grafana/tempo/pull/934) (@zalegrala)
+* [FEATURE] Add runtime config handler [#936](https://github.com/grafana/tempo/pull/936) (@mapno)
## v1.1.0 / 2021-08-26
* [CHANGE] Upgrade Cortex from v1.9.0 to v1.9.0-131-ga4bf10354 [#841](https://github.com/grafana/tempo/pull/841) (@aknuds1)
diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go
index 659dc58506e..c9b990d9014 100644
--- a/cmd/tempo/app/modules.go
+++ b/cmd/tempo/app/modules.go
@@ -102,6 +102,8 @@ func (t *App) initOverrides() (services.Service, error) {
}
t.overrides = overrides
+ t.Server.HTTP.Handle("/runtime_config", overrides.Handler())
+
return t.overrides, nil
}
@@ -282,8 +284,8 @@ func (t *App) setupModuleManager() error {
deps := map[string][]string{
// Server: nil,
- // Overrides: nil,
// Store: nil,
+ Overrides: {Server},
MemberlistKV: {Server},
QueryFrontend: {Server},
Ring: {Server, MemberlistKV},
diff --git a/modules/overrides/limits.go b/modules/overrides/limits.go
index dc294f6e779..7c4d49999e3 100644
--- a/modules/overrides/limits.go
+++ b/modules/overrides/limits.go
@@ -37,7 +37,7 @@ type Limits struct {
// Compactor enforced limits.
BlockRetention model.Duration `yaml:"block_retention" json:"block_retention"`
- // Config for overrides, convenient if it goes here.
+ // Configuration for overrides, convenient if it goes here.
PerTenantOverrideConfig string `yaml:"per_tenant_override_config" json:"per_tenant_override_config"`
PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"`
}
diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go
index a9a93724b57..2063863dfb2 100644
--- a/modules/overrides/overrides.go
+++ b/modules/overrides/overrides.go
@@ -4,8 +4,10 @@ import (
"context"
"fmt"
"io"
+ "net/http"
"time"
+ "github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/log"
"github.com/grafana/dskit/runtimeconfig"
"github.com/grafana/dskit/services"
@@ -13,10 +15,6 @@ import (
"gopkg.in/yaml.v2"
)
-// TenantLimits is a function that returns limits for given tenant, or
-// nil, if there are no tenant-specific limits.
-type TenantLimits func(userID string) *Limits
-
const wildcardTenant = "*"
// perTenantOverrides represents the overrides config file
@@ -24,6 +22,15 @@ type perTenantOverrides struct {
TenantLimits map[string]*Limits `yaml:"overrides"`
}
+// forUser returns limits for a given tenant, or nil if there are no tenant-specific limits.
+func (o *perTenantOverrides) forUser(userID string) *Limits {
+ l, ok := o.TenantLimits[userID]
+ if !ok || l == nil {
+ return nil
+ }
+ return l
+}
+
// loadPerTenantOverrides is of type runtimeconfig.Loader
func loadPerTenantOverrides(r io.Reader) (interface{}, error) {
var overrides = &perTenantOverrides{}
@@ -37,13 +44,19 @@ func loadPerTenantOverrides(r io.Reader) (interface{}, error) {
return overrides, nil
}
+// Config is a struct used to print the complete runtime config (defaults + overrides)
+type Config struct {
+ Defaults *Limits `yaml:"defaults"`
+ PerTenantOverrides perTenantOverrides `yaml:",inline"`
+}
+
// Overrides periodically fetch a set of per-user overrides, and provides convenience
// functions for fetching the correct value.
type Overrides struct {
services.Service
- defaultLimits *Limits
- tenantLimits TenantLimits
+ defaultLimits *Limits
+ runtimeConfigMgr *runtimeconfig.Manager
// Manager for subservices
subservices *services.Manager
@@ -55,7 +68,7 @@ type Overrides struct {
// are defaulted to those values. As such, the last call to NewOverrides will
// become the new global defaults.
func NewOverrides(defaults Limits) (*Overrides, error) {
- var tenantLimits TenantLimits
+ var manager *runtimeconfig.Manager
subservices := []services.Service(nil)
if defaults.PerTenantOverrideConfig != "" {
@@ -68,13 +81,13 @@ func NewOverrides(defaults Limits) (*Overrides, error) {
if err != nil {
return nil, fmt.Errorf("failed to create runtime config manager %w", err)
}
- tenantLimits = tenantLimitsFromRuntimeConfig(runtimeCfgMgr)
+ manager = runtimeCfgMgr
subservices = append(subservices, runtimeCfgMgr)
}
o := &Overrides{
- tenantLimits: tenantLimits,
- defaultLimits: &defaults,
+ runtimeConfigMgr: manager,
+ defaultLimits: &defaults,
}
if len(subservices) > 0 {
@@ -123,6 +136,65 @@ func (o *Overrides) stopping(_ error) error {
return nil
}
+func (o *Overrides) tenantOverrides() *perTenantOverrides {
+ if o.runtimeConfigMgr == nil {
+ return nil
+ }
+ cfg, ok := o.runtimeConfigMgr.GetConfig().(*perTenantOverrides)
+ if !ok || cfg == nil {
+ return nil
+ }
+
+ return cfg
+}
+
+func (o *Overrides) Handler() http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ var tenantOverrides perTenantOverrides
+ if o.tenantOverrides() != nil {
+ tenantOverrides = *o.tenantOverrides()
+ }
+ var output interface{}
+ cfg := Config{
+ Defaults: o.defaultLimits,
+ PerTenantOverrides: tenantOverrides,
+ }
+ switch r.URL.Query().Get("mode") {
+ case "diff":
+ // Default runtime config is just empty struct, but to make diff work,
+ // we set defaultLimits for every tenant that exists in runtime config.
+ defaultCfg := perTenantOverrides{TenantLimits: map[string]*Limits{}}
+ defaultCfg.TenantLimits = map[string]*Limits{}
+ for k, v := range tenantOverrides.TenantLimits {
+ if v != nil {
+ defaultCfg.TenantLimits[k] = o.defaultLimits
+ }
+ }
+
+ cfgYaml, err := util.YAMLMarshalUnmarshal(cfg.PerTenantOverrides)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ defaultCfgYaml, err := util.YAMLMarshalUnmarshal(defaultCfg)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ output, err = util.DiffConfig(defaultCfgYaml, cfgYaml)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ default:
+ output = cfg
+ }
+ util.WriteYAMLResponse(w, output)
+ }
+}
+
// IngestionRateStrategy returns whether the ingestion rate limit should be individually applied
// to each distributor instance (local) or evenly shared across the cluster (global).
func (o *Overrides) IngestionRateStrategy() string {
@@ -153,45 +225,34 @@ func (o *Overrides) MaxSearchBytesPerTrace(userID string) int {
return o.getOverridesForUser(userID).MaxSearchBytesPerTrace
}
-// IngestionRateSpans is the number of spans per second allowed for this tenant
+// IngestionRateLimitBytes is the number of spans per second allowed for this tenant
func (o *Overrides) IngestionRateLimitBytes(userID string) float64 {
return float64(o.getOverridesForUser(userID).IngestionRateLimitBytes)
}
-// IngestionBurstSize is the burst size in spans allowed for this tenant
+// IngestionBurstSizeBytes is the burst size in spans allowed for this tenant
func (o *Overrides) IngestionBurstSizeBytes(userID string) int {
return o.getOverridesForUser(userID).IngestionBurstSizeBytes
}
+// BlockRetention is the duration of the block retention for this tenant
func (o *Overrides) BlockRetention(userID string) time.Duration {
return time.Duration(o.getOverridesForUser(userID).BlockRetention)
}
func (o *Overrides) getOverridesForUser(userID string) *Limits {
- if o.tenantLimits != nil {
- l := o.tenantLimits(userID)
+ if tenantOverrides := o.tenantOverrides(); tenantOverrides != nil {
+ l := tenantOverrides.forUser(userID)
if l != nil {
return l
}
- l = o.tenantLimits(wildcardTenant)
+ l = tenantOverrides.forUser(wildcardTenant)
if l != nil {
return l
}
- }
- return o.defaultLimits
-}
-func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) TenantLimits {
- if c == nil {
- return nil
}
- return func(userID string) *Limits {
- cfg, ok := c.GetConfig().(*perTenantOverrides)
- if !ok || cfg == nil {
- return nil
- }
- return cfg.TenantLimits[userID]
- }
+ return o.defaultLimits
}