Add several metrics-generator fields to user-configurable overrides

CHANGELOG.md

@@ -2,6 +2,7 @@
 * [FEATURE] Add the `/api/status/buildinfo` endpoint [#2702](https://github.com/grafana/tempo/pull/2702) (@fabrizio-grafana)
 * [FEATURE] New encoding vParquet3 with support for dedicated attribute columns (@mapno, @stoewer) [#2649](https://github.com/grafana/tempo/pull/2649)
 * [ENHANCEMENT] Assert ingestion rate limits as early as possible [#2640](https://github.com/grafana/tempo/pull/2703) (@mghildiy)
+* [ENHANCEMENT] Add several metrics-generator fields to user-configurable overrides [#2711](https://github.com/grafana/tempo/pull/2711) (@kvrhdn)
 
 ## v2.2.0-rc0 / 2023-07-21
 

cmd/tempo/app/overrides_validation.go

@@ -3,6 +3,9 @@ package app
 import (
 	"fmt"
 
+	"golang.org/x/exp/slices"
+
+	"github.com/grafana/tempo/modules/generator"
 	"github.com/grafana/tempo/modules/overrides/userconfigurableapi"
 )
 
@@ -26,13 +29,21 @@ func NewOverridesValidator(cfg *Config) userconfigurableapi.Validator {
 }
 
 func (v *overridesValidator) Validate(limits *userconfigurableapi.UserConfigurableLimits) error {
-	if limits.Forwarders != nil {
-		for _, f := range *limits.Forwarders {
+	if forwarders, ok := limits.GetForwarders(); ok {
+		for _, f := range forwarders {
 			if _, ok := v.validForwarders[f]; !ok {
 				return fmt.Errorf("forwarder \"%s\" is not a known forwarder, contact your system administrator", f)
 			}
 		}
 	}
 
+	if processors, ok := limits.GetMetricsGenerator().GetProcessors(); ok {
+		for p := range processors.GetMap() {
+			if !slices.Contains(generator.SupportedProcessors, p) {
+				return fmt.Errorf("metrics_generator.processor \"%s\" is not a known processor, valid values: %v", p, generator.SupportedProcessors)
+			}
+		}
+	}
+
 	return nil
 }

cmd/tempo/app/overrides_validation_test.go

@@ -1,12 +1,14 @@
 package app
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/distributor/forwarder"
+	"github.com/grafana/tempo/modules/generator"
 	"github.com/grafana/tempo/modules/overrides/userconfigurableapi"
 )
 
@@ -47,6 +49,19 @@ func Test_overridesValidator(t *testing.T) {
 			},
 			expErr: "forwarder \"some-forwarder\" is not a known forwarder, contact your system administrator",
 		},
+		{
+			name: "metrics_generator.processor",
+			cfg:  Config{},
+			limits: userconfigurableapi.UserConfigurableLimits{
+				MetricsGenerator: &userconfigurableapi.UserConfigurableOverridesMetricsGenerator{
+					Processors: map[string]struct{}{
+						"service-graphs": {},
+						"span-span":      {},
+					},
+				},
+			},
+			expErr: fmt.Sprintf("metrics_generator.processor \"span-span\" is not a known processor, valid values: %v", generator.SupportedProcessors),
+		},
 	}
 
 	for _, tc := range testCases {

modules/generator/instance.go

@@ -25,7 +25,7 @@ import (
 )
 
 var (
-	allSupportedProcessors = []string{servicegraphs.Name, spanmetrics.Name, localblocks.Name}
+	SupportedProcessors = []string{servicegraphs.Name, spanmetrics.Name, localblocks.Name}
 
 	metricActiveProcessors = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "tempo",
@@ -252,7 +252,7 @@ func (i *instance) diffProcessors(desiredProcessors map[string]struct{}, desired
 			}
 		default:
 			level.Error(i.logger).Log(
-				"msg", fmt.Sprintf("processor does not exist, supported processors: [%s]", strings.Join(allSupportedProcessors, ", ")),
+				"msg", fmt.Sprintf("processor does not exist, supported processors: [%s]", strings.Join(SupportedProcessors, ", ")),
 				"processorName", processorName,
 			)
 			err = fmt.Errorf("unknown processor %s", processorName)
@@ -286,7 +286,7 @@ func (i *instance) addProcessor(processorName string, cfg ProcessorConfig) error
 		newProcessor = p
 	default:
 		level.Error(i.logger).Log(
-			"msg", fmt.Sprintf("processor does not exist, supported processors: [%s]", strings.Join(allSupportedProcessors, ", ")),
+			"msg", fmt.Sprintf("processor does not exist, supported processors: [%s]", strings.Join(SupportedProcessors, ", ")),
 			"processorName", processorName,
 		)
 		return fmt.Errorf("unknown processor %s", processorName)
@@ -319,7 +319,7 @@ func (i *instance) removeProcessor(processorName string) {
 
 // updateProcessorMetrics updates the active processor metrics. Must be called under a read lock.
 func (i *instance) updateProcessorMetrics() {
-	for _, processorName := range allSupportedProcessors {
+	for _, processorName := range SupportedProcessors {
 		isPresent := 0.0
 		if _, ok := i.processors[processorName]; ok {
 			isPresent = 1.0

modules/overrides/limits.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/grafana/tempo/pkg/sharedconfig"
 	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
+	"github.com/grafana/tempo/pkg/util/listtomap"
 	"github.com/grafana/tempo/tempodb/backend"
 )
 
@@ -62,7 +63,7 @@ type Limits struct {
 
 	// Metrics-generator config
 	MetricsGeneratorRingSize                                       int                              `yaml:"metrics_generator_ring_size" json:"metrics_generator_ring_size"`
-	MetricsGeneratorProcessors                                     ListToMap                        `yaml:"metrics_generator_processors" json:"metrics_generator_processors"`
+	MetricsGeneratorProcessors                                     listtomap.ListToMap              `yaml:"metrics_generator_processors" json:"metrics_generator_processors"`
 	MetricsGeneratorMaxActiveSeries                                uint32                           `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"`
 	MetricsGeneratorCollectionInterval                             time.Duration                    `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"`
 	MetricsGeneratorDisableCollection                              bool                             `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"`

modules/overrides/user_configurable_overrides.go

@@ -157,12 +157,12 @@ func (o *userConfigurableOverridesManager) reloadAllTenantLimits(ctx context.Con
 	return nil
 }
 
-func (o *userConfigurableOverridesManager) getTenantLimits(userID string) (*api.UserConfigurableLimits, bool) {
+// getTenantLimits returns the tenant limits for the given tenant, can be nil.
+func (o *userConfigurableOverridesManager) getTenantLimits(userID string) *api.UserConfigurableLimits {
 	o.mtx.RLock()
 	defer o.mtx.RUnlock()
 
-	tenantLimits, ok := o.tenantLimits[userID]
-	return tenantLimits, ok
+	return o.tenantLimits[userID]
 }
 
 func (o *userConfigurableOverridesManager) getAllTenantLimits() tenantLimits {
@@ -184,13 +184,54 @@ func (o *userConfigurableOverridesManager) setTenantLimit(userID string, limits
 }
 
 func (o *userConfigurableOverridesManager) Forwarders(userID string) []string {
-	tenantLimits, ok := o.getTenantLimits(userID)
-	if ok && tenantLimits.Forwarders != nil {
-		return *tenantLimits.Forwarders
+	if forwarders, ok := o.getTenantLimits(userID).GetForwarders(); ok {
+		return forwarders
 	}
 	return o.Interface.Forwarders(userID)
 }
 
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessors(userID string) map[string]struct{} {
+	if processors, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessors(); ok {
+		return processors.GetMap()
+	}
+	return o.Interface.MetricsGeneratorProcessors(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorDisableCollection(userID string) bool {
+	if disableCollection, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetDisableCollection(); ok {
+		return disableCollection
+	}
+	return o.Interface.MetricsGeneratorDisableCollection(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorServiceGraphsDimensions(userID string) []string {
+	if dimensions, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetDimensions(); ok {
+		return dimensions
+	}
+	return o.Interface.MetricsGeneratorProcessorServiceGraphsDimensions(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorServiceGraphsPeerAttributes(userID string) []string {
+	if peerAttribtues, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetPeerAttributes(); ok {
+		return peerAttribtues
+	}
+	return o.Interface.MetricsGeneratorProcessorServiceGraphsPeerAttributes(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorSpanMetricsDimensions(userID string) []string {
+	if dimensions, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetDimensions(); ok {
+		return dimensions
+	}
+	return o.Interface.MetricsGeneratorProcessorSpanMetricsDimensions(userID)
+}
+
+func (o *userConfigurableOverridesManager) MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(userID string) bool {
+	if enableTargetInfo, ok := o.getTenantLimits(userID).GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableTargetInfo(); ok {
+		return enableTargetInfo
+	}
+	return o.Interface.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(userID)
+}
+
 // statusUserConfigurableOverrides used to marshal UserConfigurableLimits for tenants
 type statusUserConfigurableOverrides struct {
 	TenantLimits tenantLimits `yaml:"user_configurable_overrides" json:"user_configurable_overrides"`

modules/overrides/user_configurable_overrides_test.go

@@ -2,6 +2,7 @@ package overrides
 
 import (
 	"context"
+	"flag"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -62,6 +63,48 @@ func TestUserConfigOverridesManager(t *testing.T) {
 	assert.Equal(t, []string{"my-forwarder"}, mgr.Forwarders(tenant2))
 }
 
+func TestUserConfigOverridesManager_allFields(t *testing.T) {
+	defaultLimits := Limits{}
+	defaultLimits.RegisterFlagsAndApplyDefaults(&flag.FlagSet{})
+	_, mgr := localUserConfigOverrides(t, defaultLimits)
+
+	assert.Empty(t, mgr.Forwarders(tenant1))
+	assert.Empty(t, mgr.MetricsGeneratorProcessors(tenant1))
+	assert.Equal(t, false, mgr.MetricsGeneratorDisableCollection(tenant1))
+	assert.Empty(t, mgr.MetricsGeneratorProcessorServiceGraphsDimensions(tenant1))
+	assert.Empty(t, mgr.MetricsGeneratorProcessorServiceGraphsPeerAttributes(tenant1))
+	assert.Empty(t, mgr.MetricsGeneratorProcessorSpanMetricsDimensions(tenant1))
+	assert.Equal(t, false, mgr.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(tenant1))
+
+	// Inject user-configurable overrides
+	mgr.tenantLimits[tenant1] = &api.UserConfigurableLimits{
+		Forwarders: &[]string{"my-forwarder"},
+		MetricsGenerator: &api.UserConfigurableOverridesMetricsGenerator{
+			Processors:        map[string]struct{}{"service-graphs": {}},
+			DisableCollection: boolPtr(true),
+			Processor: &api.UserConfigurableOverridesMetricsGeneratorProcessor{
+				ServiceGraphs: &api.UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs{
+					Dimensions:     &[]string{"sg-dimension"},
+					PeerAttributes: &[]string{"attribute"},
+				},
+				SpanMetrics: &api.UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics{
+					Dimensions:       &[]string{"sm-dimension"},
+					EnableTargetInfo: boolPtr(true),
+				},
+			},
+		},
+	}
+
+	// Verify we can get the updated overrides
+	assert.Equal(t, []string{"my-forwarder"}, mgr.Forwarders(tenant1))
+	assert.Equal(t, map[string]struct{}{"service-graphs": {}}, mgr.MetricsGeneratorProcessors(tenant1))
+	assert.Equal(t, true, mgr.MetricsGeneratorDisableCollection(tenant1))
+	assert.Equal(t, []string{"sg-dimension"}, mgr.MetricsGeneratorProcessorServiceGraphsDimensions(tenant1))
+	assert.Equal(t, []string{"attribute"}, mgr.MetricsGeneratorProcessorServiceGraphsPeerAttributes(tenant1))
+	assert.Equal(t, []string{"sm-dimension"}, mgr.MetricsGeneratorProcessorSpanMetricsDimensions(tenant1))
+	assert.Equal(t, true, mgr.MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(tenant1))
+}
+
 func TestUserConfigOverridesManager_populateFromBackend(t *testing.T) {
 	defaultLimits := Limits{
 		Forwarders: []string{"my-forwarder"},
@@ -234,3 +277,7 @@ func (b *badClient) Delete(context.Context, string, backend.Version) error {
 
 func (b badClient) Shutdown() {
 }
+
+func boolPtr(b bool) *bool {
+	return &b
+}

modules/overrides/userconfigurableapi/limits.go

@@ -1,5 +1,110 @@
 package userconfigurableapi
 
+import (
+	"github.com/grafana/tempo/pkg/util/listtomap"
+)
+
 type UserConfigurableLimits struct {
-	Forwarders *[]string `json:"forwarders" yaml:"forwarders"`
+	Forwarders *[]string `json:"forwarders"`
+
+	MetricsGenerator *UserConfigurableOverridesMetricsGenerator `json:"metrics_generator,omitempty"`
+}
+
+func (l *UserConfigurableLimits) GetForwarders() ([]string, bool) {
+	if l != nil && l.Forwarders != nil {
+		return *l.Forwarders, true
+	}
+	return nil, false
+}
+
+func (l *UserConfigurableLimits) GetMetricsGenerator() *UserConfigurableOverridesMetricsGenerator {
+	if l != nil {
+		return l.MetricsGenerator
+	}
+	return nil
+}
+
+type UserConfigurableOverridesMetricsGenerator struct {
+	Processors        listtomap.ListToMap `json:"processors,omitempty"`
+	DisableCollection *bool               `json:"disable_collection,omitempty"`
+
+	Processor *UserConfigurableOverridesMetricsGeneratorProcessor `json:"processor,omitempty"`
+}
+
+func (l *UserConfigurableOverridesMetricsGenerator) GetProcessors() (listtomap.ListToMap, bool) {
+	if l != nil && l.Processors != nil {
+		return l.Processors, true
+	}
+	return nil, false
+}
+
+func (l *UserConfigurableOverridesMetricsGenerator) GetDisableCollection() (bool, bool) {
+	if l != nil && l.DisableCollection != nil {
+		return *l.DisableCollection, true
+	}
+	return false, false
+}
+
+func (l *UserConfigurableOverridesMetricsGenerator) GetProcessor() *UserConfigurableOverridesMetricsGeneratorProcessor {
+	if l != nil {
+		return l.Processor
+	}
+	return nil
+}
+
+type UserConfigurableOverridesMetricsGeneratorProcessor struct {
+	ServiceGraphs *UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs `json:"service_graphs,omitempty"`
+	SpanMetrics   *UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics   `json:"span_metrics,omitempty"`
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessor) GetServiceGraphs() *UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs {
+	if l != nil {
+		return l.ServiceGraphs
+	}
+	return nil
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessor) GetSpanMetrics() *UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics {
+	if l != nil {
+		return l.SpanMetrics
+	}
+	return nil
+}
+
+type UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs struct {
+	Dimensions     *[]string
+	PeerAttributes *[]string `json:"peer_attributes,omitempty"`
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs) GetDimensions() ([]string, bool) {
+	if l != nil && l.Dimensions != nil {
+		return *l.Dimensions, true
+	}
+	return nil, false
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessorServiceGraphs) GetPeerAttributes() ([]string, bool) {
+	if l != nil && l.PeerAttributes != nil {
+		return *l.PeerAttributes, true
+	}
+	return nil, false
+}
+
+type UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics struct {
+	Dimensions       *[]string
+	EnableTargetInfo *bool `json:"enable_target_info,omitempty"`
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics) GetDimensions() ([]string, bool) {
+	if l != nil && l.Dimensions != nil {
+		return *l.Dimensions, true
+	}
+	return nil, false
+}
+
+func (l *UserConfigurableOverridesMetricsGeneratorProcessorSpanMetrics) GetEnableTargetInfo() (bool, bool) {
+	if l != nil && l.EnableTargetInfo != nil {
+		return *l.EnableTargetInfo, true
+	}
+	return false, false
 }

modules/overrides/list_to_map.go → pkg/util/listtomap/list_to_map.go

@@ -1,4 +1,4 @@
-package overrides
+package listtomap
 
 import (
 	"encoding/json"

modules/overrides/list_to_map_test.go → pkg/util/listtomap/list_to_map_test.go

@@ -1,4 +1,4 @@
-package overrides
+package listtomap
 
 import (
 	"encoding/json"