diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 97a13bf61de..bdf5cd5536f 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -1969,23 +1969,34 @@ }, { "kind": "field", - "name": "enable_vector_vector_binary_comparison_operations", + "name": "enable_binary_logical_operations", "required": false, - "desc": "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", + "fieldFlag": "querier.mimir-query-engine.enable-binary-logical-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_vector_scalar_binary_comparison_operations", + "name": "enable_one_to_many_and_many_to_one_binary_operations", "required": false, - "desc": "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", + "fieldFlag": "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "enable_scalars", + "required": false, + "desc": "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.", + "fieldValue": null, + "fieldDefaultValue": true, + "fieldFlag": "querier.mimir-query-engine.enable-scalars", "fieldType": "boolean", "fieldCategory": "experimental" }, @@ -2002,57 +2013,57 @@ }, { "kind": "field", - "name": "enable_binary_logical_operations", + "name": "enable_subqueries", "required": false, - "desc": "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-binary-logical-operations", + "fieldFlag": "querier.mimir-query-engine.enable-subqueries", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_scalars", + "name": "enable_vector_scalar_binary_comparison_operations", "required": false, - "desc": "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-scalars", + "fieldFlag": "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_subqueries", + "name": "enable_vector_vector_binary_comparison_operations", "required": false, - "desc": "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-subqueries", + "fieldFlag": "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_histogram_quantile_function", + "name": "disabled_aggregations", "required": false, - "desc": "Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Comma-separated list of aggregations to disable support for. Only applies if MQE is in use.", "fieldValue": null, - "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-histogram-quantile-function", - "fieldType": "boolean", + "fieldDefaultValue": "", + "fieldFlag": "querier.mimir-query-engine.disabled-aggregations", + "fieldType": "string", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_one_to_many_and_many_to_one_binary_operations", + "name": "disabled_functions", "required": false, - "desc": "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Comma-separated list of function names to disable support for. Only applies if MQE is in use.", "fieldValue": null, - "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", - "fieldType": "boolean", + "fieldDefaultValue": "", + "fieldFlag": "querier.mimir-query-engine.disabled-functions", + "fieldType": "string", "fieldCategory": "experimental" } ], diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 9cda873d2c5..f78f918010f 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -2097,12 +2097,14 @@ Usage of ./cmd/mimir/mimir: Maximum number of split (by time) or partial (by shard) queries that will be scheduled in parallel by the query-frontend for a single input query. This limit is introduced to have a fairer query scheduling and avoid a single query over a large time range saturating all available queriers. (default 14) -querier.max-samples int Maximum number of samples a single query can load into memory. This config option should be set on query-frontend too when query sharding is enabled. (default 50000000) + -querier.mimir-query-engine.disabled-aggregations comma-separated-list-of-strings + [experimental] Comma-separated list of aggregations to disable support for. Only applies if MQE is in use. + -querier.mimir-query-engine.disabled-functions comma-separated-list-of-strings + [experimental] Comma-separated list of function names to disable support for. Only applies if MQE is in use. -querier.mimir-query-engine.enable-aggregation-operations [experimental] Enable support for aggregation operations in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-binary-logical-operations [experimental] Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use. (default true) - -querier.mimir-query-engine.enable-histogram-quantile-function - [experimental] Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations [experimental] Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 63bde157959..b65a35a7889 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -1496,47 +1496,52 @@ mimir_query_engine: # CLI flag: -querier.mimir-query-engine.enable-aggregation-operations [enable_aggregation_operations: | default = true] - # (experimental) Enable support for binary comparison operations between two - # vectors in the Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations - [enable_vector_vector_binary_comparison_operations: | default = true] - - # (experimental) Enable support for binary comparison operations between a - # vector and a scalar in the Mimir query engine. Only applies if the MQE is in - # use. - # CLI flag: -querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations - [enable_vector_scalar_binary_comparison_operations: | default = true] - - # (experimental) Enable support for binary comparison operations between two - # scalars in the Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations - [enable_scalar_scalar_binary_comparison_operations: | default = true] - # (experimental) Enable support for binary logical operations in the Mimir # query engine. Only applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-binary-logical-operations [enable_binary_logical_operations: | default = true] + # (experimental) Enable support for one-to-many and many-to-one binary + # operations (group_left/group_right) in the Mimir query engine. Only applies + # if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations + [enable_one_to_many_and_many_to_one_binary_operations: | default = true] + # (experimental) Enable support for scalars in the Mimir query engine. Only # applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-scalars [enable_scalars: | default = true] + # (experimental) Enable support for binary comparison operations between two + # scalars in the Mimir query engine. Only applies if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations + [enable_scalar_scalar_binary_comparison_operations: | default = true] + # (experimental) Enable support for subqueries in the Mimir query engine. Only # applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-subqueries [enable_subqueries: | default = true] - # (experimental) Enable support for the histogram_quantile function in the - # Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-histogram-quantile-function - [enable_histogram_quantile_function: | default = true] + # (experimental) Enable support for binary comparison operations between a + # vector and a scalar in the Mimir query engine. Only applies if the MQE is in + # use. + # CLI flag: -querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations + [enable_vector_scalar_binary_comparison_operations: | default = true] - # (experimental) Enable support for one-to-many and many-to-one binary - # operations (group_left/group_right) in the Mimir query engine. Only applies - # if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations - [enable_one_to_many_and_many_to_one_binary_operations: | default = true] + # (experimental) Enable support for binary comparison operations between two + # vectors in the Mimir query engine. Only applies if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations + [enable_vector_vector_binary_comparison_operations: | default = true] + + # (experimental) Comma-separated list of aggregations to disable support for. + # Only applies if MQE is in use. + # CLI flag: -querier.mimir-query-engine.disabled-aggregations + [disabled_aggregations: | default = ""] + + # (experimental) Comma-separated list of function names to disable support + # for. Only applies if MQE is in use. + # CLI flag: -querier.mimir-query-engine.disabled-functions + [disabled_functions: | default = ""] ``` ### frontend diff --git a/pkg/querier/engine/config.go b/pkg/querier/engine/config.go index e9fbf896d0a..577c5714f6a 100644 --- a/pkg/querier/engine/config.go +++ b/pkg/querier/engine/config.go @@ -33,7 +33,7 @@ type Config struct { PromQLExperimentalFunctionsEnabled bool `yaml:"promql_experimental_functions_enabled" category:"experimental"` - MimirQueryEngine streamingpromql.FeatureToggles `yaml:"mimir_query_engine" category:"experimental"` + MimirQueryEngine streamingpromql.Features `yaml:"mimir_query_engine" category:"experimental"` } func (cfg *Config) RegisterFlags(f *flag.FlagSet) { @@ -73,8 +73,8 @@ func NewPromQLEngineOptions(cfg Config, activityTracker *activitytracker.Activit } mqeOpts := streamingpromql.EngineOpts{ - CommonOpts: commonOpts, - FeatureToggles: cfg.MimirQueryEngine, + CommonOpts: commonOpts, + Features: cfg.MimirQueryEngine, } return commonOpts, mqeOpts, cfg.PromQLExperimentalFunctionsEnabled diff --git a/pkg/streamingpromql/config.go b/pkg/streamingpromql/config.go index 3f241e7ecd4..a79892d5567 100644 --- a/pkg/streamingpromql/config.go +++ b/pkg/streamingpromql/config.go @@ -5,32 +5,35 @@ package streamingpromql import ( "flag" + "github.com/grafana/dskit/flagext" "github.com/prometheus/prometheus/promql" ) type EngineOpts struct { - CommonOpts promql.EngineOpts - FeatureToggles FeatureToggles + CommonOpts promql.EngineOpts + Features Features // When operating in pedantic mode, we panic if memory consumption is > 0 after Query.Close() // (indicating something was not returned to a pool). Pedantic bool } -type FeatureToggles struct { +type Features struct { EnableAggregationOperations bool `yaml:"enable_aggregation_operations" category:"experimental"` - EnableVectorVectorBinaryComparisonOperations bool `yaml:"enable_vector_vector_binary_comparison_operations" category:"experimental"` - EnableVectorScalarBinaryComparisonOperations bool `yaml:"enable_vector_scalar_binary_comparison_operations" category:"experimental"` - EnableScalarScalarBinaryComparisonOperations bool `yaml:"enable_scalar_scalar_binary_comparison_operations" category:"experimental"` EnableBinaryLogicalOperations bool `yaml:"enable_binary_logical_operations" category:"experimental"` + EnableOneToManyAndManyToOneBinaryOperations bool `yaml:"enable_one_to_many_and_many_to_one_binary_operations" category:"experimental"` EnableScalars bool `yaml:"enable_scalars" category:"experimental"` + EnableScalarScalarBinaryComparisonOperations bool `yaml:"enable_scalar_scalar_binary_comparison_operations" category:"experimental"` EnableSubqueries bool `yaml:"enable_subqueries" category:"experimental"` - EnableHistogramQuantileFunction bool `yaml:"enable_histogram_quantile_function" category:"experimental"` - EnableOneToManyAndManyToOneBinaryOperations bool `yaml:"enable_one_to_many_and_many_to_one_binary_operations" category:"experimental"` + EnableVectorScalarBinaryComparisonOperations bool `yaml:"enable_vector_scalar_binary_comparison_operations" category:"experimental"` + EnableVectorVectorBinaryComparisonOperations bool `yaml:"enable_vector_vector_binary_comparison_operations" category:"experimental"` + + DisabledAggregations flagext.StringSliceCSV `yaml:"disabled_aggregations" category:"experimental"` + DisabledFunctions flagext.StringSliceCSV `yaml:"disabled_functions" category:"experimental"` } // EnableAllFeatures enables all features supported by MQE, including experimental or incomplete features. -var EnableAllFeatures = FeatureToggles{ +var EnableAllFeatures = Features{ // Note that we deliberately use a keyless literal here to force a compilation error if we don't keep this in sync with new fields added to FeatureToggles. true, true, @@ -40,17 +43,20 @@ var EnableAllFeatures = FeatureToggles{ true, true, true, - true, + []string{}, + []string{}, } -func (t *FeatureToggles) RegisterFlags(f *flag.FlagSet) { +func (t *Features) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&t.EnableAggregationOperations, "querier.mimir-query-engine.enable-aggregation-operations", true, "Enable support for aggregation operations in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableVectorVectorBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", true, "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableVectorScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableScalarScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between two scalars in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableBinaryLogicalOperations, "querier.mimir-query-engine.enable-binary-logical-operations", true, "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableOneToManyAndManyToOneBinaryOperations, "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", true, "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableScalars, "querier.mimir-query-engine.enable-scalars", true, "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableScalarScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between two scalars in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableSubqueries, "querier.mimir-query-engine.enable-subqueries", true, "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableHistogramQuantileFunction, "querier.mimir-query-engine.enable-histogram-quantile-function", true, "Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableOneToManyAndManyToOneBinaryOperations, "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", true, "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableVectorScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableVectorVectorBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", true, "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.") + + f.Var(&t.DisabledAggregations, "querier.mimir-query-engine.disabled-aggregations", "Comma-separated list of aggregations to disable support for. Only applies if MQE is in use.") + f.Var(&t.DisabledFunctions, "querier.mimir-query-engine.disabled-functions", "Comma-separated list of function names to disable support for. Only applies if MQE is in use.") } diff --git a/pkg/streamingpromql/engine.go b/pkg/streamingpromql/engine.go index 23425acdf12..a8a9880e565 100644 --- a/pkg/streamingpromql/engine.go +++ b/pkg/streamingpromql/engine.go @@ -9,15 +9,18 @@ import ( "context" "errors" "fmt" + "slices" "time" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" "github.com/grafana/mimir/pkg/querier/stats" + "github.com/grafana/mimir/pkg/streamingpromql/operators/aggregations" ) const defaultLookbackDelta = 5 * time.Minute // This should be the same value as github.com/prometheus/prometheus/promql.defaultLookbackDelta. @@ -44,13 +47,29 @@ func NewEngine(opts EngineOpts, limitsProvider QueryLimitsProvider, metrics *sta return nil, errors.New("enabling delayed name removal not supported by Mimir query engine") } + // We must sort DisabledFunctions as we use a binary search on it later. + slices.Sort(opts.Features.DisabledFunctions) + + disabledAggregationsItems := make([]parser.ItemType, 0, len(opts.Features.DisabledAggregations)) + for _, agg := range opts.Features.DisabledAggregations { + item, ok := aggregations.GetAggregationItemType(agg) + if !ok { + return nil, fmt.Errorf("disabled aggregation '%s' does not exist", agg) + } + disabledAggregationsItems = append(disabledAggregationsItems, item) + } + // No point sorting DisabledAggregations earlier, as ItemType ints are not in order. + // We must sort DisabledAggregationsItems as we use a binary search on it later. + slices.Sort(disabledAggregationsItems) + return &Engine{ - lookbackDelta: lookbackDelta, - timeout: opts.CommonOpts.Timeout, - limitsProvider: limitsProvider, - activeQueryTracker: opts.CommonOpts.ActiveQueryTracker, - featureToggles: opts.FeatureToggles, - noStepSubqueryIntervalFn: opts.CommonOpts.NoStepSubqueryIntervalFn, + lookbackDelta: lookbackDelta, + timeout: opts.CommonOpts.Timeout, + limitsProvider: limitsProvider, + activeQueryTracker: opts.CommonOpts.ActiveQueryTracker, + features: opts.Features, + disabledAggregationsItems: disabledAggregationsItems, + noStepSubqueryIntervalFn: opts.CommonOpts.NoStepSubqueryIntervalFn, logger: logger, estimatedPeakMemoryConsumption: promauto.With(opts.CommonOpts.Reg).NewHistogram(prometheus.HistogramOpts{ @@ -65,11 +84,13 @@ func NewEngine(opts EngineOpts, limitsProvider QueryLimitsProvider, metrics *sta } type Engine struct { - lookbackDelta time.Duration - timeout time.Duration - limitsProvider QueryLimitsProvider - activeQueryTracker promql.QueryTracker - featureToggles FeatureToggles + lookbackDelta time.Duration + timeout time.Duration + limitsProvider QueryLimitsProvider + activeQueryTracker promql.QueryTracker + features Features + disabledAggregationsItems []parser.ItemType + noStepSubqueryIntervalFn func(rangeMillis int64) int64 logger log.Logger diff --git a/pkg/streamingpromql/engine_test.go b/pkg/streamingpromql/engine_test.go index 981bba7c580..6756d91688c 100644 --- a/pkg/streamingpromql/engine_test.go +++ b/pkg/streamingpromql/engine_test.go @@ -45,7 +45,7 @@ func init() { } func TestUnsupportedPromQLFeatures(t *testing.T) { - featureToggles := EnableAllFeatures + features := EnableAllFeatures // The goal of this is not to list every conceivable expression that is unsupported, but to cover all the // different cases and make sure we produce a reasonable error message when these cases are encountered. @@ -58,132 +58,153 @@ func TestUnsupportedPromQLFeatures(t *testing.T) { for expression, expectedError := range unsupportedExpressions { t.Run(expression, func(t *testing.T) { - requireQueryIsUnsupported(t, featureToggles, expression, expectedError) + requireQueryIsUnsupported(t, features, expression, expectedError) }) } } func TestUnsupportedPromQLFeaturesWithFeatureToggles(t *testing.T) { t.Run("aggregation operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableAggregationOperations = false + features := EnableAllFeatures + features.EnableAggregationOperations = false - requireQueryIsUnsupported(t, featureToggles, "sum by (label) (metric)", "aggregation operations") + requireQueryIsUnsupported(t, features, "sum by (label) (metric)", "aggregation operations") }) t.Run("vector/vector binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableVectorVectorBinaryComparisonOperations = false + features := EnableAllFeatures + features.EnableVectorVectorBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} > other_metric{}", "vector/vector binary expression with '>'") + requireQueryIsUnsupported(t, features, "metric{} > other_metric{}", "vector/vector binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, features, "metric{} > 1") + requireQueryIsSupported(t, features, "1 > metric{}") + requireQueryIsSupported(t, features, "2 > bool 1") + requireQueryIsSupported(t, features, "metric{} + other_metric{}") + requireQueryIsSupported(t, features, "metric{} + 1") + requireQueryIsSupported(t, features, "1 + metric{}") + requireQueryIsSupported(t, features, "2 + 1") + requireQueryIsSupported(t, features, "metric{} and other_metric{}") }) t.Run("vector/scalar binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableVectorScalarBinaryComparisonOperations = false + features := EnableAllFeatures + features.EnableVectorScalarBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} > 1", "vector/scalar binary expression with '>'") - requireQueryIsUnsupported(t, featureToggles, "1 > metric{}", "vector/scalar binary expression with '>'") + requireQueryIsUnsupported(t, features, "metric{} > 1", "vector/scalar binary expression with '>'") + requireQueryIsUnsupported(t, features, "1 > metric{}", "vector/scalar binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, features, "metric{} > other_metric{}") + requireQueryIsSupported(t, features, "2 > bool 1") + requireQueryIsSupported(t, features, "metric{} + other_metric{}") + requireQueryIsSupported(t, features, "metric{} + 1") + requireQueryIsSupported(t, features, "1 + metric{}") + requireQueryIsSupported(t, features, "2 + 1") + requireQueryIsSupported(t, features, "metric{} and other_metric{}") }) t.Run("scalar/scalar binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableScalarScalarBinaryComparisonOperations = false + features := EnableAllFeatures + features.EnableScalarScalarBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "2 > bool 1", "scalar/scalar binary expression with '>'") + requireQueryIsUnsupported(t, features, "2 > bool 1", "scalar/scalar binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, features, "metric{} > other_metric{}") + requireQueryIsSupported(t, features, "metric{} > 1") + requireQueryIsSupported(t, features, "1 > metric{}") + requireQueryIsSupported(t, features, "metric{} + other_metric{}") + requireQueryIsSupported(t, features, "metric{} + 1") + requireQueryIsSupported(t, features, "1 + metric{}") + requireQueryIsSupported(t, features, "2 + 1") + requireQueryIsSupported(t, features, "metric{} and other_metric{}") }) t.Run("binary expressions with logical operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableBinaryLogicalOperations = false + features := EnableAllFeatures + features.EnableBinaryLogicalOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} and other_metric{}", "binary expression with 'and'") - requireQueryIsUnsupported(t, featureToggles, "metric{} or other_metric{}", "binary expression with 'or'") - requireQueryIsUnsupported(t, featureToggles, "metric{} unless other_metric{}", "binary expression with 'unless'") + requireQueryIsUnsupported(t, features, "metric{} and other_metric{}", "binary expression with 'and'") + requireQueryIsUnsupported(t, features, "metric{} or other_metric{}", "binary expression with 'or'") + requireQueryIsUnsupported(t, features, "metric{} unless other_metric{}", "binary expression with 'unless'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") + requireQueryIsSupported(t, features, "metric{} + other_metric{}") + requireQueryIsSupported(t, features, "metric{} + 1") + requireQueryIsSupported(t, features, "1 + metric{}") + requireQueryIsSupported(t, features, "2 + 1") + requireQueryIsSupported(t, features, "metric{} > other_metric{}") + requireQueryIsSupported(t, features, "metric{} > 1") + requireQueryIsSupported(t, features, "1 > metric{}") + requireQueryIsSupported(t, features, "2 > bool 1") }) t.Run("scalars", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableScalars = false + features := EnableAllFeatures + features.EnableScalars = false - requireQueryIsUnsupported(t, featureToggles, "2", "scalar values") + requireQueryIsUnsupported(t, features, "2", "scalar values") }) t.Run("subqueries", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableSubqueries = false + features := EnableAllFeatures + features.EnableSubqueries = false - requireQueryIsUnsupported(t, featureToggles, "sum_over_time(metric[1m:10s])", "subquery") + requireQueryIsUnsupported(t, features, "sum_over_time(metric[1m:10s])", "subquery") }) - t.Run("histogram_quantile function", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableHistogramQuantileFunction = false + t.Run("one-to-many and many-to-one binary operations", func(t *testing.T) { + features := EnableAllFeatures + features.EnableOneToManyAndManyToOneBinaryOperations = false - requireQueryIsUnsupported(t, featureToggles, "histogram_quantile(0.5, metric)", "'histogram_quantile' function") + requireQueryIsUnsupported(t, features, "metric{} + on() group_left() other_metric{}", "binary expression with many-to-one matching") + requireQueryIsUnsupported(t, features, "metric{} + on() group_right() other_metric{}", "binary expression with one-to-many matching") }) - t.Run("one-to-many and many-to-one binary operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableOneToManyAndManyToOneBinaryOperations = false + t.Run("function disabled by name", func(t *testing.T) { + features := EnableAllFeatures + features.DisabledFunctions = []string{"histogram_quantile", "ceil", "nonexistant"} + + requireQueryIsUnsupported(t, features, "ceil(metric{})", "'ceil' function") + requireQueryIsUnsupported(t, features, "histogram_quantile(0.9, h{})", "'histogram_quantile' function") + }) + + t.Run("aggregation disabled by name", func(t *testing.T) { + features := EnableAllFeatures + features.DisabledAggregations = []string{"sum", "avg", "MAX"} - requireQueryIsUnsupported(t, featureToggles, "metric{} + on() group_left() other_metric{}", "binary expression with many-to-one matching") - requireQueryIsUnsupported(t, featureToggles, "metric{} + on() group_right() other_metric{}", "binary expression with one-to-many matching") + requireQueryIsUnsupported(t, features, "avg by (label) (metric{})", "'avg' aggregation disabled") + requireQueryIsUnsupported(t, features, "max(metric{})", "'max' aggregation disabled") + requireQueryIsUnsupported(t, features, "SUM(metric{})", "'sum' aggregation disabled") + }) + + t.Run("unknown aggregation name disabled", func(t *testing.T) { + features := EnableAllFeatures + features.DisabledAggregations = []string{"sum", "avg", "NotAnAgg"} + + opts := NewTestEngineOpts() + opts.Features = features + _, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) + require.Error(t, err) + require.EqualError(t, err, "disabled aggregation 'NotAnAgg' does not exist") }) } -func requireQueryIsUnsupported(t *testing.T, toggles FeatureToggles, expression string, expectedError string) { - requireRangeQueryIsUnsupported(t, toggles, expression, expectedError) - requireInstantQueryIsUnsupported(t, toggles, expression, expectedError) +func requireQueryIsUnsupported(t *testing.T, features Features, expression string, expectedError string) { + requireRangeQueryIsUnsupported(t, features, expression, expectedError) + requireInstantQueryIsUnsupported(t, features, expression, expectedError) } -func requireQueryIsSupported(t *testing.T, toggles FeatureToggles, expression string) { - requireRangeQueryIsSupported(t, toggles, expression) - requireInstantQueryIsSupported(t, toggles, expression) +func requireQueryIsSupported(t *testing.T, features Features, expression string) { + requireRangeQueryIsSupported(t, features, expression) + requireInstantQueryIsSupported(t, features, expression) } -func requireRangeQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, expression string, expectedError string) { +func requireRangeQueryIsUnsupported(t *testing.T, features Features, expression string, expectedError string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.Features = features engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -194,9 +215,9 @@ func requireRangeQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, require.Nil(t, qry) } -func requireInstantQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, expression string, expectedError string) { +func requireInstantQueryIsUnsupported(t *testing.T, features Features, expression string, expectedError string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.Features = features engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -207,9 +228,9 @@ func requireInstantQueryIsUnsupported(t *testing.T, featureToggles FeatureToggle require.Nil(t, qry) } -func requireRangeQueryIsSupported(t *testing.T, featureToggles FeatureToggles, expression string) { +func requireRangeQueryIsSupported(t *testing.T, features Features, expression string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.Features = features engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -217,9 +238,9 @@ func requireRangeQueryIsSupported(t *testing.T, featureToggles FeatureToggles, e require.NoError(t, err) } -func requireInstantQueryIsSupported(t *testing.T, featureToggles FeatureToggles, expression string) { +func requireInstantQueryIsSupported(t *testing.T, features Features, expression string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.Features = features engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) diff --git a/pkg/streamingpromql/operators/aggregations/common.go b/pkg/streamingpromql/operators/aggregations/common.go index 04491c72966..c671795dca2 100644 --- a/pkg/streamingpromql/operators/aggregations/common.go +++ b/pkg/streamingpromql/operators/aggregations/common.go @@ -3,6 +3,8 @@ package aggregations import ( + "strings" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/promql/parser" @@ -35,3 +37,27 @@ var AggregationGroupFactories = map[parser.ItemType]AggregationGroupFactory{ // // Invalid combinations include exponential and custom buckets, and histograms with incompatible custom buckets. var invalidCombinationOfHistograms = &histogram.FloatHistogram{} + +// The aggregation names are not exported, but their item types are. It's safe to assume the names will not change. +// (ie, "avg" will be parser.AVG). +var aggregationItemKey = map[string]parser.ItemType{ + "avg": parser.AVG, + "bottomk": parser.BOTTOMK, + "count_values": parser.COUNT_VALUES, + "count": parser.COUNT, + "group": parser.GROUP, + "limit_ratio": parser.LIMIT_RATIO, + "limitk": parser.LIMITK, + "max": parser.MAX, + "min": parser.MIN, + "quantile": parser.QUANTILE, + "stddev": parser.STDDEV, + "stdvar": parser.STDVAR, + "sum": parser.SUM, + "topk": parser.TOPK, +} + +func GetAggregationItemType(aggregation string) (parser.ItemType, bool) { + item, ok := aggregationItemKey[strings.ToLower(aggregation)] + return item, ok +} diff --git a/pkg/streamingpromql/query.go b/pkg/streamingpromql/query.go index 3c76763db20..46ca9f6eefa 100644 --- a/pkg/streamingpromql/query.go +++ b/pkg/streamingpromql/query.go @@ -169,9 +169,12 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types Stats: q.stats, }, nil case *parser.AggregateExpr: - if !q.engine.featureToggles.EnableAggregationOperations { + if !q.engine.features.EnableAggregationOperations { return nil, compat.NewNotSupportedError("aggregation operations") } + if _, found := slices.BinarySearch(q.engine.disabledAggregationsItems, e.Op); found { + return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' aggregation disabled", e.Op.String())) + } if e.Param != nil { return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' aggregation with parameter", e.Op)) @@ -203,7 +206,7 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types // We don't need to handle scalars on both sides here, as that would produce a scalar and so is handled in convertToScalarOperator. if e.LHS.Type() == parser.ValueTypeScalar || e.RHS.Type() == parser.ValueTypeScalar { - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableVectorScalarBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.features.EnableVectorScalarBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("vector/scalar binary expression with '%v'", e.Op)) } @@ -244,15 +247,15 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types } // Vectors on both sides. - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableVectorVectorBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.features.EnableVectorVectorBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("vector/vector binary expression with '%v'", e.Op)) } - if e.Op.IsSetOperator() && !q.engine.featureToggles.EnableBinaryLogicalOperations { + if e.Op.IsSetOperator() && !q.engine.features.EnableBinaryLogicalOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("binary expression with '%v'", e.Op)) } - if !e.Op.IsSetOperator() && e.VectorMatching.Card != parser.CardOneToOne && !q.engine.featureToggles.EnableOneToManyAndManyToOneBinaryOperations { + if !e.Op.IsSetOperator() && e.VectorMatching.Card != parser.CardOneToOne && !q.engine.features.EnableOneToManyAndManyToOneBinaryOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("binary expression with %v matching", e.VectorMatching.Card)) } @@ -310,11 +313,10 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types } func (q *Query) convertFunctionCallToInstantVectorOperator(e *parser.Call, timeRange types.QueryTimeRange) (types.InstantVectorOperator, error) { - // Handle special toggles for classic histograms - if !q.engine.featureToggles.EnableHistogramQuantileFunction { - if e.Func.Name == "histogram_quantile" { - return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' function", e.Func.Name)) - } + // e.Func.Name is already validated and canonicalised by the parser. Meaning we don't need to check if the function name + // refers to a function that exists, nor normalise the casing etc. before checking if it is disabled. + if _, found := slices.BinarySearch(q.engine.features.DisabledFunctions, e.Func.Name); found { + return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' function", e.Func.Name)) } factory, ok := instantVectorFunctionOperatorFactories[e.Func.Name] @@ -356,7 +358,7 @@ func (q *Query) convertToRangeVectorOperator(expr parser.Expr, timeRange types.Q return selectors.NewRangeVectorSelector(selector, q.memoryConsumptionTracker, q.stats), nil case *parser.SubqueryExpr: - if !q.engine.featureToggles.EnableSubqueries { + if !q.engine.features.EnableSubqueries { return nil, compat.NewNotSupportedError("subquery") } @@ -428,7 +430,7 @@ func (q *Query) convertToScalarOperator(expr parser.Expr, timeRange types.QueryT return nil, fmt.Errorf("cannot create scalar operator for expression that produces a %s", parser.DocumentedType(expr.Type())) } - if !q.engine.featureToggles.EnableScalars { + if !q.engine.features.EnableScalars { return nil, compat.NewNotSupportedError("scalar values") } @@ -469,7 +471,7 @@ func (q *Query) convertToScalarOperator(expr parser.Expr, timeRange types.QueryT case *parser.ParenExpr: return q.convertToScalarOperator(e.Expr, timeRange) case *parser.BinaryExpr: - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableScalarScalarBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.features.EnableScalarScalarBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("scalar/scalar binary expression with '%v'", e.Op)) } diff --git a/pkg/streamingpromql/testing.go b/pkg/streamingpromql/testing.go index 51599ecee64..c50744ec9dc 100644 --- a/pkg/streamingpromql/testing.go +++ b/pkg/streamingpromql/testing.go @@ -21,7 +21,7 @@ func NewTestEngineOpts() EngineOpts { NoStepSubqueryIntervalFn: func(int64) int64 { return time.Minute.Milliseconds() }, }, - FeatureToggles: EnableAllFeatures, - Pedantic: true, + Features: EnableAllFeatures, + Pedantic: true, } }