diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 7de981485cc..78197921627 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -1969,23 +1969,34 @@ }, { "kind": "field", - "name": "enable_vector_vector_binary_comparison_operations", + "name": "enable_binary_logical_operations", "required": false, - "desc": "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", + "fieldFlag": "querier.mimir-query-engine.enable-binary-logical-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_vector_scalar_binary_comparison_operations", + "name": "enable_one_to_many_and_many_to_one_binary_operations", "required": false, - "desc": "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", + "fieldFlag": "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "enable_scalars", + "required": false, + "desc": "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.", + "fieldValue": null, + "fieldDefaultValue": true, + "fieldFlag": "querier.mimir-query-engine.enable-scalars", "fieldType": "boolean", "fieldCategory": "experimental" }, @@ -2002,57 +2013,57 @@ }, { "kind": "field", - "name": "enable_binary_logical_operations", + "name": "enable_subqueries", "required": false, - "desc": "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-binary-logical-operations", + "fieldFlag": "querier.mimir-query-engine.enable-subqueries", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_scalars", + "name": "enable_vector_scalar_binary_comparison_operations", "required": false, - "desc": "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-scalars", + "fieldFlag": "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_subqueries", + "name": "enable_vector_vector_binary_comparison_operations", "required": false, - "desc": "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-subqueries", + "fieldFlag": "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", "fieldType": "boolean", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_histogram_quantile_function", + "name": "disabled_aggregations", "required": false, - "desc": "Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Comma-separated list of aggregations to disable. Queries using a disabled aggregation will fall back to Prometheus' query engine. Only applies if MQE is in use.", "fieldValue": null, - "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-histogram-quantile-function", - "fieldType": "boolean", + "fieldDefaultValue": "", + "fieldFlag": "querier.mimir-query-engine.disabled-aggregations", + "fieldType": "string", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_one_to_many_and_many_to_one_binary_operations", + "name": "disabled_functions", "required": false, - "desc": "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.", + "desc": "Comma-separated list of function names to disable. Queries using a disabled function will fall back to Prometheus' query engine. Only applies if MQE is in use.", "fieldValue": null, - "fieldDefaultValue": true, - "fieldFlag": "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", - "fieldType": "boolean", + "fieldDefaultValue": "", + "fieldFlag": "querier.mimir-query-engine.disabled-functions", + "fieldType": "string", "fieldCategory": "experimental" } ], diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 40b8e586b76..85d83fbf257 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -2097,12 +2097,14 @@ Usage of ./cmd/mimir/mimir: Maximum number of split (by time) or partial (by shard) queries that will be scheduled in parallel by the query-frontend for a single input query. This limit is introduced to have a fairer query scheduling and avoid a single query over a large time range saturating all available queriers. (default 14) -querier.max-samples int Maximum number of samples a single query can load into memory. This config option should be set on query-frontend too when query sharding is enabled. (default 50000000) + -querier.mimir-query-engine.disabled-aggregations comma-separated-list-of-strings + [experimental] Comma-separated list of aggregations to disable. Queries using a disabled aggregation will fall back to Prometheus' query engine. Only applies if MQE is in use. + -querier.mimir-query-engine.disabled-functions comma-separated-list-of-strings + [experimental] Comma-separated list of function names to disable. Queries using a disabled function will fall back to Prometheus' query engine. Only applies if MQE is in use. -querier.mimir-query-engine.enable-aggregation-operations [experimental] Enable support for aggregation operations in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-binary-logical-operations [experimental] Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use. (default true) - -querier.mimir-query-engine.enable-histogram-quantile-function - [experimental] Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations [experimental] Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use. (default true) -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 284442448ca..b361ba2fd1e 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -1496,47 +1496,54 @@ mimir_query_engine: # CLI flag: -querier.mimir-query-engine.enable-aggregation-operations [enable_aggregation_operations: | default = true] - # (experimental) Enable support for binary comparison operations between two - # vectors in the Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations - [enable_vector_vector_binary_comparison_operations: | default = true] - - # (experimental) Enable support for binary comparison operations between a - # vector and a scalar in the Mimir query engine. Only applies if the MQE is in - # use. - # CLI flag: -querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations - [enable_vector_scalar_binary_comparison_operations: | default = true] - - # (experimental) Enable support for binary comparison operations between two - # scalars in the Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations - [enable_scalar_scalar_binary_comparison_operations: | default = true] - # (experimental) Enable support for binary logical operations in the Mimir # query engine. Only applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-binary-logical-operations [enable_binary_logical_operations: | default = true] + # (experimental) Enable support for one-to-many and many-to-one binary + # operations (group_left/group_right) in the Mimir query engine. Only applies + # if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations + [enable_one_to_many_and_many_to_one_binary_operations: | default = true] + # (experimental) Enable support for scalars in the Mimir query engine. Only # applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-scalars [enable_scalars: | default = true] + # (experimental) Enable support for binary comparison operations between two + # scalars in the Mimir query engine. Only applies if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations + [enable_scalar_scalar_binary_comparison_operations: | default = true] + # (experimental) Enable support for subqueries in the Mimir query engine. Only # applies if the MQE is in use. # CLI flag: -querier.mimir-query-engine.enable-subqueries [enable_subqueries: | default = true] - # (experimental) Enable support for the histogram_quantile function in the - # Mimir query engine. Only applies if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-histogram-quantile-function - [enable_histogram_quantile_function: | default = true] + # (experimental) Enable support for binary comparison operations between a + # vector and a scalar in the Mimir query engine. Only applies if the MQE is in + # use. + # CLI flag: -querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations + [enable_vector_scalar_binary_comparison_operations: | default = true] - # (experimental) Enable support for one-to-many and many-to-one binary - # operations (group_left/group_right) in the Mimir query engine. Only applies - # if the MQE is in use. - # CLI flag: -querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations - [enable_one_to_many_and_many_to_one_binary_operations: | default = true] + # (experimental) Enable support for binary comparison operations between two + # vectors in the Mimir query engine. Only applies if the MQE is in use. + # CLI flag: -querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations + [enable_vector_vector_binary_comparison_operations: | default = true] + + # (experimental) Comma-separated list of aggregations to disable. Queries + # using a disabled aggregation will fall back to Prometheus' query engine. + # Only applies if MQE is in use. + # CLI flag: -querier.mimir-query-engine.disabled-aggregations + [disabled_aggregations: | default = ""] + + # (experimental) Comma-separated list of function names to disable. Queries + # using a disabled function will fall back to Prometheus' query engine. Only + # applies if MQE is in use. + # CLI flag: -querier.mimir-query-engine.disabled-functions + [disabled_functions: | default = ""] ``` ### frontend diff --git a/pkg/querier/engine/config.go b/pkg/querier/engine/config.go index e9fbf896d0a..9556ec885cf 100644 --- a/pkg/querier/engine/config.go +++ b/pkg/querier/engine/config.go @@ -33,7 +33,7 @@ type Config struct { PromQLExperimentalFunctionsEnabled bool `yaml:"promql_experimental_functions_enabled" category:"experimental"` - MimirQueryEngine streamingpromql.FeatureToggles `yaml:"mimir_query_engine" category:"experimental"` + MimirQueryEngine streamingpromql.MQEOpts `yaml:"mimir_query_engine" category:"experimental"` } func (cfg *Config) RegisterFlags(f *flag.FlagSet) { @@ -73,8 +73,8 @@ func NewPromQLEngineOptions(cfg Config, activityTracker *activitytracker.Activit } mqeOpts := streamingpromql.EngineOpts{ - CommonOpts: commonOpts, - FeatureToggles: cfg.MimirQueryEngine, + CommonOpts: commonOpts, + MQEOpts: cfg.MimirQueryEngine, } return commonOpts, mqeOpts, cfg.PromQLExperimentalFunctionsEnabled diff --git a/pkg/streamingpromql/config.go b/pkg/streamingpromql/config.go index 3f241e7ecd4..a6341f1c904 100644 --- a/pkg/streamingpromql/config.go +++ b/pkg/streamingpromql/config.go @@ -5,32 +5,35 @@ package streamingpromql import ( "flag" + "github.com/grafana/dskit/flagext" "github.com/prometheus/prometheus/promql" ) type EngineOpts struct { - CommonOpts promql.EngineOpts - FeatureToggles FeatureToggles + CommonOpts promql.EngineOpts + MQEOpts MQEOpts // When operating in pedantic mode, we panic if memory consumption is > 0 after Query.Close() // (indicating something was not returned to a pool). Pedantic bool } -type FeatureToggles struct { +type MQEOpts struct { EnableAggregationOperations bool `yaml:"enable_aggregation_operations" category:"experimental"` - EnableVectorVectorBinaryComparisonOperations bool `yaml:"enable_vector_vector_binary_comparison_operations" category:"experimental"` - EnableVectorScalarBinaryComparisonOperations bool `yaml:"enable_vector_scalar_binary_comparison_operations" category:"experimental"` - EnableScalarScalarBinaryComparisonOperations bool `yaml:"enable_scalar_scalar_binary_comparison_operations" category:"experimental"` EnableBinaryLogicalOperations bool `yaml:"enable_binary_logical_operations" category:"experimental"` + EnableOneToManyAndManyToOneBinaryOperations bool `yaml:"enable_one_to_many_and_many_to_one_binary_operations" category:"experimental"` EnableScalars bool `yaml:"enable_scalars" category:"experimental"` + EnableScalarScalarBinaryComparisonOperations bool `yaml:"enable_scalar_scalar_binary_comparison_operations" category:"experimental"` EnableSubqueries bool `yaml:"enable_subqueries" category:"experimental"` - EnableHistogramQuantileFunction bool `yaml:"enable_histogram_quantile_function" category:"experimental"` - EnableOneToManyAndManyToOneBinaryOperations bool `yaml:"enable_one_to_many_and_many_to_one_binary_operations" category:"experimental"` + EnableVectorScalarBinaryComparisonOperations bool `yaml:"enable_vector_scalar_binary_comparison_operations" category:"experimental"` + EnableVectorVectorBinaryComparisonOperations bool `yaml:"enable_vector_vector_binary_comparison_operations" category:"experimental"` + + DisabledAggregations flagext.StringSliceCSV `yaml:"disabled_aggregations" category:"experimental"` + DisabledFunctions flagext.StringSliceCSV `yaml:"disabled_functions" category:"experimental"` } -// EnableAllFeatures enables all features supported by MQE, including experimental or incomplete features. -var EnableAllFeatures = FeatureToggles{ +// MQEOptsAllFeatures enables all features supported by MQE, including experimental or incomplete features. +var MQEOptsAllFeatures = MQEOpts{ // Note that we deliberately use a keyless literal here to force a compilation error if we don't keep this in sync with new fields added to FeatureToggles. true, true, @@ -40,17 +43,20 @@ var EnableAllFeatures = FeatureToggles{ true, true, true, - true, + []string{}, + []string{}, } -func (t *FeatureToggles) RegisterFlags(f *flag.FlagSet) { +func (t *MQEOpts) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&t.EnableAggregationOperations, "querier.mimir-query-engine.enable-aggregation-operations", true, "Enable support for aggregation operations in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableVectorVectorBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", true, "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableVectorScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableScalarScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between two scalars in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableBinaryLogicalOperations, "querier.mimir-query-engine.enable-binary-logical-operations", true, "Enable support for binary logical operations in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableOneToManyAndManyToOneBinaryOperations, "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", true, "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableScalars, "querier.mimir-query-engine.enable-scalars", true, "Enable support for scalars in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableScalarScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-scalar-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between two scalars in the Mimir query engine. Only applies if the MQE is in use.") f.BoolVar(&t.EnableSubqueries, "querier.mimir-query-engine.enable-subqueries", true, "Enable support for subqueries in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableHistogramQuantileFunction, "querier.mimir-query-engine.enable-histogram-quantile-function", true, "Enable support for the histogram_quantile function in the Mimir query engine. Only applies if the MQE is in use.") - f.BoolVar(&t.EnableOneToManyAndManyToOneBinaryOperations, "querier.mimir-query-engine.enable-one-to-many-and-many-to-one-binary-operations", true, "Enable support for one-to-many and many-to-one binary operations (group_left/group_right) in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableVectorScalarBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-scalar-binary-comparison-operations", true, "Enable support for binary comparison operations between a vector and a scalar in the Mimir query engine. Only applies if the MQE is in use.") + f.BoolVar(&t.EnableVectorVectorBinaryComparisonOperations, "querier.mimir-query-engine.enable-vector-vector-binary-comparison-operations", true, "Enable support for binary comparison operations between two vectors in the Mimir query engine. Only applies if the MQE is in use.") + + f.Var(&t.DisabledAggregations, "querier.mimir-query-engine.disabled-aggregations", "Comma-separated list of aggregations to disable. Queries using a disabled aggregation will fall back to Prometheus' query engine. Only applies if MQE is in use.") + f.Var(&t.DisabledFunctions, "querier.mimir-query-engine.disabled-functions", "Comma-separated list of function names to disable. Queries using a disabled function will fall back to Prometheus' query engine. Only applies if MQE is in use.") } diff --git a/pkg/streamingpromql/engine.go b/pkg/streamingpromql/engine.go index 23425acdf12..2250ca03f81 100644 --- a/pkg/streamingpromql/engine.go +++ b/pkg/streamingpromql/engine.go @@ -49,7 +49,7 @@ func NewEngine(opts EngineOpts, limitsProvider QueryLimitsProvider, metrics *sta timeout: opts.CommonOpts.Timeout, limitsProvider: limitsProvider, activeQueryTracker: opts.CommonOpts.ActiveQueryTracker, - featureToggles: opts.FeatureToggles, + mqeOpts: opts.MQEOpts, noStepSubqueryIntervalFn: opts.CommonOpts.NoStepSubqueryIntervalFn, logger: logger, @@ -69,7 +69,7 @@ type Engine struct { timeout time.Duration limitsProvider QueryLimitsProvider activeQueryTracker promql.QueryTracker - featureToggles FeatureToggles + mqeOpts MQEOpts noStepSubqueryIntervalFn func(rangeMillis int64) int64 logger log.Logger diff --git a/pkg/streamingpromql/engine_test.go b/pkg/streamingpromql/engine_test.go index d4b9bc3d973..9e6ade8d9c5 100644 --- a/pkg/streamingpromql/engine_test.go +++ b/pkg/streamingpromql/engine_test.go @@ -45,7 +45,7 @@ func init() { } func TestUnsupportedPromQLFeatures(t *testing.T) { - featureToggles := EnableAllFeatures + mqeOpts := MQEOptsAllFeatures // The goal of this is not to list every conceivable expression that is unsupported, but to cover all the // different cases and make sure we produce a reasonable error message when these cases are encountered. @@ -58,132 +58,142 @@ func TestUnsupportedPromQLFeatures(t *testing.T) { for expression, expectedError := range unsupportedExpressions { t.Run(expression, func(t *testing.T) { - requireQueryIsUnsupported(t, featureToggles, expression, expectedError) + requireQueryIsUnsupported(t, mqeOpts, expression, expectedError) }) } } -func TestUnsupportedPromQLFeaturesWithFeatureToggles(t *testing.T) { +func TestUnsupportedPromQLFeaturesWithMQEOpts(t *testing.T) { t.Run("aggregation operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableAggregationOperations = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableAggregationOperations = false - requireQueryIsUnsupported(t, featureToggles, "sum by (label) (metric)", "aggregation operations") + requireQueryIsUnsupported(t, mqeOpts, "sum by (label) (metric)", "aggregation operations") }) t.Run("vector/vector binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableVectorVectorBinaryComparisonOperations = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableVectorVectorBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} > other_metric{}", "vector/vector binary expression with '>'") + requireQueryIsUnsupported(t, mqeOpts, "metric{} > other_metric{}", "vector/vector binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} > 1") + requireQueryIsSupported(t, mqeOpts, "1 > metric{}") + requireQueryIsSupported(t, mqeOpts, "2 > bool 1") + requireQueryIsSupported(t, mqeOpts, "metric{} + other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} + 1") + requireQueryIsSupported(t, mqeOpts, "1 + metric{}") + requireQueryIsSupported(t, mqeOpts, "2 + 1") + requireQueryIsSupported(t, mqeOpts, "metric{} and other_metric{}") }) t.Run("vector/scalar binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableVectorScalarBinaryComparisonOperations = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableVectorScalarBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} > 1", "vector/scalar binary expression with '>'") - requireQueryIsUnsupported(t, featureToggles, "1 > metric{}", "vector/scalar binary expression with '>'") + requireQueryIsUnsupported(t, mqeOpts, "metric{} > 1", "vector/scalar binary expression with '>'") + requireQueryIsUnsupported(t, mqeOpts, "1 > metric{}", "vector/scalar binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} > other_metric{}") + requireQueryIsSupported(t, mqeOpts, "2 > bool 1") + requireQueryIsSupported(t, mqeOpts, "metric{} + other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} + 1") + requireQueryIsSupported(t, mqeOpts, "1 + metric{}") + requireQueryIsSupported(t, mqeOpts, "2 + 1") + requireQueryIsSupported(t, mqeOpts, "metric{} and other_metric{}") }) t.Run("scalar/scalar binary expressions with comparison operation", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableScalarScalarBinaryComparisonOperations = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableScalarScalarBinaryComparisonOperations = false - requireQueryIsUnsupported(t, featureToggles, "2 > bool 1", "scalar/scalar binary expression with '>'") + requireQueryIsUnsupported(t, mqeOpts, "2 > bool 1", "scalar/scalar binary expression with '>'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} and other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} > other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} > 1") + requireQueryIsSupported(t, mqeOpts, "1 > metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} + other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} + 1") + requireQueryIsSupported(t, mqeOpts, "1 + metric{}") + requireQueryIsSupported(t, mqeOpts, "2 + 1") + requireQueryIsSupported(t, mqeOpts, "metric{} and other_metric{}") }) t.Run("binary expressions with logical operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableBinaryLogicalOperations = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableBinaryLogicalOperations = false - requireQueryIsUnsupported(t, featureToggles, "metric{} and other_metric{}", "binary expression with 'and'") - requireQueryIsUnsupported(t, featureToggles, "metric{} or other_metric{}", "binary expression with 'or'") - requireQueryIsUnsupported(t, featureToggles, "metric{} unless other_metric{}", "binary expression with 'unless'") + requireQueryIsUnsupported(t, mqeOpts, "metric{} and other_metric{}", "binary expression with 'and'") + requireQueryIsUnsupported(t, mqeOpts, "metric{} or other_metric{}", "binary expression with 'or'") + requireQueryIsUnsupported(t, mqeOpts, "metric{} unless other_metric{}", "binary expression with 'unless'") // Other operations should still be supported. - requireQueryIsSupported(t, featureToggles, "metric{} + other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} + 1") - requireQueryIsSupported(t, featureToggles, "1 + metric{}") - requireQueryIsSupported(t, featureToggles, "2 + 1") - requireQueryIsSupported(t, featureToggles, "metric{} > other_metric{}") - requireQueryIsSupported(t, featureToggles, "metric{} > 1") - requireQueryIsSupported(t, featureToggles, "1 > metric{}") - requireQueryIsSupported(t, featureToggles, "2 > bool 1") + requireQueryIsSupported(t, mqeOpts, "metric{} + other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} + 1") + requireQueryIsSupported(t, mqeOpts, "1 + metric{}") + requireQueryIsSupported(t, mqeOpts, "2 + 1") + requireQueryIsSupported(t, mqeOpts, "metric{} > other_metric{}") + requireQueryIsSupported(t, mqeOpts, "metric{} > 1") + requireQueryIsSupported(t, mqeOpts, "1 > metric{}") + requireQueryIsSupported(t, mqeOpts, "2 > bool 1") }) t.Run("scalars", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableScalars = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableScalars = false - requireQueryIsUnsupported(t, featureToggles, "2", "scalar values") + requireQueryIsUnsupported(t, mqeOpts, "2", "scalar values") }) t.Run("subqueries", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableSubqueries = false + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableSubqueries = false - requireQueryIsUnsupported(t, featureToggles, "sum_over_time(metric[1m:10s])", "subquery") + requireQueryIsUnsupported(t, mqeOpts, "sum_over_time(metric[1m:10s])", "subquery") }) - t.Run("histogram_quantile function", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableHistogramQuantileFunction = false + t.Run("one-to-many and many-to-one binary operations", func(t *testing.T) { + mqeOpts := MQEOptsAllFeatures + mqeOpts.EnableOneToManyAndManyToOneBinaryOperations = false - requireQueryIsUnsupported(t, featureToggles, "histogram_quantile(0.5, metric)", "'histogram_quantile' function") + requireQueryIsUnsupported(t, mqeOpts, "metric{} + on() group_left() other_metric{}", "binary expression with many-to-one matching") + requireQueryIsUnsupported(t, mqeOpts, "metric{} + on() group_right() other_metric{}", "binary expression with one-to-many matching") }) - t.Run("one-to-many and many-to-one binary operations", func(t *testing.T) { - featureToggles := EnableAllFeatures - featureToggles.EnableOneToManyAndManyToOneBinaryOperations = false + t.Run("function disabled by name", func(t *testing.T) { + mqeOpts := MQEOptsAllFeatures + mqeOpts.DisabledFunctions = []string{"ceil", "histogram_quantile", "nonexistant"} + + requireQueryIsUnsupported(t, mqeOpts, "ceil(metric{})", "'ceil' function is disabled") + requireQueryIsUnsupported(t, mqeOpts, "histogram_quantile(0.9, h{})", "'histogram_quantile' function is disabled") + }) + + t.Run("aggregation disabled by name", func(t *testing.T) { + mqeOpts := MQEOptsAllFeatures + mqeOpts.DisabledAggregations = []string{"avg", "MAX", "sum"} - requireQueryIsUnsupported(t, featureToggles, "metric{} + on() group_left() other_metric{}", "binary expression with many-to-one matching") - requireQueryIsUnsupported(t, featureToggles, "metric{} + on() group_right() other_metric{}", "binary expression with one-to-many matching") + requireQueryIsUnsupported(t, mqeOpts, "avg by (label) (metric{})", "'avg' aggregation disabled") + requireQueryIsUnsupported(t, mqeOpts, "max(metric{})", "'max' aggregation disabled") + requireQueryIsUnsupported(t, mqeOpts, "SUM(metric{})", "'sum' aggregation disabled") }) } -func requireQueryIsUnsupported(t *testing.T, toggles FeatureToggles, expression string, expectedError string) { - requireRangeQueryIsUnsupported(t, toggles, expression, expectedError) - requireInstantQueryIsUnsupported(t, toggles, expression, expectedError) +func requireQueryIsUnsupported(t *testing.T, mqeOpts MQEOpts, expression string, expectedError string) { + requireRangeQueryIsUnsupported(t, mqeOpts, expression, expectedError) + requireInstantQueryIsUnsupported(t, mqeOpts, expression, expectedError) } -func requireQueryIsSupported(t *testing.T, toggles FeatureToggles, expression string) { - requireRangeQueryIsSupported(t, toggles, expression) - requireInstantQueryIsSupported(t, toggles, expression) +func requireQueryIsSupported(t *testing.T, mqeOpts MQEOpts, expression string) { + requireRangeQueryIsSupported(t, mqeOpts, expression) + requireInstantQueryIsSupported(t, mqeOpts, expression) } -func requireRangeQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, expression string, expectedError string) { +func requireRangeQueryIsUnsupported(t *testing.T, mqeOpts MQEOpts, expression string, expectedError string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.MQEOpts = mqeOpts engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -194,9 +204,9 @@ func requireRangeQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, require.Nil(t, qry) } -func requireInstantQueryIsUnsupported(t *testing.T, featureToggles FeatureToggles, expression string, expectedError string) { +func requireInstantQueryIsUnsupported(t *testing.T, mqeOpts MQEOpts, expression string, expectedError string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.MQEOpts = mqeOpts engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -207,9 +217,9 @@ func requireInstantQueryIsUnsupported(t *testing.T, featureToggles FeatureToggle require.Nil(t, qry) } -func requireRangeQueryIsSupported(t *testing.T, featureToggles FeatureToggles, expression string) { +func requireRangeQueryIsSupported(t *testing.T, mqeOpts MQEOpts, expression string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.MQEOpts = mqeOpts engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) @@ -217,9 +227,9 @@ func requireRangeQueryIsSupported(t *testing.T, featureToggles FeatureToggles, e require.NoError(t, err) } -func requireInstantQueryIsSupported(t *testing.T, featureToggles FeatureToggles, expression string) { +func requireInstantQueryIsSupported(t *testing.T, mqeOpts MQEOpts, expression string) { opts := NewTestEngineOpts() - opts.FeatureToggles = featureToggles + opts.MQEOpts = mqeOpts engine, err := NewEngine(opts, NewStaticQueryLimitsProvider(0), stats.NewQueryMetrics(nil), log.NewNopLogger()) require.NoError(t, err) diff --git a/pkg/streamingpromql/operators/aggregations/common.go b/pkg/streamingpromql/operators/aggregations/common.go index 04491c72966..9a5f242c4c0 100644 --- a/pkg/streamingpromql/operators/aggregations/common.go +++ b/pkg/streamingpromql/operators/aggregations/common.go @@ -3,6 +3,8 @@ package aggregations import ( + "strings" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/promql/parser" @@ -35,3 +37,43 @@ var AggregationGroupFactories = map[parser.ItemType]AggregationGroupFactory{ // // Invalid combinations include exponential and custom buckets, and histograms with incompatible custom buckets. var invalidCombinationOfHistograms = &histogram.FloatHistogram{} + +func IsAggregationDisabled(op parser.ItemType, disabledAggregations []string) bool { + for _, aggName := range disabledAggregations { + switch strings.ToLower(aggName) { + case "avg": + if op == parser.AVG { + return true + } + case "count": + if op == parser.COUNT { + return true + } + case "group": + if op == parser.GROUP { + return true + } + case "max": + if op == parser.MAX { + return true + } + case "min": + if op == parser.MIN { + return true + } + case "stddev": + if op == parser.STDDEV { + return true + } + case "stdvar": + if op == parser.STDVAR { + return true + } + case "sum": + if op == parser.SUM { + return true + } + } + } + return false +} diff --git a/pkg/streamingpromql/query.go b/pkg/streamingpromql/query.go index 6f44018866c..02be6ec0915 100644 --- a/pkg/streamingpromql/query.go +++ b/pkg/streamingpromql/query.go @@ -169,9 +169,12 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types Stats: q.stats, }, nil case *parser.AggregateExpr: - if !q.engine.featureToggles.EnableAggregationOperations { + if !q.engine.mqeOpts.EnableAggregationOperations { return nil, compat.NewNotSupportedError("aggregation operations") } + if aggregations.IsAggregationDisabled(e.Op, q.engine.mqeOpts.DisabledAggregations) { + return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' aggregation disabled", e.Op.String())) + } if e.Param != nil { return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' aggregation with parameter", e.Op)) @@ -203,7 +206,7 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types // We don't need to handle scalars on both sides here, as that would produce a scalar and so is handled in convertToScalarOperator. if e.LHS.Type() == parser.ValueTypeScalar || e.RHS.Type() == parser.ValueTypeScalar { - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableVectorScalarBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.mqeOpts.EnableVectorScalarBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("vector/scalar binary expression with '%v'", e.Op)) } @@ -244,15 +247,15 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types } // Vectors on both sides. - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableVectorVectorBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.mqeOpts.EnableVectorVectorBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("vector/vector binary expression with '%v'", e.Op)) } - if e.Op.IsSetOperator() && !q.engine.featureToggles.EnableBinaryLogicalOperations { + if e.Op.IsSetOperator() && !q.engine.mqeOpts.EnableBinaryLogicalOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("binary expression with '%v'", e.Op)) } - if !e.Op.IsSetOperator() && e.VectorMatching.Card != parser.CardOneToOne && !q.engine.featureToggles.EnableOneToManyAndManyToOneBinaryOperations { + if !e.Op.IsSetOperator() && e.VectorMatching.Card != parser.CardOneToOne && !q.engine.mqeOpts.EnableOneToManyAndManyToOneBinaryOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("binary expression with %v matching", e.VectorMatching.Card)) } @@ -310,10 +313,11 @@ func (q *Query) convertToInstantVectorOperator(expr parser.Expr, timeRange types } func (q *Query) convertFunctionCallToInstantVectorOperator(e *parser.Call, timeRange types.QueryTimeRange) (types.InstantVectorOperator, error) { - // Handle special toggles for classic histograms - if !q.engine.featureToggles.EnableHistogramQuantileFunction { - if e.Func.Name == "histogram_quantile" { - return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' function", e.Func.Name)) + for _, fName := range q.engine.mqeOpts.DisabledFunctions { + // e.Func.Name is already validated by the parser. Meaning we don't need to check if the function name + // is real before checking if it is disabled. + if e.Func.Name == fName { + return nil, compat.NewNotSupportedError(fmt.Sprintf("'%s' function is disabled", e.Func.Name)) } } @@ -356,7 +360,7 @@ func (q *Query) convertToRangeVectorOperator(expr parser.Expr, timeRange types.Q return selectors.NewRangeVectorSelector(selector, q.memoryConsumptionTracker, q.stats), nil case *parser.SubqueryExpr: - if !q.engine.featureToggles.EnableSubqueries { + if !q.engine.mqeOpts.EnableSubqueries { return nil, compat.NewNotSupportedError("subquery") } @@ -428,7 +432,7 @@ func (q *Query) convertToScalarOperator(expr parser.Expr, timeRange types.QueryT return nil, fmt.Errorf("cannot create scalar operator for expression that produces a %s", parser.DocumentedType(expr.Type())) } - if !q.engine.featureToggles.EnableScalars { + if !q.engine.mqeOpts.EnableScalars { return nil, compat.NewNotSupportedError("scalar values") } @@ -469,7 +473,7 @@ func (q *Query) convertToScalarOperator(expr parser.Expr, timeRange types.QueryT case *parser.ParenExpr: return q.convertToScalarOperator(e.Expr, timeRange) case *parser.BinaryExpr: - if e.Op.IsComparisonOperator() && !q.engine.featureToggles.EnableScalarScalarBinaryComparisonOperations { + if e.Op.IsComparisonOperator() && !q.engine.mqeOpts.EnableScalarScalarBinaryComparisonOperations { return nil, compat.NewNotSupportedError(fmt.Sprintf("scalar/scalar binary expression with '%v'", e.Op)) } diff --git a/pkg/streamingpromql/testing.go b/pkg/streamingpromql/testing.go index 51599ecee64..6904337f00a 100644 --- a/pkg/streamingpromql/testing.go +++ b/pkg/streamingpromql/testing.go @@ -21,7 +21,7 @@ func NewTestEngineOpts() EngineOpts { NoStepSubqueryIntervalFn: func(int64) int64 { return time.Minute.Milliseconds() }, }, - FeatureToggles: EnableAllFeatures, - Pedantic: true, + MQEOpts: MQEOptsAllFeatures, + Pedantic: true, } }