diff --git a/CHANGELOG.md b/CHANGELOG.md index 550fb08fe4e..0c388d25c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ * [FEATURE] Continuous-test: now runable as a module with `mimir -target=continuous-test`. #7747 * [FEATURE] Store-gateway: Allow specific tenants to be enabled or disabled via `-store-gateway.enabled-tenants` or `-store-gateway.disabled-tenants` CLI flags or their corresponding YAML settings. #7653 * [FEATURE] New `-.s3.bucket-lookup-type` flag configures lookup style type, used to access bucket in s3 compatible providers. #7684 -* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.promql-engine=mimir`. #7693 #7898 #7899 #8023 #8058 #8096 #8121 #8197 #8230 #8247 #8270 #8276 #8277 #8291 #8303 #8340 #8256 #8348 #8422 #8430 +* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #7693 #7898 #7899 #8023 #8058 #8096 #8121 #8197 #8230 #8247 #8270 #8276 #8277 #8291 #8303 #8340 #8256 #8348 #8422 #8430 #8455 * [FEATURE] New `/ingester/unregister-on-shutdown` HTTP endpoint allows dynamic access to ingesters' `-ingester.ring.unregister-on-shutdown` configuration. #7739 * [FEATURE] Server: added experimental [PROXY protocol support](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). The PROXY protocol support can be enabled via `-server.proxy-protocol-enabled=true`. When enabled, the support is added both to HTTP and gRPC listening ports. #7698 * [FEATURE] mimirtool: Add `runtime-config verify` sub-command, for verifying Mimir runtime config files. #8123 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 666ffc13466..749754dcddd 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -1911,23 +1911,23 @@ }, { "kind": "field", - "name": "promql_engine", + "name": "query_engine", "required": false, - "desc": "PromQL engine to use, either 'prometheus' or 'mimir'", + "desc": "Query engine to use, either 'prometheus' or 'mimir'", "fieldValue": null, "fieldDefaultValue": "prometheus", - "fieldFlag": "querier.promql-engine", + "fieldFlag": "querier.query-engine", "fieldType": "string", "fieldCategory": "experimental" }, { "kind": "field", - "name": "enable_promql_engine_fallback", + "name": "enable_query_engine_fallback", "required": false, - "desc": "If set to true and the streaming engine is in use, fall back to using the Prometheus PromQL engine for any queries not supported by the streaming engine.", + "desc": "If set to true and the Mimir query engine is in use, fall back to using the Prometheus query engine for any queries not supported by the Mimir query engine.", "fieldValue": null, "fieldDefaultValue": true, - "fieldFlag": "querier.enable-promql-engine-fallback", + "fieldFlag": "querier.enable-query-engine-fallback", "fieldType": "boolean", "fieldCategory": "experimental" }, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 3a52513a9f3..003232153f9 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1715,8 +1715,8 @@ Usage of ./cmd/mimir/mimir: The default evaluation interval or step size for subqueries. This config option should be set on query-frontend too when query sharding is enabled. (default 1m0s) -querier.dns-lookup-period duration How often to query DNS for query-frontend or query-scheduler address. (default 10s) - -querier.enable-promql-engine-fallback - [experimental] If set to true and the streaming engine is in use, fall back to using the Prometheus PromQL engine for any queries not supported by the streaming engine. (default true) + -querier.enable-query-engine-fallback + [experimental] If set to true and the Mimir query engine is in use, fall back to using the Prometheus query engine for any queries not supported by the Mimir query engine. (default true) -querier.frontend-address string Address of the query-frontend component, in host:port format. If multiple query-frontends are running, the host should be a DNS resolving to all query-frontend instances. This option should be set only when query-scheduler component is not in use. -querier.frontend-client.backoff-max-period duration @@ -1801,10 +1801,10 @@ Usage of ./cmd/mimir/mimir: Delay before initiating requests to further ingesters when request minimization is enabled and the initially selected set of ingesters have not all responded. Ignored if -querier.minimize-ingester-requests is not enabled. (default 3s) -querier.prefer-streaming-chunks-from-store-gateways [experimental] Request store-gateways stream chunks. Store-gateways will only respond with a stream of chunks if the target store-gateway supports this, and this preference will be ignored by store-gateways that do not support this. (default true) - -querier.promql-engine string - [experimental] PromQL engine to use, either 'prometheus' or 'mimir' (default "prometheus") -querier.promql-experimental-functions-enabled [experimental] Enable experimental PromQL functions. This config option should be set on query-frontend too when query sharding is enabled. + -querier.query-engine string + [experimental] Query engine to use, either 'prometheus' or 'mimir' (default "prometheus") -querier.query-ingesters-within duration Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester. (default 13h) -querier.query-store-after duration diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index 17a7708abcb..9054af268d1 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -146,7 +146,7 @@ The following features are currently experimental: - Maximum response size for active series queries (`-querier.active-series-results-max-size-bytes`) - Enable PromQL experimental functions (`-querier.promql-experimental-functions-enabled`) - Allow streaming of `/active_series` responses to the frontend (`-querier.response-streaming-enabled`) - - Mimir query engine (`-querier.promql-engine=mimir` and `-querier.enable-promql-engine-fallback`) + - Mimir query engine (`-querier.query-engine=mimir` and `-querier.enable-query-engine-fallback`) - Maximum estimated memory consumption per query limit (`-querier.max-estimated-memory-consumption-per-query`) - Query-frontend - `-query-frontend.querier-forget-delay` diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 9c591b7477f..4ce7fec8019 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -1415,15 +1415,15 @@ store_gateway_client: # CLI flag: -querier.minimize-ingester-requests-hedging-delay [minimize_ingester_requests_hedging_delay: | default = 3s] -# (experimental) PromQL engine to use, either 'prometheus' or 'mimir' -# CLI flag: -querier.promql-engine -[promql_engine: | default = "prometheus"] - -# (experimental) If set to true and the streaming engine is in use, fall back to -# using the Prometheus PromQL engine for any queries not supported by the -# streaming engine. -# CLI flag: -querier.enable-promql-engine-fallback -[enable_promql_engine_fallback: | default = true] +# (experimental) Query engine to use, either 'prometheus' or 'mimir' +# CLI flag: -querier.query-engine +[query_engine: | default = "prometheus"] + +# (experimental) If set to true and the Mimir query engine is in use, fall back +# to using the Prometheus query engine for any queries not supported by the +# Mimir query engine. +# CLI flag: -querier.enable-query-engine-fallback +[enable_query_engine_fallback: | default = true] # The number of workers running in each querier process. This setting limits the # maximum number of concurrent queries in each querier. diff --git a/docs/sources/mimir/manage/mimir-runbooks/_index.md b/docs/sources/mimir/manage/mimir-runbooks/_index.md index c0f9ac6857e..ea0033ec9b2 100644 --- a/docs/sources/mimir/manage/mimir-runbooks/_index.md +++ b/docs/sources/mimir/manage/mimir-runbooks/_index.md @@ -2034,7 +2034,7 @@ How to **fix** it: This error occurs when execution of a query exceeds the limit on the maximum estimated memory consumed by a single query. This limit is used to protect the system’s stability from potential abuse or mistakes, when running a query fetching a huge amount of data. -This limit only applies when Mimir's query engine is used (ie. `-querier.promql-engine=mimir`). +This limit only applies when Mimir's query engine is used (ie. `-querier.query-engine=mimir`). To configure the limit on a global basis, use the `-querier.max-estimated-memory-consumption-per-query` option. To configure the limit on a per-tenant basis, set the `max_estimated_memory_consumption_per_query` per-tenant override in the runtime configuration. diff --git a/integration/querier_test.go b/integration/querier_test.go index 43b674a6a2f..0cc7c5e3792 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -521,7 +521,7 @@ func TestMimirPromQLEngine(t *testing.T) { defer s.Close() flags := mergeFlags(BlocksStorageFlags(), BlocksStorageS3Flags(), map[string]string{ - "-querier.promql-engine": "mimir", + "-querier.query-engine": "mimir", }) consul := e2edb.NewConsul() diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index c38a124951e..074597bee49 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -163,7 +163,7 @@ func TestMimir(t *testing.T) { InstanceInterfaceNames: []string{"en0", "eth0", "lo0", "lo"}, }}, Querier: querier.Config{ - PromQLEngine: "prometheus", + QueryEngine: "prometheus", }, } require.NoError(t, cfg.Server.LogLevel.Set("info")) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 6ba6c4cf684..d55af9271a5 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -57,17 +57,17 @@ type Config struct { MinimizeIngesterRequests bool `yaml:"minimize_ingester_requests" category:"advanced"` MinimiseIngesterRequestsHedgingDelay time.Duration `yaml:"minimize_ingester_requests_hedging_delay" category:"advanced"` - PromQLEngine string `yaml:"promql_engine" category:"experimental"` - EnablePromQLEngineFallback bool `yaml:"enable_promql_engine_fallback" category:"experimental"` + QueryEngine string `yaml:"query_engine" category:"experimental"` + EnableQueryEngineFallback bool `yaml:"enable_query_engine_fallback" category:"experimental"` // PromQL engine config. EngineConfig engine.Config `yaml:",inline"` } const ( - queryStoreAfterFlag = "querier.query-store-after" - prometheusPromQLEngine = "prometheus" - mimirPromQLEngine = "mimir" + queryStoreAfterFlag = "querier.query-store-after" + prometheusEngine = "prometheus" + mimirEngine = "mimir" ) // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -89,15 +89,15 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Uint64Var(&cfg.StreamingChunksPerIngesterSeriesBufferSize, "querier.streaming-chunks-per-ingester-buffer-size", 256, "Number of series to buffer per ingester when streaming chunks from ingesters.") f.Uint64Var(&cfg.StreamingChunksPerStoreGatewaySeriesBufferSize, "querier.streaming-chunks-per-store-gateway-buffer-size", 256, "Number of series to buffer per store-gateway when streaming chunks from store-gateways.") - f.StringVar(&cfg.PromQLEngine, "querier.promql-engine", prometheusPromQLEngine, fmt.Sprintf("PromQL engine to use, either '%v' or '%v'", prometheusPromQLEngine, mimirPromQLEngine)) - f.BoolVar(&cfg.EnablePromQLEngineFallback, "querier.enable-promql-engine-fallback", true, "If set to true and the streaming engine is in use, fall back to using the Prometheus PromQL engine for any queries not supported by the streaming engine.") + f.StringVar(&cfg.QueryEngine, "querier.query-engine", prometheusEngine, fmt.Sprintf("Query engine to use, either '%v' or '%v'", prometheusEngine, mimirEngine)) + f.BoolVar(&cfg.EnableQueryEngineFallback, "querier.enable-query-engine-fallback", true, "If set to true and the Mimir query engine is in use, fall back to using the Prometheus query engine for any queries not supported by the Mimir query engine.") cfg.EngineConfig.RegisterFlags(f) } func (cfg *Config) Validate() error { - if cfg.PromQLEngine != prometheusPromQLEngine && cfg.PromQLEngine != mimirPromQLEngine { - return fmt.Errorf("unknown PromQL engine '%s'", cfg.PromQLEngine) + if cfg.QueryEngine != prometheusEngine && cfg.QueryEngine != mimirEngine { + return fmt.Errorf("unknown PromQL engine '%s'", cfg.QueryEngine) } return nil @@ -160,24 +160,24 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, stor var eng promql.QueryEngine - switch cfg.PromQLEngine { - case prometheusPromQLEngine: + switch cfg.QueryEngine { + case prometheusEngine: eng = promql.NewEngine(opts) - case mimirPromQLEngine: + case mimirEngine: limitsProvider := &tenantQueryLimitsProvider{limits: limits} streamingEngine, err := streamingpromql.NewEngine(opts, limitsProvider, queryMetrics, logger) if err != nil { return nil, nil, nil, err } - if cfg.EnablePromQLEngineFallback { + if cfg.EnableQueryEngineFallback { prometheusEngine := promql.NewEngine(opts) eng = compat.NewEngineWithFallback(streamingEngine, prometheusEngine, reg, logger) } else { eng = streamingEngine } default: - panic(fmt.Sprintf("invalid config not caught by validation: unknown PromQL engine '%s'", cfg.PromQLEngine)) + panic(fmt.Sprintf("invalid config not caught by validation: unknown PromQL engine '%s'", cfg.QueryEngine)) } return NewSampleAndChunkQueryable(lazyQueryable), exemplarQueryable, eng, nil diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index d6ef75cab01..39a11af17e2 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -605,7 +605,7 @@ func TestQuerier_QueryIngestersWithinConfig(t *testing.T) { MaxSamples: 1e6, Timeout: 1 * time.Minute, }) - cfg := Config{PromQLEngine: prometheusPromQLEngine} + cfg := Config{QueryEngine: prometheusEngine} for _, c := range testCases { t.Run(c.name, func(t *testing.T) { distributor := &errDistributor{}