[query] Native PromQL integration into m3coordinator (#2343)

docs/query_engine/config/annotated_config.yaml

@@ -82,3 +82,9 @@ resultOptions:
 # for quick local setup (which this config will send data to).
 tracing:
   backend: jaeger
+
+# Query configuration.
+query:
+  timeout: <duration>
+  # The default query engine is 'prometheus' but it could be switched to 'm3query'
+  defaultEngine: <string>

docs/query_engine/config/index.md

@@ -0,0 +1,40 @@
+# Configuration
+
+## Default query engine
+
+By default M3 runs two query engines:
+- Prometheus (default) - robust and de-facto query language for metrics
+- M3 Query Engine - high-performance query engine but doesn't support all the functions yet
+
+Prometheus Query Engine (or PromQL) is the default one when calling query endpoint:
+```
+http://localhost:7201/api/v1/query?query=count(http_requests)&time=1590147165
+```
+
+But you can switch between the two in the following ways:
+- Changing default query engine in config file (see `defaultEngine` parameter in [Configuration](annotated_config.md))
+- Passing HTTP header `M3-Engine` when calling http url
+
+    ```curl -H "M3-Engine: m3query" "http://localhost:7201/api/v1/query?query=count(http_requests)&time=1590147165"```
+
+    or
+
+    ```curl -H "M3-Engine: prometheus" "http://localhost:7201/api/v1/query?query=count(http_requests)&time=1590147165"```
+
+- Passing url parameter `engine` when calling http url
+
+    ```curl "http://localhost:7201/api/v1/query?engine=m3query&query=count(http_requests)&time=1590147165"```
+
+    or
+
+    ```curl "http://localhost:7201/api/v1/query?engine=prometheus&query=count(http_requests)&time=1590147165"```
+
+- Using different URLs:
+    - /prometheus/api/v1/* - to call PromQL
+    - /m3query/api/v1/* - to call M3 Query
+
+    ```curl "http://localhost:7201/m3query/api/v1/query?query=count(http_requests)&time=1590147165"```
+
+    or
+
+    ```curl "http://localhost:7201/prometheus/api/v1/query?query=count(http_requests)&time=1590147165"```

glide.yaml

@@ -266,3 +266,9 @@ import:
 
   - package: github.com/valyala/tcplisten
     version: ceec8f93295a060cdb565ec25e4ccf17941dbd55
+
+  - package: github.com/json-iterator/go
+    version: ^1.1.9
+
+  - package: github.com/go-kit/kit
+    version: ^0.10.0

mkdocs.yml

@@ -78,15 +78,16 @@ pages:
       - "Prometheus Remote Write/Read": "coordinator/api/remote.md"
   - "Query Engine":
     - "Introduction": "query_engine/index.md"
+    - "Configuration":
+      - "Query Engine": "query_engine/config/index.md"
+      - "Annotated Config File": "query_engine/config/annotated_config.md"
     - "API":
       - "Query": "query_engine/api/index.md"
     - "Architecture":
       - "Overview": "query_engine/architecture/index.md"
       - "Blocks": "query_engine/architecture/blocks.md"
       - "Query Fanout": "query_engine/architecture/fanout.md"
       - "Function Processing": "query_engine/architecture/functions.md"
-    - "Configuration":
-      - "Annotated Config File": "query_engine/config/annotated_config.md"
   - "How-To's":
     - "M3DB Single Node Deployment": "how_to/single_node.md"
     - "M3DB Cluster Deployment, Manually": "how_to/cluster_hard_way.md"

scripts/docker-integration-tests/coordinator_noop/test.sh

@@ -44,7 +44,7 @@ if ! curl -vvvsSf localhost:7201/api/v1/services/m3coordinator/placement; then
 fi
 
 QUERY_EXP='{"error":"operation not valid for noop client"}'
-RES=$(curl "localhost:7201/api/v1/query_range?start=$(date '+%s')&end=$(date '+%s')&step=10&query=foo")
+RES=$(curl "localhost:7201/m3query/api/v1/query_range?start=$(date '+%s')&end=$(date '+%s')&step=10&query=foo")
 if [[ "$RES" != "$QUERY_EXP" ]]; then
   echo "Expected resp '$QUERY_EXP', GOT '$RES'"
   exit 1

scripts/docker-integration-tests/prometheus/test.sh

@@ -207,7 +207,7 @@ function prometheus_query_native {
   result=$(curl -s                                    \
     -H "M3-Metrics-Type: ${metrics_type}"             \
     -H "M3-Storage-Policy: ${metrics_storage_policy}" \
-    "0.0.0.0:7201/api/v1/${endpoint}?query=${query}${params_prefixed}" | jq -r "${jq_path}")
+    "0.0.0.0:7201/m3query/api/v1/${endpoint}?query=${query}${params_prefixed}" | jq -r "${jq_path}")
   test "$result" = "$expected_value"
   return $?
 }

scripts/docker-integration-tests/query_fanout/restrict.go

@@ -47,8 +47,8 @@ func main() {
 
 	requireTrue(ts > 0, "no timestamp supplied")
 	name = fmt.Sprintf("foo_%d", ts)
-	instant := fmt.Sprintf("http://0.0.0.0:7201/api/v1/query?query=%s", name)
-	rnge := fmt.Sprintf("http://0.0.0.0:7201/api/v1/query_range?query=%s"+
+	instant := fmt.Sprintf("http://0.0.0.0:7201/m3query/api/v1/query?query=%s", name)
+	rnge := fmt.Sprintf("http://0.0.0.0:7201/m3query/api/v1/query_range?query=%s"+
 		"&start=%d&end=%d&step=100", name, ts/100*100, (ts/100+1)*100)
 
 	for _, url := range []string{instant, rnge} {

scripts/docker-integration-tests/query_fanout/warning.sh

@@ -61,7 +61,7 @@ function test_instant_query {
   EXPECTED_HEADER=$3||""
   trap clean_headers EXIT
   RESPONSE=$(curl -sSL -D $HEADER_FILE -H "M3-Limit-Max-Series:$LIMIT" \
-    "http://0.0.0.0:7201/api/v1/query?query=count($METRIC_NAME)")
+    "http://0.0.0.0:7201/m3query/api/v1/query?query=count($METRIC_NAME)")
   ACTUAL=$(echo $RESPONSE | jq .data.result[0].value[1] | tr -d \" | tr -d \')
   ACTUAL_HEADER=$(cat $HEADER_FILE | grep M3-Results-Limited | cut -d' ' -f2 | tr -d "\r\n")
   test $ACTUAL = $EXPECTED && test $ACTUAL_HEADER = $EXPECTED_HEADER
@@ -79,7 +79,7 @@ function test_range_query {
   end=$(($start+9))
 
   RESPONSE=$(curl -sSL -D $HEADER_FILE -H "M3-Limit-Max-Series:$LIMIT" \
-    "http://0.0.0.0:7201/api/v1/query_range?start=$start&end=$end&step=10&query=count($METRIC_NAME)")
+    "http://0.0.0.0:7201/m3query/api/v1/query_range?start=$start&end=$end&step=10&query=count($METRIC_NAME)")
   ACTUAL=$(echo $RESPONSE | jq .data.result[0].values[0][1] | tr -d \" | tr -d \')
   ACTUAL_HEADER=$(cat $HEADER_FILE | grep M3-Results-Limited | cut -d' ' -f2 | tr -d "\r\n")
   test $ACTUAL = $EXPECTED && test $ACTUAL_HEADER = $EXPECTED_HEADER

src/cmd/services/m3query/config/config.go

@@ -206,6 +206,7 @@ type ResultOptions struct {
 // QueryConfiguration is the query configuration.
 type QueryConfiguration struct {
 	Timeout *time.Duration `yaml:"timeout"`
+	DefaultEngine string `yaml:"defaultEngine"`
 }
 
 // TimeoutOrDefault returns the configured timeout or default value.