[query] Add Prometheus engine metrics and upgrade to Prometheus 2.17.2

Makefile

@@ -33,6 +33,7 @@ thrift_output_dir    := generated/thrift/rpc
 thrift_rules_dir     := generated/thrift
 vendor_prefix        := vendor
 bad_trace_dep        := go.etcd.io/etcd/vendor/golang.org/x/net/trace
+bad_prom_vendor_dir  := github.com/prometheus/prometheus/vendor
 cache_policy         ?= recently_read
 genny_target         ?= genny-all
 
@@ -107,6 +108,8 @@ install-vendor-m3:
 	[ -d $(VENDOR) ] || make install-vendor
 	# See comment for "install-vendor-m3-remove-bad-dep" why required and the TODO.
 	make install-vendor-m3-remove-bad-dep
+	# See comment for "install-vendor-m3-remove-prometheus-vendor-dir" why required.
+	make install-vendor-m3-remove-prometheus-vendor-dir
 
 # Some deps were causing panics when using GRPC and etcd libraries were used.
 # See issue: https://github.com/etcd-io/etcd/issues/9357
@@ -126,6 +129,18 @@ install-vendor-m3:
 install-vendor-m3-remove-bad-dep:
 	([ -d $(VENDOR)/$(bad_trace_dep) ] && rm -rf $(VENDOR)/$(bad_trace_dep)) || (echo "No bad trace dep" > /dev/null)
 
+# Note: Prometheus has an entire copy of all vendored code which makes
+# it impossible to pass sub-dependencies on it since you'll get errors like:
+#   have MustRegister(... vendor/github.com/prometheus/client_golang/prometheus.Collector)
+#   want MustRegister(... vendor/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus
+# Even if you have the same deps as prometheus you can't pass the dep types to
+# it since it depends on the concrete subdirectory vendored code import path.
+# Therefore we delete the vendored code and make it rely on our own dependencies
+# we install.
+.PHONY: install-vendor-m3-remove-prometheus-vendor-dir
+install-vendor-m3-remove-prometheus-vendor-dir:
+	([ -d $(VENDOR)/$(bad_prom_vendor_dir) ] && rm -rf $(VENDOR)/$(bad_prom_vendor_dir)) || (echo "No bad prom vendor dir" > /dev/null)
+
 .PHONY: docker-dev-prep
 docker-dev-prep:
 	mkdir -p ./bin/config

glide.yaml

@@ -23,8 +23,14 @@ import:
   - package: github.com/willf/bitset
     version: e553b05586428962bf7058d1044519d87ca72d74
 
+  # NB(r): This is bad.. when we upgrade to go.mod no longer needed,
+  # but since glide does not support "/v2" syntax and some of our deps
+  # depend on it we forked xxhash and explicitly copied code into "/v2"
+  # submodule to make this package work with glide.
   - package: github.com/cespare/xxhash
-    version: 48099fad606eafc26e3a569fad19ff510fff4df6
+    version: 3f2bc324ddd8b6aafc9a91b5aabf22ff172edafb
+    repo: https://github.com/m3dbx/xxhash
+    vcs: git
 
   - package: go.etcd.io/etcd
     version: 3.4.3
@@ -167,12 +173,23 @@ import:
 
   # START_PROMETHEUS_DEPS
   - package: github.com/prometheus/prometheus
-    version: ~2.12.0
+    version: ~2.17.2
 
   # To avoid prometheus/prometheus dependencies from breaking,
-  # pin the transitive dependencies
+  # pin the transitive dependencies. See go.mod in 
+  # github.com/prometheus/prometheus for which versions to set for 
+  # relevant prometheus release pinned to above.
   - package: github.com/prometheus/common
-    version: ~0.7.0
+    version: ~0.9.1
+
+  - package: github.com/prometheus/client_golang
+    version: ~1.5.1
+
+  - package: github.com/prometheus/client_model
+    version: ~0.2.0
+
+  - package: github.com/prometheus/procfs
+    version: ~0.0.8
   # END_PROMETHEUS_DEPS
 
   # START_TALLY_PROMETHEUS_DEPS

src/cmd/services/m3comparator/main/querier.go

@@ -39,6 +39,8 @@ import (
 	"github.com/m3db/m3/src/query/storage"
 	"github.com/m3db/m3/src/query/storage/m3"
 	xtime "github.com/m3db/m3/src/x/time"
+
+	"github.com/prometheus/common/model"
 )
 
 var _ m3.Querier = (*querier)(nil)
@@ -271,7 +273,7 @@ func (q *querier) generateSingleSeriesMetrics(
 	seriesList := make([]series, 0, len(actualGens))
 	for _, gen := range actualGens {
 		tags := parser.Tags{
-			parser.NewTag("__name__", gen.name),
+			parser.NewTag(model.MetricNameLabel, gen.name),
 			parser.NewTag("foobar", "qux"),
 			parser.NewTag("name", gen.name),
 		}
@@ -305,7 +307,7 @@ func (q *querier) generateMultiSeriesMetrics(
 	seriesList := make([]series, 0, seriesCount)
 	for i := 0; i < seriesCount; i++ {
 		tags := parser.Tags{
-			parser.NewTag("__name__", metricsName),
+			parser.NewTag(model.MetricNameLabel, metricsName),
 			parser.NewTag("id", strconv.Itoa(i)),
 			parser.NewTag("parity", strconv.Itoa(i%2)),
 			parser.NewTag("const", "x"),

src/cmd/services/m3query/config/config.go

@@ -64,6 +64,8 @@ const (
 		"More information is available here: %s"
 
 	defaultQueryTimeout = 30 * time.Second
+
+	defaultPrometheusMaxSamplesPerQuery = 100000000
 )
 
 var (
@@ -205,8 +207,9 @@ type ResultOptions struct {
 
 // QueryConfiguration is the query configuration.
 type QueryConfiguration struct {
-	Timeout *time.Duration `yaml:"timeout"`
-	DefaultEngine string `yaml:"defaultEngine"`
+	Timeout       *time.Duration               `yaml:"timeout"`
+	DefaultEngine string                       `yaml:"defaultEngine"`
+	Prometheus    PrometheusQueryConfiguration `yaml:"prometheus"`
 }
 
 // TimeoutOrDefault returns the configured timeout or default value.
@@ -217,6 +220,20 @@ func (c QueryConfiguration) TimeoutOrDefault() time.Duration {
 	return defaultQueryTimeout
 }
 
+// PrometheusQueryConfiguration is the prometheus query engine configuration.
+type PrometheusQueryConfiguration struct {
+	// MaxSamplesPerQuery is the limit on fetched samples per query.
+	MaxSamplesPerQuery *int `yaml:"maxSamplesPerQuery"`
+}
+
+// MaxSamplesPerQueryOrDefault returns the max samples per query or default.
+func (c PrometheusQueryConfiguration) MaxSamplesPerQueryOrDefault() int {
+	if v := c.MaxSamplesPerQuery; v != nil {
+		return *v
+	}
+	return defaultPrometheusMaxSamplesPerQuery
+}
+
 // LimitsConfiguration represents limitations on resource usage in the query
 // instance. Limits are split between per-query and global limits.
 type LimitsConfiguration struct {

src/query/api/v1/handler/prom/common.go

@@ -26,7 +26,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/prometheus/prometheus/promql"
+	promql "github.com/prometheus/prometheus/promql/parser"
 	promstorage "github.com/prometheus/prometheus/storage"
 )
 

src/query/api/v1/handler/prom/mocks.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	kitlogzap "github.com/go-kit/kit/log/zap"
-	promlabels "github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/promql"
 	promstorage "github.com/prometheus/prometheus/storage"
 	"go.uber.org/zap/zapcore"
@@ -45,7 +45,11 @@ func (m mockSeriesSet) Next() bool             { return false }
 func (m mockSeriesSet) At() promstorage.Series { return nil }
 func (m mockSeriesSet) Err() error             { return nil }
 
-func (mockQuerier) Select(*promstorage.SelectParams, ...*promlabels.Matcher) (promstorage.SeriesSet, promstorage.Warnings, error) {
+func (mockQuerier) Select(
+	sortSeries bool,
+	hints *promstorage.SelectHints,
+	labelMatchers ...*labels.Matcher,
+) (promstorage.SeriesSet, promstorage.Warnings, error) {
 	return mockSeriesSet{}, nil, nil
 }
 
@@ -72,10 +76,9 @@ func newMockPromQLEngine() *promql.Engine {
 		instrumentOpts = instrument.NewOptions()
 		kitLogger      = kitlogzap.NewZapSugarLogger(instrumentOpts.Logger(), zapcore.InfoLevel)
 		opts           = promql.EngineOpts{
-			Logger:        log.With(kitLogger, "component", "query engine"),
-			MaxConcurrent: 10,
-			MaxSamples:    100,
-			Timeout:       1 * time.Minute,
+			Logger:     log.With(kitLogger, "component", "query engine"),
+			MaxSamples: 100,
+			Timeout:    1 * time.Minute,
 		}
 	)
 	return promql.NewEngine(opts)

src/query/api/v1/handler/prometheus/common.go

@@ -42,7 +42,7 @@ import (
 	xhttp "github.com/m3db/m3/src/x/net/http"
 
 	"github.com/golang/snappy"
-	"github.com/prometheus/prometheus/promql"
+	promql "github.com/prometheus/prometheus/promql/parser"
 )
 
 const (

src/query/api/v1/handler/prometheus/remote/read.go

@@ -50,7 +50,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/promql"
+	promql "github.com/prometheus/prometheus/promql/parser"
 	"github.com/uber-go/tally"
 	"go.uber.org/zap"
 )
@@ -302,8 +302,9 @@ func ParseExpr(r *http.Request) (*prompb.ReadRequest, *xhttp.ParseError) {
 				start = start.Add(-1 * n.Range)
 			}
 
-			offset = n.Offset
-			labelMatchers = n.LabelMatchers
+			vectorSelector := n.VectorSelector.(*promql.VectorSelector)
+			offset = vectorSelector.Offset
+			labelMatchers = vectorSelector.LabelMatchers
 		} else if n, ok := node.(*promql.VectorSelector); ok {
 			offset = n.Offset
 			labelMatchers = n.LabelMatchers

src/query/api/v1/handler/prometheus/remote/test/read.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
 
@@ -43,7 +44,7 @@ func GeneratePromReadRequest() *prompb.ReadRequest {
 				EndTimestampMs:   time.Now().UnixNano() / int64(time.Millisecond),
 				Matchers: []*prompb.LabelMatcher{
 					&prompb.LabelMatcher{
-						Name:  []byte("__name__"),
+						Name:  []byte(model.MetricNameLabel),
 						Value: []byte("first"),
 						Type:  prompb.LabelMatcher_EQ,
 					},

src/query/api/v1/handler/prometheus/remote/test/write.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 )
 
@@ -38,7 +39,7 @@ func GeneratePromWriteRequest() *prompb.WriteRequest {
 	req := &prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{{
 			Labels: []prompb.Label{
-				{Name: []byte("__name__"), Value: []byte("first")},
+				{Name: []byte(model.MetricNameLabel), Value: []byte("first")},
 				{Name: []byte("foo"), Value: []byte("bar")},
 				{Name: []byte("biz"), Value: []byte("baz")},
 			},
@@ -49,7 +50,7 @@ func GeneratePromWriteRequest() *prompb.WriteRequest {
 		},
 			{
 				Labels: []prompb.Label{
-					{Name: []byte("__name__"), Value: []byte("second")},
+					{Name: []byte(model.MetricNameLabel), Value: []byte("second")},
 					{Name: []byte("foo"), Value: []byte("qux")},
 					{Name: []byte("bar"), Value: []byte("baz")},
 				},