Report all libbeat/monitoring metrics in /debug/vars (#3550)

* beater/api: expose monitoring through /debug/vars

Change the optional /debug/vars handler to behave
exactly the same as libbeat's built-in "-httpprof"
server /debug/vars handler, so we can access libbeat
monitoring metrics.

* all: remove explicit monitoring.PublishExpvar

The libbeat built-in registries do not publish to
expvar, so they do not show up in /debug/vars by
default.

Since we're now using a custom /debug/vars handler
which exposes all metrics in the default monitoring
registry, we don't need to explicitly publish to expvar.

* Update changelog

beater/api/asset/sourcemap/handler.go

@@ -37,7 +37,7 @@ import (
 var (
 	// MonitoringMap holds a mapping for request.IDs to monitoring counters
 	MonitoringMap = request.DefaultMonitoringMapForRegistry(registry)
-	registry      = monitoring.Default.NewRegistry("apm-server.sourcemap", monitoring.PublishExpvar)
+	registry      = monitoring.Default.NewRegistry("apm-server.sourcemap")
 )
 
 // RequestDecoder is the type for a function that decodes sourcemap data from an http.Request.

beater/api/config/agent/handler.go

@@ -51,7 +51,7 @@ const (
 var (
 	// MonitoringMap holds a mapping for request.IDs to monitoring counters
 	MonitoringMap = request.DefaultMonitoringMapForRegistry(registry)
-	registry      = monitoring.Default.NewRegistry("apm-server.acm", monitoring.PublishExpvar)
+	registry      = monitoring.Default.NewRegistry("apm-server.acm")
 
 	errMsgKibanaDisabled     = errors.New(msgKibanaDisabled)
 	errMsgNoKibanaConnection = errors.New(msgNoKibanaConnection)

beater/api/expvar.go

@@ -0,0 +1,55 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package api
+
+import (
+	"expvar"
+	"fmt"
+	"net/http"
+
+	"github.com/elastic/beats/v7/libbeat/monitoring"
+)
+
+// debugVarsHandler reports expvar and all libbeat/monitoring metrics.
+//
+// TODO(axw) this is copied from libbeat/service. We should move the
+// handler to libbeat/monitoring, and export it for libbeat/service and
+// apm-server to use.
+func debugVarsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	first := true
+	report := func(key string, value interface{}) {
+		if !first {
+			fmt.Fprintf(w, ",\n")
+		}
+		first = false
+		if str, ok := value.(string); ok {
+			fmt.Fprintf(w, "%q: %q", key, str)
+		} else {
+			fmt.Fprintf(w, "%q: %v", key, value)
+		}
+	}
+
+	fmt.Fprintf(w, "{\n")
+	monitoring.Do(monitoring.Full, report)
+	expvar.Do(func(kv expvar.KeyValue) {
+		report(kv.Key, kv.Value)
+	})
+	fmt.Fprintf(w, "\n}\n")
+}

beater/api/intake/handler.go

@@ -37,7 +37,7 @@ import (
 var (
 	// MonitoringMap holds a mapping for request.IDs to monitoring counters
 	MonitoringMap = request.DefaultMonitoringMapForRegistry(registry)
-	registry      = monitoring.Default.NewRegistry("apm-server.server", monitoring.PublishExpvar)
+	registry      = monitoring.Default.NewRegistry("apm-server.server")
 )
 
 // Handler returns a request.Handler for managing intake requests for backend and rum events.

beater/api/mux.go

@@ -18,7 +18,6 @@
 package api
 
 import (
-	"expvar"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -119,7 +118,7 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu
 	if beaterConfig.Expvar.IsEnabled() {
 		path := beaterConfig.Expvar.URL
 		logger.Infof("Path %s added to request handler", path)
-		mux.Handle(path, expvar.Handler())
+		mux.Handle(path, http.HandlerFunc(debugVarsHandler))
 	}
 	return mux, nil
 }

beater/api/profile/handler.go

@@ -42,7 +42,7 @@ import (
 var (
 	// MonitoringMap holds a mapping for request.IDs to monitoring counters
 	MonitoringMap = request.DefaultMonitoringMapForRegistry(registry)
-	registry      = monitoring.Default.NewRegistry("apm-server.profile", monitoring.PublishExpvar)
+	registry      = monitoring.Default.NewRegistry("apm-server.profile")
 )
 
 const (

beater/api/root/handler.go

@@ -32,7 +32,7 @@ import (
 var (
 	// MonitoringMap holds a mapping for request.IDs to monitoring counters
 	MonitoringMap = request.DefaultMonitoringMapForRegistry(registry)
-	registry      = monitoring.Default.NewRegistry("apm-server.root", monitoring.PublishExpvar)
+	registry      = monitoring.Default.NewRegistry("apm-server.root")
 )
 
 // Handler returns error if route does not exist,

beater/jaeger/grpc.go

@@ -39,7 +39,7 @@ import (
 )
 
 var (
-	gRPCCollectorRegistry                    = monitoring.Default.NewRegistry("apm-server.jaeger.grpc.collect", monitoring.PublishExpvar)
+	gRPCCollectorRegistry                    = monitoring.Default.NewRegistry("apm-server.jaeger.grpc.collect")
 	gRPCCollectorMonitoringMap monitoringMap = request.MonitoringMapForRegistry(gRPCCollectorRegistry, monitoringKeys)
 )
 
@@ -76,7 +76,7 @@ func (c *grpcCollector) postSpans(ctx context.Context, batch model.Batch) error
 }
 
 var (
-	gRPCSamplingRegistry                    = monitoring.Default.NewRegistry("apm-server.jaeger.grpc.sampling", monitoring.PublishExpvar)
+	gRPCSamplingRegistry                    = monitoring.Default.NewRegistry("apm-server.jaeger.grpc.sampling")
 	gRPCSamplingMonitoringMap monitoringMap = request.MonitoringMapForRegistry(gRPCSamplingRegistry, monitoringKeys)
 
 	jaegerAgentPrefixes = []string{otel.AgentNameJaeger}

beater/jaeger/http.go

@@ -41,7 +41,7 @@ const (
 )
 
 var (
-	httpRegistry      = monitoring.Default.NewRegistry("apm-server.jaeger.http", monitoring.PublishExpvar)
+	httpRegistry      = monitoring.Default.NewRegistry("apm-server.jaeger.http")
 	httpMonitoringMap = request.MonitoringMapForRegistry(httpRegistry, monitoringKeys)
 )
 

beater/middleware/monitoring_middleware_test.go

@@ -29,7 +29,7 @@ import (
 )
 
 var (
-	mockMonitoringRegistry = monitoring.Default.NewRegistry("mock.monitoring", monitoring.PublishExpvar)
+	mockMonitoringRegistry = monitoring.Default.NewRegistry("mock.monitoring")
 	mockMonitoringNil      = map[request.ResultID]*monitoring.Int{}
 	mockMonitoring         = request.DefaultMonitoringMapForRegistry(mockMonitoringRegistry)
 )

beater/request/result_test.go

@@ -190,7 +190,7 @@ func TestResult_Failure(t *testing.T) {
 }
 
 func TestDefaultMonitoringMapForRegistry(t *testing.T) {
-	mockRegistry := monitoring.Default.NewRegistry("mock-default", monitoring.PublishExpvar)
+	mockRegistry := monitoring.Default.NewRegistry("mock-default")
 	m := DefaultMonitoringMapForRegistry(mockRegistry)
 	assert.Equal(t, 21, len(m))
 	for id := range m {
@@ -200,7 +200,7 @@ func TestDefaultMonitoringMapForRegistry(t *testing.T) {
 
 func TestMonitoringMapForRegistry(t *testing.T) {
 	keys := []ResultID{IDEventDroppedCount, IDResponseErrorsServiceUnavailable}
-	mockRegistry := monitoring.Default.NewRegistry("mock-with-keys", monitoring.PublishExpvar)
+	mockRegistry := monitoring.Default.NewRegistry("mock-with-keys")
 	m := MonitoringMapForRegistry(mockRegistry, keys)
 	assert.Equal(t, 2, len(m))
 	for id := range m {

changelogs/head.asciidoc

@@ -15,3 +15,4 @@ https://github.com/elastic/apm-server/compare/7.6\...master[View commits]
 [float]
 ==== Added
 * RUM V3 endpoint {pull}3328[3328], {pull}3414[3414], {pull}3512[3512]
+* We now publish libbeat metrics through the /debug/vars endpoint {pull}3550[3550]

decoder/req_decoder.go

@@ -30,7 +30,7 @@ import (
 )
 
 var (
-	decoderMetrics                = monitoring.Default.NewRegistry("apm-server.decoder", monitoring.PublishExpvar)
+	decoderMetrics                = monitoring.Default.NewRegistry("apm-server.decoder")
 	missingContentLengthCounter   = monitoring.NewInt(decoderMetrics, "missing-content-length.count")
 	deflateLengthAccumulator      = monitoring.NewInt(decoderMetrics, "deflate.content-length")
 	deflateCounter                = monitoring.NewInt(decoderMetrics, "deflate.count")

model/error/event.go

@@ -46,7 +46,7 @@ import (
 )
 
 var (
-	Metrics           = monitoring.Default.NewRegistry("apm-server.processor.error", monitoring.PublishExpvar)
+	Metrics           = monitoring.Default.NewRegistry("apm-server.processor.error")
 	transformations   = monitoring.NewInt(Metrics, "transformations")
 	stacktraceCounter = monitoring.NewInt(Metrics, "stacktraces")
 	frameCounter      = monitoring.NewInt(Metrics, "frames")

model/metricset/event.go

@@ -46,7 +46,7 @@ const (
 )
 
 var (
-	Metrics         = monitoring.Default.NewRegistry("apm-server.processor.metric", monitoring.PublishExpvar)
+	Metrics         = monitoring.Default.NewRegistry("apm-server.processor.metric")
 	transformations = monitoring.NewInt(Metrics, "transformations")
 	processorEntry  = common.MapStr{"name": processorName, "event": docType}
 )

model/sourcemap/payload.go

@@ -41,7 +41,7 @@ const (
 )
 
 var (
-	Metrics          = monitoring.Default.NewRegistry("apm-server.processor.sourcemap", monitoring.PublishExpvar)
+	Metrics          = monitoring.Default.NewRegistry("apm-server.processor.sourcemap")
 	sourcemapCounter = monitoring.NewInt(Metrics, "counter")
 
 	processorEntry = common.MapStr{"name": processorName, "event": smapDocType}

model/span/event.go

@@ -44,7 +44,7 @@ const (
 )
 
 var (
-	Metrics         = monitoring.Default.NewRegistry("apm-server.processor.span", monitoring.PublishExpvar)
+	Metrics         = monitoring.Default.NewRegistry("apm-server.processor.span")
 	transformations = monitoring.NewInt(Metrics, "transformations")
 
 	stacktraceCounter = monitoring.NewInt(Metrics, "stacktraces")

model/transaction/event.go

@@ -45,7 +45,7 @@ const (
 )
 
 var (
-	Metrics           = monitoring.Default.NewRegistry("apm-server.processor.transaction", monitoring.PublishExpvar)
+	Metrics           = monitoring.Default.NewRegistry("apm-server.processor.transaction")
 	transformations   = monitoring.NewInt(Metrics, "transformations")
 	processorEntry    = common.MapStr{"name": processorName, "event": transactionDocType}
 	cachedModelSchema = validation.CreateSchema(schema.ModelSchema, "transaction")