Skip to content

Commit

Permalink
Expose Envoy's /stats for statsd agents (#7173)
Browse files Browse the repository at this point in the history
* Expose Envoy /stats for statsd agents; Add testcases

* Remove merge conflict leftover

* Add support for prefix instead of path; Fix docstring to mirror these changes

* Add new config field to docs; Add testcases to check that /stats/prometheus is exposed as well

* Parametrize matchType (prefix or path) and value

* Update website/source/docs/connect/proxies/envoy.md

Co-Authored-By: Paul Banks <[email protected]>

Co-authored-by: Paul Banks <[email protected]>
  • Loading branch information
tpaschalis and banks authored Feb 3, 2020
1 parent 6404967 commit a335aa5
Show file tree
Hide file tree
Showing 8 changed files with 274 additions and 19 deletions.
30 changes: 22 additions & 8 deletions command/connect/envoy/bootstrap_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ type BootstrapConfig struct {
// be fixed in a future Consul version as Envoy 1.10 reaches stable release.
PrometheusBindAddr string `mapstructure:"envoy_prometheus_bind_addr"`

// StatsBindAddr configures an <ip>:<port> on which the Envoy will listen
// and expose the /stats HTTP path prefix for any agent to access. It
// does this by proxying that path prefix to the internal admin server
// which allows exposing metrics on the network without the security
// risk of exposing the full admin server API. Any other URL requested will be
// a 404.
StatsBindAddr string `mapstructure:"envoy_stats_bind_addr"`

// OverrideJSONTpl allows replacing the base template used to render the
// bootstrap. This is an "escape hatch" allowing arbitrary control over the
// proxy's configuration but will the most effort to maintain and correctly
Expand Down Expand Up @@ -188,7 +196,13 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs) error {
}
// Setup prometheus if needed. This MUST happen after the Static*JSON is set above
if c.PrometheusBindAddr != "" {
if err := c.generatePrometheusConfig(args); err != nil {
if err := c.generateMetricsListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil {
return err
}
}
// Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above
if c.StatsBindAddr != "" {
if err := c.generateMetricsListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil {
return err
}
}
Expand Down Expand Up @@ -369,10 +383,10 @@ func (c *BootstrapConfig) generateStatsConfig(args *BootstrapTplArgs) error {
return nil
}

func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error {
host, port, err := net.SplitHostPort(c.PrometheusBindAddr)
func (c *BootstrapConfig) generateMetricsListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error {
host, port, err := net.SplitHostPort(bindAddr)
if err != nil {
return fmt.Errorf("invalid prometheus_bind_addr: %s", err)
return fmt.Errorf("invalid %s bind address: %s", name, err)
}

clusterJSON := `{
Expand All @@ -390,7 +404,7 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error
]
}`
listenerJSON := `{
"name": "envoy_prometheus_metrics_listener",
"name": "` + name + `_listener",
"address": {
"socket_address": {
"address": "` + host + `",
Expand All @@ -403,7 +417,7 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error
{
"name": "envoy.http_connection_manager",
"config": {
"stat_prefix": "envoy_prometheus_metrics",
"stat_prefix": "` + name + `",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
Expand All @@ -416,11 +430,11 @@ func (c *BootstrapConfig) generatePrometheusConfig(args *BootstrapTplArgs) error
"routes": [
{
"match": {
"path": "/metrics"
"` + matchType + `": "` + matchValue + `"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/stats/prometheus"
"prefix_rewrite": "` + prefixRewrite + `"
}
},
{
Expand Down
120 changes: 120 additions & 0 deletions command/connect/envoy/bootstrap_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,77 @@ const (
}
]
}`
expectedStatsListener = `{
"name": "envoy_metrics_listener",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 9000
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"stat_prefix": "envoy_metrics",
"codec_type": "HTTP1",
"route_config": {
"name": "self_admin_route",
"virtual_hosts": [
{
"name": "self_admin",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/stats"
},
"route": {
"cluster": "self_admin",
"prefix_rewrite": "/stats"
}
},
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 404
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.router"
}
]
}
}
]
}
]
}`
expectedStatsCluster = `{
"name": "self_admin",
"connect_timeout": "5s",
"type": "STATIC",
"http_protocol_options": {},
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 19000
}
}
]
}`
)

func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
Expand Down Expand Up @@ -350,6 +421,48 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
},
wantErr: false,
},
{
name: "stats-bind-addr",
input: BootstrapConfig{
StatsBindAddr: "0.0.0.0:9000",
},
baseArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
},
wantArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: expectedStatsCluster,
// Should add a static http listener too
StaticListenersJSON: expectedStatsListener,
StatsConfigJSON: defaultStatsConfigJSON,
},
wantErr: false,
},
{
name: "stats-bind-addr-with-overrides",
input: BootstrapConfig{
StatsBindAddr: "0.0.0.0:9000",
StaticClustersJSON: `{"foo":"bar"}`,
StaticListenersJSON: `{"baz":"qux"}`,
},
baseArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
},
wantArgs: BootstrapTplArgs{
AdminBindAddress: "127.0.0.1",
AdminBindPort: "19000",
// Should add a static cluster for the self-proxy to admin
StaticClustersJSON: `{"foo":"bar"},` + expectedStatsCluster,
// Should add a static http listener too
StaticListenersJSON: `{"baz":"qux"},` + expectedStatsListener,
StatsConfigJSON: defaultStatsConfigJSON,
},
wantErr: false,
},
{
name: "stats-flush-interval",
input: BootstrapConfig{
Expand Down Expand Up @@ -379,6 +492,13 @@ func TestBootstrapConfig_ConfigureArgs(t *testing.T) {
},
wantErr: true,
},
{
name: "err-bad-stats-addr",
input: BootstrapConfig{
StatsBindAddr: "asdasdsad",
},
wantErr: true,
},
{
name: "err-bad-statsd-addr",
input: BootstrapConfig{
Expand Down
23 changes: 23 additions & 0 deletions test/integration/connect/envoy/case-stats-proxy/s1.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
services {
name = "s1"
port = 8080
connect {
sidecar_service {
proxy {
upstreams = [
{
destination_name = "s2"
local_bind_port = 5000
config {
protocol = "http"
}
}
]
config {
protocol = "http"
envoy_stats_bind_addr = "0.0.0.0:1239"
}
}
}
}
}
13 changes: 13 additions & 0 deletions test/integration/connect/envoy/case-stats-proxy/s2.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
services {
name = "s2"
port = 8181
connect {
sidecar_service {
proxy {
config {
protocol = "http"
}
}
}
}
}
6 changes: 6 additions & 0 deletions test/integration/connect/envoy/case-stats-proxy/setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

set -eEuo pipefail

gen_envoy_bootstrap s1 19000 primary
gen_envoy_bootstrap s2 19001 primary
62 changes: 62 additions & 0 deletions test/integration/connect/envoy/case-stats-proxy/verify.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env bats

load helpers

@test "s1 proxy admin is up on :19000" {
retry_default curl -f -s localhost:19000/stats -o /dev/null
}

@test "s2 proxy admin is up on :19001" {
retry_default curl -f -s localhost:19001/stats -o /dev/null
}

@test "s1 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21000 s1
}

@test "s2 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21001 s2
}

@test "s2 proxy should be healthy" {
assert_service_has_healthy_instances s2 1
}

@test "s1 upstream should have healthy endpoints for s2" {
# protocol is configured in an upstream override so the cluster name is customized here
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 1a47f6e1~s2.default.primary HEALTHY 1
}

@test "s1 upstream should be able to connect to s2 with http/1.1" {
run retry_default curl --http1.1 -s -f -d hello localhost:5000
[ "$status" -eq 0 ]
[ "$output" = "hello" ]
}

@test "s1 proxy should be exposing the /stats prefix" {
# Should have http metrics. This is just a sample one. Require the metric to
# be present not just found in a comment (anchor the regexp).
retry_default \
must_match_in_stats_proxy_response localhost:1239 \
'stats' '^http.envoy_metrics.downstream_rq_active'

# Response should include the the local cluster request.
retry_default \
must_match_in_stats_proxy_response localhost:1239 \
'stats' 'cluster.local_agent.upstream_rq_active'

# Response should include the http public listener.
retry_default \
must_match_in_stats_proxy_response localhost:1239 \
'stats' 'http.public_listener_http'

# /stats/prometheus should also be reachable and labelling the local cluster.
retry_default \
must_match_in_stats_proxy_response localhost:1239 \
'stats/prometheus' '[\{,]local_cluster="s1"[,}]'

# /stats/prometheus should also be reachable and exposing metrics.
retry_default \
must_match_in_stats_proxy_response localhost:1239 \
'stats/prometheus' 'envoy_http_downstream_rq_active'
}
Loading

0 comments on commit a335aa5

Please sign in to comment.