diff --git a/.gitignore b/.gitignore index 1d6d4d926744..68fb430b9852 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /build /docs/landing_source/.bundle +/generated diff --git a/configs/access_log_format_helper.template.json b/configs/access_log_format_helper.template.json new file mode 100644 index 000000000000..1ef69bde8745 --- /dev/null +++ b/configs/access_log_format_helper.template.json @@ -0,0 +1,19 @@ +{% macro ingress_sampled_log() %} + "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" +{% endmacro %} + +{% macro ingress_full() %} + "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" +{% endmacro %} + +{% macro ingress_error_log() %} + "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" +{% endmacro %} + +{% macro egress_error_log() %} + "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n" +{% endmacro %} + +{% macro egress_error_amazon_service() %} + "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%RESP(X-AMZN-RequestId)%\"\n" +{% endmacro %} diff --git a/configs/configgen.py b/configs/configgen.py new file mode 100755 index 000000000000..5689dedef970 --- /dev/null +++ b/configs/configgen.py @@ -0,0 +1,105 @@ +import jinja2 +import json +from collections import OrderedDict +import sys + +# +# About this script: Envoy configurations needed for a complete infrastructure are complicated. +# This script demonstrates how to programatically build Envoy configurations using jinja templates. +# This is roughly how we build our configurations at Lyft. The three configurations demonstrated +# here (front proxy, double proxy, and service to service) are also very close approximations to +# what we use at Lyft in production. They give a demonstration of how to configure most Envoy +# features. Along with the configuration guide it should be possible to modify them for different +# use cases. +# + +# This is the set of internal services that front Envoy will route to. Each cluster referenced +# in envoy_router.template.json must be specified here. It is a dictionary of dictionaries. +# Options can be specified for each cluster if needed. See make_route_internal() in +# routing_helper.template.json for the types of options supported. +front_envoy_clusters = { + 'service1': {}, + 'service2': {}, +} + +# This is the set of internal services that local Envoys will route to. All services that will be +# accessed via the 9001 egress port need to be listed here. It is a dictionary of dictionaries. +# Options can be specified for each cluster if needed. See make_route_internal() in +# routing_helper.template.json for the types of options supported. +service_to_service_envoy_clusters = { + 'ratelimit': {}, + 'service1': {}, + 'service3': {} +} + +# This is a list of external hosts that can be accessed from local Envoys. Each external service has +# its own port. This is because some SDKs don't make it easy to use host based routing. Below +# we demonstrate setting up proxying for DynamoDB. In the config, this ends up using the HTTP +# DynamoDB statistics filter, as well as generating a special access log which includes the +# X-AMZN-RequestId response header. +external_virtual_hosts = [ +{ + 'name': 'dynamodb_iad', + 'port': 9204, + 'hosts': [ + { + 'name': 'dynamodb_iad', 'domain': '*', + 'remote_address': 'dynamodb.us-east-1.amazonaws.com:443', + 'verify_subject_alt_name': 'dynamodb.us-east-1.amazonaws.com', 'ssl': True + } + ], + 'is_amzn_service': True, + 'cluster_type': 'logical_dns' +}] + +# This is the set of mongo clusters that local Envoys can talk to. Each database defines a set of +# mongos routers to talk to, and whether the global rate limit service should be called for new +# connections. Many organizations will not be interested in the mongo feature. Setting this to +# an empty dictionary will remove all mongo configuration. The configuration is a useful example +# as it demonstrates how to setup TCP proxy and the network rate limit filter. +mongos_servers = { + 'somedb': { + 'port': 27019, + 'hosts': [ + "router1.yourcompany.net:27817", + "router2.yourcompany.net:27817", + "router3.yourcompany.net:27817", + "router4.yourcompany.net:27817", + ], + 'ratelimit': True + } +} + +def generate_config(template_path, template, output_file, **context): + """ Generate a final config file based on a template and some context. """ + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path), + undefined=jinja2.StrictUndefined) + raw_output = env.get_template(template).render(**context) + # Verify valid JSON and then dump it nicely formatted to avoid jinja pain. + output = json.loads(raw_output, object_pairs_hook=OrderedDict) + with open(output_file, 'w') as fh: + json.dump(output, fh, indent=2) + +# Generate a demo config for the main front proxy. This sets up both HTTP and HTTPS listeners, +# as well as a listener for the double proxy to connect to via SSL client authentication. +generate_config('configs', 'envoy_front_proxy.template.json', + '{}/envoy_front_proxy.json'.format(sys.argv[1]), clusters=front_envoy_clusters) + +# Generate a demo config for the double proxy. This sets up both an HTTP and HTTPS listeners, +# and backhauls the traffic to the main front proxy. +generate_config('configs', 'envoy_double_proxy.template.json', + '{}/envoy_double_proxy.json'.format(sys.argv[1])) + +# Generate a demo config for the service to service (local) proxy. This sets up several different +# listeners: +# 9211: Main ingress listener for service to service traffic. +# 9001: Main egress listener for service to service traffic. Applications use this port to send +# requests to other services. +# optional external service ports: built from external_virtual_hosts above. Each external host +# that Envoy proxies to listens on its own port. +# optional mongo ports: built from mongos_servers above. +generate_config('configs', 'envoy_service_to_service.template.json', + '{}/envoy_service_to_service.json'.format(sys.argv[1]), + internal_virtual_hosts=service_to_service_envoy_clusters, + external_virtual_hosts=external_virtual_hosts, + mongos_servers=mongos_servers) diff --git a/configs/configgen.sh b/configs/configgen.sh new file mode 100755 index 000000000000..ca30f30a20f7 --- /dev/null +++ b/configs/configgen.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +SCRIPT_DIR=`dirname $0` +BUILD_DIR=build/configgen +if [ ! -d $BUILD_DIR/venv ]; then + virtualenv $BUILD_DIR/venv + $BUILD_DIR/venv/bin/pip install -r $SCRIPT_DIR/requirements.txt +fi + +mkdir -p $1 +$BUILD_DIR/venv/bin/python $SCRIPT_DIR/configgen.py $1 diff --git a/configs/envoy_double_proxy.template.json b/configs/envoy_double_proxy.template.json new file mode 100644 index 000000000000..5f51eeacaed6 --- /dev/null +++ b/configs/envoy_double_proxy.template.json @@ -0,0 +1,151 @@ +{% macro listener(port,ssl,proxy_proto) %} + { + "port": {{ port }}, + {% if ssl -%} + "ssl_context": { + "alpn_protocols": "h2,http/1.1", + "alt_alpn_protocols": "http/1.1", + "cert_chain_file": "/etc/envoy/cert.pem", + "private_key_file": "/etc/envoy/key.pem" + }, + {% endif -%} + {% if proxy_proto -%} + "use_proxy_proto": true, + {% endif -%} + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "tracing_enabled": true, + "idle_timeout_s": 840, + "access_log": [ + { + "path": "/var/log/envoy/access_error.log", + "filter": {"type": "logical_or", "filters": [ + {"type": "status_code", "op": ">=", "value": 500}, + {"type": "duration", "op": ">=", "value": 1000}, + {"type": "traceable_request"} + ] + } + }, + { + "path": "/var/log/envoy/access.log" + }], + "stat_prefix": "router", + {% if proxy_proto -%} + "use_remote_address": true, + {% endif -%} + "route_config": + { + "virtual_hosts": [ + { + "name": "all", + "domains": ["*"], + "routes": [ + { + "prefix": "/", + "cluster": "backhaul", + {# Generally allow front proxy to control timeout and use this as a backstop #} + "timeout_ms": 20000 + } + ] + } + ] + }, + "filters": [ + { "type": "both", "name": "health_check", + "config": { + "pass_through_mode": false, "endpoint": "/healthcheck" + } + }, + { "type": "decoder", "name": "buffer", + "config": { + "max_request_bytes": 5242880, + "max_request_time_s": 120 + } + }, + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] + } +{% endmacro %} + +{ + "listeners": [ + {# TCP listener for external port 443 (SSL). Assumes a TCP LB in front such as ELB which + supports proxy proto. #} + {{ listener(9300,True,True) }}, + + {# TCP listener for external port 80 (non-SSL). Assumes a TCP LB in front such as ELB which + supports proxy proto. #} + {{ listener(9301,False,True) }} + ], + + "admin": { "access_log_path": "/var/log/envoy/admin_access.log", + "port": 9901 }, + "flags_path": "/etc/envoy/flags", + "statsd_tcp_cluster_name": "statsd", + + "tracing": { + "http": { + "sinks": [ + { + "type": "lightstep", + "access_token_file": "/etc/envoy/lightstep_access_token", + "config": { + "collector_cluster": "lightstep_saas" + } + } + ] + } + }, + + "runtime": { + "symlink_root": "/srv/runtime_data/current", + "subdirectory": "envoy", + "override_subdirectory": "envoy_override" + }, + + "cluster_manager": { + "clusters": [ + { + "name": "statsd", + "connect_timeout_ms": 250, + "type": "static", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://127.0.0.1:8125"}] + }, + { + "name": "backhaul", + "connect_timeout_ms": 1000, + "type": "strict_dns", + "lb_type": "round_robin", + "features": "http2", + "max_requests_per_connection": 25000, {# There are so few connections going back + that we can get some imbalance. Until we can come + up with a better solution just limit the requests + so we can cycle and get better spread. #} + "ssl_context": { + "cert_chain_file": "/etc/envoy/envoy-double-proxy.pem", + "private_key_file": "/etc/envoy/envoy-double-proxy.key", + "verify_subject_alt_name": "front-proxy.yourcompany.com" + }, + "hosts": [{"url": "tcp://front-proxy.yourcompany.com:9400"}] + }, + { + "name": "lightstep_saas", + "ssl_context": { + "ca_cert_file": "/etc/ssl/certs/ca-certificates.crt", + "verify_subject_alt_name": "collector.lightstep.com" + }, + "connect_timeout_ms": 1000, + "type": "logical_dns", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://collector.lightstep.com:443"}] + } + ] + } +} diff --git a/configs/envoy_front_proxy.template.json b/configs/envoy_front_proxy.template.json new file mode 100644 index 000000000000..4238f362b570 --- /dev/null +++ b/configs/envoy_front_proxy.template.json @@ -0,0 +1,145 @@ +{% import 'routing_helper.template.json' as helper -%} + +{% macro listener(port) %} + { + "port": {{ port }}, + {% if kwargs['ssl'] -%} + "ssl_context": { + "alpn_protocols": "h2,http/1.1", + "alt_alpn_protocols": "http/1.1", + {% if kwargs.get('pin_double_proxy_client', False) -%} + "ca_cert_file": "/etc/envoy/envoy-ca.pem", + {# This should be the hash of the /etc/envoy/envoy-double-proxy.pem cert used in the + double proxy configuration. #} + "verify_certificate_hash": "fill me in", + {% endif -%} + "cert_chain_file": "/etc/envoy/cert.pem", + "private_key_file": "/etc/envoy/key.pem" + }, + {% endif -%} + {% if kwargs['proxy_proto'] -%} + "use_proxy_proto": true, + {% endif -%} + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "add_user_agent": true, + "tracing_enabled": true, + "idle_timeout_s": 840, + "access_log": [ + { + "path": "/var/log/envoy/access_error.log", + "filter": {"type": "logical_or", "filters": [ + {"type": "status_code", "op": ">=", "value": 500}, + {"type": "duration", "op": ">=", "value": 1000}, + {"type": "traceable_request"} + ] + } + }, + { + "path": "/var/log/envoy/access.log" + }], + "stat_prefix": "router", + {% if kwargs['proxy_proto'] -%} + "use_remote_address": true, + {% endif -%} + "route_config": {% include kwargs['router_file'] %}, + "filters": [ + { "type": "both", "name": "health_check", + "config": { + "pass_through_mode": false, "endpoint": "/healthcheck" + } + }, + { "type": "decoder", "name": "buffer", + "config": { + "max_request_bytes": 5242880, + "max_request_time_s": 120 + } + }, + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] + } +{% endmacro %} + +{ + "listeners": [ + {# TCP listeners for public HTTP/HTTPS endpoints. Assumes a TCP LB in front such as ELB which + supports proxy proto. #} + {{ listener(9300, ssl=True, proxy_proto=True, router_file='envoy_router.template.json') }}, + {{ listener(9301, ssl=False, proxy_proto=True, router_file='envoy_router.template.json') }}, + + {# TCP listener for backhaul traffic from the double proxy. + See envoy_double_proxy.template.json #} + {{ listener(9400, ssl=True, proxy_proto=False, pin_double_proxy_client=True, + router_file='envoy_router.template.json') }} + ], + + "admin": { "access_log_path": "/var/log/envoy/admin_access.log", + "port": 9901 }, + "flags_path": "/etc/envoy/flags", + "statsd_tcp_cluster_name": "statsd", + + "tracing": { + "http": { + "sinks": [ + { + "type": "lightstep", + "access_token_file": "/etc/envoy/lightstep_access_token", + "config": { + "collector_cluster": "lightstep_saas" + } + } + ] + } + }, + + "runtime": { + "symlink_root": "/srv/runtime_data/current", + "subdirectory": "envoy", + "override_subdirectory": "envoy_override" + }, + + "cluster_manager": { + "sds": { + "cluster": { + "name": "sds", + "connect_timeout_ms": 250, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://discovery.yourcompany.net:80"}] + }, + "refresh_delay_ms": 30000 + }, + + "clusters": [ + { + "name": "statsd", + "connect_timeout_ms": 250, + "type": "static", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://127.0.0.1:8125"}] + }, + { + "name": "lightstep_saas", + "ssl_context": { + "ca_cert_file": "/etc/ssl/certs/ca-certificates.crt", + "verify_subject_alt_name": "collector.lightstep.com" + }, + "connect_timeout_ms": 1000, + "type": "logical_dns", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://collector.lightstep.com:443"}] + }, + {% for service, options in clusters.iteritems() -%} + { + {{ helper.internal_cluster_definition(service, options) }} + }{% if not loop.last %},{% endif %} + {% endfor -%} + ] + } +} diff --git a/configs/envoy_router.template.json b/configs/envoy_router.template.json new file mode 100644 index 000000000000..4400f0ac360a --- /dev/null +++ b/configs/envoy_router.template.json @@ -0,0 +1,3 @@ +{ + "virtual_hosts": [] +} diff --git a/configs/envoy_service_to_service.template.json b/configs/envoy_service_to_service.template.json new file mode 100644 index 000000000000..374c5e7952de --- /dev/null +++ b/configs/envoy_service_to_service.template.json @@ -0,0 +1,372 @@ +{% import 'routing_helper.template.json' as helper -%} +{% import 'access_log_format_helper.template.json' as access_log_helper -%} + +{% macro ingress_listener(port) %} +{ + "port": {{ port }}, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "http_codec_options": "no_compression", + "tracing_enabled": true, + "idle_timeout_s": 840, + "access_log": [ + { + "path": "/var/log/envoy/ingress_http.log", + "filter": {"type": "not_healthcheck"}, + {{ access_log_helper.ingress_full() }} + }, + { + "path": "/var/log/envoy/ingress_http_error.log", + "filter": {"type": "logical_and", "filters": [ + {"type": "logical_or", "filters": [ + {"type": "status_code", "op": ">=", "value": 400}, + {"type": "status_code", "op": "=", "value": 0}, + {"type": "duration", "op": ">=", "value": 2000}, + {"type": "traceable_request"} + ] + }, + {"type": "not_healthcheck"} + ] + }, + {{ access_log_helper.ingress_error_log() }} + }, + { + "path": "/var/log/envoy/ingress_http_sampled.log", + "filter": {"type": "logical_and", "filters": [ + {"type": "not_healthcheck"}, + {"type": "runtime", "key": "access_log.ingress_http"} + ] + }, + {{ access_log_helper.ingress_sampled_log() }} + }], + "stat_prefix": "ingress_http", + "route_config": + { + "virtual_hosts": [ + { + "name": "local_service", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "content_type": "application/grpc", + "cluster": "local_service_grpc" + }, + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "local_service" + }] + } + ] + }, + "filters": [ + { "type": "both", "name": "health_check", + "config": { + "pass_through_mode": true, "cache_time_ms": 2500, "endpoint": "/healthcheck" + } + }, + { "type": "decoder", "name": "buffer", + "config": { + "max_request_bytes": 5242880, + "max_request_time_s": 120 + } + }, + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] +} +{% endmacro %} + +{ + "listeners": [ + {{ ingress_listener(9211) }}, + { + "port": 9001, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "add_user_agent": true, + "idle_timeout_s": 840, + "access_log": [ + { + "path": "/var/log/envoy/egress_http_error.log", + "filter": {"type": "logical_or", "filters": [ + {"type": "status_code", "op": ">=", "value": 400}, + {"type": "duration", "op": ">=", "value": 2000} + ] + }, + {{ access_log_helper.egress_error_log() }} + }], + "stat_prefix": "egress_http", + "use_remote_address": true, + "route_config": + { + "virtual_hosts": [ + {% for service, options in internal_virtual_hosts.iteritems() -%} + { + "name": "{{ service }}", + {# NOTE: The following domain is synthetic and is used so that envoy deals with + devbox vs. prod, etc. #} + "domains": ["{{ service }}"], + "routes": [ + { + "prefix": "/", + {{ helper.make_route_internal(service, options) }} + } + ] + }{% if not loop.last %},{% endif -%} + {% endfor -%} + ] + }, + "filters": [ + {"type": "decoder", "name": "rate_limit", + "config": { + "domain": "envoy_service_to_service" + } + }, + {"type": "both", "name": "grpc_http1_bridge", "config": {}}, + {"type": "decoder", "name": "router", "config": {}} + ] + } + }] + }{% if external_virtual_hosts|length > 0 or mongos_servers|length > 0 %},{% endif -%} + + {% for mapping in external_virtual_hosts -%} + { + "port": {{ mapping['port'] }}, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "idle_timeout_s": 840, + "access_log": [ + { + "path": "/var/log/envoy/egress_{{ mapping['name'] }}_http_error.log", + "filter": {"type": "logical_or", "filters": [ + {"type": "status_code", "op": ">=", "value": 400}, + {"type": "status_code", "op": "=", "value": 0} + {% if mapping.get('log_high_latency_requests', True) %} + ,{"type": "duration", "op": ">=", "value": 2000} + {% endif %} + ] + } + {% if mapping.get('is_amzn_service', False) -%} + ,{{ access_log_helper.egress_error_amazon_service() }} + {% else -%} + ,{{ access_log_helper.egress_error_log() }} + {% endif %} + }], + "stat_prefix": "egress_{{ mapping['name'] }}", + "route_config": + { + "virtual_hosts": [ + {% for host in mapping['hosts'] -%} + { + "name": "egress_{{ host['name'] }}", + "domains": ["{{ host['domain'] }}"], + "routes": [ + { + "prefix": "/", + "cluster": "egress_{{ host['name'] }}", + "retry_policy": { "retry_on": "connect-failure" } + {% if host.get('host_rewrite', False) -%} + ,"host_rewrite": "{{host['host_rewrite']}}" + {% endif -%} + }] + }{% if not loop.last %},{% endif -%} + {% endfor -%} + ] + }, + "filters": [ + {% if mapping['name'] in ['dynamodb_iad', 'dynamodb_legacy'] %} + { "type": "both", "name": "http_dynamo_filter", "config": {}}, + {% endif %} + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] + }{% if (mongos_servers|length > 0) or (mongos_servers|length == 0 and not loop.last ) %},{% endif -%} + {% endfor -%} + + {% for key, value in mongos_servers.iteritems() -%} + { + "port": {{ value['port'] }}, + "filters": [ + {% if value.get('ratelimit', False) %} + { + "type": "read", + "name": "ratelimit", + "config": { + "stat_prefix": "{{ key }}", + "domain": "envoy_mongo_cps", + "descriptors": [[{"key": "database", "value": "{{ key }}"}]] + } + }, + {% endif %} + { + "type": "both", + "name": "mongo_proxy", + "config": { + "stat_prefix": "{{ key }}", + "access_log": "/var/log/envoy/mongo_{{ key }}.log" + } + }, + { + "type": "read", + "name": "tcp_proxy", + "config": { + "cluster": "mongo_{{ key }}", + "stat_prefix": "mongo_{{ key }}" + } + }] + }{% if not loop.last %},{% endif -%} + {% endfor -%} + ], + + "admin": { "access_log_path": "/var/log/envoy/admin_access.log", + "port": 9901 }, + "flags_path": "/etc/envoy/flags", + "statsd_tcp_cluster_name": "statsd", + + "tracing": { + "http": { + "sinks": [ + { + "type": "lightstep", + "access_token_file": "/etc/envoy/lightstep_access_token", + "config": { + "collector_cluster": "lightstep_saas" + } + } + ] + } + }, + + "rate_limit_service": { + "type": "grpc_service", + "config": { + "cluster_name": "ratelimit" + } + }, + + "runtime": { + "symlink_root": "/srv/runtime_data/current", + "subdirectory": "envoy", + "override_subdirectory": "envoy_override" + }, + + "cluster_manager": { + "sds": { + "cluster": { + "name": "sds", + "connect_timeout_ms": 250, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://discovery.yourcompany.net:80"}] + }, + "refresh_delay_ms": 30000 + }, + + "clusters": [ + {% for service, options in internal_virtual_hosts.iteritems() -%} + { + {{ helper.internal_cluster_definition(service, options) }} + }, + {% endfor -%} + + {% for mapping in external_virtual_hosts -%} + {% for host in mapping['hosts'] -%} + { + "name": "egress_{{ host['name'] }}", + {% if host.get('ssl', False) -%} + "ssl_context": { + "ca_cert_file": "/etc/ssl/certs/ca-certificates.crt" + {% if host.get('sni', False) -%} + ,"sni": "{{ host['sni'] }}" + {% endif -%} + {% if host.get('verify_subject_alt_name', False) -%} + ,"verify_subject_alt_name": "{{ host['verify_subject_alt_name'] }}" + {% endif -%} + }, + "connect_timeout_ms": 1000, + {% else -%} + "connect_timeout_ms": 250, + {% endif -%} + "type": "{{ mapping.get("cluster_type", "strict_dns") }}", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://{{ host['remote_address'] }}"}] + }, + {% endfor -%} + {% endfor -%} + + {% for key, value in mongos_servers.iteritems() -%} + { + "name": "mongo_{{ key }}", + "connect_timeout_ms": 250, + "type": "strict_dns", + "lb_type": "random", {# We use random LB policy here because we don't HC mongo routers. If + a router drops, we want to converge on an even distribution. + Without HC, least connection would perform terribly as we would + continue to hit the bad router. #} + "hosts": [ + {% for server in value['hosts'] -%} + {% set host = server.split(':')[0] -%} + {% set port = server.split(':')[1] -%} + {"url": "tcp://{{ host }}:{{ port }}"}{% if not loop.last %},{% endif %} + {% endfor -%} + ] + }, + {% endfor -%} + { + "name": "local_service", + "connect_timeout_ms": 250, + "type": "static", + "lb_type": "round_robin", + "max_pending_requests": 30, {# Apply back pressure quickly at the local host level. NOTE: This + only is applicable with the HTTP/1.1 connection pool. #} + "max_connections": 100, + "hosts": [{"url": "tcp://127.0.0.1:8080"}] + + }, + { + "name": "local_service_grpc", + "connect_timeout_ms": 250, + "type": "static", + "lb_type": "round_robin", + "features": "http2", + "max_requests": 200, + "hosts": [{"url": "tcp://127.0.0.1:8081"}] + }, + { + "name": "statsd", + "connect_timeout_ms": 250, + "type": "static", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://127.0.0.1:8125"}] + }, + { + "name": "lightstep_saas", + "ssl_context": { + "ca_cert_file": "/etc/ssl/certs/ca-certificates.crt", + "verify_subject_alt_name": "collector.lightstep.com" + }, + "connect_timeout_ms": 1000, + "type": "logical_dns", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://collector.lightstep.com:443"}] + } + ] + } +} diff --git a/configs/requirements.txt b/configs/requirements.txt new file mode 100644 index 000000000000..31c0090e2127 --- /dev/null +++ b/configs/requirements.txt @@ -0,0 +1 @@ +jinja2==2.8 diff --git a/configs/routing_helper.template.json b/configs/routing_helper.template.json new file mode 100644 index 000000000000..672f2a92b0d9 --- /dev/null +++ b/configs/routing_helper.template.json @@ -0,0 +1,41 @@ +{% macro make_route_internal(cluster, options) %} + {% if 'timeout_ms' in options %} + "timeout_ms": {{ options['timeout_ms'] }}, + {% endif %} + "retry_policy": { + "retry_on": "{{ options.get('retry_on', 'connect-failure') }}" + }, + {% if 'global_rate_limit' in options %} + "rate_limit": { + "global": true + }, + {% endif %} + "cluster": "{{ cluster }}" +{% endmacro %} + +{% macro make_route(cluster) %} + {{ make_route_internal(cluster, clusters.get(cluster, {})) }} +{% endmacro %} + +{% macro internal_cluster_definition(service, options) %} + "name": "{{ service }}", + "connect_timeout_ms": 250, + "type": "sds", + "lb_type": "least_request", + "features": "http2", + "http_codec_options": "no_compression", + "service_name": "{{ service }}", + {% if 'max_requests' in options %} + "max_requests": {{ options['max_requests'] }}, + {% endif %} + "health_check": { + "type": "http", + "timeout_ms": 2000, + "interval_ms": 5000, + "interval_jitter_ms": 5000, + "unhealthy_threshold": 2, + "healthy_threshold": 2, + "path": "/healthcheck", + "service_name": "{{ service }}" + } +{% endmacro %} diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index 9c9322382d8f..26c6eb72ac1d 100644 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -27,3 +27,7 @@ include_directories(SYSTEM ${ENVOY_OPENSSL_INCLUDE_DIR}) set_target_properties(envoy-server PROPERTIES COTIRE_CXX_PREFIX_HEADER_INIT "../precompiled/precompiled.h") cotire(envoy-server) + +# Needed due to generated proto headers. There is probably a way to have this only depend on the +# generation of the header but I don't feel like figuring that out right now. +add_dependencies(envoy-server envoy-common) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 45b6a2603028..668dfbbe1f71 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -84,6 +84,7 @@ add_executable(envoy-test common/upstream/logical_dns_cluster_test.cc common/upstream/sds_test.cc common/upstream/upstream_impl_test.cc + example_configs_test.cc generated/helloworld.pb.cc integration/fake_upstream.cc integration/http2_integration_test.cc @@ -140,15 +141,21 @@ target_link_libraries(envoy-test pthread) target_link_libraries(envoy-test anl) target_link_libraries(envoy-test dl) +add_custom_target( + envoy.generate_example_configs configs/configgen.sh generated/configs + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} +) + add_custom_target( ${PROJECT_NAME}.check ${PROJECT_SOURCE_DIR}/test/run_envoy_tests.sh ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} - DEPENDS envoy envoy-test) + DEPENDS envoy envoy-test envoy.generate_example_configs +) if (ENVOY_CODE_COVERAGE) add_custom_target( ${PROJECT_NAME}.check-coverage - DEPENDS envoy envoy-test + DEPENDS envoy envoy-test envoy.generate_example_configs COMMAND ${PROJECT_SOURCE_DIR}/test/run_envoy_coverage.sh ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR} ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} ${ENVOY_GCOVR} ${ENVOY_GCOVR_EXTRA_ARGS} ) diff --git a/test/example_configs_test.cc b/test/example_configs_test.cc new file mode 100644 index 000000000000..173f143b8828 --- /dev/null +++ b/test/example_configs_test.cc @@ -0,0 +1,78 @@ +#include "server/configuration_impl.h" + +#include "test/integration/server.h" +#include "test/mocks/server/mocks.h" + +#include + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +class NullSslContextManager : public Ssl::ContextManager, + public Ssl::ServerContext, + public Ssl::ClientContext { +public: + Ssl::ClientContext& createSslClientContext(const std::string&, Stats::Store&, + Ssl::ContextConfig&) override { + return *this; + } + Ssl::ServerContext& createSslServerContext(const std::string&, Stats::Store&, + Ssl::ContextConfig&) override { + return *this; + } + size_t daysUntilFirstCertExpires() override { return 0; } + std::string getCaCertInformation() override { return ""; } + std::string getCertChainInformation() override { return ""; } + std::vector> getContexts() override { return {}; }; +}; + +class ConfigTest { +public: + ConfigTest(const std::string& file_path) : options_(file_path) { + ON_CALL(server_, options()).WillByDefault(ReturnRef(options_)); + ON_CALL(server_, sslContextManager()).WillByDefault(ReturnRef(ssl_context_manager_)); + ON_CALL(server_.api_, fileReadToEnd("lightstep_access_token")) + .WillByDefault(Return("access_token")); + + Server::Configuration::InitialImpl initial_config(file_path); + Server::Configuration::MainImpl main_config(server_); + + ON_CALL(server_, clusterManager()) + .WillByDefault( + Invoke([&]() -> Upstream::ClusterManager& { return main_config.clusterManager(); })); + + try { + main_config.initialize(file_path); + } catch (const EnvoyException& ex) { + throw EnvoyException(fmt::format("'{}' config failed. Error: {}", file_path, ex.what())); + } + } + + NiceMock server_; + NullSslContextManager ssl_context_manager_; + Server::TestOptionsImpl options_; +}; + +void runConfigTest(const std::string& dir_path) { + DIR* dir = opendir(dir_path.c_str()); + if (!dir) { + throw std::runtime_error("Generated configs directory not found"); + } + dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_type != DT_REG) { + continue; + } + + std::string file_name = fmt::format("{}/{}", dir_path, std::string(entry->d_name)); + Logger::Registry::getLog(Logger::Id::testing).notice("testing config: {}", file_name); + ConfigTest config(file_name); + } + + closedir(dir); +} + +TEST(ExampleConfigsTest, All) { runConfigTest("generated/configs"); } diff --git a/test/example_configs_test.h b/test/example_configs_test.h new file mode 100644 index 000000000000..c69edeb5bbd2 --- /dev/null +++ b/test/example_configs_test.h @@ -0,0 +1,6 @@ +#pragma once + +/** + * Load all configurations from a directory with sufficient mocking to be reasonably sure they work. + */ +void runConfigTest(const std::string& dir_path);