From 163d2b0652697f33915c5f486398a197cf6090f0 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 11:59:17 +0200 Subject: [PATCH 01/22] Adds jaeger config --- gateway/conf.d/jaeger.conf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 gateway/conf.d/jaeger.conf diff --git a/gateway/conf.d/jaeger.conf b/gateway/conf.d/jaeger.conf new file mode 100644 index 000000000..368c6807a --- /dev/null +++ b/gateway/conf.d/jaeger.conf @@ -0,0 +1,4 @@ +jaeger_reporter_local_agent_host_port {{ jaeger_addr | default: '127.0.0.1' }}:{{ jaeger_port | default: 6831 }}; +jaeger_service_name {{ jaeger_service_name | default: 'apicast' }}; +jaeger_sampler_type {{ jaeger_sampler_type | default: 'const' }}; +jaeger_sampler_param {{ jaeger_samples_param | default: 1 }}; From 1a1d3aa5a3410a20031e049d20688e26b541dbbd Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 11:59:39 +0200 Subject: [PATCH 02/22] Include opentracing modules and jaeger conf --- gateway/conf/nginx.conf.liquid | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 5972ed53f..233837a5c 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -5,12 +5,16 @@ env RESOLVER; env BACKEND_ENDPOINT_OVERRIDE; env OPENSSL_VERIFY; +{% if jaeger_enabled %} +load_module /opt/app-root/src/nginx-modules/ngx_http_opentracing_module.so; +load_module /opt/app-root/src/nginx-modules/ngx_http_jaeger_module.so; +{% endif %} + daemon {{ daemon | default: 'off' }}; master_process {{ master_process | default: 'on' }}; worker_processes {{ worker_processes | default: 'auto' }}; pcre_jit on; pid {{ pid | default: 'nginx.pid' }}; -timer_resolution {{ timer_resolution | default: '100ms' }}; {% for file in "main.d/*.conf" | filesystem %} {% include file %} @@ -28,6 +32,8 @@ http { tcp_nopush on; tcp_nodelay on; + + # Enabling the Lua code cache is strongly encouraged for production use # Disabling it should only be done for testing and development purposes lua_code_cache {{ lua_code_cache | default: 'on' }}; @@ -61,9 +67,13 @@ http { {% include file %} {% endfor %} + + {% if jaeger_enabled %} + {% include "conf.d/jaeger.conf" %} + {% endif %} + server { listen {{ port.management | default: 8090 }}; - server_name {{ server_name.management | default: 'management _' }}; {% include "conf.d/management.conf" %} @@ -71,7 +81,6 @@ http { server { listen {{ port.backend | default: 8081 }}; - server_name backend; {% include "conf.d/backend.conf" %} @@ -79,19 +88,17 @@ http { upstream echo { server 127.0.0.1:{{ port.echo | default: 8081 }}; - keepalive 1024; } server { listen {{ port.echo | default: 8081 }} default_server; - server_name echo _; - {% include "conf.d/echo.conf" %} } server { + access_log /dev/stdout time; listen {{ port.apicast | default: 8080 }}; From 9dde2b9fc6f4ff40649b39bb395d3a7c5722c998 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 12:00:23 +0200 Subject: [PATCH 03/22] Sets proper service_name and location_names for all servers --- gateway/conf.d/apicast.conf | 29 +++++++++++++++++++++++++++-- gateway/conf.d/backend.conf | 7 +++++++ gateway/conf.d/echo.conf | 7 +++++++ gateway/conf.d/management.conf | 6 ++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/gateway/conf.d/apicast.conf b/gateway/conf.d/apicast.conf index 7529abb96..be923ab4b 100644 --- a/gateway/conf.d/apicast.conf +++ b/gateway/conf.d/apicast.conf @@ -1,3 +1,5 @@ + + set_by_lua $user_agent 'return require("apicast.user_agent")()'; set_by_lua_block $deployment { local user_agent = require('apicast.user_agent') @@ -9,6 +11,12 @@ set_by_lua_block $deployment { # ssl_session_fetch_by_lua_block { require('apicast.executor').call() } # ssl_session_store_by_lua_block { require('apicast.executor').call() } +{% if jaeger_enabled %} + opentracing on; + opentracing_trace_locations on; + opentracing_operation_name "apicast"; +{% endif %} + location = /___http_call { internal; @@ -35,6 +43,7 @@ location = /___http_call { proxy_set_header User-Agent $user_agent; proxy_set_header X-3scale-OAuth2-Grant-Type $grant_type; proxy_set_header 3scale-options $options; + proxy_set_header uber-trace-id $http_uber_trace_id; rewrite_by_lua_block { require('resty.http_ng.backend.ngx'):resolver() @@ -43,9 +52,12 @@ location = /___http_call { location @out_of_band_authrep_action { internal; - proxy_pass_request_headers off; + {% if jaeger_enabled %} + opentracing_location_operation_name "apicast_async_authrep"; + {% endif %} + set_by_lua $original_request_time 'return ngx.var.request_time'; content_by_lua_block { @@ -61,11 +73,14 @@ location @out_of_band_authrep_action { location @upstream { internal; - rewrite_by_lua_block { require('resty.ctx').apply() } + {% if jaeger_enabled %} + opentracing_location_operation_name "apicast_upstream"; + {% endif %} + proxy_pass $proxy_pass; proxy_http_version 1.1; @@ -110,6 +125,11 @@ location @upstream { } location / { + + {% if jaeger_enabled %} + opentracing_location_operation_name "apicast_main"; + {% endif %} + set $cached_key ''; set $credentials ''; set $usage ''; @@ -149,6 +169,11 @@ location / { # This exposes a health check for kubernetes environments # This is so the healtcheck on our Google HTTP load balancers works location /_threescale/healthz { + + {% if jaeger_enabled %} + opentracing_location_operation_name "apicast_healthz"; + {% endif %} + return 200; access_log off; } diff --git a/gateway/conf.d/backend.conf b/gateway/conf.d/backend.conf index edbba1b1f..6b500b698 100644 --- a/gateway/conf.d/backend.conf +++ b/gateway/conf.d/backend.conf @@ -1,4 +1,11 @@ +{% if jaeger_enabled %} + opentracing_trace_locations off; + opentracing_operation_name "apicast_mockbackend"; + opentracing on; +{% endif %} + location /transactions/authrep.xml { + access_by_lua_block { local delay = tonumber(ngx.var.arg_delay) or 0 diff --git a/gateway/conf.d/echo.conf b/gateway/conf.d/echo.conf index bae08331a..eb8bac60d 100644 --- a/gateway/conf.d/echo.conf +++ b/gateway/conf.d/echo.conf @@ -1,3 +1,10 @@ + +{% if jaeger_enabled %} + opentracing on; + opentracing_operation_name "apicast_echo"; + opentracing_trace_locations off; +{% endif %} + location / { log_by_lua_block { if ngx.config.debug then diff --git a/gateway/conf.d/management.conf b/gateway/conf.d/management.conf index 51b620c5d..3831df1b6 100644 --- a/gateway/conf.d/management.conf +++ b/gateway/conf.d/management.conf @@ -1,3 +1,9 @@ + {% if jaeger_enabled %} + opentracing_operation_name "apicast_management"; + opentracing on; + opentracing_trace_locations off; + {% endif %} + location / { content_by_lua_block { require('apicast.management').call() } } From 925aaf410a8df4f8502e0821c5ce3bee9130bfb9 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 12:00:58 +0200 Subject: [PATCH 04/22] Read jaeger vars from ENV --- gateway/src/apicast/cli/environment.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index 1d775447a..1575956d1 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -108,7 +108,10 @@ _M.default_config = { proxy_ssl_certificate_key = env_value_ref('APICAST_PROXY_HTTPS_CERTIFICATE_KEY'), proxy_ssl_session_reuse = env_value_ref('APICAST_PROXY_HTTPS_SESSION_REUSE'), proxy_ssl_password_file = env_value_ref('APICAST_PROXY_HTTPS_PASSWORD_FILE'), - + jaeger_enabled = env_value_ref('JAEGER_ENABLED'), + jaeger_addr = env_value_ref('JAEGER_ADDR'), + jaeger_port = env_value_ref('JAEGER_PORT'), + jaeger_service_name = env_value_ref('JAEGER_SERVICE_NAME'), policy_chain = require('apicast.policy_chain').default(), nameservers = parse_nameservers(), worker_processes = cpus() or 'auto', From b1466d14883f9775004392fa9c75731bc12e0a9e Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 15:48:34 +0200 Subject: [PATCH 05/22] created opentracing directory to include individual tracer confs --- gateway/conf.d/opentracing/jaeger.conf.liquid | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 gateway/conf.d/opentracing/jaeger.conf.liquid diff --git a/gateway/conf.d/opentracing/jaeger.conf.liquid b/gateway/conf.d/opentracing/jaeger.conf.liquid new file mode 100644 index 000000000..caad679fc --- /dev/null +++ b/gateway/conf.d/opentracing/jaeger.conf.liquid @@ -0,0 +1,5 @@ +jaeger_reporter_local_agent_host_port {{ jaeger_addr | default: '127.0.0.1' }}:{{ jaeger_port | default: 6831 }}; +jaeger_service_name {{ jaeger_service_name | default: 'apicast' }}; +jaeger_sampler_type {{ jaeger_sampler_type | default: 'const' }}; +jaeger_sampler_param {{ jaeger_samples_param | default: 1 }}; +opentracing on; From 4d38495dcd814a36f95fd84a21aced83487cbef1 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 15:51:13 +0200 Subject: [PATCH 06/22] Adds new ENV vars for opentracing_enabled and opentracing_tracer --- gateway/src/apicast/cli/environment.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index 1575956d1..a06458588 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -108,10 +108,13 @@ _M.default_config = { proxy_ssl_certificate_key = env_value_ref('APICAST_PROXY_HTTPS_CERTIFICATE_KEY'), proxy_ssl_session_reuse = env_value_ref('APICAST_PROXY_HTTPS_SESSION_REUSE'), proxy_ssl_password_file = env_value_ref('APICAST_PROXY_HTTPS_PASSWORD_FILE'), - jaeger_enabled = env_value_ref('JAEGER_ENABLED'), + opentracing_enabled = env_value_ref('OPENTRACING_ENABLED'), + opentracing_tracer = env_value_ref('OPENTRACING_TRACER'), jaeger_addr = env_value_ref('JAEGER_ADDR'), jaeger_port = env_value_ref('JAEGER_PORT'), jaeger_service_name = env_value_ref('JAEGER_SERVICE_NAME'), + jaeger_sampler_type = env_value_ref('JAEGER_SAMPLER_TYPE'), + jaeger_sampler_param = env_value_ref('JAEGER_SAMPLER_PARAM'), policy_chain = require('apicast.policy_chain').default(), nameservers = parse_nameservers(), worker_processes = cpus() or 'auto', From fb9ded489ee924ee0764285559bca6659e7bb5ce Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 15:51:49 +0200 Subject: [PATCH 07/22] Remove the specific configs from each conf and put all of them into the server part in nginx.conf.liquid --- gateway/conf.d/apicast.conf | 23 ----------------------- gateway/conf.d/backend.conf | 6 ------ gateway/conf.d/jaeger.conf | 4 ---- gateway/conf.d/management.conf | 6 ------ gateway/conf/nginx.conf.liquid | 31 ++++++++++++++++++++++++++++--- 5 files changed, 28 insertions(+), 42 deletions(-) delete mode 100644 gateway/conf.d/jaeger.conf diff --git a/gateway/conf.d/apicast.conf b/gateway/conf.d/apicast.conf index be923ab4b..ef42c3a26 100644 --- a/gateway/conf.d/apicast.conf +++ b/gateway/conf.d/apicast.conf @@ -11,12 +11,6 @@ set_by_lua_block $deployment { # ssl_session_fetch_by_lua_block { require('apicast.executor').call() } # ssl_session_store_by_lua_block { require('apicast.executor').call() } -{% if jaeger_enabled %} - opentracing on; - opentracing_trace_locations on; - opentracing_operation_name "apicast"; -{% endif %} - location = /___http_call { internal; @@ -54,10 +48,6 @@ location @out_of_band_authrep_action { internal; proxy_pass_request_headers off; - {% if jaeger_enabled %} - opentracing_location_operation_name "apicast_async_authrep"; - {% endif %} - set_by_lua $original_request_time 'return ngx.var.request_time'; content_by_lua_block { @@ -77,10 +67,6 @@ location @upstream { require('resty.ctx').apply() } - {% if jaeger_enabled %} - opentracing_location_operation_name "apicast_upstream"; - {% endif %} - proxy_pass $proxy_pass; proxy_http_version 1.1; @@ -126,10 +112,6 @@ location @upstream { location / { - {% if jaeger_enabled %} - opentracing_location_operation_name "apicast_main"; - {% endif %} - set $cached_key ''; set $credentials ''; set $usage ''; @@ -169,11 +151,6 @@ location / { # This exposes a health check for kubernetes environments # This is so the healtcheck on our Google HTTP load balancers works location /_threescale/healthz { - - {% if jaeger_enabled %} - opentracing_location_operation_name "apicast_healthz"; - {% endif %} - return 200; access_log off; } diff --git a/gateway/conf.d/backend.conf b/gateway/conf.d/backend.conf index 6b500b698..8c0abf34a 100644 --- a/gateway/conf.d/backend.conf +++ b/gateway/conf.d/backend.conf @@ -1,9 +1,3 @@ -{% if jaeger_enabled %} - opentracing_trace_locations off; - opentracing_operation_name "apicast_mockbackend"; - opentracing on; -{% endif %} - location /transactions/authrep.xml { access_by_lua_block { diff --git a/gateway/conf.d/jaeger.conf b/gateway/conf.d/jaeger.conf deleted file mode 100644 index 368c6807a..000000000 --- a/gateway/conf.d/jaeger.conf +++ /dev/null @@ -1,4 +0,0 @@ -jaeger_reporter_local_agent_host_port {{ jaeger_addr | default: '127.0.0.1' }}:{{ jaeger_port | default: 6831 }}; -jaeger_service_name {{ jaeger_service_name | default: 'apicast' }}; -jaeger_sampler_type {{ jaeger_sampler_type | default: 'const' }}; -jaeger_sampler_param {{ jaeger_samples_param | default: 1 }}; diff --git a/gateway/conf.d/management.conf b/gateway/conf.d/management.conf index 3831df1b6..51b620c5d 100644 --- a/gateway/conf.d/management.conf +++ b/gateway/conf.d/management.conf @@ -1,9 +1,3 @@ - {% if jaeger_enabled %} - opentracing_operation_name "apicast_management"; - opentracing on; - opentracing_trace_locations off; - {% endif %} - location / { content_by_lua_block { require('apicast.management').call() } } diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 233837a5c..4d2102289 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -5,7 +5,7 @@ env RESOLVER; env BACKEND_ENDPOINT_OVERRIDE; env OPENSSL_VERIFY; -{% if jaeger_enabled %} +{% if opentracing_enabled %} load_module /opt/app-root/src/nginx-modules/ngx_http_opentracing_module.so; load_module /opt/app-root/src/nginx-modules/ngx_http_jaeger_module.so; {% endif %} @@ -16,6 +16,10 @@ worker_processes {{ worker_processes | default: 'auto' }}; pcre_jit on; pid {{ pid | default: 'nginx.pid' }}; +{% unless opentracing_enabled %} +timer_resolution {{ timer_resolution | default: '100ms' }}; +{% endif %} + {% for file in "main.d/*.conf" | filesystem %} {% include file %} {% endfor %} @@ -68,14 +72,19 @@ http { {% endfor %} - {% if jaeger_enabled %} - {% include "conf.d/jaeger.conf" %} + {% if opentracing_enabled == 'true' %} + {% include "conf.d/opentracing" | append opentracing_tracer | append ".conf.liquid" %} {% endif %} server { listen {{ port.management | default: 8090 }}; server_name {{ server_name.management | default: 'management _' }}; + {% if opentracing_enabled == 'true' %} + opentracing_operation_name "apicast_management"; + opentracing_trace_locations off; + {% endif %} + {% include "conf.d/management.conf" %} } @@ -83,6 +92,11 @@ http { listen {{ port.backend | default: 8081 }}; server_name backend; + {% if opentracing_enabled == 'true' %} + opentracing_operation_name "apicast_mockbackend"; + opentracing_trace_locations off; + {% endif %} + {% include "conf.d/backend.conf" %} } @@ -94,6 +108,12 @@ http { server { listen {{ port.echo | default: 8081 }} default_server; server_name echo _; + + {% if opentracing_enabled == 'true' %} + opentracing_operation_name "apicast_echo"; + opentracing_trace_locations off; + {% endif %} + {% include "conf.d/echo.conf" %} } @@ -105,6 +125,11 @@ http { server_name _; + {% if opentracing_enabled == 'true' %} + opentracing_operation_name "apicast"; + opentracing_trace_locations on; + {% endif %} + {% include "http.d/ssl.conf" %} {% for file in "apicast.d/*.conf" | filesystem %} From 63a58d798959b239137a1b9152e7292e9cc179e0 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 15:54:29 +0200 Subject: [PATCH 08/22] Checkout from master --- gateway/conf.d/apicast.conf | 6 ++---- gateway/conf.d/backend.conf | 1 - gateway/conf.d/echo.conf | 7 ------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/gateway/conf.d/apicast.conf b/gateway/conf.d/apicast.conf index ef42c3a26..7529abb96 100644 --- a/gateway/conf.d/apicast.conf +++ b/gateway/conf.d/apicast.conf @@ -1,5 +1,3 @@ - - set_by_lua $user_agent 'return require("apicast.user_agent")()'; set_by_lua_block $deployment { local user_agent = require('apicast.user_agent') @@ -37,7 +35,6 @@ location = /___http_call { proxy_set_header User-Agent $user_agent; proxy_set_header X-3scale-OAuth2-Grant-Type $grant_type; proxy_set_header 3scale-options $options; - proxy_set_header uber-trace-id $http_uber_trace_id; rewrite_by_lua_block { require('resty.http_ng.backend.ngx'):resolver() @@ -46,6 +43,7 @@ location = /___http_call { location @out_of_band_authrep_action { internal; + proxy_pass_request_headers off; set_by_lua $original_request_time 'return ngx.var.request_time'; @@ -63,6 +61,7 @@ location @out_of_band_authrep_action { location @upstream { internal; + rewrite_by_lua_block { require('resty.ctx').apply() } @@ -111,7 +110,6 @@ location @upstream { } location / { - set $cached_key ''; set $credentials ''; set $usage ''; diff --git a/gateway/conf.d/backend.conf b/gateway/conf.d/backend.conf index 8c0abf34a..edbba1b1f 100644 --- a/gateway/conf.d/backend.conf +++ b/gateway/conf.d/backend.conf @@ -1,5 +1,4 @@ location /transactions/authrep.xml { - access_by_lua_block { local delay = tonumber(ngx.var.arg_delay) or 0 diff --git a/gateway/conf.d/echo.conf b/gateway/conf.d/echo.conf index eb8bac60d..bae08331a 100644 --- a/gateway/conf.d/echo.conf +++ b/gateway/conf.d/echo.conf @@ -1,10 +1,3 @@ - -{% if jaeger_enabled %} - opentracing on; - opentracing_operation_name "apicast_echo"; - opentracing_trace_locations off; -{% endif %} - location / { log_by_lua_block { if ngx.config.debug then From f27a9d28bdaab48dba279903d5688a128762dab4 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 16:20:34 +0200 Subject: [PATCH 09/22] Replace endif to endunless --- gateway/conf/nginx.conf.liquid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 4d2102289..33032d79b 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -18,7 +18,7 @@ pid {{ pid | default: 'nginx.pid' }}; {% unless opentracing_enabled %} timer_resolution {{ timer_resolution | default: '100ms' }}; -{% endif %} +{% endunless %} {% for file in "main.d/*.conf" | filesystem %} {% include file %} From cfb69504a27054ff8ca07b53fe6763a7bf62f361 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 16:31:35 +0200 Subject: [PATCH 10/22] Use opentracing_tracer instead of a bool --- gateway/conf/nginx.conf.liquid | 18 +++++++++--------- gateway/src/apicast/cli/environment.lua | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 33032d79b..165fdbf94 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -5,7 +5,7 @@ env RESOLVER; env BACKEND_ENDPOINT_OVERRIDE; env OPENSSL_VERIFY; -{% if opentracing_enabled %} +{% if opentracing_tracer != empty %} load_module /opt/app-root/src/nginx-modules/ngx_http_opentracing_module.so; load_module /opt/app-root/src/nginx-modules/ngx_http_jaeger_module.so; {% endif %} @@ -16,7 +16,7 @@ worker_processes {{ worker_processes | default: 'auto' }}; pcre_jit on; pid {{ pid | default: 'nginx.pid' }}; -{% unless opentracing_enabled %} +{% unless opentracing_tracer != empty %} timer_resolution {{ timer_resolution | default: '100ms' }}; {% endunless %} @@ -71,16 +71,16 @@ http { {% include file %} {% endfor %} - - {% if opentracing_enabled == 'true' %} - {% include "conf.d/opentracing" | append opentracing_tracer | append ".conf.liquid" %} + {% if opentracing_tracer != empty %} + {%- capture tracer_conf %}conf.d/opentracing/{{ opentracing_tracer }}.conf.liquid{%- endcapture -%} + {% include tracer_conf %} {% endif %} server { listen {{ port.management | default: 8090 }}; server_name {{ server_name.management | default: 'management _' }}; - {% if opentracing_enabled == 'true' %} + {% if opentracing_tracer != empty %} opentracing_operation_name "apicast_management"; opentracing_trace_locations off; {% endif %} @@ -92,7 +92,7 @@ http { listen {{ port.backend | default: 8081 }}; server_name backend; - {% if opentracing_enabled == 'true' %} + {% if opentracing_tracer != empty %} opentracing_operation_name "apicast_mockbackend"; opentracing_trace_locations off; {% endif %} @@ -109,7 +109,7 @@ http { listen {{ port.echo | default: 8081 }} default_server; server_name echo _; - {% if opentracing_enabled == 'true' %} + {% if opentracing_tracer != empty %} opentracing_operation_name "apicast_echo"; opentracing_trace_locations off; {% endif %} @@ -125,7 +125,7 @@ http { server_name _; - {% if opentracing_enabled == 'true' %} + {% if opentracing_tracer != empty %} opentracing_operation_name "apicast"; opentracing_trace_locations on; {% endif %} diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index a06458588..477d09fcc 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -108,7 +108,6 @@ _M.default_config = { proxy_ssl_certificate_key = env_value_ref('APICAST_PROXY_HTTPS_CERTIFICATE_KEY'), proxy_ssl_session_reuse = env_value_ref('APICAST_PROXY_HTTPS_SESSION_REUSE'), proxy_ssl_password_file = env_value_ref('APICAST_PROXY_HTTPS_PASSWORD_FILE'), - opentracing_enabled = env_value_ref('OPENTRACING_ENABLED'), opentracing_tracer = env_value_ref('OPENTRACING_TRACER'), jaeger_addr = env_value_ref('JAEGER_ADDR'), jaeger_port = env_value_ref('JAEGER_PORT'), From 49c1898b59b5917e2b6138f69e0369b2f121be96 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 18:49:06 +0200 Subject: [PATCH 11/22] Forwared uber-trace-id header --- gateway/conf.d/apicast.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/gateway/conf.d/apicast.conf b/gateway/conf.d/apicast.conf index 7529abb96..296b4b841 100644 --- a/gateway/conf.d/apicast.conf +++ b/gateway/conf.d/apicast.conf @@ -35,6 +35,7 @@ location = /___http_call { proxy_set_header User-Agent $user_agent; proxy_set_header X-3scale-OAuth2-Grant-Type $grant_type; proxy_set_header 3scale-options $options; + proxy_set_header uber-trace-id $http_uber_trace_id; rewrite_by_lua_block { require('resty.http_ng.backend.ngx'):resolver() From fa38640544ebd416b5b99fbec254d12a9f0c61df Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Wed, 18 Apr 2018 23:40:02 +0200 Subject: [PATCH 12/22] Use opentracing load tracer for jaeger --- gateway/conf.d/opentracing/jaeger.conf.liquid | 6 ++--- .../conf.d/opentracing/jaeger.example.json | 25 +++++++++++++++++++ gateway/conf/nginx.conf.liquid | 5 ++-- gateway/src/apicast/cli/environment.lua | 6 +---- 4 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 gateway/conf.d/opentracing/jaeger.example.json diff --git a/gateway/conf.d/opentracing/jaeger.conf.liquid b/gateway/conf.d/opentracing/jaeger.conf.liquid index caad679fc..d2ec8e2ea 100644 --- a/gateway/conf.d/opentracing/jaeger.conf.liquid +++ b/gateway/conf.d/opentracing/jaeger.conf.liquid @@ -1,5 +1,3 @@ -jaeger_reporter_local_agent_host_port {{ jaeger_addr | default: '127.0.0.1' }}:{{ jaeger_port | default: 6831 }}; -jaeger_service_name {{ jaeger_service_name | default: 'apicast' }}; -jaeger_sampler_type {{ jaeger_sampler_type | default: 'const' }}; -jaeger_sampler_param {{ jaeger_samples_param | default: 1 }}; + opentracing on; +opentracing_load_tracer libjaegertracing.so conf.d/opentracing/jaeger.example.json; diff --git a/gateway/conf.d/opentracing/jaeger.example.json b/gateway/conf.d/opentracing/jaeger.example.json new file mode 100644 index 000000000..92b7b1b1f --- /dev/null +++ b/gateway/conf.d/opentracing/jaeger.example.json @@ -0,0 +1,25 @@ + { + "service_name": "apicast", + "disabled": false, + "sampler": { + "type": "const", + "param": 1 + }, + "reporter": { + "queueSize": 100, + "bufferFlushInterval": 10, + "logSpans": false, + "localAgentHostPort": "127.0.0.1:6831" + }, + "headers": { + "jaegerDebugHeader": "debug-id", + "jaegerBaggageHeader": "baggage", + "TraceContextHeaderName": "trace-id", + "traceBaggageHeaderPrefix": "testctx-" + }, + "baggage_restrictions": { + "denyBaggageOnInitializationFailure": false, + "hostPort": "127.0.0.1:5778", + "refreshInterval": 60 + } + } diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 165fdbf94..6eee015b9 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -6,8 +6,9 @@ env BACKEND_ENDPOINT_OVERRIDE; env OPENSSL_VERIFY; {% if opentracing_tracer != empty %} -load_module /opt/app-root/src/nginx-modules/ngx_http_opentracing_module.so; -load_module /opt/app-root/src/nginx-modules/ngx_http_jaeger_module.so; + {% for file in "nginx-modules/ngx_http_opentracing_module.so" | filesystem %} +load_module {{file}}; + {% endfor %} {% endif %} daemon {{ daemon | default: 'off' }}; diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index 477d09fcc..0629775b9 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -109,11 +109,7 @@ _M.default_config = { proxy_ssl_session_reuse = env_value_ref('APICAST_PROXY_HTTPS_SESSION_REUSE'), proxy_ssl_password_file = env_value_ref('APICAST_PROXY_HTTPS_PASSWORD_FILE'), opentracing_tracer = env_value_ref('OPENTRACING_TRACER'), - jaeger_addr = env_value_ref('JAEGER_ADDR'), - jaeger_port = env_value_ref('JAEGER_PORT'), - jaeger_service_name = env_value_ref('JAEGER_SERVICE_NAME'), - jaeger_sampler_type = env_value_ref('JAEGER_SAMPLER_TYPE'), - jaeger_sampler_param = env_value_ref('JAEGER_SAMPLER_PARAM'), + opentracing_config = env_value_ref('OPENTRACING_CONFIG'), policy_chain = require('apicast.policy_chain').default(), nameservers = parse_nameservers(), worker_processes = cpus() or 'auto', From 1950a752e1ff92d8294de3dafe330bc249b2ed7d Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Thu, 19 Apr 2018 19:07:11 +0200 Subject: [PATCH 13/22] Allow the user to change the config file path --- gateway/conf.d/opentracing/jaeger.conf.liquid | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gateway/conf.d/opentracing/jaeger.conf.liquid b/gateway/conf.d/opentracing/jaeger.conf.liquid index d2ec8e2ea..013559497 100644 --- a/gateway/conf.d/opentracing/jaeger.conf.liquid +++ b/gateway/conf.d/opentracing/jaeger.conf.liquid @@ -1,3 +1,2 @@ - opentracing on; -opentracing_load_tracer libjaegertracing.so conf.d/opentracing/jaeger.example.json; +opentracing_load_tracer libjaegertracing.so {{ opentracing_config | default:"conf.d/opentracing/jaeger.example.json"}}; From 597f3709268c8f5270eef609642a05e3131e7429 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Fri, 20 Apr 2018 12:48:42 +0200 Subject: [PATCH 14/22] Use uber-trace-id by default --- gateway/conf.d/opentracing/jaeger.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/conf.d/opentracing/jaeger.example.json b/gateway/conf.d/opentracing/jaeger.example.json index 92b7b1b1f..3b4efc724 100644 --- a/gateway/conf.d/opentracing/jaeger.example.json +++ b/gateway/conf.d/opentracing/jaeger.example.json @@ -14,7 +14,7 @@ "headers": { "jaegerDebugHeader": "debug-id", "jaegerBaggageHeader": "baggage", - "TraceContextHeaderName": "trace-id", + "TraceContextHeaderName": "uber-trace-id", "traceBaggageHeaderPrefix": "testctx-" }, "baggage_restrictions": { From c5febb576f63d33468782abf2c52da793bb11045 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Fri, 20 Apr 2018 16:40:59 +0200 Subject: [PATCH 15/22] Allow to set the opentracing forward header via ENV --- doc/parameters.md | 28 +++++++++++++++++++ gateway/conf.d/apicast.conf | 12 +++++++- gateway/conf.d/opentracing/jaeger.conf.liquid | 7 ++++- gateway/src/apicast/cli/environment.lua | 5 +++- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/doc/parameters.md b/doc/parameters.md index c17ebb5db..1f24ee429 100644 --- a/doc/parameters.md +++ b/doc/parameters.md @@ -243,3 +243,31 @@ URI that includes your password and portal endpoint in the following format: ` Date: Thu, 26 Apr 2018 16:31:59 +0200 Subject: [PATCH 16/22] CHANGELOG: OpenTracing support --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ffb0f538..9b2ba580e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `export()` now works correctly in policies of the local chain [PR #673](https://github.com/3scale/apicast/pull/673) - caching policy now works correctly when placed after the apicast policy in the chain [PR #674](https://github.com/3scale/apicast/pull/674) +- OpenTracing support [PR #669](https://github.com/3scale/apicast/pull/669) ### Changed From f949669a32553149607e408593545e08af0c7065 Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Thu, 26 Apr 2018 18:05:19 +0200 Subject: [PATCH 17/22] Fix CHANGELOG merge --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b2ba580e..b7a39a864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added + +- OpenTracing support [PR #669](https://github.com/3scale/apicast/pull/669) + ### Fixed - Fixed set of valid values for the exit param of the Echo policy [PR #684](https://github.com/3scale/apicast/pull/684/) From 4225d5273fe3a5b1f282d7e4894490586789d98e Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 27 Apr 2018 09:50:25 +0200 Subject: [PATCH 18/22] [nginx] search for modules in standard nginx directory --- gateway/conf/nginx.conf.liquid | 2 +- gateway/src/apicast/cli/template.lua | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 6eee015b9..18a6ecc2a 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -6,7 +6,7 @@ env BACKEND_ENDPOINT_OVERRIDE; env OPENSSL_VERIFY; {% if opentracing_tracer != empty %} - {% for file in "nginx-modules/ngx_http_opentracing_module.so" | filesystem %} + {% for file in "modules/ngx_http_opentracing_module.so" | filesystem %} load_module {{file}}; {% endfor %} {% endif %} diff --git a/gateway/src/apicast/cli/template.lua b/gateway/src/apicast/cli/template.lua index 20108d60b..76f5f3747 100644 --- a/gateway/src/apicast/cli/template.lua +++ b/gateway/src/apicast/cli/template.lua @@ -70,6 +70,11 @@ local function build_interpreter(str) return interpreter end +local function nginx_prefix() + local match = ngx.re.match(ngx.config.nginx_configure(), [[--prefix=(.+?)\s]]) + if match then return match[1] end +end + function _M:interpret(str) local interpreter = build_interpreter(str) @@ -82,7 +87,7 @@ function _M:interpret(str) local files = {} local included = {} - for _, root in ipairs({ self.root, pl.path.currentdir() }) do + for _, root in ipairs({ self.root, pl.path.currentdir(), ngx.config.prefix(), nginx_prefix() }) do for filename in fs(root) do local file = pl.path.relpath(filename, root) From e8377c1ff20582f6b6f00dfc3748c6aabe0d95a0 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 27 Apr 2018 10:22:16 +0200 Subject: [PATCH 19/22] [opentracing] ability to load jaegertracing on macOS --- gateway/conf.d/opentracing/jaeger.conf.liquid | 9 +++++++-- gateway/src/apicast/cli/template.lua | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gateway/conf.d/opentracing/jaeger.conf.liquid b/gateway/conf.d/opentracing/jaeger.conf.liquid index 4c8c7d4ab..351e0d404 100644 --- a/gateway/conf.d/opentracing/jaeger.conf.liquid +++ b/gateway/conf.d/opentracing/jaeger.conf.liquid @@ -1,7 +1,12 @@ opentracing on; {% if opentracing_config == nil or opentracing_config == empty %} - {% assign opentracing_config = "conf.d/opentracing/jaeger.example.json" %} + {% assign opentracing_config = "conf.d/opentracing/jaeger.example.json" | filesystem | first%} +{% endif %} + +{% if platform == "OSX" %} + opentracing_load_tracer libjaegertracing.dylib {{ opentracing_config }}; +{% else %} + opentracing_load_tracer libjaegertracing.so {{ opentracing_config }}; {% endif %} -opentracing_load_tracer libjaegertracing.so {{ opentracing_config }}; diff --git a/gateway/src/apicast/cli/template.lua b/gateway/src/apicast/cli/template.lua index 76f5f3747..a010f3d42 100644 --- a/gateway/src/apicast/cli/template.lua +++ b/gateway/src/apicast/cli/template.lua @@ -33,6 +33,7 @@ function _M:new(config, dir, strict) local context = setmetatable({ env = env, + platform = jit.os, }, { __index = config }) instance.root = pl.path.abspath(dir or pl.path.currentdir()) From fdbe64292eb11a32f24a332463c55c291bf34918 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 27 Apr 2018 15:39:35 +0200 Subject: [PATCH 20/22] [compose] provide jaeger all-in-one to try opentracing --- docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 99033fae6..9dad8f593 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,3 +66,15 @@ services: KEYCLOAK_LOGLEVEL: INFO ports: - "8080" + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_ZIPKIN_HTTP_PORT: 9411 + ports: + - 5775:5775/udp + - 6831:6831/udp + - 6832:6832/udp + - 5778:5778 + - 16686:16686 + - 14268:14268 + - 9411:9411 From 6a0dbc033f0575c85f45765ba5a3d49960ecd88d Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 27 Apr 2018 16:24:29 +0200 Subject: [PATCH 21/22] [t] test opentracing --- t/opentracing.t | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 t/opentracing.t diff --git a/t/opentracing.t b/t/opentracing.t new file mode 100644 index 000000000..85c1b0e79 --- /dev/null +++ b/t/opentracing.t @@ -0,0 +1,41 @@ +use lib 't'; +use Test::APIcast::Blackbox 'no_plan'; + +$ENV{OPENTRACING_TRACER} ||= 'jaeger'; + +repeat_each(1); +run_tests(); + + +__DATA__ +=== TEST 1: OpenTracing +Request passing through APIcast should publish OpenTracing info. +--- configuration + { + "services": [ + { + "proxy": { + "policy_chain": [ + { "name": "apicast.policy.upstream", + "configuration": + { + "rules": [ { "regex": "/", "url": "http://echo" } ] + } + } + ] + } + } + ] + } +--- request +GET /a_path? +--- response_body eval +qr/uber-trace-id: / +--- error_code: 200 +--- no_error_log +[error] +--- udp_listen: 6831 +--- udp_reply +--- udp_query eval +qr/jaeger.version/ +--- wait: 10 From 6e51fcd862f746003e80b8e8203e0f185deb147a Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 30 Apr 2018 18:05:14 +0200 Subject: [PATCH 22/22] [circle] bump s2i-openresty to 1.13.6.1-rover12 includes https://github.com/3scale/s2i-openresty/pull/57 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8a36e0ab..7e0967970 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: environment: S2I_VERSION: "1.1.9a-40ad911d" DOCKER_COMPOSE_VERSION: "1.16.1" - OPENRESTY_VERSION: "1.13.6.1-rover6" + OPENRESTY_VERSION: "1.13.6.1-rover12" steps: - run: apk update && apk add wget make bash curl py-pip git openssh-client - run: | @@ -40,7 +40,7 @@ jobs: make lint-schema build: docker: - - image: quay.io/3scale/s2i-openresty-centos7:1.13.6.1-rover6 + - image: quay.io/3scale/s2i-openresty-centos7:1.13.6.1-rover12 environment: TEST_NGINX_BINARY: openresty LUA_BIN_PATH: /opt/app-root/bin