OpenTracing support

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
     environment:
       S2I_VERSION: "1.1.9a-40ad911d"
       DOCKER_COMPOSE_VERSION: "1.16.1"
-      OPENRESTY_VERSION: "1.13.6.1-rover6"
+      OPENRESTY_VERSION: "1.13.6.1-rover12"
     steps:
       - run: apk update && apk add wget make bash curl py-pip git openssh-client
       - run: |
@@ -40,7 +40,7 @@ jobs:
           make lint-schema
   build:
     docker:
-      - image: quay.io/3scale/s2i-openresty-centos7:1.13.6.1-rover6
+      - image: quay.io/3scale/s2i-openresty-centos7:1.13.6.1-rover12
         environment:
           TEST_NGINX_BINARY: openresty
           LUA_BIN_PATH: /opt/app-root/bin

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+
+- OpenTracing support [PR #669](https://github.com/3scale/apicast/pull/669)
+
 ### Fixed
 
 - Fixed set of valid values for the exit param of the Echo policy [PR #684](https://github.com/3scale/apicast/pull/684/)
@@ -21,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 - `export()` now works correctly in policies of the local chain [PR #673](https://github.com/3scale/apicast/pull/673)
 - caching policy now works correctly when placed after the apicast policy in the chain [PR #674](https://github.com/3scale/apicast/pull/674)
+- OpenTracing support [PR #669](https://github.com/3scale/apicast/pull/669)
 
 ### Changed
 

doc/parameters.md

@@ -243,3 +243,31 @@ URI that includes your password and portal endpoint in the following format: `<s
 When `THREESCALE_PORTAL_ENDPOINT` environment variable is provided, the gateway will download the configuration from 3scale on initializing. The configuration includes all the settings provided on the Integration page of the API(s).
 
 It is **required** to provide either `THREESCALE_PORTAL_ENDPOINT` or `THREESCALE_CONFIG_FILE` (takes precedence) for the gateway to run successfully.
+
+
+### `OPENTRACING_TRACER`
+
+**Example** `jaeger`
+
+This environment variable controls which tracing library will be loaded, right now, there's only one opentracing tracer available, `jaeger`.
+
+If empty, opentracing support will be disabled.
+
+
+### `OPENTRACING_CONFIG`
+
+This environment variable is used to determine the config file for the opentracing tracer, if `OPENTRACING_TRACER` is not set, this variable will be ignored.
+
+Each tracer has a default configuration file:
+    * `jaeger`: `conf.d/opentracing/jaeger.example.json`
+
+You can choose to mount a different configuration than the provided by default by setting the file path using this variable.
+
+**Example**: `/tmp/jaeger/jaeger.json`
+
+### `OPENTRACING_HEADER_FORWARD`
+
+**Defaul** `uber-trace-id`
+
+This environment variable controls the HTTP header used for forwarding opentracing information, this HTTP header will be forwarded to upstream servers.
+

docker-compose.yml

@@ -66,3 +66,15 @@ services:
       KEYCLOAK_LOGLEVEL: INFO
     ports:
       - "8080"
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    environment:
+      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+    ports:
+      - 5775:5775/udp
+      - 6831:6831/udp
+      - 6832:6832/udp
+      - 5778:5778
+      - 16686:16686
+      - 14268:14268
+      - 9411:9411

gateway/conf.d/apicast.conf

@@ -36,6 +36,17 @@ location = /___http_call {
   proxy_set_header X-3scale-OAuth2-Grant-Type $grant_type;
   proxy_set_header 3scale-options $options;
 
+  # Hack for having a valid openresty config and valid liquid templating
+  #{% if opentracing_forward_header == nil or opentracing_forward_header == empty %}
+  #   {% assign opentracing_forward_header = "uber-trace-id" %}
+  #{% endif %}
+  #{% if opentracing_tracer != nil or opentracing_forward_header != nil %}
+  #   {% capture proxy_set_header_opentracing %}
+  #{#} proxy_set_header {{opentracing_forward_header}} $http_{{ opentracing_forward_header | replace: "-","_" }};
+  #   {% endcapture %}
+  #   {{ proxy_set_header_opentracing | replace: "#{#}", "" }}
+  #{% endif %}
+
   rewrite_by_lua_block {
     require('resty.http_ng.backend.ngx'):resolver()
   }

gateway/conf.d/opentracing/jaeger.conf.liquid

@@ -0,0 +1,12 @@
+opentracing on;
+
+{% if opentracing_config == nil or opentracing_config == empty %}
+    {% assign opentracing_config = "conf.d/opentracing/jaeger.example.json" | filesystem | first%}
+{% endif %}
+
+{% if platform == "OSX" %}
+    opentracing_load_tracer libjaegertracing.dylib {{ opentracing_config }};
+{% else %}
+    opentracing_load_tracer libjaegertracing.so {{ opentracing_config }};
+{% endif %}
+

gateway/conf.d/opentracing/jaeger.example.json

@@ -0,0 +1,25 @@
+ {
+    "service_name": "apicast",
+    "disabled": false,
+    "sampler": {
+      "type": "const",
+      "param": 1
+    },
+    "reporter": {
+      "queueSize": 100,
+      "bufferFlushInterval": 10,
+      "logSpans": false,
+      "localAgentHostPort": "127.0.0.1:6831"
+    },
+    "headers": {
+      "jaegerDebugHeader": "debug-id",
+      "jaegerBaggageHeader": "baggage",
+      "TraceContextHeaderName": "uber-trace-id",
+      "traceBaggageHeaderPrefix": "testctx-"
+    },
+    "baggage_restrictions": {
+        "denyBaggageOnInitializationFailure": false,
+        "hostPort": "127.0.0.1:5778",
+        "refreshInterval": 60
+    }
+ }

gateway/conf/nginx.conf.liquid

@@ -5,12 +5,21 @@ env RESOLVER;
 env BACKEND_ENDPOINT_OVERRIDE;
 env OPENSSL_VERIFY;
 
+{% if opentracing_tracer != empty %}
+  {% for file in "modules/ngx_http_opentracing_module.so" | filesystem %}
+load_module {{file}};
+  {% endfor %}
+{% endif %}
+
 daemon {{ daemon | default: 'off' }};
 master_process {{ master_process | default: 'on' }};
 worker_processes {{ worker_processes | default: 'auto' }};
 pcre_jit on;
 pid {{ pid | default: 'nginx.pid' }};
+
+{% unless opentracing_tracer != empty %}
 timer_resolution {{ timer_resolution | default: '100ms' }};
+{% endunless %}
 
 {% for file in "main.d/*.conf" | filesystem %}
   {% include file %}
@@ -28,6 +37,8 @@ http {
   tcp_nopush         on;
   tcp_nodelay        on;
 
+
+
   # Enabling the Lua code cache is strongly encouraged for production use
   # Disabling it should only be done for testing and development purposes
   lua_code_cache {{  lua_code_cache | default: 'on' }};
@@ -61,43 +72,65 @@ http {
     {% include file %}
   {% endfor %}
 
+  {% if opentracing_tracer != empty %}
+    {%- capture tracer_conf %}conf.d/opentracing/{{ opentracing_tracer }}.conf.liquid{%- endcapture -%}
+    {% include tracer_conf %}
+  {% endif %}
+
   server {
     listen {{ port.management | default: 8090 }};
-
     server_name {{ server_name.management | default: 'management _' }};
 
+    {% if opentracing_tracer != empty %}
+    opentracing_operation_name "apicast_management";
+    opentracing_trace_locations off;
+    {% endif %}
+
     {% include "conf.d/management.conf" %}
   }
 
   server {
     listen {{ port.backend | default: 8081 }};
-
     server_name backend;
 
+    {% if opentracing_tracer != empty %}
+    opentracing_operation_name "apicast_mockbackend";
+    opentracing_trace_locations off;
+    {% endif %}
+
     {% include "conf.d/backend.conf" %}
   }
 
   upstream echo {
     server 127.0.0.1:{{ port.echo | default: 8081 }};
-
     keepalive 1024;
   }
 
   server {
     listen {{ port.echo | default: 8081 }} default_server;
-
     server_name echo _;
 
+    {% if opentracing_tracer != empty %}
+    opentracing_operation_name "apicast_echo";
+    opentracing_trace_locations off;
+    {% endif %}
+
     {% include "conf.d/echo.conf" %}
   }
 
   server {
+
     access_log /dev/stdout time;
 
     listen {{ port.apicast | default: 8080 }};
 
     server_name _;
 
+    {% if opentracing_tracer != empty %}
+    opentracing_operation_name "apicast";
+    opentracing_trace_locations on;
+    {% endif %}
+
     {% include "http.d/ssl.conf" %}
 
     {% for file in "apicast.d/*.conf" | filesystem %}

gateway/src/apicast/cli/environment.lua

@@ -95,6 +95,9 @@ _M.default_environment = 'production'
 -- @tfield ?string proxy_ssl_certificate_key path to SSL certificate key
 -- @tfield ?string proxy_ssl_session_reuse whether SSL sessions can be reused
 -- @tfield ?string proxy_ssl_password_file path to a file with passphrases for the certificate keys
+-- @tfield ?string opentracing_tracer loads an opentracing tracer library, for example: jaeger
+-- @tfield ?string opentracing_config opentracing config file to load
+-- @tfield ?string opentracing_forward_header opentracing http header to forward upstream
 -- @tfield ?policy_chain policy_chain @{policy_chain} instance
 -- @tfield ?{string,...} nameservers list of nameservers
 -- @tfield ?string package.path path to load Lua files
@@ -103,12 +106,13 @@ _M.default_environment = 'production'
 
 _M.default_config = {
     ca_bundle = env_value_ref('SSL_CERT_FILE'),
-
     proxy_ssl_certificate = env_value_ref('APICAST_PROXY_HTTPS_CERTIFICATE'),
     proxy_ssl_certificate_key = env_value_ref('APICAST_PROXY_HTTPS_CERTIFICATE_KEY'),
     proxy_ssl_session_reuse = env_value_ref('APICAST_PROXY_HTTPS_SESSION_REUSE'),
     proxy_ssl_password_file = env_value_ref('APICAST_PROXY_HTTPS_PASSWORD_FILE'),
-
+    opentracing_tracer = env_value_ref('OPENTRACING_TRACER'),
+    opentracing_config = env_value_ref('OPENTRACING_CONFIG'),
+    opentracing_forward_header = env_value_ref('OPENTRACING_FORWARD_HEADER'),
     policy_chain = require('apicast.policy_chain').default(),
     nameservers = parse_nameservers(),
     worker_processes = cpus() or 'auto',

gateway/src/apicast/cli/template.lua

@@ -33,6 +33,7 @@ function _M:new(config, dir, strict)
 
     local context = setmetatable({
         env = env,
+        platform = jit.os,
     }, { __index = config })
 
     instance.root = pl.path.abspath(dir or pl.path.currentdir())
@@ -70,6 +71,11 @@ local function build_interpreter(str)
     return interpreter
 end
 
+local function nginx_prefix()
+    local match = ngx.re.match(ngx.config.nginx_configure(), [[--prefix=(.+?)\s]])
+    if match then return match[1] end
+end
+
 function _M:interpret(str)
     local interpreter = build_interpreter(str)
 
@@ -82,7 +88,7 @@ function _M:interpret(str)
         local files = {}
         local included = {}
 
-        for _, root in ipairs({ self.root, pl.path.currentdir() }) do
+        for _, root in ipairs({ self.root, pl.path.currentdir(), ngx.config.prefix(), nginx_prefix() }) do
             for filename in fs(root) do
                 local file = pl.path.relpath(filename, root)
 

t/opentracing.t

@@ -0,0 +1,41 @@
+use lib 't';
+use Test::APIcast::Blackbox 'no_plan';
+
+$ENV{OPENTRACING_TRACER} ||= 'jaeger';
+
+repeat_each(1);
+run_tests();
+
+
+__DATA__
+=== TEST 1: OpenTracing
+Request passing through APIcast should publish OpenTracing info.
+--- configuration
+    {
+        "services": [
+        {
+            "proxy": {
+            "policy_chain": [
+            { "name": "apicast.policy.upstream",
+                "configuration":
+                {
+                    "rules": [ { "regex": "/", "url": "http://echo" } ]
+                }
+            }
+            ]
+        }
+        }
+        ]
+    }
+--- request
+GET /a_path?
+--- response_body eval
+qr/uber-trace-id: /
+--- error_code: 200
+--- no_error_log
+[error]
+--- udp_listen: 6831
+--- udp_reply
+--- udp_query eval
+qr/jaeger.version/
+--- wait: 10