Skip to content

Commit

Permalink
feat(tracing): AWS API Gateway Inferred Span Support (#4837)
Browse files Browse the repository at this point in the history
* Add support for inferred spans to be created for proxies. Initially supports AWS API Gateway and creates a span when the required headers are attached on the received request.
---------

Co-authored-by: wantsui <[email protected]>
  • Loading branch information
wconti27 and wantsui authored Nov 12, 2024
1 parent b81d9d8 commit 29ff735
Show file tree
Hide file tree
Showing 4 changed files with 423 additions and 10 deletions.
4 changes: 4 additions & 0 deletions packages/dd-trace/src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,7 @@ class Config {
this._setValue(defaults, 'isTestDynamicInstrumentationEnabled', false)
this._setValue(defaults, 'logInjection', false)
this._setValue(defaults, 'lookup', undefined)
this._setValue(defaults, 'inferredProxyServicesEnabled', false)
this._setValue(defaults, 'memcachedCommandEnabled', false)
this._setValue(defaults, 'openAiLogsEnabled', false)
this._setValue(defaults, 'openaiSpanCharLimit', 128)
Expand Down Expand Up @@ -675,6 +676,7 @@ class Config {
DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH,
DD_TRACING_ENABLED,
DD_VERSION,
DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED,
OTEL_METRICS_EXPORTER,
OTEL_PROPAGATORS,
OTEL_RESOURCE_ATTRIBUTES,
Expand Down Expand Up @@ -862,6 +864,7 @@ class Config {
: !!OTEL_PROPAGATORS)
this._setBoolean(env, 'tracing', DD_TRACING_ENABLED)
this._setString(env, 'version', DD_VERSION || tags.version)
this._setBoolean(env, 'inferredProxyServicesEnabled', DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED)
}

_applyOptions (options) {
Expand Down Expand Up @@ -980,6 +983,7 @@ class Config {
this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled)
this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled)
this._setString(opts, 'version', options.version || tags.version)
this._setBoolean(opts, 'inferredProxyServicesEnabled', options.inferredProxyServicesEnabled)

// For LLMObs, we want the environment variable to take precedence over the options.
// This is reliant on environment config being set before options.
Expand Down
121 changes: 121 additions & 0 deletions packages/dd-trace/src/plugins/util/inferred_proxy.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
const log = require('../../log')
const tags = require('../../../../../ext/tags')

const RESOURCE_NAME = tags.RESOURCE_NAME
const HTTP_ROUTE = tags.HTTP_ROUTE
const SPAN_KIND = tags.SPAN_KIND
const SPAN_TYPE = tags.SPAN_TYPE
const HTTP_URL = tags.HTTP_URL
const HTTP_METHOD = tags.HTTP_METHOD

const PROXY_HEADER_SYSTEM = 'x-dd-proxy'
const PROXY_HEADER_START_TIME_MS = 'x-dd-proxy-request-time-ms'
const PROXY_HEADER_PATH = 'x-dd-proxy-path'
const PROXY_HEADER_HTTPMETHOD = 'x-dd-proxy-httpmethod'
const PROXY_HEADER_DOMAIN = 'x-dd-proxy-domain-name'
const PROXY_HEADER_STAGE = 'x-dd-proxy-stage'

const supportedProxies = {
'aws-apigateway': {
spanName: 'aws.apigateway',
component: 'aws-apigateway'
}
}

function createInferredProxySpan (headers, childOf, tracer, context) {
if (!headers) {
return null
}

if (!tracer._config?.inferredProxyServicesEnabled) {
return null
}

const proxyContext = extractInferredProxyContext(headers)

if (!proxyContext) {
return null
}

const proxySpanInfo = supportedProxies[proxyContext.proxySystemName]

log.debug(`Successfully extracted inferred span info ${proxyContext} for proxy: ${proxyContext.proxySystemName}`)

const span = tracer.startSpan(
proxySpanInfo.spanName,
{
childOf,
type: 'web',
startTime: proxyContext.requestTime,
tags: {
service: proxyContext.domainName || tracer._config.service,
component: proxySpanInfo.component,
[SPAN_KIND]: 'internal',
[SPAN_TYPE]: 'web',
[HTTP_METHOD]: proxyContext.method,
[HTTP_URL]: proxyContext.domainName + proxyContext.path,
[HTTP_ROUTE]: proxyContext.path,
stage: proxyContext.stage
}
}
)

tracer.scope().activate(span)
context.inferredProxySpan = span
childOf = span

log.debug('Successfully created inferred proxy span.')

setInferredProxySpanTags(span, proxyContext)

return childOf
}

function setInferredProxySpanTags (span, proxyContext) {
span.setTag(RESOURCE_NAME, `${proxyContext.method} ${proxyContext.path}`)
span.setTag('_dd.inferred_span', '1')
return span
}

function extractInferredProxyContext (headers) {
if (!(PROXY_HEADER_START_TIME_MS in headers)) {
return null
}

if (!(PROXY_HEADER_SYSTEM in headers && headers[PROXY_HEADER_SYSTEM] in supportedProxies)) {
log.debug(`Received headers to create inferred proxy span but headers include an unsupported proxy type ${headers}`)
return null
}

return {
requestTime: headers[PROXY_HEADER_START_TIME_MS]
? parseInt(headers[PROXY_HEADER_START_TIME_MS], 10)
: null,
method: headers[PROXY_HEADER_HTTPMETHOD],
path: headers[PROXY_HEADER_PATH],
stage: headers[PROXY_HEADER_STAGE],
domainName: headers[PROXY_HEADER_DOMAIN],
proxySystemName: headers[PROXY_HEADER_SYSTEM]
}
}

function finishInferredProxySpan (context) {
const { req } = context

if (!context.inferredProxySpan) return

if (context.inferredProxySpanFinished && !req.stream) return

// context.config.hooks.request(context.inferredProxySpan, req, res) # TODO: Do we need this??

// Only close the inferred span if one was created
if (context.inferredProxySpan) {
context.inferredProxySpan.finish()
context.inferredProxySpanFinished = true
}
}

module.exports = {
createInferredProxySpan,
finishInferredProxySpan
}
48 changes: 38 additions & 10 deletions packages/dd-trace/src/plugins/util/web.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const kinds = require('../../../../../ext/kinds')
const urlFilter = require('./urlfilter')
const { extractIp } = require('./ip_extractor')
const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../constants')
const { createInferredProxySpan, finishInferredProxySpan } = require('./inferred_proxy')

const WEB = types.WEB
const SERVER = kinds.SERVER
Expand Down Expand Up @@ -97,7 +98,7 @@ const web = {
context.span.context()._name = name
span = context.span
} else {
span = web.startChildSpan(tracer, name, req.headers)
span = web.startChildSpan(tracer, name, req)
}

context.tracer = tracer
Expand Down Expand Up @@ -253,8 +254,19 @@ const web = {
},

// Extract the parent span from the headers and start a new span as its child
startChildSpan (tracer, name, headers) {
const childOf = tracer.extract(FORMAT_HTTP_HEADERS, headers)
startChildSpan (tracer, name, req) {
const headers = req.headers
const context = contexts.get(req)
let childOf = tracer.extract(FORMAT_HTTP_HEADERS, headers)

// we may have headers signaling a router proxy span should be created (such as for AWS API Gateway)
if (tracer._config?.inferredProxyServicesEnabled) {
const proxySpan = createInferredProxySpan(headers, childOf, tracer, context)
if (proxySpan) {
childOf = proxySpan
}
}

const span = tracer.startSpan(name, { childOf })

return span
Expand All @@ -263,13 +275,21 @@ const web = {
// Validate a request's status code and then add error tags if necessary
addStatusError (req, statusCode) {
const context = contexts.get(req)
const span = context.span
const error = context.error
const hasExistingError = span.context()._tags.error || span.context()._tags[ERROR_MESSAGE]
const { span, inferredProxySpan, error } = context

const spanHasExistingError = span.context()._tags.error || span.context()._tags[ERROR_MESSAGE]
const inferredSpanContext = inferredProxySpan?.context()
const inferredSpanHasExistingError = inferredSpanContext?._tags.error || inferredSpanContext?._tags[ERROR_MESSAGE]

if (!hasExistingError && !context.config.validateStatus(statusCode)) {
const isValidStatusCode = context.config.validateStatus(statusCode)

if (!spanHasExistingError && !isValidStatusCode) {
span.setTag(ERROR, error || true)
}

if (inferredProxySpan && !inferredSpanHasExistingError && !isValidStatusCode) {
inferredProxySpan.setTag(ERROR, error || true)
}
},

// Add an error to the request
Expand Down Expand Up @@ -316,6 +336,8 @@ const web = {
web.finishMiddleware(context)

web.finishSpan(context)

finishInferredProxySpan(context)
},

obfuscateQs (config, url) {
Expand Down Expand Up @@ -426,7 +448,7 @@ function reactivate (req, fn) {
}

function addRequestTags (context, spanType) {
const { req, span, config } = context
const { req, span, inferredProxySpan, config } = context
const url = extractURL(req)

span.addTags({
Expand All @@ -443,14 +465,15 @@ function addRequestTags (context, spanType) {

if (clientIp) {
span.setTag(HTTP_CLIENT_IP, clientIp)
inferredProxySpan?.setTag(HTTP_CLIENT_IP, clientIp)
}
}

addHeaders(context)
}

function addResponseTags (context) {
const { req, res, paths, span } = context
const { req, res, paths, span, inferredProxySpan } = context

if (paths.length > 0) {
span.setTag(HTTP_ROUTE, paths.join(''))
Expand All @@ -459,6 +482,9 @@ function addResponseTags (context) {
span.addTags({
[HTTP_STATUS_CODE]: res.statusCode
})
inferredProxySpan?.addTags({
[HTTP_STATUS_CODE]: res.statusCode
})

web.addStatusError(req, res.statusCode)
}
Expand All @@ -477,18 +503,20 @@ function addResourceTag (context) {
}

function addHeaders (context) {
const { req, res, config, span } = context
const { req, res, config, span, inferredProxySpan } = context

config.headers.forEach(([key, tag]) => {
const reqHeader = req.headers[key]
const resHeader = res.getHeader(key)

if (reqHeader) {
span.setTag(tag || `${HTTP_REQUEST_HEADERS}.${key}`, reqHeader)
inferredProxySpan?.setTag(tag || `${HTTP_REQUEST_HEADERS}.${key}`, reqHeader)
}

if (resHeader) {
span.setTag(tag || `${HTTP_RESPONSE_HEADERS}.${key}`, resHeader)
inferredProxySpan?.setTag(tag || `${HTTP_RESPONSE_HEADERS}.${key}`, resHeader)
}
})
}
Expand Down
Loading

0 comments on commit 29ff735

Please sign in to comment.