From b5b61662a17b3ce303494539b77be908ebea983b Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:28:58 -0500 Subject: [PATCH 1/3] expose DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH --- README.md | 15 ++++++++++++++- src/index.ts | 6 ++++++ src/trace/listener.ts | 17 ++++++++++++++--- src/utils/tag-object.spec.ts | 33 +++++++++++++++++++++++++++++++++ src/utils/tag-object.ts | 13 +++++-------- 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b325a9119..7c896df8b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,19 @@ Follow the [configuration instructions](https://docs.datadoghq.com/serverless/co For additional tracing configuration options, check out the [official documentation for Datadog trace client](https://datadoghq.dev/dd-trace-js/). +Besides the environment variables supported by dd-trace-js, the datadog-lambda-js library added following environment variables. + +| Environment Variables | Default ValueDescription | +| -------------------- | ------------ | +| DD_ENCODE_AUTHORIZER_CONTEXT | When set to `true` for Lambda authorizers, the tracing context will be encoded into the response for propagation. Supported for NodeJS and Python. Defaults to `true`. | +| DD_DECODE_AUTHORIZER_CONTEXT | When set to `true` for Lambdas that are authorized via Lambda authorizers, it will parse and use the encoded tracing context (if found). Supported for NodeJS and Python. Defaults to `true`. | +| DD_COLD_START_TRACING | Set to `false` to disable Cold Start Tracing. Used in NodeJS and Python. Defaults to `true`. | +| DD_MIN_COLD_START_DURATION | Sets the minimum duration (in milliseconds) for a module load event to be traced via Cold Start Tracing. Number. Defaults to `3`. | +| DD_COLD_START_TRACE_SKIP_LIB | optionally skip creating Cold Start Spans for a comma-separated list of libraries. Useful to limit depth or skip known libraries. Default depends on runtime. | +| DD_CAPTURE_LAMBDA_PAYLOAD | [Captures incoming and outgoing AWS Lambda payloads][1] in the Datadog APM spans for Lambda invocations. Defaults to `false`. | +| DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH | The captured AWS Lambda payloads will become tags of the `aws.lambda` span. This sets how deep it fathoms the JSON structure. When the max depth reached, the tag's value will be the stringified value of the deeper nested items. Defaults to `10`.
For example, with input payload as 
{
  "lv1" : {
    "lv2": {
      "lv3": "val"
    }
  }
}
When set to `2`, the resulted tag's key is `function.request.lv1.lv2` and value `{\"lv3\": \"val\"}`.
When set to `0`, the the resulted tag's key is just `function.request` and value is `{\"lv1\":{\"lv2\":{\"lv3\": \"val\"}}}` | + + ## Lambda Profiling Beta Datadog's [Continuous Profiler](https://www.datadoghq.com/product/code-profiling/) is now available in beta for NodeJS in version 6.87.0 and layer version 87 and above. This optional feature is enabled by setting the `DD_PROFILING_ENABLED` environment variable to `true`. During the beta period, profiling is available at no additional cost. @@ -32,7 +45,7 @@ The first 5.x.x version was released with Lambda Layer version `69`. ### 6.x.x -The 6.x.x release introduces support for the node 16 runtime and esm modules. +The 6.x.x release introduces support for the node 16 runtime and esm modules. ### 7.x.x diff --git a/src/index.ts b/src/index.ts index e8aa40ef3..b3be30a39 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ export { TraceHeaders } from "./trace"; export const apiKeyEnvVar = "DD_API_KEY"; export const apiKeyKMSEnvVar = "DD_KMS_API_KEY"; export const captureLambdaPayloadEnvVar = "DD_CAPTURE_LAMBDA_PAYLOAD"; +export const captureLambdaPayloadMaxDepthEnvVar = "DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH"; export const traceManagedServicesEnvVar = "DD_TRACE_MANAGED_SERVICES"; export const siteURLEnvVar = "DD_SITE"; export const logLevelEnvVar = "DD_LOG_LEVEL"; @@ -70,6 +71,7 @@ export const defaultConfig: Config = { apiKeyKMS: "", autoPatchHTTP: true, captureLambdaPayload: false, + captureLambdaPayloadMaxDepth: 10, createInferredSpan: true, debugLogging: false, encodeAuthorizerContext: true, @@ -358,6 +360,10 @@ function getConfig(userConfig?: Partial): Config { config.coldStartTraceSkipLib = getEnvValue(coldStartTraceSkipLibEnvVar, "./opentracing/tracer"); } + if (userConfig === undefined || userConfig.captureLambdaPayloadMaxDepth === undefined) { + config.captureLambdaPayloadMaxDepth = Number(getEnvValue(captureLambdaPayloadMaxDepthEnvVar, "10")); + } + return config; } diff --git a/src/trace/listener.ts b/src/trace/listener.ts index be64b91a3..b91846064 100644 --- a/src/trace/listener.ts +++ b/src/trace/listener.ts @@ -10,7 +10,6 @@ import { patchHttp, unpatchHttp } from "./patch-http"; import { TraceContextService } from "./trace-context-service"; import { extractTriggerTags, extractHTTPStatusCodeTag } from "./trigger"; import { ColdStartTracerConfig, ColdStartTracer } from "./cold-start-tracer"; - import { logDebug, tagObject } from "../utils"; import { didFunctionColdStart, isProactiveInitialization } from "../utils/cold-start"; import { datadogLambdaVersion } from "../constants"; @@ -31,6 +30,12 @@ export interface TraceConfig { /** * Whether to capture the lambda payload and response in Datadog. */ + captureLambdaPayloadMaxDepth: number; + /** + * The captured AWS Lambda payloads will become tags of the `aws.lambda` span. This sets how deep + * it fathoms the JSON structure. When the max depth reached, the tag's value will be the + * stringified value of the deeper nested items. + */ captureLambdaPayload: boolean; /** * Whether to create inferred spans for managed services @@ -153,8 +158,14 @@ export class TraceListener { if (!this.tracerWrapper.currentSpan) return false; this.wrappedCurrentSpan = new SpanWrapper(this.tracerWrapper.currentSpan, {}); if (this.config.captureLambdaPayload) { - tagObject(this.tracerWrapper.currentSpan, "function.request", event); - tagObject(this.tracerWrapper.currentSpan, "function.response", result); + tagObject(this.tracerWrapper.currentSpan, "function.request", event, 0, this.config.captureLambdaPayloadMaxDepth); + tagObject( + this.tracerWrapper.currentSpan, + "function.response", + result, + 0, + this.config.captureLambdaPayloadMaxDepth, + ); } const coldStartNodes = getTraceTree(); if (coldStartNodes.length > 0) { diff --git a/src/utils/tag-object.spec.ts b/src/utils/tag-object.spec.ts index 43b3387fd..df9de4859 100644 --- a/src/utils/tag-object.spec.ts +++ b/src/utils/tag-object.spec.ts @@ -54,6 +54,39 @@ describe("tagObject", () => { ["lambda_payload.request.vals.1.thingTwo", "2"], ]); }); + it("tags reach max depth", () => { + const span = { + setTag, + }; + + tagObject( + span, + "function.request", + { + hello: "world", + level1: { + level2_dict: { + level3: 3, + }, + level2_list: [null, true, "nice", { l3: "v3" }], + level2_bool: true, + level2_int: 2, + }, + vals: [{ thingOne: 1 }, { thingTwo: 2 }], + }, + 0, + 2, + ); + expect(setTag.mock.calls).toEqual([ + ["function.request.hello", "world"], + ["function.request.level1.level2_dict", '{"level3":3}'], + ["function.request.level1.level2_list", '[null,true,"nice",{"l3":"v3"}]'], + ["function.request.level1.level2_bool", "true"], + ["function.request.level1.level2_int", "2"], + ["function.request.vals.0", '{"thingOne":1}'], + ["function.request.vals.1", '{"thingTwo":2}'], + ]); + }); it("redacts common secret keys", () => { const span = { setTag, diff --git a/src/utils/tag-object.ts b/src/utils/tag-object.ts index bd9f4417d..2b63a0479 100644 --- a/src/utils/tag-object.ts +++ b/src/utils/tag-object.ts @@ -1,15 +1,13 @@ const redactableKeys = ["authorization", "x-authorization", "password", "token"]; -const maxDepth = 10; -export function tagObject(currentSpan: any, key: string, obj: any, depth = 0): any { +export function tagObject(currentSpan: any, key: string, obj: any, depth = 0, maxDepth = 10): any { if (depth >= maxDepth) { - return; - } else { - depth += 1; + return currentSpan.setTag(key, redactVal(key, JSON.stringify(obj).substring(0, 5000))); } if (obj === null) { return currentSpan.setTag(key, obj); } + depth += 1; if (typeof obj === "string") { let parsed: string; try { @@ -18,16 +16,15 @@ export function tagObject(currentSpan: any, key: string, obj: any, depth = 0): a const redacted = redactVal(key, obj.substring(0, 5000)); return currentSpan.setTag(key, redacted); } - return tagObject(currentSpan, key, parsed, depth); + return tagObject(currentSpan, key, parsed, depth, maxDepth); } if (typeof obj === "number" || typeof obj === "boolean") { return currentSpan.setTag(key, obj.toString()); } if (typeof obj === "object") { for (const [k, v] of Object.entries(obj)) { - tagObject(currentSpan, `${key}.${k}`, v, depth); + tagObject(currentSpan, `${key}.${k}`, v, depth, maxDepth); } - return; } } From 4dead73de6e7fbe88f759d1e6f8cf875b3627181 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:56:09 -0500 Subject: [PATCH 2/3] fix --- src/trace/listener.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/trace/listener.spec.ts b/src/trace/listener.spec.ts index 022ca09ea..702aa765c 100644 --- a/src/trace/listener.spec.ts +++ b/src/trace/listener.spec.ts @@ -76,6 +76,7 @@ describe("TraceListener", () => { const defaultConfig = { autoPatchHTTP: true, captureLambdaPayload: false, + captureLambdaPayloadMaxDepth: 10, createInferredSpan: true, encodeAuthorizerContext: true, decodeAuthorizerContext: true, From eac7b63794a9dc5d7bd1b6900a1c37f442e84e14 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:36:01 -0500 Subject: [PATCH 3/3] update the readme --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7c896df8b..b0a3f8afc 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,16 @@ For additional tracing configuration options, check out the [official documentat Besides the environment variables supported by dd-trace-js, the datadog-lambda-js library added following environment variables. -| Environment Variables | Default ValueDescription | -| -------------------- | ------------ | -| DD_ENCODE_AUTHORIZER_CONTEXT | When set to `true` for Lambda authorizers, the tracing context will be encoded into the response for propagation. Supported for NodeJS and Python. Defaults to `true`. | -| DD_DECODE_AUTHORIZER_CONTEXT | When set to `true` for Lambdas that are authorized via Lambda authorizers, it will parse and use the encoded tracing context (if found). Supported for NodeJS and Python. Defaults to `true`. | -| DD_COLD_START_TRACING | Set to `false` to disable Cold Start Tracing. Used in NodeJS and Python. Defaults to `true`. | -| DD_MIN_COLD_START_DURATION | Sets the minimum duration (in milliseconds) for a module load event to be traced via Cold Start Tracing. Number. Defaults to `3`. | -| DD_COLD_START_TRACE_SKIP_LIB | optionally skip creating Cold Start Spans for a comma-separated list of libraries. Useful to limit depth or skip known libraries. Default depends on runtime. | -| DD_CAPTURE_LAMBDA_PAYLOAD | [Captures incoming and outgoing AWS Lambda payloads][1] in the Datadog APM spans for Lambda invocations. Defaults to `false`. | -| DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH | The captured AWS Lambda payloads will become tags of the `aws.lambda` span. This sets how deep it fathoms the JSON structure. When the max depth reached, the tag's value will be the stringified value of the deeper nested items. Defaults to `10`.
For example, with input payload as 
{
  "lv1" : {
    "lv2": {
      "lv3": "val"
    }
  }
}
When set to `2`, the resulted tag's key is `function.request.lv1.lv2` and value `{\"lv3\": \"val\"}`.
When set to `0`, the the resulted tag's key is just `function.request` and value is `{\"lv1\":{\"lv2\":{\"lv3\": \"val\"}}}` | + +| Environment Variables | Description | Default Value | +| -------------------- | ------------ | ------------- | +| DD_ENCODE_AUTHORIZER_CONTEXT | When set to `true` for Lambda authorizers, the tracing context will be encoded into the response for propagation. Supported for NodeJS and Python. | `true` | +| DD_DECODE_AUTHORIZER_CONTEXT | When set to `true` for Lambdas that are authorized via Lambda authorizers, it will parse and use the encoded tracing context (if found). Supported for NodeJS and Python. | `true` | +| DD_COLD_START_TRACING | Set to `false` to disable Cold Start Tracing. Used in NodeJS and Python. | `true` | +| DD_MIN_COLD_START_DURATION | Sets the minimum duration (in milliseconds) for a module load event to be traced via Cold Start Tracing. Number. | `3` | +| DD_COLD_START_TRACE_SKIP_LIB | optionally skip creating Cold Start Spans for a comma-separated list of libraries. Useful to limit depth or skip known libraries. | `./opentracing/tracer` | +| DD_CAPTURE_LAMBDA_PAYLOAD | [Captures incoming and outgoing AWS Lambda payloads][1] in the Datadog APM spans for Lambda invocations. | `false` | +| DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH | Determines the level of detail captured from AWS Lambda payloads, which are then assigned as tags for the `aws.lambda` span. It specifies the nesting depth of the JSON payload structure to process. Once the specified maximum depth is reached, the tag's value is set to the stringified value of any nested elements beyond this level.
For example, given the input payload: 
{
  "lv1" : {
    "lv2": {
      "lv3": "val"
    }
  }
}
If the depth is set to `2`, the resulting tag's key is set to `function.request.lv1.lv2` and the value is `{\"lv3\": \"val\"}`.
If the depth is set to `0`, the resulting tag's key is set to `function.request` and value is `{\"lv1\":{\"lv2\":{\"lv3\": \"val\"}}}` | `10` | ## Lambda Profiling Beta