From 02d0f45a886e8c32640ed7d515f1e87cafbfdc2c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 8 May 2023 16:57:02 -0400 Subject: [PATCH 1/4] feat: Handle provisioned concurrency and proactive initialization --- src/index.ts | 6 +++-- src/metrics/enhanced-metrics.ts | 4 ++-- src/trace/listener.ts | 7 ++++-- src/utils/cold-start.spec.ts | 39 ++++++++++++++++++++++++++++----- src/utils/cold-start.ts | 28 ++++++++++++++++++----- src/utils/index.ts | 2 +- 6 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/index.ts b/src/index.ts index 3555e1d6..e9124605 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,7 +15,7 @@ import { Logger, LogLevel, promisifiedHandler, - setColdStart, + setSandboxInit, setLogger, setLogLevel, } from "./utils"; @@ -92,6 +92,8 @@ if (getEnvValue(coldStartTracingEnvVar, "true").toLowerCase() === "true") { subscribeToDC(); } +const initTime = Date.now() + /** * Wraps your AWS lambda handler functions to add tracing/metrics support * @param handler A lambda handler function. @@ -131,8 +133,8 @@ export function datadog( let wrappedFunc: any; wrappedFunc = async (...args: any[]) => { const { event, context, responseStream } = extractArgs(isResponseStreamFunction, ...args); - setColdStart(); const startTime = new Date(); + setSandboxInit(initTime, startTime.getTime()) currentMetricsListener = metricsListener; currentTraceListener = traceListener; diff --git a/src/metrics/enhanced-metrics.ts b/src/metrics/enhanced-metrics.ts index 43366ae9..23e64ec3 100644 --- a/src/metrics/enhanced-metrics.ts +++ b/src/metrics/enhanced-metrics.ts @@ -3,7 +3,7 @@ import { sendDistributionMetric } from "../index"; import { Context } from "aws-lambda"; import { parseTagsFromARN } from "../utils/arn"; -import { getColdStartTag } from "../utils/cold-start"; +import { getSandboxInitTags } from "../utils/cold-start"; import { getProcessVersion } from "../utils/process-version"; import { writeMetricToStdout } from "./metric-log"; import { MetricsListener } from "./listener"; @@ -54,7 +54,7 @@ export function getEnhancedMetricTags(context: Context): string[] { if (context.invokedFunctionArn) { arnTags = parseTagsFromARN(context.invokedFunctionArn, context.functionVersion); } - const tags = [...arnTags, getColdStartTag(), `memorysize:${context.memoryLimitInMB}`, getVersionTag()]; + const tags = [...arnTags, ...getSandboxInitTags(), `memorysize:${context.memoryLimitInMB}`, getVersionTag()]; const runtimeTag = getRuntimeTag(); if (runtimeTag) { diff --git a/src/trace/listener.ts b/src/trace/listener.ts index a5086f0a..ec0ed67c 100644 --- a/src/trace/listener.ts +++ b/src/trace/listener.ts @@ -12,7 +12,7 @@ import { extractTriggerTags, extractHTTPStatusCodeTag } from "./trigger"; import { ColdStartTracerConfig, ColdStartTracer } from "./cold-start-tracer"; import { logDebug, tagObject } from "../utils"; -import { didFunctionColdStart } from "../utils/cold-start"; +import { didFunctionColdStart, isProactiveInitialization } from "../utils/cold-start"; import { datadogLambdaVersion } from "../constants"; import { Source, ddtraceVersion, parentSpanFinishTimeHeader, authorizingRequestIdHeader } from "./constants"; import { patchConsole } from "./patch-console"; @@ -157,7 +157,7 @@ export class TraceListener { tagObject(this.tracerWrapper.currentSpan, "function.response", result); } const coldStartNodes = getTraceTree(); - if (coldStartNodes.length > 0 && didFunctionColdStart()) { + if (coldStartNodes.length > 0 && (didFunctionColdStart() || isProactiveInitialization())) { const coldStartConfig: ColdStartTracerConfig = { tracerWrapper: this.tracerWrapper, parentSpan: this.inferredSpan || this.wrappedCurrentSpan, @@ -255,6 +255,9 @@ export class TraceListener { datadog_lambda: datadogLambdaVersion, dd_trace: ddtraceVersion, }; + if (isProactiveInitialization()) { + options.tags["proactive_initialization"] = true; + } if ( (this.contextService.traceSource === Source.Xray && this.config.mergeDatadogXrayTraces) || this.contextService.traceSource === Source.Event diff --git a/src/utils/cold-start.spec.ts b/src/utils/cold-start.spec.ts index 28d86496..6774ca13 100644 --- a/src/utils/cold-start.spec.ts +++ b/src/utils/cold-start.spec.ts @@ -1,22 +1,51 @@ -import { _resetColdStart, didFunctionColdStart, setColdStart } from "./cold-start"; +import { _resetColdStart, didFunctionColdStart, setSandboxInit, isProactiveInitialization } from "./cold-start"; beforeEach(_resetColdStart); afterAll(_resetColdStart); describe("cold-start", () => { it("identifies cold starts on the first execution", () => { - setColdStart(); + setSandboxInit(0, 1); expect(didFunctionColdStart()).toEqual(true); }); it("identifies non-cold starts on subsequent executions", () => { - setColdStart(); + + setSandboxInit(0, 1); expect(didFunctionColdStart()).toEqual(true); - setColdStart(); + setSandboxInit(0, 1); + expect(didFunctionColdStart()).toEqual(false); + + setSandboxInit(0, 1); + expect(didFunctionColdStart()).toEqual(false); + }); + + it("identifies proactive invocations on the first execution", () => { + + setSandboxInit(0, 100000); + expect(didFunctionColdStart()).toEqual(false); + expect(isProactiveInitialization()).toEqual(true); + + setSandboxInit(0, 1); + expect(didFunctionColdStart()).toEqual(false); + + setSandboxInit(0, 1); + expect(didFunctionColdStart()).toEqual(false); + }); + + it("identifies non-proactive invocations on subsequent invocations", () => { + + setSandboxInit(0, 100000); + expect(didFunctionColdStart()).toEqual(false); + expect(isProactiveInitialization()).toEqual(true); + + setSandboxInit(0, 100000); expect(didFunctionColdStart()).toEqual(false); + expect(isProactiveInitialization()).toEqual(false); - setColdStart(); + setSandboxInit(0, 100000); expect(didFunctionColdStart()).toEqual(false); + expect(isProactiveInitialization()).toEqual(false); }); }); diff --git a/src/utils/cold-start.ts b/src/utils/cold-start.ts index d0896cc6..e8087362 100644 --- a/src/utils/cold-start.ts +++ b/src/utils/cold-start.ts @@ -1,27 +1,45 @@ let functionDidColdStart = true; +let proactiveInitialization = false; let isColdStartSet = false; /** * Use global variables to determine whether the container cold started + * and if the start was proactively initialized * On the first container run, isColdStartSet and functionDidColdStart are true * For subsequent executions isColdStartSet will be true and functionDidColdStart will be false */ -export function setColdStart() { - functionDidColdStart = !isColdStartSet; +export function setSandboxInit(initTime: number, invocationStartTime: number) { + if (!isColdStartSet && (invocationStartTime - initTime) > 10_000) { + proactiveInitialization = true + functionDidColdStart = false + } else { + functionDidColdStart = !isColdStartSet; + proactiveInitialization = false; + } isColdStartSet = true; } -export function didFunctionColdStart() { +export function didFunctionColdStart(): boolean { return functionDidColdStart; } -export function getColdStartTag() { - return `cold_start:${didFunctionColdStart()}`; +export function isProactiveInitialization(): boolean { + return proactiveInitialization; +} + +export function getSandboxInitTags(): string[] { + const tags = [`cold_start:${didFunctionColdStart()}`]; + if (isProactiveInitialization()) { + tags.push("proactive_initialization:true"); + } + + return tags } // For testing, reset the globals to their original values export function _resetColdStart() { functionDidColdStart = true; + proactiveInitialization = false; isColdStartSet = false; } diff --git a/src/utils/index.ts b/src/utils/index.ts index 332f376e..b66bc6b8 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,4 +1,4 @@ -export { didFunctionColdStart, getColdStartTag, setColdStart } from "./cold-start"; +export { didFunctionColdStart, getSandboxInitTags, setSandboxInit, isProactiveInitialization } from "./cold-start"; export { wrap, promisifiedHandler } from "./handler"; export { Timer } from "./timer"; export { logError, logDebug, Logger, setLogLevel, setLogger, LogLevel } from "./log"; From 9d57d2fd85ca5afa18f0af6659ccf5fe003e06ff Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 8 May 2023 17:00:53 -0400 Subject: [PATCH 2/4] feat: lint --- src/index.ts | 4 ++-- src/utils/cold-start.spec.ts | 3 --- src/utils/cold-start.ts | 8 ++++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/index.ts b/src/index.ts index e9124605..4328181b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -92,7 +92,7 @@ if (getEnvValue(coldStartTracingEnvVar, "true").toLowerCase() === "true") { subscribeToDC(); } -const initTime = Date.now() +const initTime = Date.now(); /** * Wraps your AWS lambda handler functions to add tracing/metrics support @@ -134,7 +134,7 @@ export function datadog( wrappedFunc = async (...args: any[]) => { const { event, context, responseStream } = extractArgs(isResponseStreamFunction, ...args); const startTime = new Date(); - setSandboxInit(initTime, startTime.getTime()) + setSandboxInit(initTime, startTime.getTime()); currentMetricsListener = metricsListener; currentTraceListener = traceListener; diff --git a/src/utils/cold-start.spec.ts b/src/utils/cold-start.spec.ts index 6774ca13..d66b6781 100644 --- a/src/utils/cold-start.spec.ts +++ b/src/utils/cold-start.spec.ts @@ -10,7 +10,6 @@ describe("cold-start", () => { }); it("identifies non-cold starts on subsequent executions", () => { - setSandboxInit(0, 1); expect(didFunctionColdStart()).toEqual(true); @@ -22,7 +21,6 @@ describe("cold-start", () => { }); it("identifies proactive invocations on the first execution", () => { - setSandboxInit(0, 100000); expect(didFunctionColdStart()).toEqual(false); expect(isProactiveInitialization()).toEqual(true); @@ -35,7 +33,6 @@ describe("cold-start", () => { }); it("identifies non-proactive invocations on subsequent invocations", () => { - setSandboxInit(0, 100000); expect(didFunctionColdStart()).toEqual(false); expect(isProactiveInitialization()).toEqual(true); diff --git a/src/utils/cold-start.ts b/src/utils/cold-start.ts index e8087362..26162e44 100644 --- a/src/utils/cold-start.ts +++ b/src/utils/cold-start.ts @@ -10,9 +10,9 @@ let isColdStartSet = false; * For subsequent executions isColdStartSet will be true and functionDidColdStart will be false */ export function setSandboxInit(initTime: number, invocationStartTime: number) { - if (!isColdStartSet && (invocationStartTime - initTime) > 10_000) { - proactiveInitialization = true - functionDidColdStart = false + if (!isColdStartSet && invocationStartTime - initTime > 10_000) { + proactiveInitialization = true; + functionDidColdStart = false; } else { functionDidColdStart = !isColdStartSet; proactiveInitialization = false; @@ -34,7 +34,7 @@ export function getSandboxInitTags(): string[] { tags.push("proactive_initialization:true"); } - return tags + return tags; } // For testing, reset the globals to their original values From 72a33db9d8a3ecd4a7180dd0ca65710ecab173b8 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 8 May 2023 17:08:33 -0400 Subject: [PATCH 3/4] feat: lint --- src/trace/listener.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trace/listener.ts b/src/trace/listener.ts index ec0ed67c..333b5a67 100644 --- a/src/trace/listener.ts +++ b/src/trace/listener.ts @@ -256,7 +256,7 @@ export class TraceListener { dd_trace: ddtraceVersion, }; if (isProactiveInitialization()) { - options.tags["proactive_initialization"] = true; + options.tags.proactive_initialization = true; } if ( (this.contextService.traceSource === Source.Xray && this.config.mergeDatadogXrayTraces) || From 9bd07e64f4fc30c8880c083653fd1214dbf2f56f Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 15 May 2023 16:05:21 -0400 Subject: [PATCH 4/4] feat: lint --- src/trace/listener.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/trace/listener.ts b/src/trace/listener.ts index 1807eed0..be64b91a 100644 --- a/src/trace/listener.ts +++ b/src/trace/listener.ts @@ -160,7 +160,10 @@ export class TraceListener { if (coldStartNodes.length > 0) { const coldStartConfig: ColdStartTracerConfig = { tracerWrapper: this.tracerWrapper, - parentSpan: didFunctionColdStart() || isProactiveInitialization() ? this.inferredSpan || this.wrappedCurrentSpan : this.wrappedCurrentSpan, + parentSpan: + didFunctionColdStart() || isProactiveInitialization() + ? this.inferredSpan || this.wrappedCurrentSpan + : this.wrappedCurrentSpan, lambdaFunctionName: this.context?.functionName, currentSpanStartTime: this.wrappedCurrentSpan?.startTime(), minDuration: this.config.minColdStartTraceDuration,