From 609887999834fd3bdccbbc242cbe7414ce431da3 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <6349682+vaind@users.noreply.github.com> Date: Thu, 26 Jan 2023 16:27:16 +0100 Subject: [PATCH] ci(replay): Collect replay network metrics (#6881) Adds network info to the collected metrics - it shows the amount of data sent (Net Tx) and received (Net Rx) by the browser. It works by intercepting the network requests with a custom handler that logs the transmission stats. Because Sentry SDK is the only application code sending requests, it equals the amount of data sent & received by the SDK. You can see in the example below that because there are no errors triggered, there's no data sent/received with just the plain Sentry SDK; there's network traffic only when Replay is added. --- packages/replay/metrics/src/collector.ts | 26 +++++-- packages/replay/metrics/src/perf/network.ts | 71 +++++++++++++++++++ .../replay/metrics/src/results/analyzer.ts | 11 +++ .../metrics/src/results/metrics-stats.ts | 4 ++ packages/replay/metrics/src/scenarios.ts | 2 +- packages/replay/metrics/src/util/console.ts | 4 ++ packages/replay/metrics/src/util/github.ts | 2 +- packages/replay/metrics/src/util/json.ts | 2 + 8 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 packages/replay/metrics/src/perf/network.ts diff --git a/packages/replay/metrics/src/collector.ts b/packages/replay/metrics/src/collector.ts index d8673a8c4021..2f387fd286ac 100644 --- a/packages/replay/metrics/src/collector.ts +++ b/packages/replay/metrics/src/collector.ts @@ -3,6 +3,7 @@ import * as playwright from 'playwright'; import { CpuUsage, CpuUsageSampler, CpuUsageSerialized } from './perf/cpu.js'; import { JsHeapUsage, JsHeapUsageSampler, JsHeapUsageSerialized } from './perf/memory.js'; +import { NetworkUsage, NetworkUsageCollector, NetworkUsageSerialized } from './perf/network.js'; import { PerfMetricsSampler } from './perf/sampler.js'; import { Result } from './results/result.js'; import { Scenario, TestCase } from './scenarios.js'; @@ -28,13 +29,24 @@ const PredefinedNetworkConditions = Object.freeze({ }); export class Metrics { - constructor(public readonly vitals: WebVitals, public readonly cpu: CpuUsage, public readonly memory: JsHeapUsage) { } - - public static fromJSON(data: Partial<{ vitals: Partial, cpu: CpuUsageSerialized, memory: JsHeapUsageSerialized }>): Metrics { + constructor( + public readonly vitals: WebVitals, + public readonly cpu: CpuUsage, + public readonly memory: JsHeapUsage, + public readonly network: NetworkUsage) { } + + public static fromJSON( + data: Partial<{ + vitals: Partial, + cpu: CpuUsageSerialized, + memory: JsHeapUsageSerialized, + network: NetworkUsageSerialized + }>): Metrics { return new Metrics( WebVitals.fromJSON(data.vitals || {}), CpuUsage.fromJSON(data.cpu || {}), JsHeapUsage.fromJSON(data.memory || {}), + NetworkUsage.fromJSON(data.network || {}), ); } } @@ -131,7 +143,9 @@ export class MetricsCollector { const cpuSampler = new CpuUsageSampler(perfSampler); const memSampler = new JsHeapUsageSampler(perfSampler); + const networkCollector = await NetworkUsageCollector.create(page); const vitalsCollector = await WebVitalsCollector.create(page); + await scenario.run(browser, page); // NOTE: FID needs some interaction to actually show a value @@ -141,7 +155,11 @@ export class MetricsCollector { throw `Error logs in browser console:\n\t\t${errorLogs.join('\n\t\t')}`; } - return new Metrics(vitals, cpuSampler.getData(), memSampler.getData()); + return new Metrics( + vitals, + cpuSampler.getData(), + memSampler.getData(), + networkCollector.getData()); })(), { milliseconds: 60 * 1000, }); diff --git a/packages/replay/metrics/src/perf/network.ts b/packages/replay/metrics/src/perf/network.ts new file mode 100644 index 000000000000..f7b52614f328 --- /dev/null +++ b/packages/replay/metrics/src/perf/network.ts @@ -0,0 +1,71 @@ +import * as playwright from 'playwright'; + +export class NetworkEvent { + constructor( + public url: string | undefined, + public requestSize: number | undefined, + public responseSize: number | undefined, + public requestTimeNs: bigint | undefined, + public responseTimeNs: bigint | undefined) { } + + public static fromJSON(data: Partial): NetworkEvent { + return new NetworkEvent( + data.url as string, + data.requestSize as number, + data.responseSize as number, + data.requestTimeNs == undefined ? undefined : BigInt(data.requestTimeNs), + data.responseTimeNs == undefined ? undefined : BigInt(data.responseTimeNs), + ); + } +} + +export type NetworkUsageSerialized = Partial<{ events: Array }>; + +export class NetworkUsage { + public constructor(public events: Array) { } + + public static fromJSON(data: NetworkUsageSerialized): NetworkUsage { + return new NetworkUsage(data.events?.map(NetworkEvent.fromJSON) || []); + } +} + +export class NetworkUsageCollector { + private _events = new Array(); + + public static async create(page: playwright.Page): Promise { + const self = new NetworkUsageCollector(); + await page.route(_ => true, self._captureRequest.bind(self)); + return self; + } + + public getData(): NetworkUsage { + return new NetworkUsage(this._events); + } + + private async _captureRequest( + route: playwright.Route, request: playwright.Request): Promise { + const url = request.url(); + try { + const event = new NetworkEvent( + url, + request.postDataBuffer()?.length, + undefined, + process.hrtime.bigint(), + undefined + ); + this._events.push(event); + // Note: playwright would error out on file:/// requests. They are used to access local test app resources. + if (url.startsWith('file:///')) { + route.continue(); + } else { + const response = await route.fetch(); + const body = await response.body(); + route.fulfill({ response, body }); + event.responseTimeNs = process.hrtime.bigint(); + event.responseSize = body.length; + } + } catch (e) { + console.log(`Error when capturing request: ${request.method()} ${url} - ${e}`) + } + } +} diff --git a/packages/replay/metrics/src/results/analyzer.ts b/packages/replay/metrics/src/results/analyzer.ts index bdc2ee77864e..9740ace4a99c 100644 --- a/packages/replay/metrics/src/results/analyzer.ts +++ b/packages/replay/metrics/src/results/analyzer.ts @@ -60,6 +60,10 @@ export class ResultsAnalyzer { pushIfDefined(AnalyzerItemMetric.cpu, AnalyzerItemUnit.ratio, MetricsStats.cpu, MetricsStats.mean); pushIfDefined(AnalyzerItemMetric.memoryAvg, AnalyzerItemUnit.bytes, MetricsStats.memoryMean, MetricsStats.mean); pushIfDefined(AnalyzerItemMetric.memoryMax, AnalyzerItemUnit.bytes, MetricsStats.memoryMax, MetricsStats.max); + pushIfDefined(AnalyzerItemMetric.netTx, AnalyzerItemUnit.bytes, MetricsStats.netTx, MetricsStats.mean); + pushIfDefined(AnalyzerItemMetric.netRx, AnalyzerItemUnit.bytes, MetricsStats.netRx, MetricsStats.mean); + pushIfDefined(AnalyzerItemMetric.netCount, AnalyzerItemUnit.integer, MetricsStats.netCount, MetricsStats.mean); + pushIfDefined(AnalyzerItemMetric.netTime, AnalyzerItemUnit.ms, MetricsStats.netTime, MetricsStats.mean); return items; } @@ -69,6 +73,7 @@ export enum AnalyzerItemUnit { ms, ratio, // 1.0 == 100 % bytes, + integer, } export interface AnalyzerItemValues { @@ -115,6 +120,8 @@ class AnalyzerItemNumberValues implements AnalyzerItemValues { return filesize(value) as string; case AnalyzerItemUnit.ratio: return `${(value * 100).toFixed(2)} ${isDiff ? 'pp' : '%'}`; + case AnalyzerItemUnit.integer: + return `${value}`; default: return `${value.toFixed(2)} ${AnalyzerItemUnit[this._unit]}`; } @@ -127,6 +134,10 @@ export enum AnalyzerItemMetric { cpu, memoryAvg, memoryMax, + netTx, + netRx, + netCount, + netTime, } export interface AnalyzerItem { diff --git a/packages/replay/metrics/src/results/metrics-stats.ts b/packages/replay/metrics/src/results/metrics-stats.ts index fd23d032c11d..8dbb5249957b 100644 --- a/packages/replay/metrics/src/results/metrics-stats.ts +++ b/packages/replay/metrics/src/results/metrics-stats.ts @@ -11,6 +11,10 @@ export class MetricsStats { static cpu: NumberProvider = metrics => metrics.cpu.average; static memoryMean: NumberProvider = metrics => ss.mean(Array.from(metrics.memory.snapshots.values())); static memoryMax: NumberProvider = metrics => ss.max(Array.from(metrics.memory.snapshots.values())); + static netTx: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestSize || 0)); + static netRx: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.responseSize || 0)); + static netCount: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestTimeNs && e.responseTimeNs ? 1 : 0)); + static netTime: NumberProvider = metrics => ss.sum(metrics.network.events.map(e => e.requestTimeNs && e.responseTimeNs ? Number(e.responseTimeNs - e.requestTimeNs) / 1e6 : 0)); static mean: AnalyticsFunction = (items: Metrics[], dataProvider: NumberProvider) => { const numbers = MetricsStats._filteredValues(MetricsStats._collect(items, dataProvider)); diff --git a/packages/replay/metrics/src/scenarios.ts b/packages/replay/metrics/src/scenarios.ts index 87af4b0f3f7c..e73b367ee985 100644 --- a/packages/replay/metrics/src/scenarios.ts +++ b/packages/replay/metrics/src/scenarios.ts @@ -42,6 +42,6 @@ export class JankTestScenario implements Scenario { url = `file:///${url.replace(/\\/g, '/')}`; console.log('Navigating to ', url); await page.goto(url, { waitUntil: 'load', timeout: 60000 }); - await new Promise(resolve => setTimeout(resolve, 5000)); + await new Promise(resolve => setTimeout(resolve, 12000)); } } diff --git a/packages/replay/metrics/src/util/console.ts b/packages/replay/metrics/src/util/console.ts index c5674158fcc0..43822c9c4295 100644 --- a/packages/replay/metrics/src/util/console.ts +++ b/packages/replay/metrics/src/util/console.ts @@ -19,6 +19,10 @@ export function printStats(items: Metrics[]): void { cpu: `${((MetricsStats.mean(items, MetricsStats.cpu) || 0) * 100).toFixed(2)} %`, memoryMean: filesize(MetricsStats.mean(items, MetricsStats.memoryMean)), memoryMax: filesize(MetricsStats.max(items, MetricsStats.memoryMax)), + netTx: filesize(MetricsStats.mean(items, MetricsStats.netTx)), + netRx: filesize(MetricsStats.mean(items, MetricsStats.netRx)), + netCount: MetricsStats.mean(items, MetricsStats.netCount), + netTime: `${MetricsStats.mean(items, MetricsStats.netTime)?.toFixed(2)} ms`, }); } diff --git a/packages/replay/metrics/src/util/github.ts b/packages/replay/metrics/src/util/github.ts index 01b0e9fbe991..ccdd9b0c343f 100644 --- a/packages/replay/metrics/src/util/github.ts +++ b/packages/replay/metrics/src/util/github.ts @@ -59,7 +59,7 @@ async function tryAddOrUpdateComment(commentBuilder: PrCommentBuilder): Promise< ...defaultArgs, base: await Git.baseBranch, head: await Git.branch - })).data[0].number; + })).data[0]?.number; if (prNumber != undefined) { console.log(`Found PR number ${prNumber} based on base and head branches`); } diff --git a/packages/replay/metrics/src/util/json.ts b/packages/replay/metrics/src/util/json.ts index 095614fd115e..7b27d5d8f9f2 100644 --- a/packages/replay/metrics/src/util/json.ts +++ b/packages/replay/metrics/src/util/json.ts @@ -7,6 +7,8 @@ export function JsonStringify(object: T): string { return JSON.stringify(object, (_: unknown, value: any): unknown => { if (typeof value != 'undefined' && typeof value.toJSON == 'function') { return value.toJSON(); + } else if (typeof value == 'bigint') { + return value.toString(); } else { return value; }