Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[perf] Use Set instead of arrays for metrics aggregation #112

Merged
merged 1 commit into from
Nov 27, 2024
Merged

[perf] Use Set instead of arrays for metrics aggregation #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 55 additions & 58 deletions packages/plugins/telemetry/src/common/aggregator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,9 @@ import type { GlobalContext } from '@dd/core/types';
import type { Metric, MetricToSend, OptionsDD, Report } from '@dd/telemetry-plugin/types';

import { getMetric } from './helpers';
import { getPlugins, getLoaders } from './metrics/common';
import { addPluginMetrics, addLoaderMetrics } from './metrics/common';

const getUniversalMetrics = (globalContext: GlobalContext) => {
const metrics: Metric[] = [];
const addUniversalMetrics = (globalContext: GlobalContext, metrics: Set<Metric>) => {
const inputs = globalContext.build.inputs || [];
const outputs = globalContext.build.outputs || [];
const entries = globalContext.build.entries || [];
Expand Down Expand Up @@ -48,41 +47,40 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
}

// Counts
metrics.push(
{
metrics
.add({
metric: 'assets.count',
type: 'count',
value: outputs.length,
tags: [],
},
{
})
.add({
metric: 'entries.count',
type: 'count',
value: entries.length,
tags: [],
},
{
})
.add({
metric: 'errors.count',
type: 'count',
value: nbErrors,
tags: [],
},
{
})
.add({
metric: 'modules.count',
type: 'count',
value: inputs.length,
tags: [],
},
{
})
.add({
metric: 'warnings.count',
type: 'count',
value: nbWarnings,
tags: [],
},
);
});

if (duration) {
metrics.push({
metrics.add({
metric: 'compilation.duration',
type: 'duration',
value: duration,
Expand All @@ -106,26 +104,25 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
...assetsPerInput.get(input.filepath)!.map((assetName) => `assetName:${assetName}`),
);
}
metrics.push(
{
metrics
.add({
metric: 'modules.size',
type: 'size',
value: input.size,
tags,
},
{
})
.add({
metric: 'modules.dependencies',
type: 'count',
value: input.dependencies.size,
tags,
},
{
})
.add({
metric: 'modules.dependents',
type: 'count',
value: input.dependents.size,
tags,
},
);
});
}

// Assets
Expand All @@ -139,87 +136,87 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
.map((entryName) => `entryName:${entryName}`),
);
}
metrics.push(
{
metrics
.add({
metric: 'assets.size',
type: 'size',
value: output.size,
tags,
},
{
})
.add({
metric: 'assets.modules.count',
type: 'count',
value: output.inputs.length,
tags,
},
);
});
}

// Entries
for (const entry of entries) {
const tags = [`entryName:${entry.name}`];
metrics.push(
{
metrics
.add({
metric: 'entries.size',
type: 'size',
value: entry.size,
tags,
},
{
})
.add({
metric: 'entries.modules.count',
type: 'count',
value: entry.inputs.length,
tags,
},
{
})
.add({
metric: 'entries.assets.count',
type: 'count',
value: entry.outputs.length,
tags,
},
);
});
}

return metrics;
};

export const getMetrics = (
export const addMetrics = (
globalContext: GlobalContext,
optionsDD: OptionsDD,
metricsToSend: Set<MetricToSend>,
report?: Report,
): MetricToSend[] => {
const metrics: Metric[] = [];
): void => {
const metrics: Set<Metric> = new Set();

if (report) {
const { timings } = report;

if (timings) {
if (timings.tapables) {
metrics.push(...getPlugins(timings.tapables));
addPluginMetrics(timings.tapables, metrics);
}
if (timings.loaders) {
metrics.push(...getLoaders(timings.loaders));
addLoaderMetrics(timings.loaders, metrics);
}
}
}

metrics.push(...getUniversalMetrics(globalContext));
addUniversalMetrics(globalContext, metrics);

// Format metrics to be DD ready and apply filters
const metricsToSend: MetricToSend[] = metrics
.map((m) => {
let metric: Metric | null = m;
if (optionsDD.filters?.length) {
for (const filter of optionsDD.filters) {
// Could have been filtered out by an early filter.
if (metric) {
metric = filter(metric);
}
for (const metric of metrics) {
if (optionsDD.filters?.length) {
let filteredMetric: Metric | null = metric;
for (const filter of optionsDD.filters) {
// If it's already been filtered out, no need to keep going.
if (!filteredMetric) {
break;
}
filteredMetric = filter(metric);
}
return metric ? getMetric(metric, optionsDD) : null;
})
.filter((m) => m !== null) as MetricToSend[];

return metricsToSend;
if (filteredMetric) {
metricsToSend.add(getMetric(filteredMetric, optionsDD));
}
} else {
metricsToSend.add(getMetric(metric, optionsDD));
}
}
};
49 changes: 19 additions & 30 deletions packages/plugins/telemetry/src/common/metrics/common.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@

import type { TimingsMap, Metric } from '@dd/telemetry-plugin/types';

export const getPlugins = (plugins: TimingsMap): Metric[] => {
const metrics: Metric[] = [];

metrics.push({
export const addPluginMetrics = (plugins: TimingsMap, metrics: Set<Metric>): void => {
metrics.add({
metric: 'plugins.count',
type: 'count',
value: plugins.size,
Expand All @@ -26,67 +24,58 @@ export const getPlugins = (plugins: TimingsMap): Metric[] => {
hookDuration += duration;
pluginDuration += duration;
}
metrics.push(
{
metrics
.add({
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL set#add can be chained!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ME TOO, I didn't know that either.

metric: 'plugins.hooks.duration',
type: 'duration',
value: hookDuration,
tags: [`pluginName:${plugin.name}`, `hookName:${hook.name}`],
},
{
})
.add({
metric: 'plugins.hooks.increment',
type: 'count',
value: hook.values.length,
tags: [`pluginName:${plugin.name}`, `hookName:${hook.name}`],
},
);
});
}

metrics.push(
{
metrics
.add({
metric: 'plugins.duration',
type: 'duration',
value: pluginDuration,
tags: [`pluginName:${plugin.name}`],
},
{
})
.add({
metric: 'plugins.increment',
type: 'count',
value: pluginCount,
tags: [`pluginName:${plugin.name}`],
},
);
});
}

return metrics;
};

export const getLoaders = (loaders: TimingsMap): Metric[] => {
const metrics: Metric[] = [];

metrics.push({
export const addLoaderMetrics = (loaders: TimingsMap, metrics: Set<Metric>): void => {
metrics.add({
metric: 'loaders.count',
type: 'count',
value: loaders.size,
tags: [],
});

for (const loader of loaders.values()) {
metrics.push(
{
metrics
.add({
metric: 'loaders.duration',
type: 'duration',
value: loader.duration,
tags: [`loaderName:${loader.name}`],
},
{
})
.add({
metric: 'loaders.increment',
type: 'count',
value: loader.increment,
tags: [`loaderName:${loader.name}`],
},
);
});
}

return metrics;
};
10 changes: 5 additions & 5 deletions packages/plugins/telemetry/src/common/output/files.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ type FilesToWrite = {
[key in Files]?: { content: any };
};

export const outputFiles = async (
export const outputFiles: (
data: {
report?: Report;
metrics: MetricToSend[];
metrics: Set<MetricToSend>;
},
outputOptions: OutputOptions,
log: Logger,
cwd: string,
) => {
) => Promise<void> = async (data, outputOptions, log, cwd) => {
// Don't write any file if it's not enabled.
if (typeof outputOptions !== 'string' && typeof outputOptions !== 'object' && !outputOptions) {
return;
Expand Down Expand Up @@ -66,8 +66,8 @@ export const outputFiles = async (
};
}

if (metrics && files.metrics) {
filesToWrite.metrics = { content: metrics };
if (files.metrics) {
filesToWrite.metrics = { content: Array.from(metrics) };
}

const proms = Object.entries(filesToWrite).map(async ([filename, file]): Promise<void> => {
Expand Down
20 changes: 14 additions & 6 deletions packages/plugins/telemetry/src/common/sender.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import type { Logger } from '@dd/core/types';
import type { MetricToSend } from '@dd/telemetry-plugin/types';

export const sendMetrics = (
metrics: MetricToSend[] | undefined,
metrics: Set<MetricToSend>,
auth: { apiKey?: string; endPoint: string },
log: Logger,
) => {
Expand All @@ -16,17 +16,25 @@ export const sendMetrics = (
log.warn(`Won't send metrics to Datadog: missing API Key.`);
return;
}
if (!metrics || metrics.length === 0) {
if (!metrics.size) {
log.warn(`No metrics to send.`);
return;
}

const metricsNames = [...new Set(metrics.map((m) => m.metric))]
.sort()
.map((name) => `${name} - ${metrics.filter((m) => m.metric === name).length}`);
const metricIterations: Map<string, number> = new Map();
for (const metric of metrics) {
if (!metricIterations.has(metric.metric)) {
metricIterations.set(metric.metric, 0);
}
metricIterations.set(metric.metric, metricIterations.get(metric.metric)! + 1);
}

const metricsNames = Array.from(metricIterations.entries()).map(
([name, count]) => `${name} - ${count}`,
);

log.debug(`
Sending ${metrics.length} metrics.
Sending ${metrics.size} metrics.
Metrics:
- ${metricsNames.join('\n - ')}`);

Expand Down
Loading