Benchmark analysis

benchmark_stats.json

@@ -0,0 +1,42 @@
+{
+    "0.25.0-dev": {
+        "stable": {
+            "count": 611,
+            "mean": 120237704.94599018,
+            "median": 1011216,
+            "standardDeviation": 1266731659.1596396,
+            "min": 163759,
+            "max": 19060732880,
+            "baselineWeightedEfficiencyScore": 12425390.694599018
+        },
+        "experimental": {
+            "count": 745,
+            "mean": 1026538663.6483221,
+            "median": 8922860,
+            "standardDeviation": 3031934484.2442517,
+            "min": 84524,
+            "max": 18426067752,
+            "baselineWeightedEfficiencyScore": 105381438.76483223
+        }
+    },
+    "0.25.0-pre-bifurcation": {
+        "stable": {
+            "count": 624,
+            "mean": 101374357.53525642,
+            "median": 1040987,
+            "standardDeviation": 1192914154.3810272,
+            "min": 155606,
+            "max": 18226207933,
+            "baselineWeightedEfficiencyScore": 10543095.453525642
+        },
+        "experimental": {
+            "count": 686,
+            "mean": 1153922654.5932944,
+            "median": 8920551.5,
+            "standardDeviation": 3168782029.724511,
+            "min": 83475,
+            "max": 18422514685,
+            "baselineWeightedEfficiencyScore": 118118515.90932944
+        }
+    }
+}

benchmark_stats.md

@@ -0,0 +1,177 @@
+# Benchmark Results
+
+## Version `0.25.0-dev`
+
+<table>
+<tr>
+    <th></th>
+    <th colspan="2">Stable</th>
+    <th colspan="2">Experimental</th>
+</tr>
+<tr>
+    <th>Metric</th>
+    <th>Score</th>
+    <th>Change</th>
+    <th>Score</th>
+    <th>Change</th>
+</tr>
+    <td>Baseline Weighted Efficiency</td>
+    <td>12_425_390</td>
+    <td><span style="color: red">17.85%</span></td>
+    <td>105_381_438</td>
+    <td><span style="color: green">-10.78%</span></td>
+</tr>
+<tr>
+<td colspan="5">&nbsp;</td>
+</tr>
+<tr>
+    <th></th>
+    <th>Count</th>
+    <th></th>
+    <th>Count</th>
+    <th></th>
+</tr>
+    <tr>
+    <td>Method Executions</td>
+    <td>611</td>
+    <td><span style="color: green">-2.08%</span></td>
+    <td>745</td>
+    <td><span style="color: red">8.60%</span></td>
+</tr>
+<tr>
+<td colspan="5">&nbsp;</td>
+</tr>
+<tr>
+    <th></th>
+    <th>Number of Instructions</th>
+    <th></th>
+    <th>Number of Instructions</th>
+    <th></th>
+</tr>
+<tr>
+    <td>Mean</td>
+    <td>120_237_704</td>
+    <td><span style="color: red">18.61%</span></td>
+    <td>1_026_538_663</td>
+    <td><span style="color: green">-11.04%</span></td>
+</tr>
+<tr>
+    <td>Median</td>
+    <td>1_011_216</td>
+    <td><span style="color: green">-2.86%</span></td>
+    <td>8_922_860</td>
+    <td><span style="color: red">0.03%</span></td>
+</tr>
+<tr>
+    <td>Standard Deviation</td>
+    <td>1_266_731_659</td>
+    <td><span style="color: red">6.19%</span></td>
+    <td>3_031_934_484</td>
+    <td><span style="color: green">-4.32%</span></td>
+</tr>
+<tr>
+    <td>Min</td>
+    <td>163_759</td>
+    <td><span style="color: red">5.24%</span></td>
+    <td>84_524</td>
+    <td><span style="color: red">1.26%</span></td>
+</tr>
+<tr>
+    <td>Max</td>
+    <td>19_060_732_880</td>
+    <td><span style="color: red">4.58%</span></td>
+    <td>18_426_067_752</td>
+    <td><span style="color: red">0.02%</span></td>
+</tr>
+
+</table>
+
+## Version `0.25.0-pre-bifurcation`
+
+<table>
+<tr>
+    <th></th>
+    <th colspan="2">Stable</th>
+    <th colspan="2">Experimental</th>
+</tr>
+<tr>
+    <th>Metric</th>
+    <th>Score</th>
+    <th>Change</th>
+    <th>Score</th>
+    <th>Change</th>
+</tr>
+    <td>Baseline Weighted Efficiency</td>
+    <td>10_543_095</td>
+    <td>0.00%</td>
+    <td>118_118_515</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+<td colspan="5">&nbsp;</td>
+</tr>
+<tr>
+    <th></th>
+    <th>Count</th>
+    <th></th>
+    <th>Count</th>
+    <th></th>
+</tr>
+    <tr>
+    <td>Method Executions</td>
+    <td>624</td>
+    <td>0.00%</td>
+    <td>686</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+<td colspan="5">&nbsp;</td>
+</tr>
+<tr>
+    <th></th>
+    <th>Number of Instructions</th>
+    <th></th>
+    <th>Number of Instructions</th>
+    <th></th>
+</tr>
+<tr>
+    <td>Mean</td>
+    <td>101_374_357</td>
+    <td>0.00%</td>
+    <td>1_153_922_654</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+    <td>Median</td>
+    <td>1_040_987</td>
+    <td>0.00%</td>
+    <td>8_920_551</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+    <td>Standard Deviation</td>
+    <td>1_192_914_154</td>
+    <td>0.00%</td>
+    <td>3_168_782_029</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+    <td>Min</td>
+    <td>155_606</td>
+    <td>0.00%</td>
+    <td>83_475</td>
+    <td>0.00%</td>
+</tr>
+<tr>
+    <td>Max</td>
+    <td>18_226_207_933</td>
+    <td>0.00%</td>
+    <td>18_422_514_685</td>
+    <td>0.00%</td>
+</tr>
+
+</table>
+
+---
+
+_Report generated automatically by Azle benchmark tools_

scripts/analyze_benchmarks/extractor.ts

@@ -0,0 +1,88 @@
+import { readFile } from 'fs/promises';
+
+export type BenchmarkEntry = {
+    instructions: { __bigint__: string };
+    method_name: string;
+    timestamp: { __bigint__: string };
+};
+
+type CanisterBenchmark = {
+    current?: {
+        version: string;
+        benchmarks: BenchmarkEntry[];
+    };
+    previous: {
+        version: string;
+        benchmarks: BenchmarkEntry[];
+    };
+};
+
+type BenchmarksJson = {
+    [canisterName: string]: CanisterBenchmark;
+};
+
+/**
+ * Extracts benchmark entries from multiple files and groups them by version
+ * @param files Array of file paths to process
+ * @returns A record mapping version strings to arrays of benchmark entries
+ */
+export async function extractBenchmarksEntriesFromFiles(
+    files: string[]
+): Promise<Record<string, BenchmarkEntry[]>> {
+    const versionEntriesArrays = await Promise.all(
+        files.map(extractBenchmarkEntries)
+    );
+    const versionEntries = versionEntriesArrays.flat();
+
+    return groupEntriesByVersion(versionEntries);
+}
+
+/**
+ * Extracts benchmark entries from a single file
+ * @param file Path to the benchmark file
+ * @returns Array of tuples containing version and benchmark entry pairs
+ */
+async function extractBenchmarkEntries(
+    file: string
+): Promise<Array<[string, BenchmarkEntry]>> {
+    const data: BenchmarksJson = JSON.parse(await readFile(file, 'utf-8'));
+
+    return Object.values(data).flatMap((canisterData) => {
+        const currentEntries = canisterData.current
+            ? canisterData.current.benchmarks.map(
+                  (benchmark) =>
+                      [canisterData.current!.version, benchmark] as [
+                          string,
+                          BenchmarkEntry
+                      ]
+              )
+            : [];
+
+        const previousEntries = canisterData.previous.benchmarks.map(
+            (benchmark) =>
+                [canisterData.previous.version, benchmark] as [
+                    string,
+                    BenchmarkEntry
+                ]
+        );
+
+        return [...currentEntries, ...previousEntries];
+    });
+}
+
+/**
+ * Groups benchmark entries by their version
+ * @param entries Array of version and benchmark entry pairs
+ * @returns A record mapping version strings to arrays of benchmark entries
+ */
+function groupEntriesByVersion(
+    entries: Array<[string, BenchmarkEntry]>
+): Record<string, BenchmarkEntry[]> {
+    return entries.reduce(
+        (acc, [version, entry]) => ({
+            ...acc,
+            [version]: [...(acc[version] ?? []), entry]
+        }),
+        {} as Record<string, BenchmarkEntry[]>
+    );
+}

scripts/analyze_benchmarks/file_finder.ts

@@ -0,0 +1,37 @@
+import { readdir, stat } from 'fs/promises';
+import { join } from 'path';
+
+/**
+ * Recursively finds all benchmark.json files in a directory and its subdirectories
+ * @param dir Directory path to search
+ * @returns Array of full paths to benchmark.json files
+ */
+export async function findBenchmarkFiles(dir: string): Promise<string[]> {
+    if (dir.includes('node_modules')) {
+        return [];
+    }
+    const items = await readdir(dir);
+    const itemResults = await Promise.all(
+        items.map((item) => processDirectoryItem(dir, item))
+    );
+    return itemResults.flat();
+}
+
+/**
+ * Processes a single item in a directory to determine if it's a benchmark file
+ * @param dir Parent directory path
+ * @param item Name of the item to process
+ * @returns Array of paths to benchmark files (empty if item is not a benchmark file)
+ */
+async function processDirectoryItem(
+    dir: string,
+    item: string
+): Promise<string[]> {
+    const fullPath = join(dir, item);
+
+    const statInfo = await stat(fullPath);
+    if (statInfo.isDirectory()) {
+        return findBenchmarkFiles(fullPath);
+    }
+    return item === 'benchmarks.json' ? [fullPath] : [];
+}