benchmark: rewrite benchmark in TS (#3559)

.eslintrc.yml

@@ -692,6 +692,7 @@ overrides:
     rules:
       internal-rules/only-ascii: [error, { allowEmoji: true }]
       node/no-sync: off
+      import/no-namespace: off
       import/no-unresolved: off
       import/no-nodejs-modules: off
       no-console: off
@@ -736,6 +737,3 @@ overrides:
       # Ignore docusarus related webpack aliases
       import/no-unresolved:
         ['error', { 'ignore': ['^@theme', '^@docusaurus', '^@generated'] }]
-  - files: 'benchmark/benchmark.js'
-    parserOptions:
-      sourceType: script

benchmark/benchmark.js → benchmark/benchmark.ts

@@ -1,10 +1,8 @@
-'use strict';
-
-const os = require('os');
-const fs = require('fs');
-const path = require('path');
-const assert = require('assert');
-const cp = require('child_process');
+import * as assert from 'node:assert';
+import * as cp from 'node:child_process';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
 
 const NS_PER_SEC = 1e9;
 const LOCAL = 'local';
@@ -25,11 +23,11 @@ if (require.main === module) {
   });
 }
 
-function localDir(...paths) {
+function localDir(...paths: ReadonlyArray<string>) {
   return path.join(__dirname, '..', ...paths);
 }
 
-function exec(command, options = {}) {
+function exec(command: string, options = {}) {
   const result = cp.execSync(command, {
     encoding: 'utf-8',
     stdio: ['inherit', 'pipe', 'inherit'],
@@ -38,9 +36,16 @@ function exec(command, options = {}) {
   return result?.trimEnd();
 }
 
+interface BenchmarkProject {
+  revision: string;
+  projectPath: string;
+}
+
 // Build a benchmark-friendly environment for the given revision
 // and returns path to its 'dist' directory.
-function prepareBenchmarkProjects(revisionList) {
+function prepareBenchmarkProjects(
+  revisionList: ReadonlyArray<string>,
+): Array<BenchmarkProject> {
   const tmpDir = path.join(os.tmpdir(), 'graphql-js-benchmark');
   fs.rmSync(tmpDir, { recursive: true, force: true });
   fs.mkdirSync(tmpDir);
@@ -74,7 +79,7 @@ function prepareBenchmarkProjects(revisionList) {
     return { revision, projectPath };
   });
 
-  function prepareNPMPackage(revision) {
+  function prepareNPMPackage(revision: string) {
     if (revision === LOCAL) {
       const repoDir = localDir();
       const archivePath = path.join(tmpDir, 'graphql-local.tgz');
@@ -100,7 +105,7 @@ function prepareBenchmarkProjects(revisionList) {
     return archivePath;
   }
 
-  function buildNPMArchive(repoDir) {
+  function buildNPMArchive(repoDir: string) {
     exec('npm --quiet run build:npm', { cwd: repoDir });
 
     const distDir = path.join(repoDir, 'npmDist');
@@ -109,34 +114,45 @@ function prepareBenchmarkProjects(revisionList) {
   }
 }
 
-async function collectSamples(modulePath) {
+async function collectSamples(modulePath: string) {
   const samples = [];
 
   // If time permits, increase sample size to reduce the margin of error.
   const start = Date.now();
   while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
-    const { clocked, memUsed } = await sampleModule(modulePath);
-    assert(clocked > 0);
-    assert(memUsed > 0);
-    samples.push({ clocked, memUsed });
+    const sample = await sampleModule(modulePath);
+    assert(sample.clocked > 0);
+    assert(sample.memUsed > 0);
+    samples.push(sample);
   }
   return samples;
 }
 
 // T-Distribution two-tailed critical values for 95% confidence.
 // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
 // prettier-ignore
-const tTable = {
+const tTable: { [v: number]: number } = {
   '1':  12.706, '2':  4.303, '3':  3.182, '4':  2.776, '5':  2.571, '6':  2.447,
   '7':  2.365,  '8':  2.306, '9':  2.262, '10': 2.228, '11': 2.201, '12': 2.179,
   '13': 2.16,   '14': 2.145, '15': 2.131, '16': 2.12,  '17': 2.11,  '18': 2.101,
   '19': 2.093,  '20': 2.086, '21': 2.08,  '22': 2.074, '23': 2.069, '24': 2.064,
   '25': 2.06,   '26': 2.056, '27': 2.052, '28': 2.048, '29': 2.045, '30': 2.042,
-  infinity: 1.96,
 };
+const tTableInfinity = 1.96;
+
+interface BenchmarkComputedStats {
+  name: string;
+  memPerOp: number;
+  ops: number;
+  deviation: number;
+  numSamples: number;
+}
 
 // Computes stats on benchmark results.
-function computeStats(samples) {
+function computeStats(
+  name: string,
+  samples: ReadonlyArray<BenchmarkSample>,
+): BenchmarkComputedStats {
   assert(samples.length > 1);
 
   // Compute the sample mean (estimate of the population mean).
@@ -166,7 +182,7 @@ function computeStats(samples) {
   const df = samples.length - 1;
 
   // Compute the critical value.
-  const critical = tTable[df] || tTable.infinity;
+  const critical = tTable[df] ?? tTableInfinity;
 
   // Compute the margin of error.
   const moe = sem * critical;
@@ -175,14 +191,15 @@ function computeStats(samples) {
   const rme = (moe / mean) * 100 || 0;
 
   return {
+    name,
     memPerOp: Math.floor(meanMemUsed),
     ops: NS_PER_SEC / mean,
     deviation: rme,
     numSamples: samples.length,
   };
 }
 
-function beautifyBenchmark(results) {
+function beautifyBenchmark(results: ReadonlyArray<BenchmarkComputedStats>) {
   const nameMaxLen = maxBy(results, ({ name }) => name.length);
   const opsTop = maxBy(results, ({ ops }) => ops);
   const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length);
@@ -195,7 +212,7 @@ function beautifyBenchmark(results) {
     printBench(result);
   }
 
-  function printBench(bench) {
+  function printBench(bench: BenchmarkComputedStats) {
     const { name, memPerOp, ops, deviation, numSamples } = bench;
     console.log(
       '  ' +
@@ -234,22 +251,25 @@ function beautifyBenchmark(results) {
   }
 }
 
-function beautifyBytes(bytes) {
+function beautifyBytes(bytes: number) {
   const sizes = ['Bytes', 'KB', 'MB', 'GB'];
   const i = Math.floor(Math.log2(bytes) / 10);
   return beautifyNumber(bytes / 2 ** (i * 10)) + ' ' + sizes[i];
 }
 
-function beautifyNumber(num) {
+function beautifyNumber(num: number) {
   return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
 }
 
-function maxBy(array, fn) {
+function maxBy<T>(array: ReadonlyArray<T>, fn: (obj: T) => number) {
   return Math.max(...array.map(fn));
 }
 
 // Prepare all revisions and run benchmarks matching a pattern against them.
-async function runBenchmarks(benchmarks, benchmarkProjects) {
+async function runBenchmarks(
+  benchmarks: ReadonlyArray<string>,
+  benchmarkProjects: ReadonlyArray<BenchmarkProject>,
+) {
   for (const benchmark of benchmarks) {
     const results = [];
     for (let i = 0; i < benchmarkProjects.length; ++i) {
@@ -264,11 +284,7 @@ async function runBenchmarks(benchmarks, benchmarkProjects) {
       try {
         const samples = await collectSamples(modulePath);
 
-        results.push({
-          name: revision,
-          samples,
-          ...computeStats(samples),
-        });
+        results.push(computeStats(revision, samples));
         process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
       } catch (error) {
         console.log('  ' + revision + ': ' + red(String(error)));
@@ -281,10 +297,10 @@ async function runBenchmarks(benchmarks, benchmarkProjects) {
   }
 }
 
-function getArguments(argv) {
+function getArguments(argv: ReadonlyArray<string>) {
   const revsIndex = argv.indexOf('--revs');
   const revisions = revsIndex === -1 ? [] : argv.slice(revsIndex + 1);
-  const benchmarks = revsIndex === -1 ? argv : argv.slice(0, revsIndex);
+  const benchmarks = revsIndex === -1 ? [...argv] : argv.slice(0, revsIndex);
 
   switch (revisions.length) {
     case 0:
@@ -315,31 +331,37 @@ function findAllBenchmarks() {
     .map((name) => path.join('benchmark', name));
 }
 
-function bold(str) {
+function bold(str: string | number) {
   return '\u001b[1m' + str + '\u001b[0m';
 }
 
-function red(str) {
+function red(str: string | number) {
   return '\u001b[31m' + str + '\u001b[0m';
 }
 
-function green(str) {
+function green(str: string | number) {
   return '\u001b[32m' + str + '\u001b[0m';
 }
 
-function yellow(str) {
+function yellow(str: string | number) {
   return '\u001b[33m' + str + '\u001b[0m';
 }
 
-function cyan(str) {
+function cyan(str: string | number) {
   return '\u001b[36m' + str + '\u001b[0m';
 }
 
-function grey(str) {
+function grey(str: string | number) {
   return '\u001b[90m' + str + '\u001b[0m';
 }
 
-function sampleModule(modulePath) {
+interface BenchmarkSample {
+  name: string;
+  clocked: number;
+  memUsed: number;
+}
+
+function sampleModule(modulePath: string): Promise<BenchmarkSample> {
   const sampleCode = `
     import assert from 'assert';
     assert(global.gc);
@@ -371,7 +393,7 @@ function sampleModule(modulePath) {
 
   return new Promise((resolve, reject) => {
     const child = cp.spawn(
-      process.argv[0],
+      process.execPath,
       [
         '--no-concurrent-sweeping',
         '--predictable',
@@ -386,8 +408,8 @@ function sampleModule(modulePath) {
       },
     );
 
-    let message;
-    let error;
+    let message: any;
+    let error: any;
 
     child.on('message', (msg) => (message = msg));
     child.on('error', (e) => (error = e));

package.json

@@ -32,7 +32,7 @@
     "version": "node resources/gen-version.js && npm test && git add src/version.ts",
     "fuzzonly": "mocha --full-trace src/**/__tests__/**/*-fuzz.ts",
     "changelog": "node resources/gen-changelog.js",
-    "benchmark": "node benchmark/benchmark.js",
+    "benchmark": "ts-node benchmark/benchmark.ts",
     "test": "npm run lint && npm run check && npm run testonly:cover && npm run prettier:check && npm run check:spelling && npm run check:integrations",
     "lint": "eslint --cache --max-warnings 0 .",
     "check": "tsc --pretty",

tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "include": ["src/**/*", "integrationTests/*"],
+  "include": ["src/**/*", "integrationTests/*", "benchmark/benchmark.ts"],
   "compilerOptions": {
     "lib": ["es2020"],
     "target": "es2020",