misc: add compare-timings script (#9723)

.gitignore

@@ -32,6 +32,7 @@ last-run-results.html
 
 latest-run
 lantern-data
+timings-data
 
 closure-error.log
 yarn-error.log

lighthouse-core/scripts/compare-timings.js

@@ -0,0 +1,166 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+// Example:
+//     node lighthouse-core/scripts/compare-timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
+//     node lighthouse-core/scripts/compare-timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'
+
+const fs = require('fs');
+const {execSync} = require('child_process');
+const yargs = require('yargs');
+
+const LH_ROOT = `${__dirname}/../..`;
+const ROOT_OUTPUT_DIR = `${LH_ROOT}/timings-data`;
+
+const argv = yargs
+  .help('help')
+  .describe({
+    'name': 'Unique identifier, makes the folder for storing LHRs. Not a path',
+    // --collect
+    'collect': 'Saves LHRs to disk',
+    'lh-flags': 'Lighthouse flags',
+    'urls': 'Urls to run',
+    'n': 'Number of times to run',
+    // --summarize
+    'summarize': 'Prints statistics report',
+    'measure-filter': 'Regex filter of measures to report. Optional',
+    'output': 'table, json',
+  })
+  .string('measure-filter')
+  .default('output', 'table')
+  .array('urls')
+  .string('lh-flags')
+  .default('lh-flags', '')
+  .wrap(yargs.terminalWidth())
+.argv;
+
+const outputDir = `${ROOT_OUTPUT_DIR}/${argv.name}`;
+
+/**
+ * @param {number[]} values
+ */
+function sum(values) {
+  return values.reduce((sum, value) => sum + value);
+}
+
+/**
+ * @param {number[]} values
+ */
+function average(values) {
+  return sum(values) / values.length;
+}
+
+/**
+ * @param {number[]} values
+ */
+function sampleStdev(values) {
+  const mean = average(values);
+  const variance = sum(values.map(value => (value - mean) ** 2)) / (values.length - 1);
+  return Math.sqrt(variance);
+}
+
+/**
+ * Round to the tenth.
+ * @param {number} value
+ */
+function round(value) {
+  return Math.round(value * 10) / 10;
+}
+
+function collect() {
+  if (!fs.existsSync(ROOT_OUTPUT_DIR)) fs.mkdirSync(ROOT_OUTPUT_DIR);
+  if (fs.existsSync(outputDir)) throw new Error(`folder already exists: ${outputDir}`);
+  fs.mkdirSync(outputDir);
+
+  for (const url of argv.urls) {
+    for (let i = 0; i < argv.n; i++) {
+      const cmd = [
+        'node',
+        `${LH_ROOT}/lighthouse-cli`,
+        url,
+        `--output-path=${outputDir}/lhr-${url.replace(/[^a-zA-Z0-9]/g, '_')}-${i}.json`,
+        '--output=json',
+        argv.lhFlags,
+      ].join(' ');
+      execSync(cmd, {stdio: 'ignore'});
+    }
+  }
+}
+
+function summarize() {
+  // `${url}@@@${entry.name}` -> duration
+  /** @type {Map<string, number[]>} */
+  const durationsMap = new Map();
+  /** @type {RegExp|null} */
+  const measureFilter = argv.measureFilter ? new RegExp(argv.measureFilter, 'i') : null;
+
+  for (const lhrPath of fs.readdirSync(outputDir)) {
+    const lhrJson = fs.readFileSync(`${outputDir}/${lhrPath}`, 'utf-8');
+    /** @type {LH.Result} */
+    const lhr = JSON.parse(lhrJson);
+
+    // Group the durations of each entry of the same name.
+    /** @type {Record<string, number[]>} */
+    const durationsByName = {};
+    for (const entry of lhr.timing.entries) {
+      if (measureFilter && !measureFilter.test(entry.name)) {
+        continue;
+      }
+
+      const durations = durationsByName[entry.name] = durationsByName[entry.name] || [];
+      durations.push(entry.duration);
+    }
+
+    // Push the aggregate time of each unique (by name) entry.
+    for (const [name, durationsForSingleRun] of Object.entries(durationsByName)) {
+      const key = `${lhr.requestedUrl}@@@${name}`;
+      let durations = durationsMap.get(key);
+      if (!durations) {
+        durations = [];
+        durationsMap.set(key, durations);
+      }
+      durations.push(sum(durationsForSingleRun));
+    }
+  }
+
+  const results = [...durationsMap].map(([key, durations]) => {
+    const [url, entryName] = key.split('@@@');
+    const mean = average(durations);
+    const min = Math.min(...durations);
+    const max = Math.max(...durations);
+    const stdev = sampleStdev(durations);
+    return {
+      measure: entryName,
+      url,
+      n: durations.length,
+      mean: round(mean),
+      stdev: round(stdev),
+      min: round(min),
+      max: round(max),
+    };
+  }).sort((a, b) => {
+    // sort by {measure, url}
+    const measureComp = a.measure.localeCompare(b.measure);
+    if (measureComp !== 0) return measureComp;
+    return a.url.localeCompare(b.url);
+  });
+
+  if (argv.output === 'table') {
+    // eslint-disable-next-line no-console
+    console.table(results);
+  } else if (argv.output === 'json') {
+    // eslint-disable-next-line no-console
+    console.log(JSON.stringify(results, null, 2));
+  }
+}
+
+function main() {
+  if (argv.collect) collect();
+  if (argv.summarize) summarize();
+}
+
+main();