Skip to content

Commit

Permalink
Better benchmark isolation
Browse files Browse the repository at this point in the history
  • Loading branch information
@texodus
texodus committed May 27, 2023
1 parent 9217cca commit d3c6d91
Show file tree
Hide file tree
Showing 4 changed files with 314 additions and 194 deletions.
2 changes: 1 addition & 1 deletion packages/perspective/src/js/perspective.node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,9 @@ module.exports.sync_module = async () => {
};

const DEFAULT_ASSETS = [
path.join(__dirname, "../../../../tools/perspective-bench/dist"),
"@finos/perspective-test",
"@finos/perspective/dist/cdn",
"@finos/perspective-bench/dist",
"@finos/perspective-workspace/dist/cdn",
"@finos/perspective-workspace/dist/css",
"@finos/perspective-viewer/dist/cdn",
Expand Down
1 change: 1 addition & 0 deletions tools/perspective-bench/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
"microtime": "^3.0.0"
},
"dependencies": {
"perspective-2-1-0": "npm:@finos/[email protected]",
"perspective-2-0-0": "npm:@finos/[email protected]",
"perspective-1-9-0": "npm:@finos/[email protected]",
"perspective-1-8-0": "npm:@finos/[email protected]",
Expand Down
252 changes: 59 additions & 193 deletions tools/perspective-bench/src/js/bench.js
View file
Original file line number Diff line number Diff line change
@@ -1,90 +1,34 @@
/******************************************************************************
*
* Copyright (c) 2019, the Perspective Authors.
*
* This file is part of the Perspective library, distributed under the terms of
* the Apache License 2.0. The full license can be found in the LICENSE file.
*
*/

const fs = require("fs");
const microtime = require("microtime");
const psp = require("@finos/perspective");
const cp = require("child_process");
const path = require("path");

const PKG_DEPS = Object.keys(
JSON.parse(fs.readFileSync(path.join(__dirname, "../../package.json")))
.dependencies
);

process.setMaxListeners(PKG_DEPS.length + 1);

const VERSIONS = ["@finos/perspective", ...PKG_DEPS];

function load_version(path, i) {
const module = require(path);
let { version } = JSON.parse(
fs.readFileSync(require.resolve(`${path}/package.json`))
);

if (i === 0) {
version = `${version} (master)`;
}

return { version, perspective: module.default || module };
}

const MODULES = VERSIONS.map(load_version);

const SUPERSTORE_ARROW = fs.readFileSync(
require.resolve("superstore-arrow/superstore.arrow")
).buffer;

const ITERATIONS = 10;
const WARM_UP_ITERATIONS = 1;

Object.defineProperty(Array.prototype, "push_if", {
value: function (x) {
if (x !== undefined) {
this.push(x);
}
},
});

Object.defineProperty(Array.prototype, "sum", {
value: function () {
return this.reduce((x, y) => x + y, 0);
},
});

async function benchmark({ name, before, before_all, test, after, after_all }) {
let obs_records = [];
console.log(`${name}`);
for (let j = 0; j < MODULES.length; j++) {
let { version, perspective } = MODULES[j];
perspective.version = version.split(".").map((x) => parseInt(x));
const args = [];
args.push_if(await before_all?.(perspective));
const observations = [];
for (let i = 0; i < ITERATIONS + WARM_UP_ITERATIONS; i++) {
const args2 = args.slice();
args2.push_if(await before?.(perspective, ...args2));
const start = microtime.now();
args2.push_if(await test(perspective, ...args2));
if (i >= WARM_UP_ITERATIONS) {
observations.push(microtime.now() - start);
}

await after?.(perspective, ...args2);
}

const avg = observations.sum() / observations.length / 1000;
console.log(` - ${version} ${avg.toFixed(3)}ms`);
await after_all?.(perspective, ...args);

obs_records = obs_records.concat(
observations.map((obs) => ({
version: version,
version_idx: j,
time: obs,
benchmark: name,
}))
);
}
/**
* Convert a list to arrow and write it to disk. `@finos/perspective` is
* imported in this scope to prevent interpreter-wide side effects of the
* library from impacting forked processes, based on an observation that some
* runs inline had anomalies across many observations that couldn't be explained
* by contemporary system load.
* @param {Array} obs_records an array of records to persist
*/
async function persist_to_arrow(obs_records) {
const psp = require("@finos/perspective");
const table = await psp.table({
version: "string",
time: "float",
version_idx: "integer",
benchmark: "string",
});

const table = await OBS_TABLE;
const view = await OBS_VIEW;
const view = await table.view();
await table.update(obs_records);
const arrow = await view.to_arrow();
fs.writeFileSync(
Expand All @@ -94,124 +38,46 @@ async function benchmark({ name, before, before_all, test, after, after_all }) {
);
}

const OBS_TABLE = psp.table({
version: "string",
time: "float",
version_idx: "integer",
benchmark: "string",
});

const OBS_VIEW = OBS_TABLE.then((table) => table.view());

async function to_data_suite() {
async function before_all(perspective) {
const table = await perspective.table(SUPERSTORE_ARROW.slice());
const view = await table.view();
return { table, view };
}

async function after_all(perspective, { table, view }) {
await view.delete();
await table.delete();
}

await benchmark({
name: `.to_arrow()`,
before_all,
after_all,
async test(_perspective, { view }) {
const _arrow = await view.to_arrow();
},
});

await benchmark({
name: `.to_csv()`,
before_all,
after_all,
async test(_perspective, { view }) {
const _csv = await view.to_csv();
},
/**
* Run the benchmarks in a forked process and colelct the observations.
* @param {{version: string, i: number}} version the versions spec to send to
* the child process.
* @returns an array of observation records for this version.
*/
async function benchmark_version(version) {
let obs_records = [];
const worker = cp.fork("./src/js/worker.js");
let cont;
worker.on("message", (details) => {
if (details.finished) {
cont();
} else {
obs_records = obs_records.concat(details.obs_records);
}
});

await benchmark({
name: `.to_columns()`,
before_all,
after_all,
async test(_perspective, { view }) {
const _columns = await view.to_columns();
},
worker.send(version);
await new Promise((r) => {
cont = r;
});

await benchmark({
name: `.to_json()`,
before_all,
after_all,
async test(_perspective, { view }) {
const _json = await view.to_json();
},
});
worker.kill();
return obs_records;
}

async function view_suite() {
async function before_all(perspective) {
const table = await perspective.table(SUPERSTORE_ARROW.slice());
for (let i = 0; i < 4; i++) {
await table.update(SUPERSTORE_ARROW.slice());
}

return table;
}

async function after_all(perspective, table) {
await table.delete();
}
async function main() {
const pkg_path = path.join(__dirname, "../../package.json");
const pkg_json = fs.readFileSync(pkg_path);
const pkg_deps = Object.keys(JSON.parse(pkg_json).dependencies);
const versions = ["@finos/perspective", ...pkg_deps];

async function after(perspective, table, view) {
await view.delete();
let obs_records = [];
for (let i = 0; i < versions.length; i++) {
const batch = await benchmark_version({ path: versions[i], i });
obs_records = obs_records.concat(batch);
}

await benchmark({
name: `.view()`,
before_all,
after_all,
after,
async test(_perspective, table) {
return await table.view();
},
});

await benchmark({
name: `.view({group_by})`,
before_all,
after_all,
after,
async test(perspective, table) {
const [M, m, _] = perspective.version;
if ((M === 1 && m >= 2) || M === 2) {
return await table.view({ group_by: ["Product Name"] });
} else {
return await table.view({ row_pivots: ["Product Name"] });
}
},
});
}

async function table_suite() {
await benchmark({
name: `.table(arrow)`,
async after(_perspective, table) {
await table.delete();
},
async test(perspective) {
return await perspective.table(SUPERSTORE_ARROW.slice());
},
});
}

async function bench_all() {
await to_data_suite();
await view_suite();
await table_suite();
await persist_to_arrow(obs_records);
}

bench_all();
main();
Loading

0 comments on commit d3c6d91

Please sign in to comment.