From 1dd47b928d31b59c42ddb18309ea70d8e8f6f11d Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 10:11:01 +0900 Subject: [PATCH 1/7] fix(benchmark): run benchmark suites sequentially --- packages/vitest/src/node/config.ts | 3 ++- packages/vitest/src/runtime/runners/benchmark.ts | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 9d1c96a18fa3..26d82b7fa1c1 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -119,7 +119,8 @@ export function resolveConfig( if (resolved.minWorkers) resolved.minWorkers = Number(resolved.minWorkers) - resolved.fileParallelism ??= true + // run benchmark sequentially by default + resolved.fileParallelism ??= mode !== 'benchmark' if (!resolved.fileParallelism) { // ignore user config, parallelism cannot be implemented without limiting workers diff --git a/packages/vitest/src/runtime/runners/benchmark.ts b/packages/vitest/src/runtime/runners/benchmark.ts index 87838b007153..a72a6a232ef1 100644 --- a/packages/vitest/src/runtime/runners/benchmark.ts +++ b/packages/vitest/src/runtime/runners/benchmark.ts @@ -35,8 +35,9 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { benchmarkSuiteGroup.push(task) } - if (benchmarkSuiteGroup.length) - await Promise.all(benchmarkSuiteGroup.map(subSuite => runBenchmarkSuite(subSuite, runner))) + // run sub suites sequentially + for (const subSuite of benchmarkSuiteGroup) + await runBenchmarkSuite(subSuite, runner) if (benchmarkGroup.length) { const defer = createDefer() From 9016b3f6a27712ed05551e437d28093e6473d10a Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 11:07:57 +0900 Subject: [PATCH 2/7] test: add test --- pnpm-lock.yaml | 59 ++++++++++--------- .../fixtures/sequential/f1.bench.ts | 26 ++++++++ .../fixtures/sequential/f2.bench.ts | 9 +++ test/benchmark2/fixtures/sequential/helper.ts | 11 ++++ test/benchmark2/fixtures/sequential/setup.ts | 6 ++ .../fixtures/sequential/vitest.config.ts | 7 +++ test/benchmark2/package.json | 11 ++++ .../__snapshots__/sequential.test.ts.snap | 20 +++++++ test/benchmark2/test/sequential.test.ts | 11 ++++ test/benchmark2/vitest.config.ts | 3 + 10 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 test/benchmark2/fixtures/sequential/f1.bench.ts create mode 100644 test/benchmark2/fixtures/sequential/f2.bench.ts create mode 100644 test/benchmark2/fixtures/sequential/helper.ts create mode 100644 test/benchmark2/fixtures/sequential/setup.ts create mode 100644 test/benchmark2/fixtures/sequential/vitest.config.ts create mode 100644 test/benchmark2/package.json create mode 100644 test/benchmark2/test/__snapshots__/sequential.test.ts.snap create mode 100644 test/benchmark2/test/sequential.test.ts create mode 100644 test/benchmark2/vitest.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ee182d27ca4..97bf82c97a65 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1538,6 +1538,12 @@ importers: specifier: workspace:* version: link:../../packages/vitest + test/benchmark2: + devDependencies: + vitest: + specifier: workspace:* + version: link:../../packages/vitest + test/browser: devDependencies: '@vitejs/plugin-basic-ssl': @@ -5810,7 +5816,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 jest-message-util: 27.5.1 jest-util: 27.5.1 @@ -5831,7 +5837,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.8.1 @@ -5868,7 +5874,7 @@ packages: dependencies: '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 dev: true @@ -5885,7 +5891,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@sinonjs/fake-timers': 8.1.0 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-message-util: 27.5.1 jest-mock: 27.5.1 jest-util: 27.5.1 @@ -5914,7 +5920,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -6045,7 +6051,7 @@ packages: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.11.27 + '@types/node': 20.11.30 '@types/yargs': 16.0.7 chalk: 4.1.2 dev: true @@ -9391,7 +9397,7 @@ packages: resolution: {integrity: sha512-N7UDG0/xiPQa2D/XrVJXjkWbpqHCd2sBaB32ggRF2l83RhPfamgKGF8gwwqyksS95qUS5ZYF9aF+lLPRlwI2UA==} dependencies: '@types/connect': 3.4.37 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/braces@3.0.1: @@ -9412,7 +9418,7 @@ packages: /@types/connect@3.4.37: resolution: {integrity: sha512-zBUSRqkfZ59OcwXon4HVxhx5oWCJmc0OtBTK05M+p0dYjgN6iTwIL2T/WbsQZrEsdnwaF9cWQ+azOnpPvIqY3Q==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/cookie@0.4.1: @@ -9479,7 +9485,7 @@ packages: /@types/express-serve-static-core@4.17.39: resolution: {integrity: sha512-BiEUfAiGCOllomsRAZOiMFP7LAnrifHpt56pc4Z7l9K6ACyN06Ns1JLMBxwkfLOjJRlSf06NwWsT7yzfpaVpyQ==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 '@types/qs': 6.9.9 '@types/range-parser': 1.2.6 '@types/send': 0.17.3 @@ -9540,7 +9546,7 @@ packages: /@types/graceful-fs@4.1.8: resolution: {integrity: sha512-NhRH7YzWq8WiNKVavKPBmtLYZHxNY19Hh+az28O/phfp68CF45pMFud+ZzJ8ewnxnC5smIdF3dqFeiSUQ5I+pw==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/hast@2.3.4: @@ -9700,7 +9706,7 @@ packages: /@types/node-fetch@2.6.7: resolution: {integrity: sha512-lX17GZVpJ/fuCjguZ5b3TjEbSENxmEk1B2z02yoXSK9WMEWRivhdSY73wWMn6bpcCDAOh6qAdktpKHIlkDk2lg==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 form-data: 4.0.0 dev: true @@ -9727,8 +9733,8 @@ packages: undici-types: 5.26.5 dev: true - /@types/node@20.11.27: - resolution: {integrity: sha512-qyUZfMnCg1KEz57r7pzFtSGt49f6RPkPBis3Vo4PbS7roQEDn22hiHzl/Lo1q4i4hDEgBJmBF/NTNg2XR0HbFg==} + /@types/node@20.11.30: + resolution: {integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw==} dependencies: undici-types: 5.26.5 dev: true @@ -9892,7 +9898,7 @@ packages: /@types/resolve@1.17.1: resolution: {integrity: sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/resolve@1.20.2: @@ -9910,7 +9916,7 @@ packages: resolution: {integrity: sha512-/7fKxvKUoETxjFUsuFlPB9YndePpxxRAOfGC/yJdc9kTjTeP5kRCTzfnE8kPUKCeyiyIZu0YQ76s50hCedI1ug==} dependencies: '@types/mime': 1.3.4 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/serve-static@1.15.4: @@ -9918,7 +9924,7 @@ packages: dependencies: '@types/http-errors': 2.0.3 '@types/mime': 3.0.3 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/set-cookie-parser@2.4.2: @@ -18937,7 +18943,7 @@ packages: '@jest/environment': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 co: 4.6.0 dedent: 0.7.0 @@ -19072,7 +19078,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 jest-util: 27.5.1 jsdom: 16.7.0 @@ -19090,7 +19096,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 jest-util: 27.5.1 dev: true @@ -19134,7 +19140,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@types/graceful-fs': 4.1.8 - '@types/node': 20.11.27 + '@types/node': 20.11.30 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -19174,7 +19180,7 @@ packages: '@jest/source-map': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 co: 4.6.0 expect: 27.5.1 @@ -19254,7 +19260,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /jest-pnp-resolver@1.2.3(jest-resolve@27.5.1): @@ -19315,7 +19321,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 emittery: 0.8.1 graceful-fs: 4.2.11 @@ -19380,7 +19386,7 @@ packages: resolution: {integrity: sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 graceful-fs: 4.2.11 dev: true @@ -19431,7 +19437,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -19468,7 +19474,7 @@ packages: dependencies: '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 ansi-escapes: 4.3.2 chalk: 4.1.2 jest-util: 27.5.1 @@ -27591,7 +27597,6 @@ packages: /workbox-google-analytics@7.0.0: resolution: {integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg==} - deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained dependencies: workbox-background-sync: 7.0.0 workbox-core: 7.0.0 diff --git a/test/benchmark2/fixtures/sequential/f1.bench.ts b/test/benchmark2/fixtures/sequential/f1.bench.ts new file mode 100644 index 000000000000..ade39b672687 --- /dev/null +++ b/test/benchmark2/fixtures/sequential/f1.bench.ts @@ -0,0 +1,26 @@ +import { bench, describe } from "vitest" +import { appendLog, benchOptions, sleep } from "./helper"; + +bench("B1", async () => { + await appendLog("F1 / B1") + await sleep(10); +}, benchOptions) + +describe("S1", () => { + bench("B1", async () => { + await appendLog("F1 / S1 / B1") + await sleep(10); + }, benchOptions) + + bench("B2", async () => { + await appendLog("F1 / S1 / B2") + await sleep(10); + }, benchOptions) +}) + +describe("S2", () => { + bench("B1", async () => { + await appendLog("F1 / S2 / B1") + await sleep(10); + }, benchOptions) +}) diff --git a/test/benchmark2/fixtures/sequential/f2.bench.ts b/test/benchmark2/fixtures/sequential/f2.bench.ts new file mode 100644 index 000000000000..8e5762bdca4f --- /dev/null +++ b/test/benchmark2/fixtures/sequential/f2.bench.ts @@ -0,0 +1,9 @@ +import { bench, describe } from "vitest" +import { appendLog, benchOptions, sleep } from "./helper"; + +describe("S1", () => { + bench("B1", async () => { + await appendLog("F2 / S1 / B1") + await sleep(10); + }, benchOptions) +}) diff --git a/test/benchmark2/fixtures/sequential/helper.ts b/test/benchmark2/fixtures/sequential/helper.ts new file mode 100644 index 000000000000..209740e421b4 --- /dev/null +++ b/test/benchmark2/fixtures/sequential/helper.ts @@ -0,0 +1,11 @@ +import fs from "node:fs"; + +export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +export const testLogFile = new URL("./test.log", import.meta.url); + +export async function appendLog(data: string) { + await fs.promises.appendFile(testLogFile, data + "\n"); +} + +export const benchOptions = { time: 0, iterations: 3, warmupIterations: 0, warmupTime: 0 } diff --git a/test/benchmark2/fixtures/sequential/setup.ts b/test/benchmark2/fixtures/sequential/setup.ts new file mode 100644 index 000000000000..9a929fa54cad --- /dev/null +++ b/test/benchmark2/fixtures/sequential/setup.ts @@ -0,0 +1,6 @@ +import fs from "node:fs"; +import { testLogFile } from "./helper"; + +export default async function setup() { + await fs.promises.rm(testLogFile, { force: true }); +} diff --git a/test/benchmark2/fixtures/sequential/vitest.config.ts b/test/benchmark2/fixtures/sequential/vitest.config.ts new file mode 100644 index 000000000000..50eef9c93888 --- /dev/null +++ b/test/benchmark2/fixtures/sequential/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + globalSetup: ["./setup.ts"] + } +}); diff --git a/test/benchmark2/package.json b/test/benchmark2/package.json new file mode 100644 index 000000000000..5566080b1da6 --- /dev/null +++ b/test/benchmark2/package.json @@ -0,0 +1,11 @@ +{ + "name": "@vitest/benchmark-sequential", + "type": "module", + "private": true, + "scripts": { + "test": "vitest" + }, + "devDependencies": { + "vitest": "workspace:*" + } +} diff --git a/test/benchmark2/test/__snapshots__/sequential.test.ts.snap b/test/benchmark2/test/__snapshots__/sequential.test.ts.snap new file mode 100644 index 000000000000..6c7ec14b6d45 --- /dev/null +++ b/test/benchmark2/test/__snapshots__/sequential.test.ts.snap @@ -0,0 +1,20 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sequential 1`] = ` +"F1 / S1 / B1 +F1 / S1 / B1 +F1 / S1 / B1 +F1 / S1 / B2 +F1 / S1 / B2 +F1 / S1 / B2 +F1 / S2 / B1 +F1 / S2 / B1 +F1 / S2 / B1 +F1 / B1 +F1 / B1 +F1 / B1 +F2 / S1 / B1 +F2 / S1 / B1 +F2 / S1 / B1 +" +`; diff --git a/test/benchmark2/test/sequential.test.ts b/test/benchmark2/test/sequential.test.ts new file mode 100644 index 000000000000..4106d83498cc --- /dev/null +++ b/test/benchmark2/test/sequential.test.ts @@ -0,0 +1,11 @@ +import fs from 'node:fs' +import { expect, it } from 'vitest' +import * as pathe from 'pathe' +import { runVitest } from '../../test-utils' + +it('sequential', async () => { + const root = pathe.join(import.meta.dirname, '../fixtures/sequential') + await runVitest({ root }, [], 'benchmark') + const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8') + expect(testLog).toMatchSnapshot() +}) diff --git a/test/benchmark2/vitest.config.ts b/test/benchmark2/vitest.config.ts new file mode 100644 index 000000000000..abed6b2116e1 --- /dev/null +++ b/test/benchmark2/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) From 1c7e5c501481d545e932adef5f5969e070bccb27 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 11:16:53 +0900 Subject: [PATCH 3/7] test: refactor --- test/benchmark2/fixtures/sequential/f1.bench.ts | 10 +++++----- test/benchmark2/fixtures/sequential/f2.bench.ts | 4 ++-- test/benchmark2/fixtures/sequential/helper.ts | 12 ++++++++++-- test/benchmark2/fixtures/sequential/vitest.config.ts | 3 +++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/test/benchmark2/fixtures/sequential/f1.bench.ts b/test/benchmark2/fixtures/sequential/f1.bench.ts index ade39b672687..acf940fae47b 100644 --- a/test/benchmark2/fixtures/sequential/f1.bench.ts +++ b/test/benchmark2/fixtures/sequential/f1.bench.ts @@ -1,26 +1,26 @@ import { bench, describe } from "vitest" -import { appendLog, benchOptions, sleep } from "./helper"; +import { appendLog, benchOptions, sleepBench } from "./helper"; bench("B1", async () => { await appendLog("F1 / B1") - await sleep(10); + await sleepBench(); }, benchOptions) describe("S1", () => { bench("B1", async () => { await appendLog("F1 / S1 / B1") - await sleep(10); + await sleepBench(); }, benchOptions) bench("B2", async () => { await appendLog("F1 / S1 / B2") - await sleep(10); + await sleepBench(); }, benchOptions) }) describe("S2", () => { bench("B1", async () => { await appendLog("F1 / S2 / B1") - await sleep(10); + await sleepBench(); }, benchOptions) }) diff --git a/test/benchmark2/fixtures/sequential/f2.bench.ts b/test/benchmark2/fixtures/sequential/f2.bench.ts index 8e5762bdca4f..3968708f7a8b 100644 --- a/test/benchmark2/fixtures/sequential/f2.bench.ts +++ b/test/benchmark2/fixtures/sequential/f2.bench.ts @@ -1,9 +1,9 @@ import { bench, describe } from "vitest" -import { appendLog, benchOptions, sleep } from "./helper"; +import { appendLog, benchOptions, sleepBench } from "./helper"; describe("S1", () => { bench("B1", async () => { await appendLog("F2 / S1 / B1") - await sleep(10); + await sleepBench(); }, benchOptions) }) diff --git a/test/benchmark2/fixtures/sequential/helper.ts b/test/benchmark2/fixtures/sequential/helper.ts index 209740e421b4..b609d7c9375c 100644 --- a/test/benchmark2/fixtures/sequential/helper.ts +++ b/test/benchmark2/fixtures/sequential/helper.ts @@ -1,6 +1,9 @@ import fs from "node:fs"; -export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) +const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10); +const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3); + +export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS)) export const testLogFile = new URL("./test.log", import.meta.url); @@ -8,4 +11,9 @@ export async function appendLog(data: string) { await fs.promises.appendFile(testLogFile, data + "\n"); } -export const benchOptions = { time: 0, iterations: 3, warmupIterations: 0, warmupTime: 0 } +export const benchOptions = { + time: 0, + iterations: BENCH_ITERATIONS, + warmupIterations: 0, + warmupTime: 0, +} diff --git a/test/benchmark2/fixtures/sequential/vitest.config.ts b/test/benchmark2/fixtures/sequential/vitest.config.ts index 50eef9c93888..3be3d3371ac6 100644 --- a/test/benchmark2/fixtures/sequential/vitest.config.ts +++ b/test/benchmark2/fixtures/sequential/vitest.config.ts @@ -1,5 +1,8 @@ import { defineConfig } from "vitest/config" +// to see the difference better, increase sleep time and iterations e.g. by +// SLEEP_BENCH_MS=100 pnpm -C test/benchmark2 test bench -- --root fixtures/sequential --fileParallelism + export default defineConfig({ test: { globalSetup: ["./setup.ts"] From b5e299fca408ff098bbad439230740b8580f0de8 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 12:16:20 +0900 Subject: [PATCH 4/7] test: move to benchmark2 --- .gitignore | 2 +- test/benchmark2/fixtures/basic/base.bench.ts | 59 +++ test/benchmark2/fixtures/basic/bench.json | 388 ++++++++++++++++++ test/benchmark2/fixtures/basic/mode.bench.ts | 15 + test/benchmark2/fixtures/basic/only.bench.ts | 76 ++++ .../fixtures/basic/vitest.config.ts | 3 + test/benchmark2/test/basic.test.ts | 38 ++ test/benchmark2/vitest.config.ts | 6 +- 8 files changed, 585 insertions(+), 2 deletions(-) create mode 100644 test/benchmark2/fixtures/basic/base.bench.ts create mode 100644 test/benchmark2/fixtures/basic/bench.json create mode 100644 test/benchmark2/fixtures/basic/mode.bench.ts create mode 100644 test/benchmark2/fixtures/basic/only.bench.ts create mode 100644 test/benchmark2/fixtures/basic/vitest.config.ts create mode 100644 test/benchmark2/test/basic.test.ts diff --git a/.gitignore b/.gitignore index e6fecff012e2..67e90a1a3058 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ dist ltex* .DS_Store bench/test/*/*/ -**/benchmark/bench.json +**/bench.json **/browser/browser.json cypress/videos cypress/downloads diff --git a/test/benchmark2/fixtures/basic/base.bench.ts b/test/benchmark2/fixtures/basic/base.bench.ts new file mode 100644 index 000000000000..9d2fc8851478 --- /dev/null +++ b/test/benchmark2/fixtures/basic/base.bench.ts @@ -0,0 +1,59 @@ +import { bench, describe } from 'vitest' + +describe('sort', () => { + bench('normal', () => { + const x = [1, 5, 4, 2, 3] + x.sort((a, b) => { + return a - b + }) + }, { iterations: 5, time: 0 }) + + bench('reverse', () => { + const x = [1, 5, 4, 2, 3] + x.reverse().sort((a, b) => { + return a - b + }) + }, { iterations: 5, time: 0 }) + + // TODO: move to failed tests + // should not be collected + // it('test', () => { + // expect(1 + 1).toBe(3) + // }) +}) + +function timeout(time: number) { + return new Promise((resolve) => { + setTimeout(resolve, time) + }) +} + +describe('timeout', () => { + bench('timeout100', async () => { + await timeout(100) + }, { + setup() { + + }, + teardown() { + + }, + }) + + bench('timeout75', async () => { + await timeout(75) + }) + + bench('timeout50', async () => { + await timeout(50) + }) + + bench('timeout25', async () => { + await timeout(25) + }) + + // TODO: move to failed tests + // test('reduce', () => { + // expect(1 - 1).toBe(2) + // }) +}) diff --git a/test/benchmark2/fixtures/basic/bench.json b/test/benchmark2/fixtures/basic/bench.json new file mode 100644 index 000000000000..51720b4d70e3 --- /dev/null +++ b/test/benchmark2/fixtures/basic/bench.json @@ -0,0 +1,388 @@ +{ + "numTotalTestSuites": 17, + "numTotalTests": 21, + "testResults": { + "sort": [ + { + "name": "normal", + "rank": 2, + "rme": 63.75003460579095, + "samples": [ + 0.00027100000011159864, + 0.00036100000011174416, + 0.0003710000000864966, + 0.0008010000001377193, + 0.0008510000000114815 + ], + "totalTime": 0.00265500000045904, + "min": 0.00027100000011159864, + "max": 0.0008510000000114815, + "hz": 1883239.171049159, + "period": 0.0005310000000918081, + "mean": 0.0005310000000918081, + "variance": 7.434999998950333e-8, + "sd": 0.0002726719640694718, + "sem": 0.00012194260944354383, + "df": 4, + "critical": 2.776, + "moe": 0.00033851268381527764, + "p75": 0.0008010000001377193, + "p99": 0.0008510000000114815, + "p995": 0.0008510000000114815, + "p999": 0.0008510000000114815 + }, + { + "name": "reverse", + "rank": 1, + "rme": 51.05144487442421, + "samples": [ + 0.00026000000002568413, + 0.0003299999998489511, + 0.0004209999999602587, + 0.000450999999884516, + 0.0007319999999708671 + ], + "totalTime": 0.002193999999690277, + "min": 0.00026000000002568413, + "max": 0.0007319999999708671, + "hz": 2278942.570968934, + "period": 0.0004387999999380554, + "mean": 0.0004387999999380554, + "variance": 3.2559700001299255e-8, + "sd": 0.000180443065816615, + "sem": 0.00008069659224688396, + "df": 4, + "critical": 2.776, + "moe": 0.00022401374007734984, + "p75": 0.000450999999884516, + "p99": 0.0007319999999708671, + "p995": 0.0007319999999708671, + "p999": 0.0007319999999708671 + } + ], + "timeout": [ + { + "name": "timeout100", + "rank": 4, + "rme": 0.06773510108852708, + "samples": [ + 100.18396200000007, + 100.1845430000003, + 100.1901640000001, + 100.19649600000002, + 100.19721700000036, + 100.20109400000001, + 100.20602299999973, + 100.2201600000003, + 100.22907699999996, + 100.4975830000003 + ], + "totalTime": 1002.3063190000012, + "min": 100.18396200000007, + "max": 100.4975830000003, + "hz": 9.97698987867998, + "period": 100.23063190000012, + "mean": 100.23063190000012, + "variance": 0.009008309566777879, + "sd": 0.0949121149631483, + "sem": 0.03001384608272968, + "df": 9, + "critical": 2.262, + "moe": 0.06789131983913455, + "p75": 100.2201600000003, + "p99": 100.4975830000003, + "p995": 100.4975830000003, + "p999": 100.4975830000003 + }, + { + "name": "timeout75", + "rank": 3, + "rme": 0.03451459334004675, + "samples": [ + 75.14856899999995, + 75.17159199999969, + 75.17959800000017, + 75.20059700000002, + 75.20394299999998, + 75.21693700000014, + 75.2215470000001, + 75.225414, + 75.24683400000049, + 75.27161100000012 + ], + "totalTime": 752.0866420000007, + "min": 75.14856899999995, + "max": 75.27161100000012, + "hz": 13.296340396908672, + "period": 75.20866420000007, + "mean": 75.20866420000007, + "variance": 0.001316909924630703, + "sd": 0.03628925356948945, + "sem": 0.01147566958669821, + "df": 9, + "critical": 2.262, + "moe": 0.02595796460511135, + "p75": 75.225414, + "p99": 75.27161100000012, + "p995": 75.27161100000012, + "p999": 75.27161100000012 + }, + { + "name": "timeout50", + "rank": 2, + "rme": 0.02195374873672317, + "samples": [ + 50.13126999999986, + 50.1318510000001, + 50.137392000000546, + 50.14171999999962, + 50.14517599999999, + 50.15144800000053, + 50.15398300000015, + 50.158502000000226, + 50.15953300000001, + 50.18201600000066 + ], + "totalTime": 501.4928910000017, + "min": 50.13126999999986, + "max": 50.18201600000066, + "hz": 19.94046212710913, + "period": 50.14928910000017, + "mean": 50.14928910000017, + "variance": 0.00023689818832729718, + "sd": 0.015391497273731921, + "sem": 0.004867218798526497, + "df": 9, + "critical": 2.262, + "moe": 0.011009648922266937, + "p75": 50.158502000000226, + "p99": 50.18201600000066, + "p995": 50.18201600000066, + "p999": 50.18201600000066 + }, + { + "name": "timeout25", + "rank": 1, + "rme": 0.1397838127019474, + "samples": [ + 25.131234000000404, + 25.133326999999554, + 25.137445000000298, + 25.140301000000363, + 25.141693000000487, + 25.143606999999975, + 25.146702000000005, + 25.14819499999976, + 25.15596000000005, + 25.16253300000062, + 25.171509000000697, + 25.183271999999306, + 25.195714999999836, + 25.200943999999254, + 25.205203000000438, + 25.2344379999995, + 25.23504899999989, + 25.249085000000377, + 25.33510699999988, + 25.428031999999803 + ], + "totalTime": 503.8793510000005, + "min": 25.131234000000404, + "max": 25.428031999999803, + "hz": 39.69204127993723, + "period": 25.193967550000025, + "mean": 25.193967550000025, + "variance": 0.0056623724760347036, + "sd": 0.0752487373716975, + "sem": 0.016826129198414447, + "df": 19, + "critical": 2.093, + "moe": 0.03521708841228144, + "p75": 25.205203000000438, + "p99": 25.428031999999803, + "p995": 25.428031999999803, + "p999": 25.428031999999803 + } + ], + "a0": [ + { + "name": "0", + "rank": 1, + "rme": 0, + "samples": [ + 0.00024999999993724487 + ], + "totalTime": 0.00024999999993724487, + "min": 0.00024999999993724487, + "max": 0.00024999999993724487, + "hz": 4000000.001004082, + "period": 0.00024999999993724487, + "mean": 0.00024999999993724487, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.00024999999993724487, + "p99": 0.00024999999993724487, + "p995": 0.00024999999993724487, + "p999": 0.00024999999993724487 + } + ], + "c1": [ + { + "name": "1", + "rank": 1, + "rme": 0, + "samples": [ + 0.00029099999994741665 + ], + "totalTime": 0.00029099999994741665, + "min": 0.00029099999994741665, + "max": 0.00029099999994741665, + "hz": 3436426.117459446, + "period": 0.00029099999994741665, + "mean": 0.00029099999994741665, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.00029099999994741665, + "p99": 0.00029099999994741665, + "p995": 0.00029099999994741665, + "p999": 0.00029099999994741665 + } + ], + "a2": [ + { + "name": "2", + "rank": 1, + "rme": 0, + "samples": [ + 0.00022100000001046283 + ], + "totalTime": 0.00022100000001046283, + "min": 0.00022100000001046283, + "max": 0.00022100000001046283, + "hz": 4524886.877613832, + "period": 0.00022100000001046283, + "mean": 0.00022100000001046283, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.00022100000001046283, + "p99": 0.00022100000001046283, + "p995": 0.00022100000001046283, + "p999": 0.00022100000001046283 + } + ], + "b3": [ + { + "name": "3", + "rank": 1, + "rme": 0, + "samples": [ + 0.00022999999987405317 + ], + "totalTime": 0.00022999999987405317, + "min": 0.00022999999987405317, + "max": 0.00022999999987405317, + "hz": 4347826.089337369, + "period": 0.00022999999987405317, + "mean": 0.00022999999987405317, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.00022999999987405317, + "p99": 0.00022999999987405317, + "p995": 0.00022999999987405317, + "p999": 0.00022999999987405317 + } + ], + "b4": [ + { + "name": "4", + "rank": 1, + "rme": 0, + "samples": [ + 0.00024099999995996768 + ], + "totalTime": 0.00024099999995996768, + "min": 0.00024099999995996768, + "max": 0.00024099999995996768, + "hz": 4149377.594050246, + "period": 0.00024099999995996768, + "mean": 0.00024099999995996768, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.00024099999995996768, + "p99": 0.00024099999995996768, + "p995": 0.00024099999995996768, + "p999": 0.00024099999995996768 + } + ], + "": [ + { + "name": "visited", + "rank": 1, + "rme": 0, + "samples": [ + 0.0013429999999061693 + ], + "totalTime": 0.0013429999999061693, + "min": 0.0013429999999061693, + "max": 0.0013429999999061693, + "hz": 744601.6381756266, + "period": 0.0013429999999061693, + "mean": 0.0013429999999061693, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.0013429999999061693, + "p99": 0.0013429999999061693, + "p995": 0.0013429999999061693, + "p999": 0.0013429999999061693 + }, + { + "name": "visited2", + "rank": 2, + "rme": 0, + "samples": [ + 0.0016129999999066058 + ], + "totalTime": 0.0016129999999066058, + "min": 0.0016129999999066058, + "max": 0.0016129999999066058, + "hz": 619962.8022677625, + "period": 0.0016129999999066058, + "mean": 0.0016129999999066058, + "variance": 0, + "sd": 0, + "sem": 0, + "df": 0, + "critical": 12.71, + "moe": 0, + "p75": 0.0016129999999066058, + "p99": 0.0016129999999066058, + "p995": 0.0016129999999066058, + "p999": 0.0016129999999066058 + } + ] + } +} \ No newline at end of file diff --git a/test/benchmark2/fixtures/basic/mode.bench.ts b/test/benchmark2/fixtures/basic/mode.bench.ts new file mode 100644 index 000000000000..0967332de750 --- /dev/null +++ b/test/benchmark2/fixtures/basic/mode.bench.ts @@ -0,0 +1,15 @@ +import { bench, describe } from 'vitest' + +describe.todo('unimplemented suite') + +describe.skip('skipped', () => { + bench('skipped', () => { + throw new Error('should be skipped') + }) + + bench.todo('unimplemented test') +}) + +bench.skip('skipped', () => { + throw new Error('should be skipped') +}) diff --git a/test/benchmark2/fixtures/basic/only.bench.ts b/test/benchmark2/fixtures/basic/only.bench.ts new file mode 100644 index 000000000000..0ecd4faa688e --- /dev/null +++ b/test/benchmark2/fixtures/basic/only.bench.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert' +import { bench, describe, expect } from 'vitest' + +const run = [false, false, false, false, false] + +describe('a0', () => { + bench.only('0', () => { + run[0] = true + }, { iterations: 1, time: 0 }) + bench('s0', () => { + expect(true).toBe(false) + }) +}) + +describe('a1', () => { + describe('b1', () => { + describe('c1', () => { + bench.only('1', () => { + run[1] = true + }, { iterations: 1, time: 0 }) + }) + bench('s1', () => { + expect(true).toBe(false) + }) + }) +}) + +describe.only('a2', () => { + bench('2', () => { + run[2] = true + }, { iterations: 1, time: 0 }) +}) + +bench('s2', () => { + expect(true).toBe(false) +}) + +describe.only('a3', () => { + describe('b3', () => { + bench('3', () => { + run[3] = true + }, { iterations: 1, time: 0 }) + }) + bench.skip('s3', () => { + expect(true).toBe(false) + }) +}) + +describe('a4', () => { + describe.only('b4', () => { + bench('4', () => { + run[4] = true + }, { iterations: 1, time: 0 }) + }) + describe('sb4', () => { + bench('s4', () => { + expect(true).toBe(false) + }) + }) +}) + +bench.only( + 'visited', + () => { + assert.deepEqual(run, [true, true, true, true, true]) + }, + { iterations: 1, time: 0 }, +) + +bench.only( + 'visited2', + () => { + assert.deepEqual(run, [true, true, true, true, true]) + }, + { iterations: 1, time: 0 }, +) diff --git a/test/benchmark2/fixtures/basic/vitest.config.ts b/test/benchmark2/fixtures/basic/vitest.config.ts new file mode 100644 index 000000000000..abed6b2116e1 --- /dev/null +++ b/test/benchmark2/fixtures/basic/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) diff --git a/test/benchmark2/test/basic.test.ts b/test/benchmark2/test/basic.test.ts new file mode 100644 index 000000000000..46021e7483f4 --- /dev/null +++ b/test/benchmark2/test/basic.test.ts @@ -0,0 +1,38 @@ +import fs from 'node:fs' +import { expect, it } from 'vitest' +import * as pathe from 'pathe' +import { runVitest } from '../../test-utils' + +it('basic', { timeout: 60_000 }, async () => { + const root = pathe.join(import.meta.dirname, '../fixtures/basic') + const benchFile = pathe.join(root, 'bench.json') + fs.rmSync(benchFile, { force: true }) + + await runVitest({ + root, + allowOnly: true, + benchmark: { + reporters: 'json', + outputFile: 'bench.json', + }, + }, [], 'benchmark') + + const benchResult = await fs.promises.readFile(benchFile, 'utf-8') + const resultJson = JSON.parse(benchResult) + + expect(Object.keys(resultJson.testResults)).toEqual( + expect.arrayContaining([ + 'sort', + 'timeout', + 'a0', + 'c1', + 'a2', + 'b3', + 'b4', + ]), + ) + + const skipped = ['skip', 's0', 's1', 's2', 's3', 'sb4', 's4', 'unimplemented suite', 'unimplemented test'] + for (const b of skipped) + expect(benchResult).not.toContain(b) +}) diff --git a/test/benchmark2/vitest.config.ts b/test/benchmark2/vitest.config.ts index abed6b2116e1..edda33039507 100644 --- a/test/benchmark2/vitest.config.ts +++ b/test/benchmark2/vitest.config.ts @@ -1,3 +1,7 @@ import { defineConfig } from 'vitest/config' -export default defineConfig({}) +export default defineConfig({ + test: { + fileParallelism: false, + }, +}) From 9fbaf46049a370a9893612f2c65d78ae4e1d162c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 12:18:00 +0900 Subject: [PATCH 5/7] test: move to test/benchmark --- pnpm-lock.yaml | 6 - .../{test => fixtures/basic}/base.bench.ts | 0 .../{test => fixtures/basic}/mode.bench.ts | 0 .../{test => fixtures/basic}/only.bench.ts | 0 .../fixtures/basic/vitest.config.ts | 0 .../fixtures/sequential/f1.bench.ts | 0 .../fixtures/sequential/f2.bench.ts | 0 .../fixtures/sequential/helper.ts | 0 .../fixtures/sequential/setup.ts | 0 .../fixtures/sequential/vitest.config.ts | 0 test/benchmark/package.json | 6 +- test/benchmark/specs/runner.test.mjs | 40 -- .../__snapshots__/sequential.test.ts.snap | 0 .../test/basic.test.ts | 0 .../test/sequential.test.ts | 0 test/benchmark/vitest.config.ts | 20 +- test/benchmark2/fixtures/basic/base.bench.ts | 59 --- test/benchmark2/fixtures/basic/bench.json | 388 ------------------ test/benchmark2/fixtures/basic/mode.bench.ts | 15 - test/benchmark2/fixtures/basic/only.bench.ts | 76 ---- test/benchmark2/package.json | 11 - test/benchmark2/vitest.config.ts | 7 - 22 files changed, 3 insertions(+), 625 deletions(-) rename test/benchmark/{test => fixtures/basic}/base.bench.ts (100%) rename test/benchmark/{test => fixtures/basic}/mode.bench.ts (100%) rename test/benchmark/{test => fixtures/basic}/only.bench.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/basic/vitest.config.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/sequential/f1.bench.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/sequential/f2.bench.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/sequential/helper.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/sequential/setup.ts (100%) rename test/{benchmark2 => benchmark}/fixtures/sequential/vitest.config.ts (100%) delete mode 100644 test/benchmark/specs/runner.test.mjs rename test/{benchmark2 => benchmark}/test/__snapshots__/sequential.test.ts.snap (100%) rename test/{benchmark2 => benchmark}/test/basic.test.ts (100%) rename test/{benchmark2 => benchmark}/test/sequential.test.ts (100%) delete mode 100644 test/benchmark2/fixtures/basic/base.bench.ts delete mode 100644 test/benchmark2/fixtures/basic/bench.json delete mode 100644 test/benchmark2/fixtures/basic/mode.bench.ts delete mode 100644 test/benchmark2/fixtures/basic/only.bench.ts delete mode 100644 test/benchmark2/package.json delete mode 100644 test/benchmark2/vitest.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bed7c7c48030..237ecbf9f03e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1541,12 +1541,6 @@ importers: specifier: workspace:* version: link:../../packages/vitest - test/benchmark2: - devDependencies: - vitest: - specifier: workspace:* - version: link:../../packages/vitest - test/browser: devDependencies: '@vitejs/plugin-basic-ssl': diff --git a/test/benchmark/test/base.bench.ts b/test/benchmark/fixtures/basic/base.bench.ts similarity index 100% rename from test/benchmark/test/base.bench.ts rename to test/benchmark/fixtures/basic/base.bench.ts diff --git a/test/benchmark/test/mode.bench.ts b/test/benchmark/fixtures/basic/mode.bench.ts similarity index 100% rename from test/benchmark/test/mode.bench.ts rename to test/benchmark/fixtures/basic/mode.bench.ts diff --git a/test/benchmark/test/only.bench.ts b/test/benchmark/fixtures/basic/only.bench.ts similarity index 100% rename from test/benchmark/test/only.bench.ts rename to test/benchmark/fixtures/basic/only.bench.ts diff --git a/test/benchmark2/fixtures/basic/vitest.config.ts b/test/benchmark/fixtures/basic/vitest.config.ts similarity index 100% rename from test/benchmark2/fixtures/basic/vitest.config.ts rename to test/benchmark/fixtures/basic/vitest.config.ts diff --git a/test/benchmark2/fixtures/sequential/f1.bench.ts b/test/benchmark/fixtures/sequential/f1.bench.ts similarity index 100% rename from test/benchmark2/fixtures/sequential/f1.bench.ts rename to test/benchmark/fixtures/sequential/f1.bench.ts diff --git a/test/benchmark2/fixtures/sequential/f2.bench.ts b/test/benchmark/fixtures/sequential/f2.bench.ts similarity index 100% rename from test/benchmark2/fixtures/sequential/f2.bench.ts rename to test/benchmark/fixtures/sequential/f2.bench.ts diff --git a/test/benchmark2/fixtures/sequential/helper.ts b/test/benchmark/fixtures/sequential/helper.ts similarity index 100% rename from test/benchmark2/fixtures/sequential/helper.ts rename to test/benchmark/fixtures/sequential/helper.ts diff --git a/test/benchmark2/fixtures/sequential/setup.ts b/test/benchmark/fixtures/sequential/setup.ts similarity index 100% rename from test/benchmark2/fixtures/sequential/setup.ts rename to test/benchmark/fixtures/sequential/setup.ts diff --git a/test/benchmark2/fixtures/sequential/vitest.config.ts b/test/benchmark/fixtures/sequential/vitest.config.ts similarity index 100% rename from test/benchmark2/fixtures/sequential/vitest.config.ts rename to test/benchmark/fixtures/sequential/vitest.config.ts diff --git a/test/benchmark/package.json b/test/benchmark/package.json index 2e72a0df4901..5566080b1da6 100644 --- a/test/benchmark/package.json +++ b/test/benchmark/package.json @@ -1,11 +1,9 @@ { - "name": "@vitest/benchmark", + "name": "@vitest/benchmark-sequential", "type": "module", "private": true, "scripts": { - "test": "node --test specs/* && echo '1'", - "bench:json": "vitest bench --reporter=json", - "bench": "vitest bench" + "test": "vitest" }, "devDependencies": { "vitest": "workspace:*" diff --git a/test/benchmark/specs/runner.test.mjs b/test/benchmark/specs/runner.test.mjs deleted file mode 100644 index 0a53b7ab05a4..000000000000 --- a/test/benchmark/specs/runner.test.mjs +++ /dev/null @@ -1,40 +0,0 @@ -import { existsSync, rmSync } from 'node:fs' -import * as assert from 'node:assert' -import { readFile } from 'node:fs/promises' -import test from 'node:test' -import { startVitest } from 'vitest/node' - -if (existsSync('./bench.json')) - rmSync('./bench.json') - -try { - await startVitest('benchmark', ['base.bench', 'mode.bench', 'only.bench'], { - watch: false, - }) -} -catch (error) { - console.error(error) - process.exit(1) -} - -const benchResult = await readFile('./bench.json', 'utf-8') -const resultJson = JSON.parse(benchResult) - -await test('benchmarks are actually running', async () => { - assert.ok(resultJson.testResults.sort, 'sort is in results') - assert.ok(resultJson.testResults.timeout, 'timeout is in results') - assert.ok(resultJson.testResults.a0, 'a0 is in results') - assert.ok(resultJson.testResults.c1, 'c1 is in results') - assert.ok(resultJson.testResults.a2, 'a2 is in results') - assert.ok(resultJson.testResults.b3, 'b3 is in results') - assert.ok(resultJson.testResults.b4, 'b4 is in results') -}) - -await test('doesn\'t have skipped tests', () => { - assert.doesNotMatch(benchResult, /skip/, 'contains skipped benchmarks') - - const skippedBenches = ['s0', 's1', 's2', 's3', 'sb4', 's4'] - const todoBenches = ['unimplemented suite', 'unimplemented test'] - - assert.ok(skippedBenches.concat(todoBenches).every(b => !benchResult.includes(b)), 'contains skipped benchmarks') -}) diff --git a/test/benchmark2/test/__snapshots__/sequential.test.ts.snap b/test/benchmark/test/__snapshots__/sequential.test.ts.snap similarity index 100% rename from test/benchmark2/test/__snapshots__/sequential.test.ts.snap rename to test/benchmark/test/__snapshots__/sequential.test.ts.snap diff --git a/test/benchmark2/test/basic.test.ts b/test/benchmark/test/basic.test.ts similarity index 100% rename from test/benchmark2/test/basic.test.ts rename to test/benchmark/test/basic.test.ts diff --git a/test/benchmark2/test/sequential.test.ts b/test/benchmark/test/sequential.test.ts similarity index 100% rename from test/benchmark2/test/sequential.test.ts rename to test/benchmark/test/sequential.test.ts diff --git a/test/benchmark/vitest.config.ts b/test/benchmark/vitest.config.ts index 790c4797b8e3..edda33039507 100644 --- a/test/benchmark/vitest.config.ts +++ b/test/benchmark/vitest.config.ts @@ -1,25 +1,7 @@ import { defineConfig } from 'vitest/config' -function noop() {} - export default defineConfig({ test: { - update: false, - allowOnly: true, - benchmark: { - outputFile: './bench.json', - reporters: ['json', { - onInit: noop, - onPathsCollected: noop, - onCollected: noop, - onFinished: noop, - onTaskUpdate: noop, - onTestRemoved: noop, - onWatcherStart: noop, - onWatcherRerun: noop, - onServerRestart: noop, - onUserConsoleLog: noop, - }], - }, + fileParallelism: false, }, }) diff --git a/test/benchmark2/fixtures/basic/base.bench.ts b/test/benchmark2/fixtures/basic/base.bench.ts deleted file mode 100644 index 9d2fc8851478..000000000000 --- a/test/benchmark2/fixtures/basic/base.bench.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('sort', () => { - bench('normal', () => { - const x = [1, 5, 4, 2, 3] - x.sort((a, b) => { - return a - b - }) - }, { iterations: 5, time: 0 }) - - bench('reverse', () => { - const x = [1, 5, 4, 2, 3] - x.reverse().sort((a, b) => { - return a - b - }) - }, { iterations: 5, time: 0 }) - - // TODO: move to failed tests - // should not be collected - // it('test', () => { - // expect(1 + 1).toBe(3) - // }) -}) - -function timeout(time: number) { - return new Promise((resolve) => { - setTimeout(resolve, time) - }) -} - -describe('timeout', () => { - bench('timeout100', async () => { - await timeout(100) - }, { - setup() { - - }, - teardown() { - - }, - }) - - bench('timeout75', async () => { - await timeout(75) - }) - - bench('timeout50', async () => { - await timeout(50) - }) - - bench('timeout25', async () => { - await timeout(25) - }) - - // TODO: move to failed tests - // test('reduce', () => { - // expect(1 - 1).toBe(2) - // }) -}) diff --git a/test/benchmark2/fixtures/basic/bench.json b/test/benchmark2/fixtures/basic/bench.json deleted file mode 100644 index 51720b4d70e3..000000000000 --- a/test/benchmark2/fixtures/basic/bench.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "numTotalTestSuites": 17, - "numTotalTests": 21, - "testResults": { - "sort": [ - { - "name": "normal", - "rank": 2, - "rme": 63.75003460579095, - "samples": [ - 0.00027100000011159864, - 0.00036100000011174416, - 0.0003710000000864966, - 0.0008010000001377193, - 0.0008510000000114815 - ], - "totalTime": 0.00265500000045904, - "min": 0.00027100000011159864, - "max": 0.0008510000000114815, - "hz": 1883239.171049159, - "period": 0.0005310000000918081, - "mean": 0.0005310000000918081, - "variance": 7.434999998950333e-8, - "sd": 0.0002726719640694718, - "sem": 0.00012194260944354383, - "df": 4, - "critical": 2.776, - "moe": 0.00033851268381527764, - "p75": 0.0008010000001377193, - "p99": 0.0008510000000114815, - "p995": 0.0008510000000114815, - "p999": 0.0008510000000114815 - }, - { - "name": "reverse", - "rank": 1, - "rme": 51.05144487442421, - "samples": [ - 0.00026000000002568413, - 0.0003299999998489511, - 0.0004209999999602587, - 0.000450999999884516, - 0.0007319999999708671 - ], - "totalTime": 0.002193999999690277, - "min": 0.00026000000002568413, - "max": 0.0007319999999708671, - "hz": 2278942.570968934, - "period": 0.0004387999999380554, - "mean": 0.0004387999999380554, - "variance": 3.2559700001299255e-8, - "sd": 0.000180443065816615, - "sem": 0.00008069659224688396, - "df": 4, - "critical": 2.776, - "moe": 0.00022401374007734984, - "p75": 0.000450999999884516, - "p99": 0.0007319999999708671, - "p995": 0.0007319999999708671, - "p999": 0.0007319999999708671 - } - ], - "timeout": [ - { - "name": "timeout100", - "rank": 4, - "rme": 0.06773510108852708, - "samples": [ - 100.18396200000007, - 100.1845430000003, - 100.1901640000001, - 100.19649600000002, - 100.19721700000036, - 100.20109400000001, - 100.20602299999973, - 100.2201600000003, - 100.22907699999996, - 100.4975830000003 - ], - "totalTime": 1002.3063190000012, - "min": 100.18396200000007, - "max": 100.4975830000003, - "hz": 9.97698987867998, - "period": 100.23063190000012, - "mean": 100.23063190000012, - "variance": 0.009008309566777879, - "sd": 0.0949121149631483, - "sem": 0.03001384608272968, - "df": 9, - "critical": 2.262, - "moe": 0.06789131983913455, - "p75": 100.2201600000003, - "p99": 100.4975830000003, - "p995": 100.4975830000003, - "p999": 100.4975830000003 - }, - { - "name": "timeout75", - "rank": 3, - "rme": 0.03451459334004675, - "samples": [ - 75.14856899999995, - 75.17159199999969, - 75.17959800000017, - 75.20059700000002, - 75.20394299999998, - 75.21693700000014, - 75.2215470000001, - 75.225414, - 75.24683400000049, - 75.27161100000012 - ], - "totalTime": 752.0866420000007, - "min": 75.14856899999995, - "max": 75.27161100000012, - "hz": 13.296340396908672, - "period": 75.20866420000007, - "mean": 75.20866420000007, - "variance": 0.001316909924630703, - "sd": 0.03628925356948945, - "sem": 0.01147566958669821, - "df": 9, - "critical": 2.262, - "moe": 0.02595796460511135, - "p75": 75.225414, - "p99": 75.27161100000012, - "p995": 75.27161100000012, - "p999": 75.27161100000012 - }, - { - "name": "timeout50", - "rank": 2, - "rme": 0.02195374873672317, - "samples": [ - 50.13126999999986, - 50.1318510000001, - 50.137392000000546, - 50.14171999999962, - 50.14517599999999, - 50.15144800000053, - 50.15398300000015, - 50.158502000000226, - 50.15953300000001, - 50.18201600000066 - ], - "totalTime": 501.4928910000017, - "min": 50.13126999999986, - "max": 50.18201600000066, - "hz": 19.94046212710913, - "period": 50.14928910000017, - "mean": 50.14928910000017, - "variance": 0.00023689818832729718, - "sd": 0.015391497273731921, - "sem": 0.004867218798526497, - "df": 9, - "critical": 2.262, - "moe": 0.011009648922266937, - "p75": 50.158502000000226, - "p99": 50.18201600000066, - "p995": 50.18201600000066, - "p999": 50.18201600000066 - }, - { - "name": "timeout25", - "rank": 1, - "rme": 0.1397838127019474, - "samples": [ - 25.131234000000404, - 25.133326999999554, - 25.137445000000298, - 25.140301000000363, - 25.141693000000487, - 25.143606999999975, - 25.146702000000005, - 25.14819499999976, - 25.15596000000005, - 25.16253300000062, - 25.171509000000697, - 25.183271999999306, - 25.195714999999836, - 25.200943999999254, - 25.205203000000438, - 25.2344379999995, - 25.23504899999989, - 25.249085000000377, - 25.33510699999988, - 25.428031999999803 - ], - "totalTime": 503.8793510000005, - "min": 25.131234000000404, - "max": 25.428031999999803, - "hz": 39.69204127993723, - "period": 25.193967550000025, - "mean": 25.193967550000025, - "variance": 0.0056623724760347036, - "sd": 0.0752487373716975, - "sem": 0.016826129198414447, - "df": 19, - "critical": 2.093, - "moe": 0.03521708841228144, - "p75": 25.205203000000438, - "p99": 25.428031999999803, - "p995": 25.428031999999803, - "p999": 25.428031999999803 - } - ], - "a0": [ - { - "name": "0", - "rank": 1, - "rme": 0, - "samples": [ - 0.00024999999993724487 - ], - "totalTime": 0.00024999999993724487, - "min": 0.00024999999993724487, - "max": 0.00024999999993724487, - "hz": 4000000.001004082, - "period": 0.00024999999993724487, - "mean": 0.00024999999993724487, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.00024999999993724487, - "p99": 0.00024999999993724487, - "p995": 0.00024999999993724487, - "p999": 0.00024999999993724487 - } - ], - "c1": [ - { - "name": "1", - "rank": 1, - "rme": 0, - "samples": [ - 0.00029099999994741665 - ], - "totalTime": 0.00029099999994741665, - "min": 0.00029099999994741665, - "max": 0.00029099999994741665, - "hz": 3436426.117459446, - "period": 0.00029099999994741665, - "mean": 0.00029099999994741665, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.00029099999994741665, - "p99": 0.00029099999994741665, - "p995": 0.00029099999994741665, - "p999": 0.00029099999994741665 - } - ], - "a2": [ - { - "name": "2", - "rank": 1, - "rme": 0, - "samples": [ - 0.00022100000001046283 - ], - "totalTime": 0.00022100000001046283, - "min": 0.00022100000001046283, - "max": 0.00022100000001046283, - "hz": 4524886.877613832, - "period": 0.00022100000001046283, - "mean": 0.00022100000001046283, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.00022100000001046283, - "p99": 0.00022100000001046283, - "p995": 0.00022100000001046283, - "p999": 0.00022100000001046283 - } - ], - "b3": [ - { - "name": "3", - "rank": 1, - "rme": 0, - "samples": [ - 0.00022999999987405317 - ], - "totalTime": 0.00022999999987405317, - "min": 0.00022999999987405317, - "max": 0.00022999999987405317, - "hz": 4347826.089337369, - "period": 0.00022999999987405317, - "mean": 0.00022999999987405317, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.00022999999987405317, - "p99": 0.00022999999987405317, - "p995": 0.00022999999987405317, - "p999": 0.00022999999987405317 - } - ], - "b4": [ - { - "name": "4", - "rank": 1, - "rme": 0, - "samples": [ - 0.00024099999995996768 - ], - "totalTime": 0.00024099999995996768, - "min": 0.00024099999995996768, - "max": 0.00024099999995996768, - "hz": 4149377.594050246, - "period": 0.00024099999995996768, - "mean": 0.00024099999995996768, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.00024099999995996768, - "p99": 0.00024099999995996768, - "p995": 0.00024099999995996768, - "p999": 0.00024099999995996768 - } - ], - "": [ - { - "name": "visited", - "rank": 1, - "rme": 0, - "samples": [ - 0.0013429999999061693 - ], - "totalTime": 0.0013429999999061693, - "min": 0.0013429999999061693, - "max": 0.0013429999999061693, - "hz": 744601.6381756266, - "period": 0.0013429999999061693, - "mean": 0.0013429999999061693, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.0013429999999061693, - "p99": 0.0013429999999061693, - "p995": 0.0013429999999061693, - "p999": 0.0013429999999061693 - }, - { - "name": "visited2", - "rank": 2, - "rme": 0, - "samples": [ - 0.0016129999999066058 - ], - "totalTime": 0.0016129999999066058, - "min": 0.0016129999999066058, - "max": 0.0016129999999066058, - "hz": 619962.8022677625, - "period": 0.0016129999999066058, - "mean": 0.0016129999999066058, - "variance": 0, - "sd": 0, - "sem": 0, - "df": 0, - "critical": 12.71, - "moe": 0, - "p75": 0.0016129999999066058, - "p99": 0.0016129999999066058, - "p995": 0.0016129999999066058, - "p999": 0.0016129999999066058 - } - ] - } -} \ No newline at end of file diff --git a/test/benchmark2/fixtures/basic/mode.bench.ts b/test/benchmark2/fixtures/basic/mode.bench.ts deleted file mode 100644 index 0967332de750..000000000000 --- a/test/benchmark2/fixtures/basic/mode.bench.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { bench, describe } from 'vitest' - -describe.todo('unimplemented suite') - -describe.skip('skipped', () => { - bench('skipped', () => { - throw new Error('should be skipped') - }) - - bench.todo('unimplemented test') -}) - -bench.skip('skipped', () => { - throw new Error('should be skipped') -}) diff --git a/test/benchmark2/fixtures/basic/only.bench.ts b/test/benchmark2/fixtures/basic/only.bench.ts deleted file mode 100644 index 0ecd4faa688e..000000000000 --- a/test/benchmark2/fixtures/basic/only.bench.ts +++ /dev/null @@ -1,76 +0,0 @@ -import assert from 'node:assert' -import { bench, describe, expect } from 'vitest' - -const run = [false, false, false, false, false] - -describe('a0', () => { - bench.only('0', () => { - run[0] = true - }, { iterations: 1, time: 0 }) - bench('s0', () => { - expect(true).toBe(false) - }) -}) - -describe('a1', () => { - describe('b1', () => { - describe('c1', () => { - bench.only('1', () => { - run[1] = true - }, { iterations: 1, time: 0 }) - }) - bench('s1', () => { - expect(true).toBe(false) - }) - }) -}) - -describe.only('a2', () => { - bench('2', () => { - run[2] = true - }, { iterations: 1, time: 0 }) -}) - -bench('s2', () => { - expect(true).toBe(false) -}) - -describe.only('a3', () => { - describe('b3', () => { - bench('3', () => { - run[3] = true - }, { iterations: 1, time: 0 }) - }) - bench.skip('s3', () => { - expect(true).toBe(false) - }) -}) - -describe('a4', () => { - describe.only('b4', () => { - bench('4', () => { - run[4] = true - }, { iterations: 1, time: 0 }) - }) - describe('sb4', () => { - bench('s4', () => { - expect(true).toBe(false) - }) - }) -}) - -bench.only( - 'visited', - () => { - assert.deepEqual(run, [true, true, true, true, true]) - }, - { iterations: 1, time: 0 }, -) - -bench.only( - 'visited2', - () => { - assert.deepEqual(run, [true, true, true, true, true]) - }, - { iterations: 1, time: 0 }, -) diff --git a/test/benchmark2/package.json b/test/benchmark2/package.json deleted file mode 100644 index 5566080b1da6..000000000000 --- a/test/benchmark2/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@vitest/benchmark-sequential", - "type": "module", - "private": true, - "scripts": { - "test": "vitest" - }, - "devDependencies": { - "vitest": "workspace:*" - } -} diff --git a/test/benchmark2/vitest.config.ts b/test/benchmark2/vitest.config.ts deleted file mode 100644 index edda33039507..000000000000 --- a/test/benchmark2/vitest.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - fileParallelism: false, - }, -}) From e34c1b451bbadbd578cdda05a2eb960b007ea73e Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 28 Mar 2024 12:18:58 +0900 Subject: [PATCH 6/7] chore: comment --- test/benchmark/fixtures/sequential/vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/fixtures/sequential/vitest.config.ts b/test/benchmark/fixtures/sequential/vitest.config.ts index 3be3d3371ac6..fe63b73cf264 100644 --- a/test/benchmark/fixtures/sequential/vitest.config.ts +++ b/test/benchmark/fixtures/sequential/vitest.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "vitest/config" // to see the difference better, increase sleep time and iterations e.g. by -// SLEEP_BENCH_MS=100 pnpm -C test/benchmark2 test bench -- --root fixtures/sequential --fileParallelism +// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism export default defineConfig({ test: { From 77bf5a29c0fc52d4c11cb609f31bddbcc8a154d8 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Mon, 1 Apr 2024 11:18:37 +0900 Subject: [PATCH 7/7] chore: cleanup config --- test/benchmark/vitest.config.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/benchmark/vitest.config.ts b/test/benchmark/vitest.config.ts index edda33039507..abed6b2116e1 100644 --- a/test/benchmark/vitest.config.ts +++ b/test/benchmark/vitest.config.ts @@ -1,7 +1,3 @@ import { defineConfig } from 'vitest/config' -export default defineConfig({ - test: { - fileParallelism: false, - }, -}) +export default defineConfig({})