feat(runner)!: support describe(..., { shuffle: boolean }) and inhe…

…rit from parent suite (#6670)

docs/api/index.md

@@ -906,10 +906,23 @@ Vitest provides a way to run all tests in random order via CLI flag [`--sequence
 ```ts
 import { describe, test } from 'vitest'
 
+// or describe('suite', { shuffle: true }, ...)
 describe.shuffle('suite', () => {
   test('random test 1', async () => { /* ... */ })
   test('random test 2', async () => { /* ... */ })
   test('random test 3', async () => { /* ... */ })
+
+  // `shuffle` is inherited
+  describe('still random', () => {
+    test('random 4.1', async () => { /* ... */ })
+    test('random 4.2', async () => { /* ... */ })
+  })
+
+  // disable shuffle inside
+  describe('not random', { shuffle: false }, () => {
+    test('in order 5.1', async () => { /* ... */ })
+    test('in order 5.2', async () => { /* ... */ })
+  })
 })
 // order depends on sequence.seed option in config (Date.now() by default)
 ```

packages/runner/src/collect.ts

@@ -32,6 +32,7 @@ export async function collectTests(
     const testLocations = typeof spec === 'string' ? undefined : spec.testLocations
 
     const file = createFileTask(filepath, config.root, config.name, runner.pool)
+    file.shuffle = config.sequence.shuffle
 
     runner.onCollectStart?.(file)
 

packages/runner/src/run.ts

@@ -412,7 +412,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
           }
           else {
             const { sequence } = runner.config
-            if (sequence.shuffle || suite.shuffle) {
+            if (suite.shuffle) {
               // run describe block independently from tests
               const suites = tasksGroup.filter(
                 group => group.type === 'suite',

packages/runner/src/suite.ts

@@ -207,8 +207,7 @@ export function getRunner(): VitestRunner {
 
 function createDefaultSuite(runner: VitestRunner) {
   const config = runner.config.sequence
-  const api = config.shuffle ? suite.shuffle : suite
-  return api('', { concurrent: config.concurrent }, () => {})
+  return suite('', { concurrent: config.concurrent }, () => {})
 }
 
 export function clearCollectorContext(
@@ -292,7 +291,6 @@ function createSuiteCollector(
   name: string,
   factory: SuiteFactory = () => {},
   mode: RunMode,
-  shuffle?: boolean,
   each?: boolean,
   suiteOptions?: TestOptions,
 ) {
@@ -331,9 +329,7 @@ function createSuiteCollector(
     ) {
       task.concurrent = true
     }
-    if (shuffle) {
-      task.shuffle = true
-    }
+    task.shuffle = suiteOptions?.shuffle
 
     const context = createTestContext(task, runner)
     // create test context
@@ -425,7 +421,7 @@ function createSuiteCollector(
       mode,
       each,
       file: undefined!,
-      shuffle,
+      shuffle: suiteOptions?.shuffle,
       tasks: [],
       meta: Object.create(null),
       concurrent: suiteOptions?.concurrent,
@@ -523,8 +519,10 @@ function createSuite() {
     const isSequentialSpecified = options.sequential || this.sequential || options.concurrent === false
 
     // inherit options from current suite
-    if (currentSuite?.options) {
-      options = { ...currentSuite.options, ...options }
+    options = {
+      ...currentSuite?.options,
+      ...options,
+      shuffle: this.shuffle ?? options.shuffle ?? currentSuite?.options?.shuffle ?? runner?.config.sequence.shuffle,
     }
 
     // inherit concurrent / sequential from suite
@@ -537,7 +535,6 @@ function createSuite() {
       formatName(name),
       factory,
       mode,
-      this.shuffle,
       this.each,
       options,
     )

packages/runner/src/types/tasks.ts

@@ -286,16 +286,16 @@ interface EachFunctionReturn<T extends any[]> {
   (
     name: string | Function,
     fn: (...args: T) => Awaitable<void>,
-    options: TestOptions
+    options: TestCollectorOptions
   ): void
   (
     name: string | Function,
     fn: (...args: T) => Awaitable<void>,
-    options?: number | TestOptions
+    options?: number | TestCollectorOptions
   ): void
   (
     name: string | Function,
-    options: TestOptions,
+    options: TestCollectorOptions,
     fn: (...args: T) => Awaitable<void>
   ): void
 }
@@ -316,7 +316,7 @@ interface TestForFunctionReturn<Arg, Context> {
   ): void
   (
     name: string | Function,
-    options: TestOptions,
+    options: TestCollectorOptions,
     fn: (args: Arg, context: Context) => Awaitable<void>
   ): void
 }
@@ -347,16 +347,16 @@ interface TestCollectorCallable<C = object> {
   <ExtraContext extends C>(
     name: string | Function,
     fn: TestFunction<ExtraContext>,
-    options: TestOptions
+    options: TestCollectorOptions
   ): void
   <ExtraContext extends C>(
     name: string | Function,
     fn?: TestFunction<ExtraContext>,
-    options?: number | TestOptions
+    options?: number | TestCollectorOptions
   ): void
   <ExtraContext extends C>(
     name: string | Function,
-    options?: TestOptions,
+    options?: TestCollectorOptions,
     fn?: TestFunction<ExtraContext>
   ): void
 }
@@ -370,6 +370,8 @@ type ChainableTestAPI<ExtraContext = object> = ChainableFunction<
   }
 >
 
+type TestCollectorOptions = Omit<TestOptions, 'shuffle'>
+
 export interface TestOptions {
   /**
    * Test timeout.
@@ -399,6 +401,10 @@ export interface TestOptions {
    * Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`.
    */
   sequential?: boolean
+  /**
+   * Whether the tasks of the suite run in a random order.
+   */
+  shuffle?: boolean
   /**
    * Whether the test should be skipped.
    */

test/config/fixtures/shuffle/basic.test.ts

@@ -0,0 +1,27 @@
+import { afterAll, describe, expect, test } from 'vitest'
+
+const numbers: number[] = []
+
+test.for([1, 2, 3, 4, 5])('test %s', (v) => {
+  numbers.push(10 + v)
+})
+
+describe("inherit shuffle", () => {
+  test.for([1, 2, 3, 4, 5])('test %s', (v) => {
+    numbers.push(20 + v)
+  })
+})
+
+describe('unshuffle', { shuffle: false }, () => {
+  test.for([1, 2, 3, 4, 5])('test %s', (v) => {
+    numbers.push(30 + v)
+  })
+})
+
+afterAll(() => {
+  expect(numbers).toEqual([
+    11, 14, 13, 15, 12,
+    31, 32, 33, 34, 35,
+    21, 24, 23, 25, 22
+  ])
+})

test/config/fixtures/shuffle/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    sequence: {
+      seed: 101,
+      shuffle: true,
+    }
+  }
+})

test/config/test/shuffle-options.test.ts

@@ -47,3 +47,18 @@ test.each([
   const { ctx } = await run({ shuffle, sequencer: CustomSequencer })
   expect(ctx?.config.sequence.sequencer.name).toBe('CustomSequencer')
 })
+
+test('shuffle', async () => {
+  const { stderr, ctx } = await runVitest({
+    root: './fixtures/shuffle',
+  })
+  expect(stderr).toBe('')
+  expect(ctx?.state.getFiles().map(f => [f.name, f.result?.state])).toMatchInlineSnapshot(`
+    [
+      [
+        "basic.test.ts",
+        "pass",
+      ],
+    ]
+  `)
+})

test/core/test/random.test.ts

@@ -1,18 +1,31 @@
-import { afterAll, describe, expect, test } from 'vitest'
+import { describe, expect, test } from 'vitest'
 
 // tests use seed of 101, so they have deterministic random order
 const numbers: number[] = []
 
 describe.shuffle('random tests', () => {
-  describe('inside', () => {
-    // shuffle is not inherited from parent
-
+  describe('suite unshuffle', { shuffle: false }, () => {
     test('inside 1', () => {
       numbers.push(1)
     })
+    test('inside 1.5', () => {
+      numbers.push(1.5)
+    })
     test('inside 2', () => {
       numbers.push(2)
     })
+
+    describe('suite shuffle', { shuffle: true }, () => {
+      test('inside 2.1', () => {
+        numbers.push(2.1)
+      })
+      test('inside 2.2', () => {
+        numbers.push(2.2)
+      })
+      test('inside 2.3', () => {
+        numbers.push(2.3)
+      })
+    })
   })
 
   test('test 1', () => {
@@ -24,8 +37,20 @@ describe.shuffle('random tests', () => {
   test('test 3', () => {
     numbers.push(5)
   })
+})
 
-  afterAll(() => {
-    expect(numbers).toStrictEqual([4, 5, 3, 1, 2])
-  })
+test('assert', () => {
+  expect(numbers).toMatchInlineSnapshot(`
+      [
+        4,
+        5,
+        3,
+        1,
+        1.5,
+        2,
+        2.2,
+        2.3,
+        2.1,
+      ]
+    `)
 })