Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!(runner): support concurrent suites #5491

Merged
merged 17 commits into from
May 14, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions packages/runner/src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
else {
for (let tasksGroup of partitionSuiteChildren(suite)) {
if (tasksGroup[0].concurrent === true) {
const mutex = limit(runner.config.maxConcurrency)
await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
await Promise.all(tasksGroup.map(c => runSuiteChild(c, runner)))
}
else {
const { sequence } = runner.config
Expand Down Expand Up @@ -379,15 +378,19 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
}
}

let limitMaxConcurrency: ReturnType<typeof limit>

async function runSuiteChild(c: Task, runner: VitestRunner) {
if (c.type === 'test' || c.type === 'custom')
return runTest(c, runner)
return limitMaxConcurrency(() => runTest(c, runner))

else if (c.type === 'suite')
return runSuite(c, runner)
}

export async function runFiles(files: File[], runner: VitestRunner) {
limitMaxConcurrency ??= limit(runner.config.maxConcurrency)

for (const file of files) {
if (!file.tasks.length && !runner.config.passWithNoTests) {
if (!file.result?.errors?.length) {
Expand Down
1 change: 1 addition & 0 deletions packages/runner/src/suite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
tasks: [],
meta: Object.create(null),
projectName: '',
concurrent: suiteOptions?.concurrentSuite,
}

if (runner && includeLocation && runner.config.includeTaskLocation) {
Expand Down
4 changes: 4 additions & 0 deletions packages/runner/src/types/tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,10 @@ export interface TestOptions {
* Tests inherit `concurrent` from `describe()` and nested `describe()` will inherit from parent's `concurrent`.
*/
concurrent?: boolean
/**
* TODO: suite only options?
*/
concurrentSuite?: boolean
sheremet-va marked this conversation as resolved.
Show resolved Hide resolved
/**
* Whether tests run sequentially.
* Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`.
Expand Down
1 change: 1 addition & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

41 changes: 41 additions & 0 deletions test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { describe, test, vi } from 'vitest'
import { createDefer } from 'vitest/dist/utils.js'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is it importing from dist 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, I didn't notice but it seems typescript got that one from autocomplete.
I installed @vitest/utils and it looks okay now.


// 3 tests depend on each other,
// so they will deadlock when maxConcurrency < 3
//
// [a] [b] [c]
// * ->
// * ->
// <- *
// <------

vi.setConfig({ maxConcurrency: 2 })

describe('wrapper', { concurrent: true, timeout: 500 }, () => {
const defers = [
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
]

describe('1st suite', () => {
test('a', async () => {
defers[0].resolve()
await defers[2]
})

test('b', async () => {
await defers[0]
defers[1].resolve()
await defers[2]
})
})

describe('2nd suite', () => {
test('c', async () => {
await defers[1]
defers[2].resolve()
})
})
})
37 changes: 37 additions & 0 deletions test/cli/fixtures/fails/concurrent-test-deadlock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, test, vi } from 'vitest'
import { createDefer } from 'vitest/dist/utils.js'

// 3 tests depend on each other,
// so they will deadlock when maxConcurrency < 3
//
// [a] [b] [c]
// * ->
// * ->
// <- *
// <------

vi.setConfig({ maxConcurrency: 2 })

describe('wrapper', { concurrent: true, timeout: 500 }, () => {
const defers = [
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
]

test('a', async () => {
defers[0].resolve()
await defers[2]
})

test('b', async () => {
await defers[0]
defers[1].resolve()
await defers[2]
})

test('c', async () => {
await defers[1]
defers[2].resolve()
})
})
4 changes: 4 additions & 0 deletions test/cli/test/__snapshots__/fails.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;

exports[`should fail concurrent-suite-deadlock.test.ts > concurrent-suite-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;

exports[`should fail concurrent-test-deadlock.test.ts > concurrent-test-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;

exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;

exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;
Expand Down
131 changes: 131 additions & 0 deletions test/core/test/concurrent-suite.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import { createDefer } from '@vitest/utils'
import { afterAll, describe, expect, test } from 'vitest'

describe('basic', () => {
const defers = [
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
]

afterAll(async () => {
await defers[3]
})

describe('1st suite', { concurrentSuite: true }, () => {
test('0', async () => {
defers[0].resolve()
})

test('1', async () => {
await defers[2] // this would deadlock if sequential
defers[1].resolve()
})
})

describe('2nd suite', { concurrentSuite: true }, () => {
test('2', async () => {
await defers[0]
defers[2].resolve()
})
test('3', async () => {
await defers[1]
defers[3].resolve()
})
})
})

describe('inherits option', { concurrentSuite: true }, () => {
const defers = [
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
]

afterAll(async () => {
await defers[3]
})

describe('1st suite', () => {
test('0', async () => {
defers[0].resolve()
})

test('1', async () => {
await defers[2] // this would deadlock if sequential
defers[1].resolve()
})
})

describe('2nd suite', () => {
test('2', async () => {
await defers[0]
defers[2].resolve()
})
test('3', async () => {
await defers[1]
defers[3].resolve()
})
})
})

describe('works with describe.each', () => {
const defers = [
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
createDefer<void>(),
]

afterAll(async () => {
await defers[3]
})

describe.each(['1st suite', '2nd suite'])('%s', { concurrentSuite: true }, (s) => {
if (s === '1st suite') {
test('0', async () => {
defers[0].resolve()
})

test('1', async () => {
await defers[2] // this would deadlock if sequential
defers[1].resolve()
})
}

if (s === '2nd suite') {
test('2', async () => {
await defers[0]
defers[2].resolve()
})
test('3', async () => {
await defers[1]
defers[3].resolve()
})
}
})
})

const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))

describe('tests are sequential', () => {
describe('1st suite', { concurrentSuite: true }, () => {
let done = false

test('0', async () => {
await sleep(200)
expect(done).toBe(false)
})

test('1', async () => {
await sleep(100)
done = true
})

test('2', () => {
expect(done).toBe(true)
})
})
})
Loading