mirror of
https://github.com/tester-army/e2e.git
synced 2026-11-21 03:05:26 +08:00
1386 lines
56 KiB
Plaintext
1386 lines
56 KiB
Plaintext
---
|
|
title: test
|
|
description: Tests, setup tests, groups, hooks, options, and fixtures.
|
|
"og:image": "https://e2e.com/docs/images/og/reference/test.png?v=d1ad3b01c3"
|
|
"twitter:image": "https://e2e.com/docs/images/og/reference/test.png?v=d1ad3b01c3"
|
|
---
|
|
|
|
`test` declares tests, setup tests, groups, and hooks. Declarations run
|
|
synchronously while the test file is imported. Use `test.extend` to add
|
|
fixtures and `test.skip` to skip a test at registration or during its body.
|
|
|
|
```ts title="tests/checkout.e2e.ts"
|
|
import { beforeEach, describe, expect, test } from 'e2e';
|
|
|
|
describe('checkout', { tags: ['billing'] }, () => {
|
|
beforeEach(async ({ app }) => {
|
|
await app.open('/checkout');
|
|
});
|
|
|
|
test('pays with a saved card', { retries: 2 }, async ({ screen }) => {
|
|
await screen.getByRole('button', 'Pay').tap();
|
|
await expect(screen.getByText('Payment received')).toBeVisible();
|
|
});
|
|
});
|
|
```
|
|
|
|
## Reference
|
|
|
|
### Members
|
|
|
|
| Member | Returns |
|
|
| --- | --- |
|
|
| [`test`](#test) | `TestCase` |
|
|
| [`test.skip`](#test-skip) | `TestCase \| void` |
|
|
| [`test.skipIf`](#test-skipif) | `TestRegistrar` |
|
|
| [`test.runIf`](#test-runif) | `TestRegistrar` |
|
|
| [`test.each`](#test-each) | `EachRegistrar` |
|
|
| [`test.for`](#test-for) | `EachRegistrar` |
|
|
| [`test.fail`](#test-fail) | `TestCase \| void` |
|
|
| [`test.setTimeout`](#test-settimeout) | `void` |
|
|
| [`test.only`](#test-only) | `TestCase` |
|
|
| [`test.setup`](#test-setup) | `TestCase` |
|
|
| [`test.extend`](#test-extend) | `TestAPI` |
|
|
| [`describe`](#describe) | `void` |
|
|
| [Hooks](#hooks) | `void` |
|
|
|
|
`describe` and the four hooks are also top-level exports, the same functions
|
|
as `test.describe`, `test.beforeEach`, and the rest:
|
|
`import { describe, beforeEach, test } from 'e2e'`.
|
|
|
|
- A per-test hook imported from `e2e` is typed with the [built-in fixtures](#fixtures)
|
|
every engine has.
|
|
- `@e2e-dev/web` and `@e2e-dev/mobile` export `test`, `describe`, and all
|
|
four hooks typed with their `browser` or `device` fixture, so a test file
|
|
registers from one import.
|
|
- A `test` you extended with your own fixtures registers hooks that see them
|
|
through `test.beforeEach`.
|
|
|
|
Registering a test, group, or hook outside module evaluation throws
|
|
`COLLECTION_ERROR`. The full declarations are under [Types](#types).
|
|
|
|
#### `test`
|
|
|
|
Registers one test, with or without [options](#options).
|
|
|
|
```ts
|
|
import { describe, test } from '@e2e-dev/web';
|
|
import { expect } from 'e2e';
|
|
|
|
test('app opens', async ({ app, browser }) => {
|
|
await app.open();
|
|
await expect(browser).toHaveURL('/');
|
|
});
|
|
|
|
test('checks out', { tags: ['billing'], retries: 2, timeout: 60_000 }, async ({ app }) => {
|
|
await app.open('/checkout');
|
|
});
|
|
```
|
|
|
|
A test or group title is 1 through 512 UTF-8 bytes after NFC normalization,
|
|
with no NUL character.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | Two tests in one file share a title path: the group titles, then the test's own. |
|
|
|
|
#### `test.skip`
|
|
|
|
Registers a skipped test, or skips the running test.
|
|
|
|
```ts
|
|
test.skip('exports a PDF', async ({ app }) => {}); // at registration
|
|
test.skip(organizations.length < 2, 'a single organization'); // inside a body
|
|
test.skip('not on this tenant'); // inside a body, always
|
|
```
|
|
|
|
- A skipped test does not affect the exit code by default.
|
|
- To skip at collection by option, use the [`skip`](#skip) option.
|
|
- At registration it returns a `TestCase`; inside a body, `void`.
|
|
|
|
Call `test.skip(condition, reason)` inside a test for a condition you learn
|
|
at runtime, such as a disabled feature flag:
|
|
|
|
```ts
|
|
test('switches organizations', { session: 'admin' }, async ({ app, agent }) => {
|
|
const organizations = await listOrganizations();
|
|
test.skip(organizations.length < 2, 'the demo tenant has a single organization');
|
|
await app.open('/backend');
|
|
await agent.act(`switch to the organization "${organizations[1].name}"`);
|
|
});
|
|
```
|
|
|
|
- When the condition is true, the call throws, as Playwright's does. Nothing
|
|
after the call runs.
|
|
- With no failure before it, the test is reported `skipped` with `reason`,
|
|
with no retry.
|
|
- Steps before it stay in the report as they ran.
|
|
- A failed `expect.soft` before the call still fails the attempt, as in
|
|
Playwright. The attempt is `failed`, with the soft failures as its error and
|
|
the skip reason in its `skip`. It retries like any failure.
|
|
- Hooks and fixtures still run their teardown.
|
|
- When the condition is a precondition, call it first, before any step.
|
|
- A failed attempt followed by a skip on retry leaves the test `skipped`. The
|
|
CLI lists it under `Skipped After Failure` with its original error. Set
|
|
[`failOnSkippedFailure: true`](/reference/config) to fail the run with exit
|
|
code 1 in that case. The default is `false`.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `INVALID_ARGUMENT` | Called in a setup test. A setup test cannot skip because other tests depend on its sessions. |
|
|
| `COLLECTION_ERROR` | The conditional form, outside a test body. |
|
|
|
|
#### `test.skipIf`
|
|
|
|
Returns a function that registers one test, like `test(title, fn)`, skipped
|
|
when the condition is truthy.
|
|
|
|
```ts
|
|
test.skipIf(process.platform === 'win32')('watches files', async ({ app }) => {});
|
|
test.skipIf(process.env.CI)('opens the system dialog', async ({ app }) => {});
|
|
```
|
|
|
|
- The condition is read one time, at the call, while the file is collected.
|
|
Use a value known then, such as `process.platform` or an environment
|
|
variable. For a condition you learn at runtime, call
|
|
[`test.skip(condition, reason)`](#test-skip) inside the body.
|
|
- Any truthy value skips, as in Vitest.
|
|
- A skipped test is the same as `test(title, { skip: true }, fn)`: its body
|
|
never runs, and its other options are kept.
|
|
- A [`skip`](#skip) option skips the test whatever the condition, and its
|
|
reason string is the reported reason.
|
|
- Outside a serial group, a falsy condition registers the test as `test`
|
|
would.
|
|
- It takes the same two forms as [`test`](#test) and has the same fixtures,
|
|
on an extended `test` too.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | Called inside a serial group, whatever the condition. Set [`skip`](#skip) on the serial `describe` instead. |
|
|
| `COLLECTION_ERROR` | The condition is a Promise, which is also a type error. Await it before the call. |
|
|
|
|
#### `test.runIf`
|
|
|
|
Returns a function that registers one test, like `test(title, fn)`, skipped
|
|
when the condition is falsy.
|
|
|
|
```ts
|
|
test.runIf(process.platform === 'linux')('reads inotify events', async ({ app }) => {});
|
|
```
|
|
|
|
`test.runIf(condition)` is `test.skipIf(!condition)`.
|
|
Everything in [`test.skipIf`](#test-skipif) applies.
|
|
|
|
#### `test.each`
|
|
|
|
Returns a function that registers one test per row, like `test(title, fn)`,
|
|
in row order. Use it for the same steps with different data.
|
|
|
|
```ts
|
|
test.each([{ user: 'Ada' }, { user: 'Grace' }])('signs in as $user', async ({ user }, { app }) => {
|
|
await app.open();
|
|
// ...
|
|
});
|
|
|
|
test.each([
|
|
['standard', '5.00'],
|
|
['express', '15.00'],
|
|
])('ships %s for %s', async (speed, price, { screen }) => {});
|
|
```
|
|
|
|
- The body gets the row first and the fixtures last. When every row is an
|
|
array, each row is spread, so `[speed, price]` arrives as
|
|
`(speed, price, fixtures)`. When any row is not an array, every row
|
|
arrives whole, as `(row, fixtures)`, as in Vitest and Jest.
|
|
- The body names the [fixtures](#fixtures) it uses by destructuring the
|
|
parameter after the row, and e2e sets up only those. A body that uses none
|
|
can leave the parameter out.
|
|
- The title is formatted from the row:
|
|
|
|
| In the title | Becomes |
|
|
| --- | --- |
|
|
| `$name`, `$name.path` | That key of an object row. A string shows as written, any other value as Node's `inspect` shows it. |
|
|
| `%s` | The next item, shown the same way. |
|
|
| `%o` | The next item as Node's `inspect` shows it, so a string is quoted. |
|
|
| `%d` | The next item as a number (`Number(item)`), except a `BigInt` keeps its `n` suffix and a `Symbol` shows as `NaN`. |
|
|
| `%i`, `%f` | The next item read as an integer or a float, like `parseInt` / `parseFloat`: `'42px'` becomes `42`. A `BigInt` keeps its `n` suffix under `%i`. |
|
|
| `%j` | The next item as JSON. A circular value shows as `[Circular]`; a function, a `Symbol`, or `undefined` as `undefined`. |
|
|
| `%#`, `%$` | The row index, from 0, and the row number, from 1. |
|
|
| `%%` | A percent sign. |
|
|
|
|
The items are an array row's items in order; any other row is the one
|
|
item. A placeholder with no item left, or `$name` on a row that is not an
|
|
object, stays as written. A value other than a string always shows on one
|
|
line; a string shows as written, line breaks included. An `Error` shows
|
|
as its name and message, without its stack.
|
|
- A title is part of the test id. `%#` and `%$` change when rows are added or
|
|
moved, so prefer `$name` or a value placeholder for ids that stay stable.
|
|
- Titles differ from Vitest in a few places: `$0` and `$1` do not read an
|
|
array row, `$name` is not read from an array row's first item, `%s` shows
|
|
an object with `inspect`, not its own `toString` (a `URL` is not shown as
|
|
its href), `%O` and `%c` are not placeholders, and a table written as a template
|
|
literal is not read. The body differs too:
|
|
Vitest's `test.each` body gets only the row, while here the fixtures
|
|
come after it.
|
|
- Each row is a separate test, with its own id, report entry, and retries.
|
|
Options apply to every row, so a [`skip`](#skip) or [`tags`](#tags)
|
|
option skips or tags them all.
|
|
- It takes the same two forms as [`test`](#test) and has the same fixtures,
|
|
on an extended `test` too.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | The rows are not an array, the array is empty, or the rows are a template literal table. |
|
|
| `COLLECTION_ERROR` | Two rows format to the same title. Put a placeholder in the title that tells the rows apart. |
|
|
| `COLLECTION_ERROR` | A `%j` item cannot be turned into JSON for a reason other than a cycle, such as a `BigInt`. The message names the row. |
|
|
| `COLLECTION_ERROR` | The body takes the fixtures whole (`async (row, fixtures) => {}`) or in a rest parameter (`async (...args) => {}`), or breaks another [fixture naming rule](#fixtures). |
|
|
|
|
#### `test.for`
|
|
|
|
[`test.each`](#test-each), except an array row is not spread: the body
|
|
always takes `(row, fixtures)`. Titles are formatted the same way.
|
|
|
|
```ts
|
|
test.for([
|
|
['standard', '5.00'],
|
|
['express', '15.00'],
|
|
])('ships %s for %s', async ([speed, price], { screen }) => {});
|
|
```
|
|
|
|
#### `test.fail`
|
|
|
|
Registers a test that is expected to fail, or marks the running test.
|
|
|
|
```ts
|
|
test.fail('shows the cart badge', async ({ app, screen }) => {}); // at registration
|
|
test.fail(platform === 'web', 'tap on a display: contents element'); // inside a body
|
|
test.fail('known gap'); // inside a body, always
|
|
test('exports a PDF', { fail: 'PDF export is broken' }, async ({ app }) => {}); // by option
|
|
```
|
|
|
|
- The body keeps running after the call. A `beforeEach` hook can mark the
|
|
test too.
|
|
- A failed assertion, action, thrown error, or timeout makes the test
|
|
`passed`. The report keeps the reason and the error.
|
|
- A pass makes the test `failed` with `UNEXPECTED_PASS`. Remove the mark
|
|
once the cause is fixed.
|
|
- Infrastructure errors, interrupts, a failed `beforeAll` or `afterAll`, and
|
|
API misuse such as `INVALID_ARGUMENT` still fail the test.
|
|
- An expected failure is not retried. An unexpected pass is.
|
|
- A skip wins over the mark.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `INVALID_ARGUMENT` | A true condition in a setup test, a serial group member, an `afterEach` hook, or a fixture teardown. |
|
|
| `COLLECTION_ERROR` | The conditional form, outside a test body, or the `fail` option inside a serial group. |
|
|
|
|
#### `test.setTimeout`
|
|
|
|
Changes the running test's timeout from its body or a `beforeEach` hook, or a
|
|
`beforeAll` hook's own timeout from inside it.
|
|
|
|
```ts
|
|
test('imports a large file', async ({ app }) => {
|
|
test.setTimeout(180_000);
|
|
await app.open('/import');
|
|
});
|
|
```
|
|
|
|
- The new timeout counts from the start of the attempt, like the
|
|
[`timeout`](#timeout) option, and replaces it for this attempt. A retry
|
|
starts from the option again.
|
|
- A timeout shorter than the time already spent times the test out at once.
|
|
- In a `beforeAll` hook, it replaces that hook's timeout, the config
|
|
`timeout`, counted from the hook's start. The tests keep their own.
|
|
- `afterEach` and `afterAll` hooks and fixture teardowns run on
|
|
`cleanupTimeout`, so they cannot change it.
|
|
- A session from [`host.openSession`](/reference/runner#host-opensession) runs
|
|
no e2e test, so it cannot either. Pass `timeout` to `host.openSession`.
|
|
- There is no `0` for no timeout. Pass a large value instead.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `INVALID_ARGUMENT` | A timeout that is not a positive safe integer of milliseconds, one above 2147483647, or a call from an `afterEach` or `afterAll` hook, a fixture teardown, or a hosted session; or from code a test or hook left running after it ended. |
|
|
| `COLLECTION_ERROR` | A call outside a running test or hook. Use the `timeout` option there. |
|
|
|
|
#### `test.only`
|
|
|
|
Registers one focused local test.
|
|
|
|
```ts
|
|
test.only('pays', async ({ app }) => {
|
|
await app.open('/checkout');
|
|
});
|
|
```
|
|
|
|
Other tests are reported as `filtered`. A CI run with a focused test fails
|
|
with `ONLY_IN_CI`.
|
|
|
|
#### `test.setup`
|
|
|
|
Registers a setup test that declares the sessions it saves, and must save
|
|
each exactly once.
|
|
|
|
```ts
|
|
import { credentials } from 'e2e';
|
|
|
|
test.setup('authenticate', { sessions: ['member'] }, async ({ app, screen, session }) => {
|
|
await app.open('/login');
|
|
await screen.getByLabel('Email').fill(credentials.user('member').username);
|
|
await screen.getByLabel('Password').fill(credentials.user('member').password);
|
|
await screen.getByRole('button', 'Sign in').tap();
|
|
await session.save('member');
|
|
});
|
|
```
|
|
|
|
- `test.setup` must be top-level, not inside `describe`.
|
|
- The [`sessions`](#sessions) option declares the names.
|
|
- A test consumes a saved session with the [`session`](#session) option.
|
|
- The setup body gets one extra fixture, `session`:
|
|
|
|
```ts
|
|
export interface SetupSession {
|
|
/** Saves state under a setup-declared name. */
|
|
save(name: string): Promise<void>;
|
|
}
|
|
```
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `SESSION_CONTRACT` | At runtime, a declared session was not saved, or was saved twice. |
|
|
|
|
#### `test.extend`
|
|
|
|
Returns a new `test` typed with more fixtures. The one it was called on is
|
|
unchanged.
|
|
|
|
```ts
|
|
import { test as base } from 'e2e';
|
|
import type { Device } from '@e2e-dev/mobile';
|
|
|
|
export const test = base.extend<{ device: Device }>(); // types only
|
|
```
|
|
|
|
- With no argument, `extend` refines only the types: it returns the same
|
|
runtime `test`, typed with the built-in fixtures an engine contributes. They still
|
|
resolve from the target's engine at runtime. This types a device, desktop,
|
|
or web surface without a global `declare module` augmentation.
|
|
- With definitions, it adds fixtures of your own. See
|
|
[Your own fixtures](#your-own-fixtures).
|
|
- The first type argument types test fixtures. The second types
|
|
[worker fixtures](#worker-fixtures), which `beforeAll` and `afterAll` see
|
|
too. `@e2e-dev/mobile` types `device`, `app`, and `screen` there.
|
|
|
|
#### `describe`
|
|
|
|
Declares a group of tests, with or without options.
|
|
|
|
```ts
|
|
import { describe, test } from 'e2e';
|
|
|
|
describe('checkout', { tags: ['smoke'], serial: true }, () => {
|
|
test('adds an item', async ({ screen }) => {});
|
|
test('pays', async ({ screen }) => {});
|
|
});
|
|
```
|
|
|
|
- A group takes the [options](#options) a test takes, except `only`, plus
|
|
[`serial`](#serial).
|
|
- The body is synchronous.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | An `async` body. It is also a type error. |
|
|
| `COLLECTION_ERROR` | `test.setup` inside the body. |
|
|
|
|
### Hooks
|
|
|
|
| Hook | Runs |
|
|
| --- | --- |
|
|
| [`beforeEach`](#beforeeach) | Before each test attempt |
|
|
| [`afterEach`](#aftereach) | After each test attempt |
|
|
| [`beforeAll`](#beforeall) | Once per suite instance |
|
|
| [`afterAll`](#afterall) | After the scope's last test |
|
|
|
|
Hook order follows scope nesting, not position in the file, so a hook
|
|
declared after a test still applies to it.
|
|
|
|
#### `beforeEach`
|
|
|
|
Registers a test-attempt setup hook.
|
|
|
|
```ts
|
|
test.beforeEach(async ({ app }) => {
|
|
await app.open('/');
|
|
});
|
|
```
|
|
|
|
- Runs outermost group first, then in declaration order within a group.
|
|
- `beforeEach` and the body share the test timeout.
|
|
- Receives the test fixtures. The ones it names are set up for every test it
|
|
applies to. See [Your own fixtures](#your-own-fixtures).
|
|
|
|
#### `afterEach`
|
|
|
|
Registers a test-attempt teardown hook.
|
|
|
|
```ts
|
|
test.afterEach(async ({ app }) => {
|
|
await app.screenshot('after');
|
|
});
|
|
```
|
|
|
|
- Runs innermost group first, in reverse declaration order, after a failure
|
|
or timeout too.
|
|
- Each `afterEach` hook gets its own `cleanupTimeout` budget with working
|
|
fixtures, so teardown can still drive the app after the body timed out.
|
|
- Receives the test fixtures. The ones it names are set up before the first
|
|
`beforeEach`, on the test timeout, never when the hook starts.
|
|
- A hook that names a fixture whose setup failed is skipped.
|
|
|
|
#### `beforeAll`
|
|
|
|
Registers a suite-instance setup hook. It runs once per suite instance, on
|
|
first entry into the scope.
|
|
|
|
```ts
|
|
test.beforeAll(async ({ database, device }) => {
|
|
await database.seed();
|
|
await device.setPermission('camera', 'deny'); // a mobile target
|
|
});
|
|
```
|
|
|
|
The module is re-imported for every retry, serial group, and setup test, so
|
|
`beforeAll` runs again in each.
|
|
|
|
It receives [worker fixtures](#worker-fixtures) only:
|
|
|
|
- The [built-in fixtures](#fixtures) whose scope is worker on the target:
|
|
`platform`, and on a mobile target `device`, `app`, and `screen`.
|
|
- The worker fixtures of the `test` it was registered through. The ones it
|
|
names are set up, and the ones declared with `auto: true`.
|
|
|
|
On a browser target, `browser`, `app`, and `screen` are test fixtures. Put UI
|
|
setup in `beforeEach`.
|
|
|
|
On a mobile target:
|
|
|
|
- The `beforeAll` hooks a test enters run in one short engine attempt of
|
|
their own, which launches nothing. So do the `afterAll` hooks of the scopes
|
|
a test leaves.
|
|
- The attempt opens for the first hook that names `device`, `app`, or
|
|
`screen`, or a worker fixture that names one. Hooks that name none open no
|
|
attempt.
|
|
- In a serial group, or a test an `e2e mcp` session runs, the hooks run on
|
|
the attempt already open.
|
|
- When that attempt cannot open, or the run was interrupted, the hooks still
|
|
run. Reading `device`, `app`, or `screen` then throws with the reason. A
|
|
failed open is one run error in phase `launch`.
|
|
|
|
The steps a suite hook takes are in no test's record. A failure names the
|
|
hook's scope.
|
|
|
|
Each `beforeAll` hook runs on the config `timeout`.
|
|
[`test.setTimeout`](#test-settimeout) inside it changes that hook's, and
|
|
the worker fixtures it sets up run on the changed timeout too.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `HOOK_FAILED` | The hook throws. The suite's tests are skipped with cause `hook-failed`. Inside a serial group it skips every remaining member and fails the group without a retry. |
|
|
| `COLLECTION_ERROR` | The hook names a test fixture. The message names it and the hook's line, and no test runs. |
|
|
|
|
#### `afterAll`
|
|
|
|
Registers a suite-instance teardown hook. It runs as soon as the scope's last
|
|
test in the realm is done.
|
|
|
|
```ts
|
|
afterAll(async () => {
|
|
await resetDatabase();
|
|
});
|
|
```
|
|
|
|
A realm that a failure discards runs `afterAll` for every scope whose
|
|
`beforeAll` started. It receives the same fixtures as
|
|
[`beforeAll`](#beforeall) and has `cleanupTimeout` to finish. A throw is
|
|
`HOOK_FAILED`, as for `beforeAll`.
|
|
|
|
#### Blocked event loop
|
|
|
|
A test that blocks its worker's event loop (`while (true) {}` in a fixture,
|
|
`beforeEach`, the body, or `afterEach`) stops every timer in that worker, so
|
|
the worker cannot time it out.
|
|
|
|
- When the worker has not responded `cleanupTimeout` (at least 5 s) after the
|
|
test timeout, the runner kills it.
|
|
- That attempt is reported `timed-out` with `TEST_TIMEOUT` and no artifacts,
|
|
after the attempts before it, and is not retried.
|
|
- The rest of its file runs on a new worker. In a serial group, the members
|
|
after it are skipped.
|
|
- A `beforeAll` or `afterAll` that blocks still hangs the run.
|
|
|
|
<Note>
|
|
A worker with a debugger attached when the test starts (an editor's
|
|
auto-attach, or `NODE_OPTIONS=--inspect=0`) is never killed this way, so a
|
|
breakpoint can stay paused. A programmatic run with an inline config or
|
|
in-memory tests runs in the runner's own process and has no worker to kill: a
|
|
blocked event loop hangs it.
|
|
</Note>
|
|
|
|
### Options
|
|
|
|
| Option | Type | Default |
|
|
| --- | --- | --- |
|
|
| [`timeout`](#timeout) | `number` | `config.timeout` |
|
|
| [`retries`](#retries) | `number` | `config.retries` |
|
|
| [`tags`](#tags) | `readonly string[]` | - |
|
|
| [`skip`](#skip) | `boolean \| string` | - |
|
|
| [`fail`](#fail) | `boolean \| string` | - |
|
|
| [`only`](#only) | `boolean` | - |
|
|
| [`platforms`](#platforms) | `readonly string[]` | - |
|
|
| [`requires`](#requires) | `readonly Capability[]` | - |
|
|
| [`session`](#session) | `string` | - |
|
|
| [`agentContext`](#agentcontext) | `string` | - |
|
|
| [`agent`](#agent) | `string \| readonly string[]` | the run's agents |
|
|
| [`trace`](#trace) | `RecordingMode` | the run's |
|
|
| [`video`](#video) | `RecordingMode` | the run's |
|
|
| [`serial`](#serial) | `boolean` | - |
|
|
| [`sessions`](#sessions) | `readonly string[]` | required |
|
|
|
|
`test` takes every option but `serial` and `sessions`. `describe` takes
|
|
every option but `only`, `fail`, and `sessions`. `test.setup` requires
|
|
`sessions` and takes no `session`, `only`, `skip`, or `fail`.
|
|
|
|
Resolution order: test, nearest group, outer groups, config, built-in
|
|
default. Inside a [serial group](#serial), the group's `retries`, `trace`, and
|
|
`video` apply.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | An option key the call does not take (`{ timout: 1000 }`). The error names the nearest known key when it is a likely typo, and every key the call takes otherwise. |
|
|
|
|
#### `timeout`
|
|
|
|
The attempt deadline in milliseconds. Innermost wins.
|
|
|
|
```ts
|
|
test('imports a large file', { timeout: 300_000 }, async ({ app }) => {}); // default: config.timeout
|
|
```
|
|
|
|
`timeout` must be a positive safe integer, at most 2147483647 (about 24 days).
|
|
|
|
#### `retries`
|
|
|
|
How many times a failed test is retried.
|
|
|
|
```ts
|
|
test('checks out', { retries: 2 }, async ({ app }) => {}); // default: config.retries
|
|
```
|
|
|
|
An integer from 0 through 10.
|
|
|
|
#### `tags`
|
|
|
|
Tags that `--tag` selects on. Every layer's tags are unioned.
|
|
|
|
```ts
|
|
describe('checkout', { tags: ['billing', 'smoke'] }, () => {});
|
|
```
|
|
|
|
Tags are distinct names, none blank, with no comma and no leading or
|
|
trailing whitespace.
|
|
|
|
#### `skip`
|
|
|
|
Skips the test at collection. A string is the reason. Any truthy layer skips.
|
|
Skip reasons, including runtime `test.skip` reasons, are secret-redacted,
|
|
sanitized, and limited to 8,192 UTF-8 bytes before reporting.
|
|
|
|
```ts
|
|
test('exports a PDF', { skip: 'PDF export is off on staging' }, async ({ app }) => {});
|
|
```
|
|
|
|
#### `fail`
|
|
|
|
Expects the test to fail; see [`test.fail`](#test-fail). A string is the
|
|
reason. Not allowed inside a serial group.
|
|
|
|
```ts
|
|
test('exports a PDF', { fail: 'PDF export is broken' }, async ({ app }) => {});
|
|
```
|
|
|
|
#### `only`
|
|
|
|
Focuses the test locally. CI rejects it with `ONLY_IN_CI`.
|
|
|
|
```ts
|
|
test('pays', { only: true }, async ({ app }) => {});
|
|
```
|
|
|
|
#### `platforms`
|
|
|
|
Platforms the test runs on. Other targets skip it. Innermost wins.
|
|
|
|
```ts
|
|
test('uses Face ID', { platforms: ['ios'] }, async ({ app }) => {});
|
|
```
|
|
|
|
#### `requires`
|
|
|
|
Capabilities the target must have. A target that lacks one skips the test at
|
|
selection. Innermost wins.
|
|
|
|
```ts
|
|
test('onboarding asks for the camera', { requires: ['native-app'] }, async ({ app, agent }) => {
|
|
await app.open();
|
|
await agent.assert('The app asks for camera access');
|
|
});
|
|
```
|
|
|
|
Accepted values:
|
|
|
|
- The harness capabilities of the target's engine: `observation`, `actions`,
|
|
`location`, `pointer`, `keyboard`, `computed-style`, `state`, `artifacts`.
|
|
- One per built-in fixture an engine contributes: `browser`, `device`. See
|
|
[the engine contract](/reference/engine#engine).
|
|
- The kind of app the target declares: `browser` for an `app.url`,
|
|
`native-app` for an `app.bundleId` or `app.appPath`.
|
|
|
|
#### `session`
|
|
|
|
The session a setup test saved, restored before the body runs.
|
|
|
|
```ts
|
|
test('sees the dashboard', { session: 'member' }, async ({ app }) => {});
|
|
```
|
|
|
|
See [`test.setup`](#test-setup).
|
|
|
|
#### `agentContext`
|
|
|
|
Extra context for agent steps. Layers are concatenated outermost first.
|
|
|
|
```ts
|
|
describe('pricing', { agentContext: 'Prices are shown in EUR.' }, () => {});
|
|
```
|
|
|
|
#### `agent`
|
|
|
|
The configured agent (`agents.<name>`) this test or group runs with, in
|
|
place of the run's agents.
|
|
|
|
```ts
|
|
test('checks out', { agent: ['fast', 'smart'] }, async ({ agent }) => {}); // one result per agent
|
|
```
|
|
|
|
- A list runs the test once per agent named, as one result each.
|
|
- Innermost wins. A call's own `agent` option wins over it.
|
|
- `--agent` narrows a list to the names both name, and never overrides a pin
|
|
the flag does not name.
|
|
- A setup test runs once per target, so it pins at most one agent.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | At selection, before any test runs: a name that `agents` does not define. |
|
|
|
|
#### `trace`
|
|
|
|
Whether the test keeps a trace page, in place of the run's.
|
|
|
|
```ts
|
|
test('checkout', { trace: 'retain-on-failure' }, async ({ agent }) => {});
|
|
```
|
|
|
|
- Takes a [recording mode](/reference/config#recording), the same as the
|
|
config's `trace`: `off`, `on`, `retain-on-failure`, `on-first-retry`, or
|
|
`on-all-retries`.
|
|
- A test's or group's own value wins over `--trace`, the target's, and the
|
|
config's. One test can record on a run that records nothing, or opt out of
|
|
one that records everything.
|
|
|
|
#### `video`
|
|
|
|
Which of the test's attempts record a video, in place of the run's.
|
|
|
|
```ts
|
|
test('checkout', { video: 'on' }, async ({ agent }) => {
|
|
await agent.act('Buy the first product in the catalog');
|
|
});
|
|
```
|
|
|
|
- Takes the same [recording modes](/reference/config#recording) as `trace`
|
|
and the config's `video`.
|
|
- A test's or group's own value wins over `--video`, the target's, and the
|
|
config's.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `UNSUPPORTED_ARTIFACT` | Before any test starts, naming the test: the mode records on some attempt and the target's engine cannot record video. |
|
|
|
|
#### `serial`
|
|
|
|
`describe` only. Runs the group as one ordered retry unit with shared app
|
|
state.
|
|
|
|
```ts
|
|
describe('checkout', { serial: true }, () => {
|
|
test('adds an item', async ({ screen }) => {});
|
|
test('pays', async ({ screen }) => {});
|
|
});
|
|
```
|
|
|
|
- The group also shares secret state: a value one member resolves stays
|
|
redacted for the rest of the group's attempt. A retry starts fresh.
|
|
- Members take the `trace` and `video` on the serial `describe` or a group
|
|
around it. The group records one video per group attempt.
|
|
- A group attempt may not start because its file fails to load or its session
|
|
fails to launch. Then the first member fails with that error, and the rest
|
|
skip with cause `serial-predecessor-failed`. When the run is interrupted
|
|
during the launch, every member skips instead.
|
|
- If a retry is interrupted or never reaches its members, they keep the
|
|
verdicts and durations of the last attempt that ran them.
|
|
- The exit code follows the error's category, as for any other test.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | A member sets `retries`, `trace`, `video`, `session`, `platforms`, `requires`, `skip`, `only`, or a nested `serial`. |
|
|
| `COLLECTION_ERROR` | A member registers through `test.skipIf` or `test.runIf`. |
|
|
|
|
#### `sessions`
|
|
|
|
`test.setup` only. The session names the setup test must save.
|
|
|
|
```ts
|
|
test.setup('authenticate', { sessions: ['member', 'admin'] }, async ({ session }) => {});
|
|
```
|
|
|
|
A static, non-empty array of unique names. Each name matches
|
|
`/^[A-Za-z0-9_.-]{1,128}$/`. See [`test.setup`](#test-setup).
|
|
|
|
### Fixtures
|
|
|
|
Built-in fixtures are the fixtures a `test` has without `test.extend`: five
|
|
on every engine, and the ones the target's engine contributes. Each has a
|
|
scope on each engine.
|
|
|
|
| Built-in fixture | Type | Web | Mobile |
|
|
| --- | --- | --- | --- |
|
|
| [`agent`](/reference/agent) | `Agent` | test | test |
|
|
| [`app`](/reference/app) | `App` | test | worker |
|
|
| [`screen`](/reference/screen) | `Screen` | test | worker |
|
|
| `platform` | `string` | worker | worker |
|
|
| [`session`](#test-setup) | `SetupSession` | test | test |
|
|
| [`browser`](/reference/web) | `Browser` | test | - |
|
|
| [`device`](/reference/mobile#device) | `Device` | - | worker |
|
|
|
|
The scope decides which code can use a built-in fixture:
|
|
|
|
- A test fixture belongs to one test attempt. Test bodies, `beforeEach`, and
|
|
`afterEach` get it.
|
|
- A worker fixture outlives a test. `beforeAll`, `afterAll`, and
|
|
[worker fixtures](#worker-fixtures) get it too.
|
|
|
|
A browser test gets a new context, so on web `app`, `screen`, and `browser`
|
|
are test fixtures. A device keeps its state between tests, so on mobile `app`,
|
|
`screen`, and `device` are worker fixtures. An engine of your own sets this
|
|
with [`isolation`](/reference/engine#fixture-isolation).
|
|
|
|
`session` reaches only `test.setup` bodies. [`TestFixtures`](#testfixtures)
|
|
describes each core one. Every other fixture is one of
|
|
[your own](#your-own-fixtures).
|
|
|
|
A test body, a hook, and a fixture function name the fixtures they use by
|
|
destructuring their first parameter: `async ({ app, screen }) => {}`. e2e
|
|
reads the names from the function's source. A [`test.each`](#test-each) or
|
|
[`test.for`](#test-for) body names them in the parameter after its row.
|
|
|
|
| Throws | When |
|
|
| --- | --- |
|
|
| `COLLECTION_ERROR` | A test that runs on a target, or one of its hooks, names a fixture the target does not have. No test runs. Declare `requires: ['browser']`, `platforms`, or `test.skipIf` to skip such tests on that target instead. |
|
|
| `COLLECTION_ERROR` | A test body, a hook, or a fixture function takes the whole object (`async (fixtures) => {}`), a rest property (`{ app, ...rest }`), or a computed property, or is a bound function (`fn.bind()`), whose parameters cannot be read. |
|
|
|
|
<Note>
|
|
Built-in fixtures are lazy. Destructuring in the parameter list acquires
|
|
them before the body's first statement. [Your own
|
|
fixtures](#your-own-fixtures) are set up before the first `beforeEach`.
|
|
</Note>
|
|
|
|
#### Your own fixtures
|
|
|
|
`test.extend(fixtures)` defines resources set up before each test that names
|
|
them and cleaned up afterward, including after a failure. In each fixture
|
|
function, code before `await use(value)` sets up the resource. The test
|
|
receives `value`, and code after `use` tears it down. For a guide with a full
|
|
suite, see [Writing fixtures](/fixtures).
|
|
|
|
```ts
|
|
import { test as base } from '@e2e-dev/web';
|
|
|
|
export const test = base.extend<{ workspace: Workspace }>({
|
|
workspace: async ({ browser }, use) => {
|
|
const workspace = await createWorkspace();
|
|
await signIn(browser, workspace);
|
|
await use(workspace); // the hooks and the body run here
|
|
await workspace.cleanup(); // teardown, after the body even when it failed
|
|
},
|
|
});
|
|
|
|
test('renames the workspace', async ({ workspace, screen }) => {
|
|
await screen.getByLabel('Name').fill(`${workspace.name} (renamed)`);
|
|
});
|
|
```
|
|
|
|
A fixture is set up on demand: for a test that names it, or that names a
|
|
fixture which names it. The fixtures a function names in its first parameter
|
|
are its dependencies:
|
|
|
|
- A definition sees the fixtures of the `test` it extends and the ones
|
|
declared beside it in the same call, in any order. It does not see a
|
|
fixture a later `extend` declares.
|
|
- A definition that needs no fixture takes `({}, use)`. ESLint's
|
|
`no-empty-pattern` accepts it with `allowObjectPatternsAsParameters: true`.
|
|
- `[fn, { auto: true }]` sets a fixture up for every test of that `test`,
|
|
named or not. Use it for a fixture that tests need and never read.
|
|
|
|
Per attempt:
|
|
|
|
1. The fixtures to set up are decided before anything runs: the ones the
|
|
body and every `beforeEach` and `afterEach` hook that applies name, the
|
|
ones those fixtures name, and the `auto` ones. A fixture nothing names
|
|
does not run.
|
|
2. They run in declaration order, each after the fixtures it names, until
|
|
each calls `use`. Setup counts against the test `timeout`. A failure is
|
|
reported in phase `beforeEach`, before any `beforeEach` hook has run.
|
|
3. `beforeEach` hooks, the body, then `afterEach` hooks run with the same
|
|
fixtures. Each receives the ones it names. The fixture set is the test's
|
|
own chain: a hook that names
|
|
`workspace` around a test registered through `base` fails with
|
|
`COLLECTION_ERROR`.
|
|
4. Teardowns run in reverse order after the last `afterEach`, whether or not
|
|
the body passed, each with its own `cleanupTimeout` budget. An error there
|
|
fails a passing attempt in phase `afterEach` and joins `secondaryErrors`
|
|
after a body failure.
|
|
|
|
A mistake that fails at collection is one run error with the file and line of
|
|
the code, and no test runs. A test that does not run on a target is not
|
|
checked against that target's fixtures.
|
|
|
|
| Mistake | Fails |
|
|
| --- | --- |
|
|
| A value that is not a plain object of functions or `[fn, { scope, auto }]` pairs, a scope other than `'test'` or `'worker'`, an `auto` that is not a boolean, an option other than `scope` and `auto`, or an empty name | At import, `COLLECTION_ERROR` naming the fixture |
|
|
| A function that does not destructure its first parameter, takes a rest or computed property there, or is a bound function | At import, `COLLECTION_ERROR` naming the fixture |
|
|
| Fixtures that name each other in a cycle | At import, `COLLECTION_ERROR` naming the cycle |
|
|
| A name the target has no fixture for, in a test that runs there, its hooks, or a fixture they need | At collection, `COLLECTION_ERROR` |
|
|
| A `scope` or an `auto` that differs from that of the [fixture it extends](#extending-a-fixture) | At import when it is the same on every engine, else at collection, `COLLECTION_ERROR` naming both |
|
|
| A fixture that names one a later `extend` declares | At collection, `COLLECTION_ERROR` |
|
|
| A definition that returns without calling `use` | The attempt, in `beforeEach`, `TEST_SETUP_FAILED` |
|
|
| A definition that calls `use` twice | The attempt, at teardown, `TEST_SETUP_FAILED` |
|
|
| A definition that throws before `use` | The attempt, in `beforeEach`, with that error. An `afterEach` hook that names the fixture is skipped. |
|
|
| A definition that throws after `use` | The attempt, in `afterEach`, with that error |
|
|
|
|
<Note>
|
|
A body or hook that hits the test `timeout` is abandoned, not cancelled, so
|
|
teardown starts while it may still be running. A setup abandoned this way
|
|
that reaches `use` later gets an immediate resolve. Its teardown runs
|
|
detached from the run, so what it allocated is still released. Later
|
|
fixtures, hooks, and the test body do not start after an abandoned setup or
|
|
hook finishes.
|
|
</Note>
|
|
|
|
#### Worker fixtures
|
|
|
|
A worker fixture has one instance per worker and target. Declare it as
|
|
`[fn, { scope: 'worker' }]` and type it with the second type argument of
|
|
`extend`:
|
|
|
|
```ts
|
|
export const test = base.extend<{}, { database: Database }>({
|
|
database: [async ({}, use) => {
|
|
const database = await startDatabase();
|
|
await use(database);
|
|
await database.stop();
|
|
}, { scope: 'worker' }],
|
|
});
|
|
```
|
|
|
|
- Setup runs the first time a test or suite hook names it, or names a
|
|
fixture that names it, inside that test's or hook's timeout. A failed setup
|
|
fails that test or hook, and the next one sets the fixture up again.
|
|
- `[fn, { scope: 'worker', auto: true }]` sets it up for the first test or
|
|
suite hook registered through its `test`, named or not.
|
|
- Teardown runs when the worker ends, before the engine is disposed, last set
|
|
up first. Each teardown has `cleanupTimeout`. One that throws is a run error
|
|
in phase `cleanup`.
|
|
- It sees worker fixtures only: the [built-in fixtures](#fixtures) whose
|
|
scope is worker on the target, the worker fixtures of the `test` it extends,
|
|
and the ones declared beside it. Naming a test fixture is
|
|
`COLLECTION_ERROR`.
|
|
- On a mobile target its `device`, `app`, and `screen` act in the test or
|
|
hook that is running when it calls them. Its teardown can still drive the
|
|
device: it runs in a short engine attempt when the fixture names one of
|
|
them, and without one when it names none.
|
|
- A worker keeps one instance per definition. A test file is imported again
|
|
after a failed test and for each serial group and setup test, so a worker
|
|
fixture the file declares gets a new instance each time, and the old one
|
|
stays set up until the worker ends.
|
|
Declare worker fixtures in a module the file imports.
|
|
- `extend({ ... })` without type arguments infers test fixtures only.
|
|
|
|
#### Extending a fixture
|
|
|
|
A definition named like a fixture the `test` already has extends that fixture.
|
|
It can be a [built-in fixture](#fixtures) or a fixture of an earlier `extend`.
|
|
|
|
```ts
|
|
export const test = base.extend({
|
|
app: async ({ app }, use) => {
|
|
await app.open('/dashboard');
|
|
await use(app); // tests, hooks, and other fixtures get this value
|
|
},
|
|
});
|
|
```
|
|
|
|
- Inside the function, its own name is the fixture it extends: the built-in,
|
|
or the earlier fixture of that name. A function that does not name itself
|
|
replaces the fixture.
|
|
- Tests, hooks, and other fixtures that name the fixture get the value passed
|
|
to `use`. What the runner does with a built-in fixture does not change.
|
|
- It is set up when the fixture it extends would be: on demand, for code that
|
|
names the fixture, or for every test when it extends an `auto` fixture. An
|
|
`auto` in the options must be the `auto` of the fixture it extends.
|
|
- It keeps the scope of the fixture it extends on the target, as the
|
|
[built-in fixtures](#fixtures) table gives it. `device` is a worker fixture,
|
|
so its override runs one time per worker and `beforeAll` gets it. A `scope`
|
|
in the options must be that scope.
|
|
- It keeps the type. Define it with no type argument: the type arguments of
|
|
`extend` are for new fixtures.
|
|
- On a target that has no fixture of that name, nothing fails until code that
|
|
runs there names it. Then the run fails at collection: the target has no
|
|
such fixture.
|
|
|
|
## Types
|
|
|
|
### `TestAPI`
|
|
|
|
```ts
|
|
/** The two call forms of `test`, shared with what `test.skipIf` and `test.runIf` return. */
|
|
export interface TestRegistrar<Fixtures = TestFixtures & WorkerFixtures> {
|
|
/** Registers one test synchronously during module evaluation. */
|
|
(title: string, fn: TestFn<Fixtures>): TestCase;
|
|
/** Registers one configured test synchronously during module evaluation. */
|
|
(title: string, options: TestOptions, fn: TestFn<Fixtures>): TestCase;
|
|
}
|
|
|
|
/**
|
|
* The two call forms of `test` for `test.each` and `test.for`: one test per
|
|
* row, each titled from the row. `Args` are the body's arguments.
|
|
*/
|
|
export interface EachRegistrar<Args extends readonly unknown[]> {
|
|
/** Registers one test per row synchronously during module evaluation. */
|
|
(title: string, fn: EachFn<Args>): readonly TestCase[];
|
|
/** Registers one configured test per row synchronously during module evaluation. */
|
|
(title: string, options: TestOptions, fn: EachFn<Args>): readonly TestCase[];
|
|
}
|
|
|
|
export interface TestAPI<Fixtures = TestFixtures, Worker = WorkerFixtures> extends TestRegistrar<Fixtures & Worker> {
|
|
/** Registers one skipped test. */
|
|
skip(title: string, fn: TestFn<Fixtures & Worker>): TestCase;
|
|
/**
|
|
* Inside a running test: skips it when `condition` is true (or always,
|
|
* with no condition) by throwing, so nothing after the call runs. The
|
|
* attempt is reported `skipped` with `reason`; steps that already ran stay
|
|
* in the report without deciding the status, and teardown still runs. A
|
|
* setup test cannot skip: its sessions are owed.
|
|
* Outside a test body this form is `COLLECTION_ERROR`; use the `skip`
|
|
* option to skip at collection.
|
|
*/
|
|
skip(condition: boolean, reason?: string): void;
|
|
skip(reason?: string): void;
|
|
/**
|
|
* Returns a function that registers one test, like `test(title, fn)`:
|
|
* skipped when `condition` is truthy, as if it had the `skip` option. The
|
|
* condition is read once, at this call, so it can only depend on what is
|
|
* known while the file is collected (`process.platform`, `process.env.CI`):
|
|
*
|
|
* test.skipIf(process.platform === 'win32')('file watcher', async () => {});
|
|
*
|
|
* A `skip` option skips the test whatever the condition. A serial group
|
|
* rejects the call: set `skip` on the serial `describe` instead. A Promise
|
|
* condition also fails the call: await it first.
|
|
*/
|
|
skipIf<Condition>(condition: SynchronousCondition<Condition>): TestRegistrar<Fixtures & Worker>;
|
|
/**
|
|
* Returns a function that registers one test, like `test(title, fn)`:
|
|
* skipped when `condition` is falsy, `test.skipIf(!condition)`.
|
|
*/
|
|
runIf<Condition>(condition: SynchronousCondition<Condition>): TestRegistrar<Fixtures & Worker>;
|
|
/**
|
|
* Returns a function that registers one test per row, like `test(title,
|
|
* fn)`, in row order. The title is formatted from the row: `$name` and
|
|
* `$name.path` read an object row, and `%s`, `%d`, `%i`, `%f`, `%j`, `%o`
|
|
* take an array row's items in order (any other row is the one item). `%#`
|
|
* is the row index, `%$` the row number, `%%` a percent sign. When every
|
|
* row is an array, each row is spread into the body's arguments; otherwise
|
|
* every row is the first one. The fixtures come last:
|
|
*
|
|
* test.each([{ user: 'Ada' }, { user: 'Grace' }])('signs in as $user', async ({ user }, { app }) => {});
|
|
* test.each([[1, 2, 3]])('%i + %i = %i', async (a, b, sum, { app }) => {});
|
|
*
|
|
* Rows must be a non-empty array. Two rows that format to the same title
|
|
* fail as duplicate titles. As in a test body, destructure the fixtures
|
|
* (`{ app }`): e2e sets up only the ones named there.
|
|
*/
|
|
each<Row extends readonly unknown[] | [unknown]>(rows: readonly Row[]): EachRegistrar<[...Row, Fixtures & Worker]>;
|
|
each<Row>(rows: readonly Row[]): EachRegistrar<EachArgs<Row, Fixtures & Worker>>;
|
|
/**
|
|
* `test.each` that never spreads: the body takes `(row, fixtures)` for
|
|
* every row, an array row included. Titles format as in `test.each`.
|
|
*/
|
|
for<Row extends readonly unknown[] | [unknown]>(rows: readonly Row[]): EachRegistrar<[Row, Fixtures & Worker]>;
|
|
for<Row>(rows: readonly Row[]): EachRegistrar<[Row, Fixtures & Worker]>;
|
|
/** Registers one test that is expected to fail. */
|
|
fail(title: string, fn: TestFn<Fixtures & Worker>): TestCase;
|
|
/**
|
|
* Inside a running test: marks it expected to fail when `condition` is
|
|
* true (or always, with no condition). The body keeps running. A test
|
|
* failure then passes the test, keeping the error and `reason` in the
|
|
* report; a pass fails it with `UNEXPECTED_PASS`. Infrastructure failures
|
|
* and interrupts are not covered. A setup test or a serial group member
|
|
* cannot be marked. Outside a test body this form is `COLLECTION_ERROR`;
|
|
* use `test.fail(title, fn)` or the `fail` option to mark at collection.
|
|
*/
|
|
fail(condition: boolean, reason?: string): void;
|
|
fail(reason?: string): void;
|
|
/** Registers one focused local test. CI rejects focused tests. */
|
|
only(title: string, fn: TestFn<Fixtures & Worker>): TestCase;
|
|
/** Registers one setup test with statically declared session outputs. */
|
|
setup(title: string, options: SetupOptions, fn: SetupFn<Fixtures & Worker>): TestCase;
|
|
/**
|
|
* Returns the same runtime `test`, typed with an engine's contributed
|
|
* fixtures. A pure type refinement, the fixtures still resolve from the
|
|
* target's engine at runtime, so a project types its device/desktop/web
|
|
* surface without a global `declare module` augmentation:
|
|
*
|
|
* export const test = base.extend<{ device: Device }>();
|
|
*/
|
|
extend<Extra = {}, ExtraWorker = {}>(): TestAPI<Fixtures & Extra, Worker & ExtraWorker>;
|
|
/**
|
|
* Returns a new `test` whose registrations extend fixtures it already
|
|
* has, built-in ones included. Inside the function the fixture's own name
|
|
* is the fixture it extends; the value it hands to `use` is what tests,
|
|
* hooks, and other fixtures receive under that name. It keeps the type and
|
|
* the scope of the fixture it extends:
|
|
*
|
|
* export const test = base.extend({
|
|
* app: async ({ app }, use) => {
|
|
* await app.open('/dashboard');
|
|
* await use(app);
|
|
* },
|
|
* });
|
|
*/
|
|
extend(overrides: FixtureOverrides<Fixtures & Worker, Worker>): TestAPI<Fixtures, Worker>;
|
|
/**
|
|
* Returns a new `test` whose registrations carry these fixture
|
|
* definitions on top of the ones it already had. Each attempt sets up the
|
|
* ones its body and its `beforeEach` and `afterEach` hooks name, and the
|
|
* ones those fixtures name, in declaration order with each after the
|
|
* fixtures it names; it runs the hooks and the body with them, and tears
|
|
* them down in reverse order after `afterEach`. A fixture nothing names is
|
|
* not set up:
|
|
*
|
|
* export const test = base.extend<{ workspace: Workspace }>({
|
|
* workspace: async ({ browser }, use) => {
|
|
* const workspace = await createWorkspace();
|
|
* await use(workspace);
|
|
* await workspace.cleanup();
|
|
* },
|
|
* });
|
|
*
|
|
* A worker fixture (the second type argument, `[fn, { scope: 'worker' }]`)
|
|
* is set up once per worker and target, the first time a test or suite
|
|
* hook of the chain names it, and torn down when the worker ends:
|
|
*
|
|
* export const test = base.extend<{}, { database: Database }>({
|
|
* database: [async ({}, use) => {
|
|
* const database = await startDatabase();
|
|
* await use(database);
|
|
* await database.stop();
|
|
* }, { scope: 'worker' }],
|
|
* });
|
|
*
|
|
* `auto: true` sets a fixture up whether or not anything names it: for
|
|
* every test (`[fn, { auto: true }]`), or once per worker
|
|
* (`[fn, { scope: 'worker', auto: true }]`). A suite hook sets up the auto
|
|
* worker fixtures of its chain, never a test fixture.
|
|
*/
|
|
extend<Extra = {}, ExtraWorker = {}>(
|
|
fixtures: FixtureDefinitions<Fixtures & Worker, Extra, Worker, ExtraWorker>,
|
|
): TestAPI<Fixtures & Extra, Worker & ExtraWorker>;
|
|
/** Declares a group synchronously. */
|
|
describe<Result>(title: string, body: SynchronousBody<Result>): void;
|
|
/** Declares a configured group synchronously. */
|
|
describe<Result>(
|
|
title: string,
|
|
options: DescribeOptions,
|
|
body: SynchronousBody<Result>,
|
|
): void;
|
|
/** Registers a test-attempt setup hook: it runs after the test's fixtures are set up. */
|
|
beforeEach(fn: TestHookFn<Fixtures & Worker>): void;
|
|
/** Registers a test-attempt teardown hook: it runs before the test's fixtures are torn down. */
|
|
afterEach(fn: TestHookFn<Fixtures & Worker>): void;
|
|
/** Registers a suite setup hook, run once per describe between attempts, with the worker fixtures. */
|
|
beforeAll(fn: TestHookFn<Worker>): void;
|
|
/** Registers a suite teardown hook, run once per describe between attempts, with the worker fixtures. */
|
|
afterAll(fn: TestHookFn<Worker>): void;
|
|
}
|
|
|
|
export const test: TestAPI;
|
|
export const describe: TestAPI['describe'];
|
|
export const beforeEach: TestAPI['beforeEach'];
|
|
export const afterEach: TestAPI['afterEach'];
|
|
export const beforeAll: TestAPI['beforeAll'];
|
|
export const afterAll: TestAPI['afterAll'];
|
|
```
|
|
|
|
### `TestOptions`
|
|
|
|
```ts
|
|
export interface TestOptions {
|
|
/** Attempt deadline in milliseconds; defaults to `config.timeout`. Innermost wins. */
|
|
timeout?: number;
|
|
/** Retries, 0 through 10; defaults to `config.retries`. Inside a serial group the group's value applies. */
|
|
retries?: number;
|
|
/** Tags for `--tag`: distinct names, none blank, with no comma and no leading or trailing whitespace; every layer's tags are unioned. */
|
|
tags?: readonly string[];
|
|
/** Skips the test; a string is the reason. Any truthy layer skips. */
|
|
skip?: boolean | string;
|
|
/**
|
|
* Expects the test to fail; a string is the reason. A test failure then
|
|
* passes it, and a pass fails it with `UNEXPECTED_PASS`. Not allowed inside
|
|
* a serial group.
|
|
*/
|
|
fail?: boolean | string;
|
|
/** Focuses the test locally. CI rejects it with `ONLY_IN_CI`. */
|
|
only?: boolean;
|
|
/** Platforms the test runs on; other targets skip it. Innermost wins. */
|
|
platforms?: readonly string[];
|
|
/** Capabilities the target must have; otherwise the test is skipped at selection. Innermost wins. */
|
|
requires?: readonly Capability[];
|
|
/** Session a setup test saved, restored before the body runs. */
|
|
session?: string;
|
|
/** Extra context for agent steps; layers are concatenated outermost first. */
|
|
agentContext?: string;
|
|
/**
|
|
* The configured agent (`agents.<name>`) this test or group runs with, in
|
|
* place of the run's agents. A list runs the test once per agent named, as
|
|
* one result each. Innermost wins; a call's own `agent` option wins over
|
|
* it. `--agent` narrows a list to the names both name and never overrides
|
|
* a pin the flag does not name.
|
|
*/
|
|
agent?: string | readonly string[];
|
|
/**
|
|
* Which of the test's attempts keep a trace, in place of the run's: the
|
|
* same modes as the config's `trace`. Innermost wins, over `--trace` too;
|
|
* inside a serial group the group's value applies, since the group runs as
|
|
* one unit.
|
|
*/
|
|
trace?: RecordingMode;
|
|
/**
|
|
* Which of the test's attempts record a video, in place of the run's:
|
|
* the same modes as the config's `video`. Innermost wins, over `--video`
|
|
* too; inside a serial group the group's value applies, since the group
|
|
* records as one unit. A mode set here is required of the target's engine.
|
|
*/
|
|
video?: RecordingMode;
|
|
}
|
|
|
|
export interface DescribeOptions extends Omit<TestOptions, 'only' | 'fail'> {
|
|
/** Runs the group as one ordered retry unit with shared app state. Nesting one serial group in another is a `COLLECTION_ERROR`. */
|
|
serial?: boolean;
|
|
}
|
|
|
|
export interface SetupOptions extends Omit<TestOptions, 'session' | 'only' | 'skip' | 'fail' | 'agent'> {
|
|
/** Session names this setup test MUST save. */
|
|
sessions: readonly string[];
|
|
/** A setup test runs once per target, so it pins at most one agent. */
|
|
agent?: string;
|
|
}
|
|
```
|
|
|
|
### `TestFixtures`
|
|
|
|
```ts
|
|
export interface TestFixtures {
|
|
/** Agent steps. Acquiring the built-in agent without a configured model is `MODEL_UNAVAILABLE`. */
|
|
readonly agent: Agent;
|
|
/** App lifecycle: open, restart, clear state, back, screenshot, recording. */
|
|
readonly app: App;
|
|
/** Semantic queries and the actions on their matches. */
|
|
readonly screen: Screen;
|
|
/** The target's platform label: `web`, `ios`, `android`, or an engine's own string. */
|
|
readonly platform: string;
|
|
}
|
|
|
|
export interface SetupFixtures extends TestFixtures {
|
|
/** Saves the sessions the setup test declared. */
|
|
readonly session: SetupSession;
|
|
}
|
|
|
|
/**
|
|
* How long one fixture instance lives. `test`, the default, is one instance
|
|
* per test attempt that names it, shared by its `beforeEach` hooks, its
|
|
* body, and its `afterEach` hooks. `worker` is one instance per worker and
|
|
* target, set up the first time a test or suite hook names it and torn down
|
|
* when the worker ends.
|
|
*/
|
|
export type FixtureScope = 'test' | 'worker';
|
|
|
|
/**
|
|
* The fixtures that outlive a test: what `beforeAll`, `afterAll`, and a
|
|
* worker-scoped `test.extend()` fixture receive. An engine whose state
|
|
* outlives an attempt (a device keeps permissions and installed apps) adds
|
|
* its own fixtures, `app`, and `screen` here; on an engine that gives each
|
|
* attempt a fresh surface (a browser context) they are test fixtures.
|
|
*/
|
|
export interface WorkerFixtures {
|
|
/** The target's platform label: `web`, `ios`, `android`, or an engine's own string. */
|
|
readonly platform: string;
|
|
}
|
|
|
|
/** @deprecated `beforeAll` and `afterAll` receive the worker fixtures: use `WorkerFixtures`. */
|
|
export type SuiteFixtures = WorkerFixtures;
|
|
```
|
|
|
|
### `FixtureDefinitions`
|
|
|
|
```ts
|
|
/**
|
|
* Defines one fixture: everything before `await use(value)` is its setup,
|
|
* everything after is its teardown. The fixtures it destructures in its
|
|
* first parameter are the ones it depends on, set up before it; one that
|
|
* needs none writes `async ({}, use) => {}`. The body, its hooks, and the
|
|
* fixtures that name it run while `use` is pending; it resolves once they
|
|
* are done, whether or not the body failed.
|
|
*/
|
|
export type FixtureFn<Fixtures, Value> = (
|
|
fixtures: Fixtures,
|
|
use: (value: Value) => Promise<void>,
|
|
) => Promise<void>;
|
|
|
|
/**
|
|
* A test fixture: its function, or the function and its options. With
|
|
* `auto: true` it is set up for every test of its chain, whether or not
|
|
* anything names it.
|
|
*/
|
|
export type TestFixtureDefinition<Fixtures, Value> =
|
|
| FixtureFn<Fixtures, Value>
|
|
| readonly [FixtureFn<Fixtures, Value>, { readonly scope?: 'test'; readonly auto?: boolean }];
|
|
/**
|
|
* A worker fixture: its function and `{ scope: 'worker' }`. It sees worker
|
|
* fixtures only. With `auto: true` it is set up once per worker, before the
|
|
* first test or suite hook of its chain, whether or not anything names it.
|
|
*/
|
|
export type WorkerFixtureDefinition<Worker, Value> = readonly [
|
|
FixtureFn<Worker, Value>,
|
|
{ readonly scope: 'worker'; readonly auto?: boolean },
|
|
];
|
|
/**
|
|
* One definition per fixture `test.extend()` adds: `Extra` holds the test
|
|
* fixtures, `ExtraWorker` the worker fixtures. A definition sees the fixtures
|
|
* of the `test` it extends and the ones declared beside it, in any order: it
|
|
* is set up after the ones it names. A test fixture sees every fixture a
|
|
* test sees; a worker fixture sees worker fixtures only, since it outlives
|
|
* every test fixture.
|
|
*
|
|
* A name the base already has is not a new fixture, so declaring it in
|
|
* `Extra` or `ExtraWorker` is a type error. Defined beside them, it extends
|
|
* that fixture, a built-in one included (`FixtureOverrides`): inside its
|
|
* function its own name is the fixture it extends, and it keeps that
|
|
* fixture's type and scope:
|
|
*
|
|
* base.extend({ app: async ({ app }, use) => { await app.open('/'); await use(app); } })
|
|
*/
|
|
export type FixtureDefinitions<Base, Extra, Worker = {}, ExtraWorker = {}> = {
|
|
readonly [K in keyof Extra]: K extends keyof Base
|
|
? never
|
|
: TestFixtureDefinition<Base & NoInfer<Extra> & NoInfer<ExtraWorker>, Extra[K]>;
|
|
} & {
|
|
// Not inferred: an `extend({ ... })` without type arguments infers its test
|
|
// fixtures from the definitions, as before worker fixtures; a worker
|
|
// fixture is declared through the second type argument.
|
|
readonly [K in keyof NoInfer<ExtraWorker>]: K extends keyof Base | keyof Extra
|
|
? never
|
|
: WorkerFixtureDefinition<Worker & NoInfer<ExtraWorker>, NoInfer<ExtraWorker>[K]>;
|
|
} & FixtureOverrides<Base & NoInfer<Extra> & NoInfer<ExtraWorker>, Worker & NoInfer<ExtraWorker>, Base, Worker>;
|
|
/**
|
|
* Fixtures that extend ones the base already has, each optional. `Sees` and
|
|
* `WorkerSees` are what an override's function receives: the base's
|
|
* fixtures, and the ones declared beside it.
|
|
*/
|
|
export type FixtureOverrides<Sees, WorkerSees, Base = Sees, Worker = WorkerSees> = {
|
|
readonly [K in keyof Base]?: FixtureOverride<Base, Worker, K, Sees, WorkerSees>;
|
|
};
|
|
/**
|
|
* A fixture that extends `K`, one the base already has: it hands over the
|
|
* same type and keeps the scope, so the override of a worker fixture
|
|
* receives worker fixtures (`WorkerSees`) only.
|
|
*/
|
|
export type FixtureOverride<Base, Worker, K extends keyof Base, Sees, WorkerSees> = K extends keyof Worker
|
|
? FixtureFn<WorkerSees, Base[K]> | WorkerFixtureDefinition<WorkerSees, Base[K]>
|
|
: TestFixtureDefinition<Sees, Base[K]>;
|
|
```
|
|
|
|
### Function types
|
|
|
|
```ts
|
|
/**
|
|
* A test body. It names the fixtures it uses by destructuring its first
|
|
* parameter (`async ({ app, screen }) => {}`), and the runner sets up only
|
|
* those; a body that takes the whole object (`async (fixtures) => {}`) or a
|
|
* rest property is a `COLLECTION_ERROR`. The same holds for a hook and for
|
|
* the first parameter of a `test.extend()` fixture.
|
|
*/
|
|
export type TestFn<Fixtures = TestFixtures> = (fixtures: Fixtures) => void | Promise<void>;
|
|
|
|
/** A setup test body; `session.save` is the extra fixture. */
|
|
export type SetupFn<Fixtures = TestFixtures> = (
|
|
fixtures: Fixtures & SetupFixtures,
|
|
) => void | Promise<void>;
|
|
|
|
/** A hook body: `beforeEach` and `afterEach` get the test's fixtures, `beforeAll` and `afterAll` its worker fixtures. */
|
|
export type TestHookFn<Fixtures = TestFixtures> = (fixtures: Fixtures) => void | Promise<void>;
|
|
|
|
/** @deprecated use `TestHookFn<WorkerFixtures>`. */
|
|
export type SuiteHookFn = TestHookFn<WorkerFixtures>;
|
|
|
|
/** A `describe` body. An async body is a type error. */
|
|
export type SynchronousBody<Result> = Extract<Result, PromiseLike<unknown>> extends never
|
|
? () => Result
|
|
: never;
|
|
|
|
/**
|
|
* The `test.each` body arguments for the rows' type: array rows spread, then
|
|
* the fixtures. Rows that are not all arrays are passed whole. Not
|
|
* distributive, so a mixed table is decided once, as at run time. An
|
|
* `unknown` row may be either at run time, so only the fixtures, last, are known.
|
|
*/
|
|
export type EachArgs<Row, Fixtures> = [Row] extends [readonly unknown[]]
|
|
? [...Row, Fixtures]
|
|
: unknown extends Row
|
|
? [...unknown[], Fixtures]
|
|
: [Row, Fixtures];
|
|
|
|
/** A `test.each` or `test.for` body: the row's arguments, then the fixtures. */
|
|
export type EachFn<Args extends readonly unknown[]> = (...args: Args) => void | Promise<void>;
|
|
|
|
/** A `test.skipIf` or `test.runIf` condition. A Promise is a type error. */
|
|
export type SynchronousCondition<Condition> = Extract<Condition, PromiseLike<unknown>> extends never
|
|
? Condition
|
|
: never;
|
|
|
|
/** Opaque handle of a registered test. */
|
|
export interface TestCase {
|
|
readonly [testCaseBrand]: true;
|
|
}
|
|
```
|