mirror of
https://github.com/tester-army/e2e.git
synced 2026-11-22 18:19:48 +08:00
Co-authored-by: Oskar Kwaśniewski <oskar@okwasniewski.com>
620 lines
26 KiB
Plaintext
620 lines
26 KiB
Plaintext
---
|
||
title: "e2e/runner"
|
||
description: "Run a test suite or an exploration, list a selection, or host e2e sessions in another test runner."
|
||
"og:image": "https://e2e.com/docs/images/og/reference/runner.png?v=a4609b13d7"
|
||
"twitter:image": "https://e2e.com/docs/images/og/reference/runner.png?v=a4609b13d7"
|
||
---
|
||
|
||
`e2e/runner` runs a suite from code without invoking the CLI. `run` executes
|
||
tests and returns their report, `explore` runs one exploration and returns
|
||
its findings, `list` returns a selection, and `openHost` hosts e2e sessions
|
||
in another test runner. Importing `e2e` from a test file does not load the
|
||
runner.
|
||
|
||
<Note>
|
||
`run`, `list`, `explore`, and `openHost` need Node.js. On Bun, they throw
|
||
`NODE_UNSUPPORTED`; run tests with the [CLI](/reference/cli#bun) there.
|
||
</Note>
|
||
|
||
```ts title="scripts/run-e2e.ts"
|
||
import type { E2EConfig } from 'e2e';
|
||
import { run } from 'e2e/runner';
|
||
import { web } from '@e2e-dev/web';
|
||
|
||
const config: E2EConfig = {
|
||
targets: [{ name: 'web', engine: web(), app: { url: 'http://localhost:3000' } }],
|
||
};
|
||
const controller = new AbortController();
|
||
const result = await run({
|
||
config,
|
||
files: ['tests/signup.e2e.ts'],
|
||
interruptSignal: controller.signal,
|
||
onEvent(event) {
|
||
if (event.type === 'test-started') console.log(event.title);
|
||
},
|
||
});
|
||
|
||
console.log(result.status, result.report.run.summary);
|
||
```
|
||
|
||
## Reference
|
||
|
||
### Functions
|
||
|
||
| Function | Returns |
|
||
| --- | --- |
|
||
| [`run`](#run) | `Promise<RunResult>` |
|
||
| [`explore`](#explore) | `Promise<ExploreResult>` |
|
||
| [`list`](#list) | `Promise<ListResult>` |
|
||
| [`openHost`](#openhost) | `Promise<E2EHost<Fixtures>>` |
|
||
| [`isE2EError`](#ise2eerror) | `boolean` |
|
||
|
||
#### `run`
|
||
|
||
`run(options?: RunOptions)` executes a complete suite. It finishes cleanup
|
||
before resolving and never exits the caller's process.
|
||
|
||
```ts
|
||
const result = await run({ files: ['tests/signup.e2e.ts'] });
|
||
process.exitCode = result.exitCode;
|
||
```
|
||
|
||
##### Options
|
||
|
||
| Option | Example | Type | Default |
|
||
| --- | --- | --- | --- |
|
||
| `cwd` | `'apps/web'` | `string` | the current working directory |
|
||
| `config` | `'e2e.ci.config.ts'` | `string \| E2EConfig` | discovered `e2e.config.ts` |
|
||
| `files` | `['tests/signup.e2e.ts:12']` | `readonly string[]` | the configured suite |
|
||
| `changedSince` | `'origin/main'` | `string` | - |
|
||
| `interruptSignal` | `controller.signal` | `AbortSignal` | - |
|
||
| `onEvent` | `(event) => {}` | `RunEventSink` | - |
|
||
|
||
- `files` takes paths, globs, bare names, or `file:line`.
|
||
- `changedSince` works as [`--changed-since`](/filters).
|
||
- Aborting `interruptSignal` interrupts tests and executes teardown.
|
||
- `onEvent` receives typed progress events. Terminal progress is off. It
|
||
must not block. A throwing or rejecting callback is disabled without
|
||
failing the run.
|
||
- Put execution settings, such as retries and workers, in the config.
|
||
- A runtime config (an `E2EConfig` value) runs with one worker in the
|
||
caller's process. A file config supports the configured worker processes.
|
||
|
||
##### Result
|
||
|
||
| Field | Meaning |
|
||
| --- | --- |
|
||
| `exitCode` | The CLI-equivalent exit code: `0`, `1`, `2`, `3`, `4`, or `130`. |
|
||
| `status` | `passed`, `failed`, `blocked`, `error`, or `interrupted`. |
|
||
| `report` | The complete report. Read `report.run.results`, `report.run.errors`, and `report.run.summary` for outcomes. |
|
||
| `reportPath` | Absolute path to `report.json`, or `undefined` if it could not be written. |
|
||
| `aiTracePath` | AI trace path, or `undefined` when no AI trace was written. |
|
||
|
||
Test failures, config failures, and interruption resolve with their status
|
||
and errors in the report.
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `RUNNER_BUSY` | Another suite is running, or a host is open. Close every host before running a suite, and await the suite before opening a host. |
|
||
|
||
- Helper modules your test files import stay cached between `run` calls in
|
||
one process. An edit to them needs a new process.
|
||
- Each run that returns a report sends one anonymous
|
||
[`e2e_run_completed`](/telemetry) event, with the same opt-outs as the CLI.
|
||
The first call can print the telemetry notice on stderr.
|
||
- Suite output uses UTF-8 text redaction and does not preserve binary data.
|
||
Your application's logs and progress callback output stay under your
|
||
control.
|
||
|
||
#### `explore`
|
||
|
||
`explore(options?: ExploreOptions)` runs one [exploration](/explore), the run
|
||
`e2e explore` starts. It finishes cleanup before resolving and never exits
|
||
the caller's process.
|
||
|
||
```ts title="scripts/explore-e2e.ts"
|
||
import type { E2EConfig } from 'e2e';
|
||
import { explore } from 'e2e/runner';
|
||
import { web } from '@e2e-dev/web';
|
||
import { gateway } from 'ai';
|
||
|
||
const config: E2EConfig = {
|
||
agents: { default: { model: gateway('openai/gpt-6-luna-fast') } },
|
||
targets: [{ name: 'web', engine: web(), app: { url: 'http://localhost:3000' } }],
|
||
};
|
||
const controller = new AbortController();
|
||
const result = await explore({
|
||
config,
|
||
goal: 'Explore the checkout flow like a first-time buyer and report anything off',
|
||
maxSteps: 4,
|
||
interruptSignal: controller.signal,
|
||
onEvent(event) {
|
||
if (event.type === 'explore' && event.progress.phase === 'finding') console.log(event.progress.finding.title);
|
||
},
|
||
});
|
||
|
||
console.log(result.status, result.explore.ended, result.explore.findings.length);
|
||
```
|
||
|
||
##### Options
|
||
|
||
| Option | Example | Type | Default |
|
||
| --- | --- | --- | --- |
|
||
| `cwd` | `'apps/web'` | `string` | the current working directory |
|
||
| `config` | `'e2e.ci.config.ts'` | `string \| E2EConfig` | discovered `e2e.config.ts` |
|
||
| `goal` | `'Explore the login flow'` | `string` | `'Explore the app and find bugs'`; with `changedSince`, `'Explore what this change affects and find bugs in it'` |
|
||
| `target` | `'web'` | `string` | the first configured target |
|
||
| `agent` | `'smart'` | `string` | `agents.default` |
|
||
| `session` | `'loggedIn'` | `string` | - |
|
||
| `changedSince` | `'origin/main'` | `string` | - |
|
||
| `maxSteps` | `4` | `number` | `8` |
|
||
| `timeoutMs` | `300_000` | `number` | `600_000` |
|
||
| `interruptSignal` | `controller.signal` | `AbortSignal` | - |
|
||
| `onEvent` | `(event) => {}` | `RunEventSink` | - |
|
||
|
||
- `goal` is at most 2,000 characters.
|
||
- `session` names a session a [setup test](/authentication) saves. That setup
|
||
runs first, and the exploration starts from it, as `--session` does.
|
||
- `changedSince` explores the change since that ref, as
|
||
[`--changed-since`](/reference/cli#e2e-explore) does.
|
||
- `maxSteps` is from 1 to 100. `timeoutMs` is from `180000` to `7200000`.
|
||
- `onEvent` receives the same events as for `run`, plus an `explore` event
|
||
for each planning decision, step, and finding. What `e2e explore` prints
|
||
before the run starts, such as the target it chose, arrives as a `notice`
|
||
event with target `run`, before the `plan` event.
|
||
- Recording and output settings, such as `video`, `trace`, and `output`,
|
||
come from the config.
|
||
|
||
##### Result
|
||
|
||
`ExploreResult` has every [`RunResult`](#run) field, plus `explore`, the
|
||
record `report.run.explore` also holds.
|
||
|
||
| Field | Meaning |
|
||
| --- | --- |
|
||
| `explore.goal`, `explore.budgets` | The goal, with secret values redacted, and the step and time budgets it ran with. |
|
||
| `explore.ended` | Why exploration stopped: `finished`, `skipped` (with `changedSince`, the planner judged the change has nothing a user can see; the run passes), `step-limit`, `time`, `stuck`, or `aborted`. |
|
||
| `explore.summary` | The agent's closing assessment, when it gave one. |
|
||
| `explore.change` | With `changedSince`, the change it planned from: `base`, the merge base, and `files`, how many changed. |
|
||
| `explore.steps` | Each step the agent planned and ran, with its status: `passed`, `failed`, `blocked`, or `exhausted`. |
|
||
| `explore.findings` | Each finding: `kind`, `severity` from 1 to 5, `title`, `expected`, `actual`, and `reproduction`. An `issue` fails the run; a `warning` does not. |
|
||
|
||
Once the exploration starts, findings, failures, and interruption resolve
|
||
with their status in the report. Options or a config that cannot start one
|
||
reject first, as the matching `e2e explore` flags do. Recognize them with
|
||
[`isE2EError`](#ise2eerror).
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `INVALID_CONFIG` | `goal`, `maxSteps`, or `timeoutMs` is out of range, or `agent` names no configured agent. |
|
||
| `UNKNOWN_TARGET` | `target` names no configured target. |
|
||
| `MODEL_UNAVAILABLE` | The agent has no model. |
|
||
| `INVALID_CHANGED_SINCE` | `changedSince` is empty, starts with `-`, or names a ref git can't measure a change from. |
|
||
| `RUNNER_BUSY` | A suite or an exploration is running, or a host is open. |
|
||
| The codes `run` records | The config is missing, or the loader refused it. |
|
||
|
||
Each exploration that returns a report sends one anonymous
|
||
[`e2e_run_completed`](/telemetry) event, as `run` does.
|
||
|
||
#### `list`
|
||
|
||
`list(options?: ListOptions)` collects and selects the way
|
||
[`e2e list`](/reference/cli#e2e-list) does, and stops there. No app process,
|
||
engine, or worker starts, and nothing is written.
|
||
|
||
```ts
|
||
import { isE2EError, list } from 'e2e/runner';
|
||
|
||
const { pairs, unmatched, targets } = await list({
|
||
files: ['tests/signup.e2e.ts'],
|
||
grep: [/checkout/],
|
||
});
|
||
|
||
for (const pair of pairs) {
|
||
console.log(`${pair.id} ${pair.disposition} ${pair.titlePath.join(' › ')}`);
|
||
}
|
||
```
|
||
|
||
Each call imports the test files again, so a later `list()` in the same
|
||
process sees an edit.
|
||
|
||
##### Options
|
||
|
||
| Option | Example | Type | Default |
|
||
| --- | --- | --- | --- |
|
||
| `config` | `'e2e.ci.config.ts'` | `string \| E2EConfig` | discovered from `cwd`, the same walk `e2e list` does |
|
||
| `targets` | `['web']` | `readonly string[]` | - |
|
||
| `cwd` | `'apps/web'` | `string` | the current working directory |
|
||
| `files` | `['tests/', 'signup.e2e.ts:12']` | `readonly string[]` | the configured suite |
|
||
| `tags`, `tagMode`, `excludeTags` | `{ tags: ['smoke'], tagMode: 'all' }` | `readonly string[]`, `'any' \| 'all'` | - |
|
||
| `grep`, `grepInvert` | `[/checkout/]` | `readonly RegExp[]` | - |
|
||
| `lastFailed` | `true` | `boolean` | - |
|
||
| `shard` | `{ index: 1, total: 4 }` | `{ index, total }` | - |
|
||
| `changedSince` | `'origin/main'` | `string` | - |
|
||
| `passWithNoTests` | `true` | `boolean` | - |
|
||
| `output` | `'.e2e'` | `string` | the config's `output` |
|
||
| `env` | `{ ...process.env, CI: '1' }` | `NodeJS.ProcessEnv` | `process.env` |
|
||
|
||
- `targets` names targets, as `--target` does.
|
||
- `files` is the positional file selection: paths, globs, bare names, and
|
||
`file:line`.
|
||
- `tags`, `tagMode`, and `excludeTags` are the `--tag`, `--tag-mode`, and
|
||
`--exclude-tag` filters.
|
||
- `grep` and `grepInvert`: a test's describe titles and title, joined by
|
||
spaces, must match one `grep` pattern and no `grepInvert` pattern.
|
||
- `lastFailed` keeps only tests the previous run did not pass, read from
|
||
`report.json` under `output`.
|
||
- `shard` is the `--shard index/total` slice.
|
||
- `changedSince` works as `--changed-since`. The [test filters](/filters)
|
||
run in the calling process; `list` prints none of their lines, `e2e list`
|
||
does.
|
||
- `passWithNoTests` resolves with no runnable test instead of throwing
|
||
`NO_TESTS`.
|
||
- `output` is the results directory.
|
||
- `env` is the environment the config resolves against.
|
||
|
||
##### Result
|
||
|
||
`ListResult` is `{ pairs, unmatched, targets }`.
|
||
|
||
- `pairs` is one `ListedPair` per test, target, and agent, in selection
|
||
order, including pairs a filter removed. Repeats are not expanded.
|
||
- `unmatched` is the positional arguments, as written, that selected no
|
||
discovered file.
|
||
- `targets` is the selected target names, in config order.
|
||
|
||
| Field | Meaning |
|
||
| --- | --- |
|
||
| `id` | The result id `report.json` stores for repeat 0. |
|
||
| `file`, `title`, `titlePath`, `kind`, `tags` | The collected test. `kind` is `test` or `setup`. `tags` is `[]` when the test declares none. |
|
||
| `source` | The declaration, project-relative: `{ file, line, column }`. `undefined` when the test recorded none. |
|
||
| `session` | The session the test consumes, from the test or its describe chain. `undefined` when it consumes none. |
|
||
| `sessions` | Session names a setup produces. `[]` on an ordinary test. |
|
||
| `serialId` | Set when the test belongs to a serial group, otherwise `undefined`. |
|
||
| `target`, `agent` | The target name and the configured agent this pair runs as. |
|
||
| `disposition` | `run`, `skip`, or `filtered`. |
|
||
| `reason` | Why the pair is skipped or filtered, such as `title does not match --grep`. Omitted when it runs. |
|
||
| `filter` | The name of the test filter that left the pair out. A pair a selection flag filtered has none. |
|
||
|
||
`e2e list` prints the `run` and `skip` pairs, and the pairs a test filter
|
||
left out. `list` returns every `filtered` pair.
|
||
|
||
##### Errors
|
||
|
||
A failure is the same classified error `e2e list` exits with. Recognize one
|
||
with [`isE2EError`](#ise2eerror) and branch on `error.code`.
|
||
|
||
```ts
|
||
import { isE2EError, list } from 'e2e/runner';
|
||
|
||
try {
|
||
await list({ targets: ['missing'] });
|
||
} catch (error) {
|
||
if (isE2EError(error) && error.code === 'UNKNOWN_TARGET') {
|
||
console.error(error.message);
|
||
}
|
||
}
|
||
```
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `NO_TESTS` | The selection is empty and `passWithNoTests` is not set. A `ConfigurationError`. When the test filters leave nothing to run, `list` resolves instead, with every pair `filtered` or `skip`. |
|
||
| `UNKNOWN_TARGET` | A name in `targets` is not a configured target. |
|
||
| `INVALID_CHANGED_SINCE` | `changedSince` is empty or starts with `-`, or the test filters run and it names a ref git can't measure a change from. |
|
||
| The codes `run` records | The config is missing, the loader refused it, or collection failed. |
|
||
|
||
#### `openHost`
|
||
|
||
`openHost<Fixtures>(options?: OpenHostOptions)` prepares e2e for Playwright
|
||
Test, Vitest, Jest, or your own harness, and returns a [host](#host).
|
||
|
||
```ts
|
||
import { openHost } from 'e2e/runner';
|
||
|
||
await using host = await openHost({ target: 'web' });
|
||
await using session = await host.openSession({ name: 'checkout' });
|
||
await session.fixtures.app.open('/checkout');
|
||
```
|
||
|
||
- Your runner owns collection, retries, and reporting.
|
||
- The host prepares the target's engine, starts app processes, and
|
||
initializes the engine once per worker lifetime. Use a worker-scoped host
|
||
and a test-scoped session; see the
|
||
[Playwright Test example](#playwright-test-fixtures).
|
||
- Each [`host.openSession`](#host-opensession) creates a test-lifetime attempt
|
||
with fresh fixtures, a deadline, steps, and artifacts.
|
||
- If opening fails, e2e tears down what started before throwing the error.
|
||
|
||
##### Options
|
||
|
||
| Option | Example | Type | Default |
|
||
| --- | --- | --- | --- |
|
||
| `config` | `'e2e.config.ts'` | `string \| E2EConfig` | nearest `e2e.config.ts`, `.mts`, `.js`, or `.mjs` |
|
||
| `cwd` | `'apps/web'` | `string` | `process.cwd()` |
|
||
| `target` | `'android'` | `string` | the only target |
|
||
| `env` | `process.env` | `NodeJS.ProcessEnv` | `process.env` |
|
||
| `signal` | `controller.signal` | `AbortSignal` | - |
|
||
| `headed` | `true` | `boolean` | `false` |
|
||
| `onNotice` | `(target, message) => {}` | `(target: string, message: string) => void` | - |
|
||
|
||
- `config` is a config file path relative to `cwd`, or an `E2EConfig` value.
|
||
Omit it to discover the config from `cwd`, the same walk `list` does.
|
||
- `cwd` is the directory for config discovery, and the project root of a
|
||
config value.
|
||
- `target` is required when the config declares several targets.
|
||
- `env` is the environment the config, engines, and providers read.
|
||
- `signal` cancels host provisioning, a pending session open, or live
|
||
fixture calls.
|
||
- `headed` runs a browser engine headed.
|
||
- `onNotice` receives progress outside a step, such as provisioning or an
|
||
app process. It must not throw.
|
||
|
||
A host given a config path loads the file and its project-file imports fresh. Package
|
||
imports stay shared, so their singleton state is not recreated per host. A
|
||
config value uses its own engine instances. Each instance belongs to one
|
||
host until that host closes, so a second host must use another instance.
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `UNKNOWN_TARGET` | `target` names no configured target. |
|
||
| `TARGET_REQUIRED` | The config declares several targets and `target` names none. |
|
||
| `RUNNER_BUSY` | A suite is running. Await it before opening a host. |
|
||
|
||
Opening failures are classified, redacted errors; raw engine causes are not
|
||
exposed. Match them with [`isE2EError`](#ise2eerror) and `error.code`.
|
||
|
||
#### `isE2EError`
|
||
|
||
A type guard that recognizes a classified e2e error. Branch on its
|
||
`error.code`.
|
||
|
||
```ts
|
||
if (isE2EError(error) && error.code === 'NO_TESTS') process.exitCode = 0;
|
||
```
|
||
|
||
`instanceof` can miss an e2e error across copies of the package, so use
|
||
`isE2EError`. `ConfigurationError` and `isE2EError` are both exported
|
||
from `e2e/runner`.
|
||
|
||
### Host
|
||
|
||
`E2EHost<Fixtures>` is what [`openHost`](#openhost) returns.
|
||
|
||
| Member | Returns |
|
||
| --- | --- |
|
||
| `runId`, `target` | `string` |
|
||
| [`host.openSession`](#host-opensession) | `Promise<E2ESession<Fixtures>>` |
|
||
| [`host.close`](#host-close) | `Promise<readonly SerializedError[]>` |
|
||
|
||
`runId` and `target` are the identity the host's sessions share.
|
||
|
||
Only one hosted attempt may be opening, live, or closing per process, even
|
||
across different hosts and whether the config is a file or a value.
|
||
|
||
- An overlapping open is rejected before engine attempt state starts. Await
|
||
`session.close()` before opening the next session on the same host.
|
||
- A launch that fails or is cancelled ends what it opened and leaves the host
|
||
usable, as a run keeps its worker for the retry.
|
||
- A session whose engine cleanup failed closes the host permanently, since
|
||
its isolation is uncertain; open a new host.
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `WORKER_PROTOCOL` | An overlapping hosted attempt, or a call on a closed host. |
|
||
|
||
#### `host.openSession`
|
||
|
||
`host.openSession(options: OpenSessionOptions)` opens a fresh
|
||
[session](#session) on this host's target. A session is one test attempt with
|
||
no retry loop. A retry opens a new session on the same worker host.
|
||
|
||
```ts
|
||
const session = await host.openSession({
|
||
name: 'checkout',
|
||
timeout: 60_000, // default: the config's timeout
|
||
onStep: (step) => {
|
||
if (step.phase === 'end') console.log(`${step.api} ${step.label}: ${step.status}`);
|
||
},
|
||
});
|
||
```
|
||
|
||
##### Options
|
||
|
||
| Option | Example | Type | Default |
|
||
| --- | --- | --- | --- |
|
||
| `name` | `'checkout'` | `string` | required |
|
||
| `timeout` | `60_000` | `number` | the config's `timeout` |
|
||
| `signal` | `controller.signal` | `AbortSignal` | - |
|
||
| `agent` | `'smart'` | `string` | the first configured agent |
|
||
| `onStep` | `(step) => {}` | `(progress: StepProgress) => void` | - |
|
||
|
||
- `name` is the test name for step and trace reporting.
|
||
- `timeout` is the deadline for the session's operations in milliseconds,
|
||
positive and finite, counted from when the session opens. It does not bound
|
||
provisioning or opening, and it does not close the session.
|
||
- `signal` cancels opening or live fixture calls. The host's `signal` also
|
||
applies. Provisioning happens in `openHost`, so only the host's `signal`
|
||
cancels it; either signal cancels a launch.
|
||
- `agent` is the agent the `agent` fixture uses when a call names none.
|
||
- `onStep` receives live `StepProgress`, the same feed reporters receive. It
|
||
must not block or throw. Its `phase` is `start`, `event`, `activity`,
|
||
`replay`, or `end`. An `end` event includes `status`, `durationMs`,
|
||
`modelCalls`, and a redacted `error` when the step failed.
|
||
- `timeout`, `agent`, and `onStep` belong to the session, not `openHost`.
|
||
|
||
| Throws | When |
|
||
| --- | --- |
|
||
| `INVALID_CONFIG` | `name` is not a string, or `timeout` is not positive and finite. |
|
||
| `CANCELLED` | The session's or the host's `signal` aborted before or during opening. |
|
||
| `WORKER_PROTOCOL` | Another hosted attempt is opening, live, or closing, or the host is closed. |
|
||
|
||
#### `host.close`
|
||
|
||
`host.close()` cancels any active or pending session, waits for it to close,
|
||
then disposes the engine, stops app processes, and releases provisioning.
|
||
|
||
```ts
|
||
const failures = await host.close();
|
||
if (failures.length > 0) throw new AggregateError(failures, 'e2e host close failed');
|
||
```
|
||
|
||
- Idempotent: it disposes and releases resources only once.
|
||
- A host is `AsyncDisposable`: `await using host = await openHost()` closes
|
||
it at scope exit and throws an `AggregateError` when closing reports
|
||
failures.
|
||
|
||
### Session
|
||
|
||
`E2ESession<Fixtures>` is what [`host.openSession`](#host-opensession) returns.
|
||
|
||
| Member | Returns |
|
||
| --- | --- |
|
||
| `fixtures` | `TestFixtures & Fixtures` |
|
||
| `runId`, `attemptId`, `target` | `string` |
|
||
| `steps()` | `readonly StepRecord[]` |
|
||
| `report()` | `Report \| undefined` |
|
||
| `artifactsDir` | `string` |
|
||
| `tracePath` | `string` |
|
||
| [`session.close`](#session-close) | `Promise<readonly SerializedError[]>` |
|
||
|
||
- `fixtures` holds `agent`, `app`, `screen`, `platform`, and the engine's
|
||
fixtures.
|
||
- `runId` and `target` are the host's run and target; `attemptId` is unique
|
||
to this attempt.
|
||
- `steps()` returns read-only `StepRecord`s for steps run so far, as the
|
||
[report](/reference/reporters) records them.
|
||
- `report()` returns the session `Report` the trace is rendered from, after
|
||
close and a successful trace write; otherwise `undefined`. Its status
|
||
describes the SDK session, not your runner's test-body verdict.
|
||
- `artifactsDir` is the absolute directory for this attempt's screenshots,
|
||
recordings, and trace.
|
||
- `tracePath` is the absolute path to `trace.md`; attach or read it after
|
||
`close()` settles.
|
||
|
||
#### `session.close`
|
||
|
||
`session.close()` cancels remaining operations, ends the attempt, and writes
|
||
its trace. It returns soft expectation, abandoned-operation (an unsettled
|
||
poll or step), and cleanup failures instead of throwing them.
|
||
|
||
```ts
|
||
const failures = await session.close();
|
||
if (failures.length > 0) throw new AggregateError(failures, 'e2e session close failed');
|
||
```
|
||
|
||
- Idempotent: later calls return the same failures.
|
||
- The host stays open for the next test.
|
||
- A session is `AsyncDisposable`:
|
||
`await using session = await host.openSession({ name: 'checkout' })`
|
||
closes it at scope exit and throws an `AggregateError` of the returned
|
||
failures.
|
||
|
||
#### Deadline and soft failures
|
||
|
||
`expect.soft` collects failures for the session's whole lifetime. Check
|
||
`close()`'s result to fail the host runner's test.
|
||
|
||
- `expect.poll` uses the session's remaining budget, even when its own
|
||
timeout is longer.
|
||
- Contributed fixture operations also honor that budget, including
|
||
operations that disable their own timeout.
|
||
- When the deadline passes, a poll or operation that was watching it reports
|
||
its own cut. Everything else still running, including an operation an
|
||
engine or a model never answered, fails with `TEST_TIMEOUT`.
|
||
- Either way the session's outcome is timed out and `close()` returns the
|
||
timeout. Later operations fail with `TEST_TIMEOUT`; close the session.
|
||
|
||
#### Artifacts
|
||
|
||
Artifacts live under `<output>/results/host/<resultSegment>/`. At close,
|
||
e2e writes `trace.md` there through the same redacted formatting reports
|
||
use.
|
||
|
||
- e2e generates a safe `resultSegment` from the name and the unique attempt
|
||
identity. The name never becomes a raw path component.
|
||
- Repeated names and retries get separate directories.
|
||
- With provider-backed secrets, the directory uses an identity-only segment,
|
||
since unresolved values cannot be redacted from a title slug. The trace
|
||
still shows the redacted name.
|
||
- Artifact paths are relative to `<output>/results`, not
|
||
`session.artifactsDir`.
|
||
- Video follows the target's mode for a first attempt. Failure-only video is
|
||
kept for soft failures and for failed, blocked, timed-out, or cancelled
|
||
steps.
|
||
|
||
<Note>
|
||
A hosted session writes its trace, not `report.json`. For your runner's
|
||
reporting, use `onStep`, `steps()`, `report()`, and `close()`'s failures.
|
||
Your runner's test-body failures are not passed to e2e.
|
||
|
||
The one-shot attempt `e2e mcp` uses stays internal. There is no top-level
|
||
`openSession` export.
|
||
</Note>
|
||
|
||
## Types
|
||
|
||
`e2e/runner` exports these types beside the functions.
|
||
|
||
| Type | Describes |
|
||
| --- | --- |
|
||
| `RunOptions`, `RunResult` | [`run`](#run)'s options and result |
|
||
| `ExploreOptions`, `ExploreResult` | [`explore`](#explore)'s options and result |
|
||
| `RunEventSink` | The `onEvent` callback |
|
||
| `ListOptions`, `ListResult`, `ListedPair` | [`list`](#list)'s options, result, and one pair |
|
||
| `OpenHostOptions`, `E2EHost` | [`openHost`](#openhost)'s options and the [host](#host) |
|
||
| `OpenSessionOptions`, `E2ESession` | [`openSession`](#host-opensession)'s options and the [session](#session) |
|
||
| `StepProgress`, `StepRecord` | One `onStep` progress event, and one step `steps()` returns |
|
||
| `SerializedError` | One failure `close()` returns |
|
||
|
||
## Examples
|
||
|
||
### Playwright Test fixtures
|
||
|
||
A worker-scoped host and a test-scoped session:
|
||
|
||
```ts title="tests/fixtures.ts"
|
||
import { test as base } from '@playwright/test';
|
||
import { openHost, type E2EHost, type E2ESession } from 'e2e/runner';
|
||
import type { Device } from '@e2e-dev/mobile';
|
||
|
||
type Fixtures = { device: Device };
|
||
|
||
const test = base.extend<
|
||
{ e2e: E2ESession<Fixtures> },
|
||
{ e2eHost: E2EHost<Fixtures> }
|
||
>({
|
||
e2eHost: [async ({}, use) => {
|
||
const host = await openHost<Fixtures>({ target: 'android' });
|
||
try {
|
||
await use(host);
|
||
} finally {
|
||
const failures = await host.close();
|
||
if (failures.length > 0) throw new AggregateError(failures, 'e2e host close failed');
|
||
}
|
||
}, { scope: 'worker' }],
|
||
e2e: async ({ e2eHost }, use, testInfo) => {
|
||
const session = await e2eHost.openSession({
|
||
name: testInfo.titlePath.join(' › '),
|
||
// Playwright's 0 disables its timeout; e2e then falls back to the config's.
|
||
timeout: testInfo.timeout || undefined,
|
||
onStep: (step) => {
|
||
if (step.phase === 'end') console.log(`${step.api} ${step.label}: ${step.status}`);
|
||
},
|
||
});
|
||
try {
|
||
await use(session);
|
||
} finally {
|
||
const failures = await session.close();
|
||
await testInfo.attach('e2e trace', { path: session.tracePath, contentType: 'text/markdown' });
|
||
if (failures.length > 0) throw new AggregateError(failures, 'e2e session close failed');
|
||
}
|
||
},
|
||
});
|
||
|
||
test('turns on dark mode', async ({ e2e }) => {
|
||
const { agent, app } = e2e.fixtures;
|
||
await agent.act('open settings and turn on dark mode');
|
||
await app.screenshot();
|
||
});
|
||
```
|