fix(run): a worker crash during a retry keeps the attempts before it (#837)

This commit is contained in:
Oskar Kwaśniewski
2026-10-05 16:09:43 +02:00
committed by GitHub
parent df8d421bee
commit 338378ed48
12 changed files with 1071 additions and 129 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"e2e": patch
---
A worker that crashes during a retry no longer drops the attempts before it. Before, a test whose first attempt failed and whose retry crashed the worker showed a single attempt 0 with `WORKER_CRASH`, and the first attempt's error was lost. The report now keeps every finished attempt and adds the crash as the next one. A crash after the last attempt (in an `afterAll`) is recorded on that attempt instead of as a retry. For a serial group, the finished group attempts and the members that finished in the crashed attempt stay in the report, the member that was running fails with `WORKER_CRASH`, and a crash after a member finished fails the group attempt without blaming that member. A forced interrupt keeps the attempts that finished and the verdict they reach, `flaky` included.
+1 -1
View File
@@ -119,7 +119,7 @@ These appear in terminal output and in the report as plain string codes.
| `TRACE_WITHHELD` | A trace from a session that knows a secret value could not be rewritten, so it was deleted, or the engine returned an archive path outside the attempt's artifact directory, so nothing was registered. Recorded in the attempt's `secondaryErrors`; never fails the attempt. | infrastructure | 3 |
| `BROWSER_INSTALL_FAILED` | Browser provisioning failed | infrastructure | 3 |
| `WORKER_INIT_FAILED` | A worker process never became ready | infrastructure | 3 |
| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped, a serial group's finished members included, since their results wait for the whole group. An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 |
| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped. Attempts that finished before the exit stay in the report, a retry's earlier attempts and a serial group's finished members included, and the crash is recorded as the attempt after them, or on the last attempt when it came after it (an `afterAll`). An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 |
| `WORKER_CRASH` | A worker exited during a specific test | infrastructure | 3 |
| `WORKER_ENTRY_MISSING` | The worker entry module could not be resolved. Packaging fault. | infrastructure | 3 |
| `NODE_UNSUPPORTED` | The runtime cannot run e2e's TypeScript loader: a Node.js outside `^22.22.3 \|\| >=24.8.0`, Bun (no `module.registerHooks`), or Deno. The CLI checks this before it starts and exits 2 with the same sentence; the loader checks it again where it registers its hooks, for a host that runs e2e's runner without the CLI | infrastructure | 3 |
+38 -15
View File
@@ -46,10 +46,11 @@ import type {
ResultStatus,
RunError,
SerialGroupRecord,
SerialMemberRecord,
} from './records.ts';
import { isFailedStatus } from './records.ts';
import { runWithRetries } from './retry.ts';
import { runSerialUnit, type SerialHost, type SharedSerialSession } from './serial.ts';
import { runSerialUnit, type SerialAttemptRun, type SerialAttemptStart, type SerialHost, type SharedSerialSession } from './serial.ts';
import { interruptedSkip, pairKey, pairResult, repeatSegment, unstartedResult } from './units.ts';
import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, redactsRecordings, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.ts';
import { isSecret } from '../secrets.ts';
@@ -64,6 +65,14 @@ import type { SetupFn } from '../types.ts';
export interface ExecutionEvents {
onResult?(result: ResultRecord): void;
onSerialGroup?(group: SerialGroupRecord): void;
/** An attempt of an ordinary or setup pair begins, its realm and `beforeAll` hooks included. */
onAttemptStart?(pair: TestTargetPair, attemptIndex: number): void;
/** One finished attempt of an ordinary or setup pair, ahead of the pair's result. */
onAttempt?(pair: TestTargetPair, attempt: AttemptRecord): void;
/** One member that finished in a serial group attempt, ahead of the group. */
onSerialMember?(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
/** One finished serial group attempt, ahead of the group. */
onSerialAttempt?(groupId: string, run: SerialAttemptRun): void;
/** Fires before a runnable pair starts; a serial unit announces each member as it begins. */
onPairStart?(pair: TestTargetPair): void;
/** Live step progress of one running attempt, for reporters. */
@@ -222,6 +231,16 @@ export class TargetExecutor implements SerialHost {
this.options.events?.onSerialGroup?.(group);
}
/** Streams one member finished in a serial group attempt (SerialHost). */
serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void {
this.options.events?.onSerialMember?.(groupId, attempt, member);
}
/** Streams one finished serial group attempt (SerialHost). */
serialAttemptFinished(groupId: string, run: SerialAttemptRun): void {
this.options.events?.onSerialAttempt?.(groupId, run);
}
/** Builds one engine operation context. */
private op(attemptId: string, timeoutMs: number, signal: AbortSignal): OperationContext {
return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' };
@@ -423,6 +442,7 @@ export class TargetExecutor implements SerialHost {
pair.options.retries + 1,
this.interruptSignal,
async (attemptIndex) => {
this.options.events?.onAttemptStart?.(pair, attemptIndex);
if (realm === null) realm = await this.realms.create(file);
const registered = findRegistered(realm, pair.test);
if (registered === undefined) {
@@ -437,6 +457,7 @@ export class TargetExecutor implements SerialHost {
kind: 'ordinary',
});
attempts.push(attempt);
this.options.events?.onAttempt?.(pair, attempt);
if (isFailedStatus(attempt.status)) {
// A failed realm is never reused, but afterAll still runs for every
// scope whose beforeAll started in it. A body that skipped itself
@@ -501,6 +522,7 @@ export class TargetExecutor implements SerialHost {
pair.options.retries + 1,
this.interruptSignal,
async (attemptIndex) => {
this.options.events?.onAttemptStart?.(pair, attemptIndex);
const realm =
attemptIndex === 0 && freshRegistration !== undefined
? this.realms.adopt(freshRegistration, file)
@@ -520,23 +542,24 @@ export class TargetExecutor implements SerialHost {
kind: 'setup',
staging,
});
const missing = attempt.status === 'passed' ? staging.missing() : [];
if (missing.length > 0) {
attempt.status = 'failed';
attempt.error = serializeError(
new E2EError(
'test',
'SESSION_CONTRACT',
`setup must save each declared session exactly once; missing: [${missing.join(', ')}]`,
),
{ phase: 'body' },
);
}
attempts.push(attempt);
// Sent before teardown and the session saves, as in runOrdinaryPair:
// a crash in either then lands on this attempt instead of erasing it.
this.options.events?.onAttempt?.(pair, attempt);
await this.realms.leave(realm);
if (attempt.status === 'passed') {
const missing = staging.missing();
if (missing.length > 0) {
attempt.status = 'failed';
attempt.error = serializeError(
new E2EError(
'test',
'SESSION_CONTRACT',
`setup must save each declared session exactly once; missing: [${missing.join(', ')}]`,
),
{ phase: 'body' },
);
return attempt;
}
for (const [name, saved] of staging.entries()) {
await this.options.sessionStore.save(name, this.sessionIdentity, saved);
}
+27 -8
View File
@@ -30,19 +30,38 @@ export async function runWithRetries(
interruptSignal: AbortSignal,
runOnce: (attemptIndex: number) => Promise<RetryAttempt | undefined>,
): Promise<ResultStatus> {
let finalStatus: ResultStatus = 'failed';
let attemptCount = 0;
const attempts: RetryAttempt[] = [];
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
if (interruptSignal.aborted) break;
const attempt = await runOnce(attemptIndex);
if (attempt === undefined) break;
attemptCount += 1;
if (attempt.status === 'passed') return attemptCount > 1 ? 'flaky' : 'passed';
if (attempt.status === 'interrupted') return attemptCount > 1 ? finalStatus : 'interrupted';
attempts.push(attempt);
// A body that skipped itself has decided; a retry would only ask again.
if (attempt.status === 'skipped') return 'skipped';
finalStatus = attempt.status;
if (attempt.status === 'passed' || attempt.status === 'interrupted' || attempt.status === 'skipped') break;
if (!isRetryEligible(attempt)) break;
}
return finalStatus;
return retryVerdict(attempts);
}
/**
* The verdict a sequence of finished attempts reaches: a pass is `flaky`
* after a failure, an interrupt during a retry keeps the verdict before it,
* and a self-skip decides. Also the verdict of attempts a worker finished
* before the run's interrupt stopped it, so both read alike.
*/
export function retryVerdict(attempts: readonly RetryAttempt[]): ResultStatus {
let verdict: ResultStatus = 'failed';
for (const [index, attempt] of attempts.entries()) {
switch (attempt.status) {
case 'passed':
return index > 0 ? 'flaky' : 'passed';
case 'interrupted':
return index > 0 ? verdict : 'interrupted';
case 'skipped':
return 'skipped';
default:
verdict = attempt.status;
}
}
return verdict;
}
+126 -16
View File
@@ -14,7 +14,9 @@ import type { AiTraceSnapshot } from '../internal/ai-trace.ts';
import type { DebugSnapshot } from '../internal/debug.ts';
import { InfrastructureError, serializeError } from '../internal/errors.ts';
import { timestamp, uuidv7 } from '../internal/ids.ts';
import type { ResultRecord, RunError, SerialGroupRecord } from './records.ts';
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord } from './records.ts';
import { retryVerdict } from './retry.ts';
import { SerialGroupProgress, serialGroupId } from './serial.ts';
import type { StepProgress } from './steps.ts';
import type { SpawnUnitRunner, UnitRunner } from './unit-runner.ts';
import {
@@ -114,6 +116,24 @@ interface TargetState {
failed: boolean;
}
/** The attempts of one pair heard ahead of its result. */
interface PairAttempts {
readonly finished: AttemptRecord[];
/** The index of the attempt that started and has not finished: a crash now is that attempt's. */
running: number | undefined;
}
/** The result of a pair from the attempts it finished, with the verdict they reach. */
function finishedResult(pair: TestTargetPair, attempts: AttemptRecord[]): ResultRecord {
const last = attempts.at(-1);
return pairResult(pair, {
status: retryVerdict(attempts),
selected: true,
attempts,
...(last?.status === 'skipped' ? { skip: last.skip } : {}),
});
}
/**
* One worker and the unit it is executing. Owns its own lifecycle so the
* scheduler never has to keep a separate record in sync with the transport.
@@ -128,8 +148,10 @@ class SchedulerWorker {
readonly reported = new Set<string>();
/** The pair (`pairKey`) executing now, when one is. */
inFlightPair: string | undefined;
/** Pairs (`pairKey`) that started in `unit`; a serial member reports only once its whole group is done. */
readonly started = new Set<string>();
/** Attempts of `unit`'s pairs heard ahead of the pair's result, by `pairKey`: what a crash keeps. */
readonly attempts = new Map<string, PairAttempts>();
/** What was heard of `unit`'s serial groups still running, by group report id: what a crash keeps. */
readonly serialGroups = new Map<string, SerialGroupProgress>();
sawFailure = false;
becameReady = false;
/** Told to tear down at once by a forced interrupt; its exit is then the one asked for. */
@@ -159,7 +181,8 @@ class SchedulerWorker {
this.state = 'busy';
this.unit = unit;
this.reported.clear();
this.started.clear();
this.attempts.clear();
this.serialGroups.clear();
this.inFlightPair = undefined;
this.sawFailure = false;
const pairs: WirePair[] = unit.pairs.map((pair) => ({
@@ -205,6 +228,26 @@ class SchedulerWorker {
this.killTimer.unref();
}
/** The attempts heard of one of `unit`'s pairs, started on first word of them. */
pairAttempts(key: string): PairAttempts {
let attempts = this.attempts.get(key);
if (attempts === undefined) {
attempts = { finished: [], running: undefined };
this.attempts.set(key, attempts);
}
return attempts;
}
/** The progress of one of `unit`'s serial groups, started on first word of it. */
serialGroup(groupId: string): SerialGroupProgress {
let progress = this.serialGroups.get(groupId);
if (progress === undefined) {
progress = new SerialGroupProgress();
this.serialGroups.set(groupId, progress);
}
return progress;
}
clearKillTimer(): void {
if (this.killTimer === undefined) return;
clearTimeout(this.killTimer);
@@ -584,7 +627,11 @@ class Scheduler {
}
case 'pair-start': {
worker.inFlightPair = pairKey(message.testId, message.agent, message.repeat);
worker.started.add(worker.inFlightPair);
if (message.serialId !== undefined) {
worker
.serialGroup(serialGroupId(message.serialId, worker.targetName, message.agent, message.repeat))
.memberStarted(message.testId);
}
const { type: _type, ...start } = message;
this.options.events.onTestStart?.(start, worker.targetName);
break;
@@ -609,7 +656,9 @@ class Scheduler {
case 'result': {
const state = this.targetState(worker);
const result = decodeResult(message.result, state.target);
worker.reported.add(pairKey(result.test.id, result.agent, result.repeat));
const key = pairKey(result.test.id, result.agent, result.repeat);
worker.reported.add(key);
worker.attempts.delete(key);
if (result.status !== 'passed' && result.status !== 'flaky' && result.status !== 'skipped') {
worker.sawFailure = true;
}
@@ -618,9 +667,28 @@ class Scheduler {
break;
}
case 'serial-group': {
worker.serialGroups.delete(message.group.id);
this.options.events.onSerialGroup(message.group);
break;
}
case 'attempt-start': {
worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat)).running = message.index;
break;
}
case 'attempt': {
const attempts = worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat));
attempts.finished.push(message.attempt);
attempts.running = undefined;
break;
}
case 'serial-member': {
worker.serialGroup(message.groupId).memberFinished(message.attempt, message.member);
break;
}
case 'serial-attempt': {
worker.serialGroup(message.groupId).attemptFinished(message.run);
break;
}
case 'unit-done': {
for (const runError of message.runErrors) this.options.events.onRunError(runError);
if (message.debug !== undefined) this.options.events.onDebug?.(message.debug);
@@ -724,18 +792,53 @@ class Scheduler {
return this.signalledWave.has(worker);
}
/** Emits records for a unit whose worker died before reporting it done. */
/**
* Emits records for a unit whose worker died before reporting it done.
* Attempts the worker finished stand. The test in flight gets a
* `WORKER_CRASH` attempt after them when the crash landed during an
* attempt; one between attempts (an `afterAll` after the last) is recorded
* on the last attempt instead, whose cleanup it forced. A serial group in
* flight gets a failed group attempt after its finished ones. A worker the
* run's interrupt stopped charges no crash: a test keeps the attempts it
* finished and the verdict they reach, as a retry the interrupt cut short
* does, and a serial group's attempt in flight is recorded interrupted.
*/
private synthesizeCrashResults(
state: TargetState,
worker: SchedulerWorker,
unit: WorkUnit,
): void {
const interrupted = this.interrupting;
const interrupted = this.interrupting && (worker.terminated || worker.killed);
const crash = serializeError(new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'));
const settledGroups = new Set<string>();
for (const pair of unit.pairs) {
const key = pairKey(pair.test.id, pair.agent, pair.repeat);
if (worker.reported.has(key)) continue;
const { serialId } = pair.test;
if (serialId !== undefined) {
const groupId = serialGroupId(serialId, worker.targetName, pair.agent, pair.repeat);
if (settledGroups.has(groupId)) continue;
const members = unit.pairs.filter(
(member) => member.test.serialId === serialId && member.agent === pair.agent && member.repeat === pair.repeat,
);
const progress = worker.serialGroups.get(groupId);
const settled = interrupted
? progress?.interrupted(members, state.target, this.stopSkip ?? INTERRUPTED_BEFORE_START)
: progress?.crashed(members, state.target, crash);
if (settled !== undefined) {
settledGroups.add(groupId);
this.options.events.onSerialGroup(settled.group);
for (const result of settled.results) this.report(result);
continue;
}
}
const { finished, running } = worker.attempts.get(key) ?? { finished: [], running: undefined };
if (interrupted) {
this.report(unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START));
this.report(
finished.length === 0
? unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START)
: finishedResult(pair, finished),
);
continue;
}
// A crashed setup never persisted its sessions; dependents must skip.
@@ -748,29 +851,36 @@ class Scheduler {
this.report(
unstartedResult(pair, {
cause: 'infrastructure-unavailable',
reason: worker.started.has(key)
? 'worker process exited before this serial group finished'
: 'worker process exited before this test started',
reason: 'worker process exited before this test started',
}),
);
continue;
}
const last = finished.at(-1);
if (last !== undefined && running === undefined) {
this.report(
finishedResult(pair, [
...finished.slice(0, -1),
{ ...last, secondaryErrors: [...last.secondaryErrors, crash], cleanup: 'forced' },
]),
);
continue;
}
this.report(
pairResult(pair, {
status: 'failed',
selected: true,
attempts: [
...finished,
{
id: uuidv7(),
index: 0,
index: running ?? finished.length,
status: 'failed',
startedAt: timestamp(),
durationMs: 0,
steps: [],
artifacts: [],
error: serializeError(
new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'),
),
error: crash,
secondaryErrors: [],
cleanup: 'forced',
},
+255 -67
View File
@@ -20,7 +20,7 @@ import type { StepRecord } from './steps.ts';
import { findRegistered, type FileRef, type Realm, RealmManager } from './realm.ts';
import type { AttemptRecord, ResultRecord, ResultStatus, SerialAttemptRecord, SerialGroupRecord, SerialMemberRecord, FailedStatus } from './records.ts';
import { isFailedStatus } from './records.ts';
import { runWithRetries } from './retry.ts';
import { isRetryEligible, retryVerdict, runWithRetries } from './retry.ts';
import { interruptedSkip, pairResult, repeatSegment } from './units.ts';
/**
@@ -87,6 +87,34 @@ export interface SerialHost {
* steps, duration, and error from the group it has already seen.
*/
emitSerialGroup(group: SerialGroupRecord): void;
/**
* Streams one member that finished in the group attempt running now, so a
* worker that dies before the group ends does not take it along.
*/
serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
/** Streams one finished group attempt, for the same reason. */
serialAttemptFinished(groupId: string, run: SerialAttemptRun): void;
}
/** The identity of a group attempt, known from its start. */
export type SerialAttemptStart = Pick<SerialAttemptRecord, 'id' | 'index' | 'startedAt'>;
/** One group attempt as it ended, and whether it got as far as its members. */
export interface SerialAttemptRun {
readonly record: SerialAttemptRecord;
/**
* False when the attempt stopped before any member ran (its file would not
* load, its session would not launch). Such a retry keeps the members'
* verdicts from the attempt before it.
*/
readonly reachedMembers: boolean;
}
/** The report id of one serial group variant: a group runs once per agent and per repeat. */
export function serialGroupId(serialId: string, targetName: string, agent: string, repeat: number): string {
return canonicalDigest(
repeat === 0 ? { serialId, targetId: targetName, agent } : { serialId, targetId: targetName, agent, repeat },
);
}
/** Runs one serial unit to completion and emits every member result. */
@@ -95,46 +123,20 @@ export async function runSerialUnit(
members: readonly TestTargetPair[],
file: FileRef,
): Promise<SerialGroupRecord> {
const first = members[0]!;
const serialId = first.test.serialId!;
// Every member of a unit runs as the same agent (selection guarantees it),
// so the group is one variant of the flow and is identified as such.
const agent = first.agent;
const { repeat } = first;
const groupRecordId = canonicalDigest(
repeat === 0 ? { serialId, targetId: host.target.name, agent } : { serialId, targetId: host.target.name, agent, repeat },
);
const group: SerialGroupRecord = {
id: groupRecordId,
serialId,
declarationIndex: first.test.declarationIndex,
file: first.test.file,
titlePath: serialTitlePath(first.test),
targetId: host.target.name,
platform: host.target.platform,
agent,
repeat,
memberTestIds: members.map((member) => member.test.id),
status: 'failed',
attempts: [],
};
const memberFinalStatus = new Map<string, SerialMemberRecord>();
const group = newSerialGroup(members, host.target);
const runs: SerialAttemptRun[] = [];
const finalStatus = await runWithRetries(
first.options.retries + 1,
members[0]!.options.retries + 1,
host.interruptSignal,
async (attemptIndex) => {
const { record: attempt, reachedMembers } = await runSerialAttempt(host, members, file, attemptIndex);
group.attempts.push(attempt);
// A retry the run interrupted, or one that never reached its members,
// keeps the verdict before it, so each member keeps what that attempt said.
const cutRetry = attemptIndex > 0 && (attempt.status === 'interrupted' || !reachedMembers);
if (!cutRetry) for (const member of attempt.members) memberFinalStatus.set(member.testId, member);
const run = await runSerialAttempt(host, group.id, members, file, attemptIndex);
runs.push(run);
group.attempts.push(run.record);
host.serialAttemptFinished(group.id, run);
// A beforeAll failure is not retry-eligible: the
// attempt stands as recorded and the retry loop stops here.
if (attempt.error?.code === 'HOOK_FAILED') return undefined;
return attempt;
if (run.record.error?.code === 'HOOK_FAILED') return undefined;
return run.record;
},
);
@@ -146,8 +148,51 @@ export async function runSerialUnit(
}
host.emitSerialGroup(group);
for (const member of members) {
const memberRecord = memberFinalStatus.get(member.test.id);
for (const result of memberResults(group, members, runs)) host.emit(result);
return group;
}
/** A group record with no attempts yet. */
function newSerialGroup(members: readonly TestTargetPair[], target: ResolvedTarget): SerialGroupRecord {
const first = members[0]!;
const serialId = first.test.serialId!;
// Every member of a unit runs as the same agent (selection guarantees it),
// so the group is one variant of the flow and is identified as such.
const { agent, repeat } = first;
return {
id: serialGroupId(serialId, target.name, agent, repeat),
serialId,
declarationIndex: first.test.declarationIndex,
file: first.test.file,
titlePath: serialTitlePath(first.test),
targetId: target.name,
platform: target.platform,
agent,
repeat,
memberTestIds: members.map((member) => member.test.id),
status: 'failed',
attempts: [],
};
}
/**
* Each member's result once its group is decided. Members carry no attempts
* of their own; the verdict is the member's record in the last attempt that
* counts. A retry the run interrupted, or one that never reached its
* members, keeps the verdict before it.
*/
function memberResults(
group: SerialGroupRecord,
members: readonly TestTargetPair[],
runs: readonly SerialAttemptRun[],
): ResultRecord[] {
const verdicts = new Map<string, SerialMemberRecord>();
for (const [attemptIndex, run] of runs.entries()) {
const cutRetry = attemptIndex > 0 && (run.record.status === 'interrupted' || !run.reachedMembers);
if (!cutRetry) for (const member of run.record.members) verdicts.set(member.testId, member);
}
return members.map((member) => {
const memberRecord = verdicts.get(member.test.id);
let status: ResultStatus;
let skip: SkipInfo | undefined;
// A member that skipped itself stays skipped whatever the group did: a
@@ -163,28 +208,160 @@ export async function runSerialUnit(
} else {
status = memberRecord.status;
}
host.emit(
pairResult(member, {
status,
selected: true,
...(skip !== undefined ? { skip } : {}),
attempts: [],
serialGroupId: groupRecordId,
}),
);
return pairResult(member, {
status,
selected: true,
...(skip !== undefined ? { skip } : {}),
attempts: [],
serialGroupId: group.id,
});
});
}
/** A serial group decided without its worker's word: the record and its members' results. */
interface SettledSerialGroup {
readonly group: SerialGroupRecord;
readonly results: readonly ResultRecord[];
}
/**
* What the runner heard of one serial group a worker is running: the
* attempts it finished, and the members finished so far in the attempt
* running now. When the worker dies before reporting the group, this is
* what the report keeps.
*/
export class SerialGroupProgress {
private readonly finished: SerialAttemptRun[] = [];
private current: SerialAttemptStart | undefined;
private currentMembers: SerialMemberRecord[] = [];
/** The member whose body is running now, by test id, until its record arrives. */
private running: string | undefined;
/** A member of the group announced its start. */
memberStarted(testId: string): void {
this.running = testId;
}
return group;
/** A member finished in the attempt running now. */
memberFinished(attempt: SerialAttemptStart, member: SerialMemberRecord): void {
this.current = attempt;
this.currentMembers.push(member);
if (this.running === member.testId) this.running = undefined;
}
/** A group attempt finished; the next one starts empty. */
attemptFinished(run: SerialAttemptRun): void {
this.finished.push(run);
this.current = undefined;
this.currentMembers = [];
this.running = undefined;
}
/**
* The group after its worker died with `error`, or undefined when the
* runner heard nothing of it running. The finished attempts stand, and the
* attempt in flight fails: the member whose body was running fails with
* `error` and the ones after it skip. A crash between members (a hook,
* the session closing) fails the attempt and leaves its finished members
* as they were; one in a retry before any member ran fails the first, as a
* launch failure does. A crash after the last attempt, when no retry
* follows it, is recorded on that attempt.
*/
crashed(members: readonly TestTargetPair[], target: ResolvedTarget, error: SerializedError): SettledSerialGroup | undefined {
const last = this.finished.at(-1);
if (!this.inFlight) {
if (last === undefined) return undefined;
if (!retryFollows(last.record, this.finished.length, members)) {
const closed: SerialAttemptRun = {
...last,
record: { ...last.record, secondaryErrors: [...last.record.secondaryErrors, error], cleanup: 'forced' },
};
return this.settled([...this.finished.slice(0, -1), closed], members, target);
}
const record = this.attemptInFlight('failed', error);
endBeforeMembers(record, members, error, NEVER_INTERRUPTED);
return this.settled([...this.finished, { record, reachedMembers: false }], members, target);
}
const record = this.attemptInFlight('failed', error);
const next = this.currentMembers.length;
const wasRunning = this.running !== undefined && this.running === members[next]?.test.id;
const skip: SkipInfo = wasRunning
? predecessorFailed(next)
: { cause: 'infrastructure-unavailable', reason: 'worker process exited during this group attempt' };
for (const [memberIndex, member] of members.entries()) {
if (memberIndex < next) continue;
record.members.push(
wasRunning && memberIndex === next
? memberFailedWith(record, memberIndex, member.test.id, error)
: skippedMember(record.id, memberIndex, member.test.id, skip),
);
}
return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
}
/**
* The group after the run's interrupt stopped its worker, or undefined
* when the runner heard nothing of it running. The finished attempts
* stand, and an attempt in flight is interrupted: the members it finished
* keep their records and the rest skip with `skip`. The verdict is the one
* those attempts reach, as for a retry the interrupt cut short.
*/
interrupted(members: readonly TestTargetPair[], target: ResolvedTarget, skip: SkipInfo): SettledSerialGroup | undefined {
if (!this.inFlight) return this.finished.length === 0 ? undefined : this.settled(this.finished, members, target);
const record = this.attemptInFlight('interrupted', undefined);
for (const [memberIndex, member] of members.entries()) {
if (memberIndex >= this.currentMembers.length) record.members.push(skippedMember(record.id, memberIndex, member.test.id, skip));
}
return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
}
/** Whether a member of the attempt in flight started or finished. */
private get inFlight(): boolean {
return this.currentMembers.length > 0 || this.running !== undefined;
}
/** The record of the attempt in flight as it stood, with the members it finished. */
private attemptInFlight(status: SerialAttemptRecord['status'], error: SerializedError | undefined): SerialAttemptRecord {
return {
id: this.current?.id ?? uuidv7(),
index: this.finished.length,
status,
startedAt: this.current?.startedAt ?? timestamp(),
durationMs: 0,
members: [...this.currentMembers],
artifacts: [],
...(error === undefined ? {} : { error }),
secondaryErrors: [],
cleanup: 'forced',
};
}
private settled(
runs: readonly SerialAttemptRun[],
members: readonly TestTargetPair[],
target: ResolvedTarget,
): SettledSerialGroup {
const group = newSerialGroup(members, target);
group.status = retryVerdict(runs.map((run) => run.record));
group.attempts = runs.map((run) => run.record);
return { group, results: memberResults(group, members, runs) };
}
}
/** One group attempt as it ended, and whether it got as far as its members. */
interface SerialAttemptRun {
readonly record: SerialAttemptRecord;
readonly reachedMembers: boolean;
/**
* Whether the group retries after `record`, its attempt number `attempts`:
* the retry policy allows it, and a hook failure never retries.
*/
function retryFollows(record: SerialAttemptRecord, attempts: number, members: readonly TestTargetPair[]): boolean {
return attempts < members[0]!.options.retries + 1 && isRetryEligible(record) && record.error?.code !== 'HOOK_FAILED';
}
/** An abort signal that never fires. */
const NEVER_INTERRUPTED = new AbortController().signal;
async function runSerialAttempt(
host: SerialHost,
groupId: string,
members: readonly TestTargetPair[],
file: FileRef,
attemptIndex: number,
@@ -222,6 +399,12 @@ async function runSerialAttempt(
secondaryErrors: [],
cleanup: 'complete',
};
const start: SerialAttemptStart = { id: attemptId, index: attemptIndex, startedAt };
/** Records one member and streams it, so a worker that dies later in the attempt does not take it along. */
const finishMember = (member: SerialMemberRecord): void => {
memberRecords.push(member);
host.serialMemberFinished(groupId, start, member);
};
let realm: Realm;
try {
@@ -258,12 +441,12 @@ async function runSerialAttempt(
skipRemaining = interruptedMemberSkip(host.interruptSignal);
}
if (skipRemaining !== undefined) {
memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
continue;
}
const registered = findRegistered(realm, member.test);
if (registered === undefined) {
memberRecords.push({
finishMember({
id: memberId,
index: memberIndex,
testId: member.test.id,
@@ -285,7 +468,7 @@ async function runSerialAttempt(
hookFailure = await host.realms.enterScopes(realm, registered);
if (hookFailure !== undefined) {
skipRemaining = { cause: 'hook-failed', reason: hookFailure.message };
memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
continue;
}
host.pairStarted(member);
@@ -294,7 +477,7 @@ async function runSerialAttempt(
shared,
});
shared.priorSteps.push(...memberAttempt.steps);
memberRecords.push({
finishMember({
id: memberId,
index: memberIndex,
testId: member.test.id,
@@ -367,22 +550,27 @@ function endBeforeMembers(
members.forEach((member, memberIndex) => {
record.members.push(
memberIndex === 0
? {
id: `${record.id}:member:0`,
index: 0,
testId: member.test.id,
status: 'failed',
startedAt: record.startedAt,
durationMs: 0,
steps: [],
error,
secondaryErrors: [],
}
? memberFailedWith(record, 0, member.test.id, error)
: skippedMember(record.id, memberIndex, member.test.id, predecessorFailed(0)),
);
});
}
/** A member that failed with an error of its group attempt's, having run no step of its own. */
function memberFailedWith(record: SerialAttemptRecord, index: number, testId: string, error: SerializedError): SerialMemberRecord {
return {
id: `${record.id}:member:${index}`,
index,
testId,
status: 'failed',
startedAt: record.startedAt,
durationMs: 0,
steps: [],
error,
secondaryErrors: [],
};
}
function isFailedMember(member: SerialMemberRecord): member is SerialMemberRecord & { status: FailedStatus } {
return isFailedStatus(member.status);
}
+16 -13
View File
@@ -39,20 +39,23 @@ let outbox: Promise<void> = Promise.resolve();
/** How long an exit waits for the outbox: a channel whose runner is gone may never acknowledge. */
const FLUSH_GRACE_MS = 2_000;
/** Queues one message; resolves once the channel has taken it (or is gone). */
/**
* Hands one message to the channel at once; resolves once the channel has
* taken it and every message before it (or is gone). The channel keeps the
* order. Handing it over synchronously is what lets a `pair-start` reach the
* runner when the test body exits the process right after it.
*/
function send(message: WorkerToMain): Promise<void> {
outbox = outbox.then(
() =>
new Promise<void>((resolve) => {
try {
if (process.send === undefined || !process.connected) resolve();
else process.send(message, undefined, undefined, () => resolve());
} catch {
// channel already closed; nothing left to deliver
resolve();
}
}),
);
const taken = new Promise<void>((resolve) => {
try {
if (process.send === undefined || !process.connected) resolve();
else process.send(message, undefined, undefined, () => resolve());
} catch {
// channel already closed; nothing left to deliver
resolve();
}
});
outbox = outbox.then(() => taken);
return outbox;
}
+52 -1
View File
@@ -13,7 +13,8 @@ import type { CliOverrides, PortAssignments, ResolvedTarget } from '../../config
import type { AiTraceSnapshot } from '../../internal/ai-trace.ts';
import type { DebugSnapshot } from '../../internal/debug.ts';
import type { SerializedError } from '../../internal/errors.ts';
import type { ResultRecord, RunError, SerialGroupRecord, WireResultRecord } from '../records.ts';
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord, WireResultRecord } from '../records.ts';
import type { SerialAttemptRun, SerialAttemptStart } from '../serial.ts';
import type { StepProgress } from '../steps.ts';
/** One runnable pair on the wire; the worker resolves the test function. */
@@ -190,6 +191,52 @@ export interface SerialGroupMessage {
readonly group: SerialGroupRecord;
}
/**
* An attempt of an ordinary or setup pair begins, its realm and `beforeAll`
* hooks included. With `attempt`, it tells a crash during an attempt from
* one between attempts (an `afterAll` after the last one): only the first
* is charged a new attempt, with this index.
*/
export interface AttemptStartMessage {
readonly type: 'attempt-start';
readonly testId: string;
readonly agent: string;
readonly repeat: number;
readonly index: number;
}
/**
* One finished attempt of an ordinary or setup pair. A pair's result waits
* for its last attempt, so each attempt also goes out as it ends: a worker
* that dies during a retry must not take the attempts before it along. The
* result still carries every attempt; the scheduler keeps these only until
* it arrives.
*/
export interface AttemptMessage {
readonly type: 'attempt';
readonly testId: string;
readonly agent: string;
readonly repeat: number;
readonly attempt: AttemptRecord;
}
/** One member that finished in the serial group attempt running now; see `AttemptMessage`. */
export interface SerialMemberMessage {
readonly type: 'serial-member';
/** The group's report id (`SerialGroupRecord.id`). */
readonly groupId: string;
readonly attempt: SerialAttemptStart;
readonly member: SerialMemberRecord;
}
/** One finished serial group attempt; see `AttemptMessage`. */
export interface SerialAttemptMessage {
readonly type: 'serial-attempt';
/** The group's report id (`SerialGroupRecord.id`). */
readonly groupId: string;
readonly run: SerialAttemptRun;
}
export interface UnitDoneMessage {
readonly type: 'unit-done';
readonly unitId: string;
@@ -236,6 +283,10 @@ export type WorkerToMain =
| NoticeMessage
| ResultMessage
| SerialGroupMessage
| AttemptStartMessage
| AttemptMessage
| SerialMemberMessage
| SerialAttemptMessage
| UnitDoneMessage
| ShutdownDoneMessage
| FatalMessage
+6
View File
@@ -123,6 +123,12 @@ export class TargetWorker {
if (countsTowardFailureLimit(result.status)) this.countFailure();
},
onSerialGroup: (group) => this.host.emit({ type: 'serial-group', group }),
onAttemptStart: (pair, index) =>
this.host.emit({ type: 'attempt-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, index }),
onAttempt: (pair, attempt) =>
this.host.emit({ type: 'attempt', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, attempt }),
onSerialMember: (groupId, attempt, member) => this.host.emit({ type: 'serial-member', groupId, attempt, member }),
onSerialAttempt: (groupId, run) => this.host.emit({ type: 'serial-attempt', groupId, run }),
onPairStart: (pair) => {
this.inFlight = { testId: pair.test.id, agent: pair.agent, repeat: pair.repeat };
this.host.emit({
@@ -270,15 +270,20 @@ test.describe('wizard', { serial: true }, () => {
{ 'tests/serial-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const group = outcome.report.run.serialGroups[0]!;
expect(group.status).toBe('failed');
expect(group.attempts[0]!.members.map((member) => [member.status, member.error?.code])).toEqual([
['passed', undefined],
['failed', 'WORKER_CRASH'],
['skipped', undefined],
]);
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
const crashed = resultByTitle(outcome, 'step 2 crashes the worker');
expect(crashed.status).toBe('failed');
expect(crashed.attempts[0]?.error?.code).toBe('WORKER_CRASH');
const finished = resultByTitle(outcome, 'step 1 passes');
expect(finished.status).toBe('skipped');
expect(finished.skip?.reason).toBe('worker process exited before this serial group finished');
expect(crashed.serialGroupId).toBe(group.id);
const unreached = resultByTitle(outcome, 'step 3 never starts');
expect(unreached.status).toBe('skipped');
expect(unreached.skip?.reason).toBe('worker process exited before this test started');
expect(unreached.skip?.cause).toBe('serial-predecessor-failed');
expect(outcome.exitCode).toBe(3);
expect(outcome.report.run.summary.failed).toBe(1);
assertValidReport(outcome.report);
@@ -287,6 +292,292 @@ test.describe('wizard', { serial: true }, () => {
120_000,
);
it(
'keeps the failed attempt before a retry that crashed the worker',
async () => {
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-crash-')), 'failed-once');
const file = `import { existsSync, writeFileSync } from 'node:fs';
import { test } from 'e2e';
test('fails, then crashes on retry', { retries: 1 }, async () => {
if (!existsSync(${JSON.stringify(marker)})) {
writeFileSync(${JSON.stringify(marker)}, '');
throw new Error('first attempt fails');
}
process.exit(7);
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/retry-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const result = resultByTitle(outcome, 'fails, then crashes on retry');
expect(result.status).toBe('failed');
expect(result.attempts.map((attempt) => [attempt.index, attempt.status, attempt.error?.message])).toEqual([
[0, 'failed', 'first attempt fails'],
[1, 'failed', 'worker process exited during this test'],
]);
expect(result.attempts[1]!.error?.code).toBe('WORKER_CRASH');
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'keeps the failed group attempt before a serial retry that crashed the worker',
async () => {
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-crash-')), 'failed-once');
const file = `import { existsSync, writeFileSync } from 'node:fs';
import { test } from 'e2e';
test.describe('wizard', { serial: true, retries: 1 }, () => {
test('step 1 passes', async () => {});
test('step 2 fails, then crashes on retry', async () => {
if (!existsSync(${JSON.stringify(marker)})) {
writeFileSync(${JSON.stringify(marker)}, '');
throw new Error('first attempt fails');
}
process.exit(7);
});
test('step 3 never passes', async () => {});
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/serial-retry-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const group = outcome.report.run.serialGroups[0]!;
expect(group.status).toBe('failed');
expect(
group.attempts.map((attempt) => [attempt.index, attempt.members.map((member) => [member.status, member.error?.code])]),
).toEqual([
[0, [['passed', undefined], ['failed', 'ERROR'], ['skipped', undefined]]],
[1, [['passed', undefined], ['failed', 'WORKER_CRASH'], ['skipped', undefined]]],
]);
expect(group.attempts[0]!.members[1]!.error?.message).toBe('first attempt fails');
expect(group.attempts[1]!.error?.code).toBe('WORKER_CRASH');
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
const crashed = resultByTitle(outcome, 'step 2 fails, then crashes on retry');
expect(crashed.status).toBe('failed');
expect(crashed.serialGroupId).toBe(group.id);
expect(resultByTitle(outcome, 'step 3 never passes').skip?.cause).toBe('serial-predecessor-failed');
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'records a crash in afterAll after a failed attempt on that attempt, not as a retry',
async () => {
const file = `import { test } from 'e2e';
test.afterAll(() => {
process.exit(7);
});
test('always fails', { retries: 1 }, async () => {
throw new Error('fails every time');
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/teardown-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const result = resultByTitle(outcome, 'always fails');
expect(result.status).toBe('failed');
expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.message, attempt.cleanup])).toEqual([
[0, 'fails every time', 'forced'],
]);
expect(result.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'keeps a passed setup attempt when afterAll crashes the worker, and skips its dependents',
async () => {
const setupFile = `import { test } from 'e2e';
test.afterAll(() => {
process.exit(7);
});
test.setup('seed storage', { sessions: ['seeded'] }, async ({ app, session }) => {
await app.open('/storage');
await session.save('seeded');
});
`;
const consumerFile = `import { test } from 'e2e';
test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => {
await app.open('/storage');
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/auth.setup.e2e.ts': setupFile, 'tests/consumer.e2e.ts': consumerFile },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const setup = resultByTitle(outcome, 'seed storage');
expect(setup.status).toBe('passed');
expect(setup.attempts.map((attempt) => [attempt.index, attempt.status, attempt.cleanup])).toEqual([
[0, 'passed', 'forced'],
]);
expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
expect(resultByTitle(outcome, 'starts with the seeded state').skip?.cause).toBe('setup-failed');
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'keeps the SESSION_CONTRACT failure of a setup attempt when afterAll crashes the worker',
async () => {
const setupFile = `import { test } from 'e2e';
test.afterAll(() => {
process.exit(7);
});
test.setup('never saves', { sessions: ['seeded'] }, async ({ app }) => {
await app.open('/storage');
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{
'tests/auth.setup.e2e.ts': setupFile,
'tests/consumer.e2e.ts': `import { test } from 'e2e';
test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => {
await app.open('/storage');
});
`,
},
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const setup = resultByTitle(outcome, 'never saves');
expect(setup.status).toBe('failed');
expect(setup.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.cleanup])).toEqual([
[0, 'SESSION_CONTRACT', 'forced'],
]);
expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'charges a crash in a retry beforeAll to the retry, not to the attempt before it',
async () => {
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-hook-crash-')), 'failed-once');
const file = `import { existsSync, writeFileSync } from 'node:fs';
import { test } from 'e2e';
test.beforeAll(() => {
if (existsSync(${JSON.stringify(marker)})) process.exit(7);
});
test('fails once', { retries: 1 }, async () => {
writeFileSync(${JSON.stringify(marker)}, '');
throw new Error('first attempt fails');
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/retry-hook-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const result = resultByTitle(outcome, 'fails once');
expect(result.status).toBe('failed');
expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.secondaryErrors.length])).toEqual([
[0, 'ERROR', 0],
[1, 'WORKER_CRASH', 0],
]);
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'fails the first member when a serial retry crashes the worker before any member runs',
async () => {
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-hook-crash-')), 'failed-once');
const file = `import { existsSync, writeFileSync } from 'node:fs';
import { test } from 'e2e';
test.describe('wizard', { serial: true, retries: 1 }, () => {
test.beforeAll(() => {
if (existsSync(${JSON.stringify(marker)})) process.exit(7);
});
test('step 1 passes', async () => {});
test('step 2 fails once', async () => {
writeFileSync(${JSON.stringify(marker)}, '');
throw new Error('first attempt fails');
});
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/serial-retry-hook-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const group = outcome.report.run.serialGroups[0]!;
expect(group.status).toBe('failed');
expect(
group.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.members.map((member) => [member.status, member.error?.code])]),
).toEqual([
[0, 'ERROR', [['passed', undefined], ['failed', 'ERROR']]],
[1, 'WORKER_CRASH', [['failed', 'WORKER_CRASH'], ['skipped', undefined]]],
]);
// The retry never reached its members, so they keep attempt 0's verdicts.
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
expect(resultByTitle(outcome, 'step 2 fails once').status).toBe('failed');
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'charges a crash after the last serial member to the group attempt, not to a member',
async () => {
const file = `import { test } from 'e2e';
test.describe('wizard', { serial: true }, () => {
test.afterAll(() => {
process.exit(7);
});
test('step 1 passes', async () => {});
test('step 2 passes', async () => {});
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/serial-teardown-crash.e2e.ts': file },
{ appUrl: app.url, configSource: workerConfigSource(1) },
);
const group = outcome.report.run.serialGroups[0]!;
expect(group.status).toBe('failed');
expect(group.attempts).toHaveLength(1);
expect(group.attempts[0]!.error?.code).toBe('WORKER_CRASH');
expect(group.attempts[0]!.members.map((member) => member.status)).toEqual(['passed', 'passed']);
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
expect(resultByTitle(outcome, 'step 2 passes').status).toBe('passed');
expect(outcome.exitCode).toBe(3);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);
it(
'interrupts workers, reports 130, and lists the file the one worker never reached',
async () => {
+17 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { isRetryEligible, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts';
import { isRetryEligible, retryVerdict, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts';
import type { SerializedError } from '../../src/internal/errors.ts';
function error(category: SerializedError['category']): SerializedError {
@@ -126,3 +126,19 @@ describe('runWithRetries', () => {
expect(status).toBe('failed');
});
});
describe('retryVerdict', () => {
const failed: RetryAttempt = { status: 'failed', error: error('test') };
it.each<[readonly RetryAttempt[], string]>([
[[], 'failed'],
[[{ status: 'passed' }], 'passed'],
[[failed, { status: 'passed' }], 'flaky'],
[[failed, { status: 'timed-out' }], 'timed-out'],
[[{ status: 'interrupted' }], 'interrupted'],
[[{ status: 'timed-out' }, { status: 'interrupted' }], 'timed-out'],
[[failed, { status: 'skipped' }], 'skipped'],
])('reads %j as %s', (attempts, verdict) => {
expect(retryVerdict(attempts)).toBe(verdict);
});
});
+232 -2
View File
@@ -15,8 +15,9 @@ import type {
import type { ResolvedTarget } from '../../src/config/resolve.ts';
import { defineEngine, type EngineHandle } from '../../src/engine/index.ts';
import { classifyError, serializeError } from '../../src/internal/errors.ts';
import type { ResultRecord, RunError, SerialGroupRecord } from '../../src/run/records.ts';
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord } from '../../src/run/records.ts';
import { runUnits } from '../../src/run/scheduler.ts';
import { serialGroupId, type SerialAttemptRun } from '../../src/run/serial.ts';
import { buildWorkPlans, type TargetWorkPlan } from '../../src/run/units.ts';
import type { SpawnUnitRunner, UnitRunner, UnitRunnerEvents } from '../../src/run/unit-runner.ts';
import type { MainToWorker, RunUnitMessage } from '../../src/run/worker/protocol.ts';
@@ -142,6 +143,16 @@ interface FakeBehaviour {
readonly neverReady?: readonly string[];
/** Unit ids whose worker never finishes: a test that will not end on its own. */
readonly hangOn?: readonly string[];
/** Attempts the unit's first pair finishes before the worker hangs: a test caught between attempts. */
readonly attemptsBeforeHang?: readonly AttemptRecord[];
/** The hung pair starts its next attempt after `attemptsBeforeHang`, and the worker exits on its own once interrupted. */
readonly exitDuringNextAttemptOnInterrupt?: boolean;
/** Serial group attempts the unit's pairs (one group) finish before the worker hangs. */
readonly serialRunsBeforeHang?: readonly SerialAttemptRun[];
/** Members the group's next attempt finishes before the worker hangs. */
readonly serialMembersBeforeHang?: readonly SerialMemberRecord[];
/** Instead of hanging, the worker exits on its own once it sent what came before. */
readonly exitInsteadOfHang?: boolean;
/** Workers ignore `terminate` and have to be killed. */
readonly ignoreTerminate?: boolean;
/**
@@ -152,6 +163,31 @@ interface FakeBehaviour {
readonly signalledInit?: { readonly workers: number; readonly signal: NodeJS.Signals };
}
const FAILED_ATTEMPT: AttemptRecord = {
id: 'attempt-0',
index: 0,
status: 'failed',
startedAt: '2026-01-01T00:00:00.000Z',
durationMs: 5,
steps: [],
artifacts: [],
error: serializeError(classifyError(new Error('first attempt fails'))),
secondaryErrors: [],
cleanup: 'complete',
};
const PASSED_RETRY: AttemptRecord = {
id: 'attempt-1',
index: 1,
status: 'passed',
startedAt: '2026-01-01T00:00:01.000Z',
durationMs: 5,
steps: [],
artifacts: [],
secondaryErrors: [],
cleanup: 'complete',
};
class FakeFleet {
readonly spawned: { targetName: string; workerSlot: number }[] = [];
readonly unitsByWorker: RunUnitMessage[][] = [];
@@ -221,6 +257,10 @@ class FakeRunner implements UnitRunner {
if (this.exited) return;
if (message.type !== 'run-unit') this.fleet.controlMessages.push(message.type);
if (message.type === 'interrupt') this.fleet.interruptSkips.push(message.skip);
if (message.type === 'interrupt' && this.behaviour.exitDuringNextAttemptOnInterrupt === true) {
setTimeout(() => this.end('code 7, signal null'), 0);
return;
}
if (message.type === 'shutdown') {
setTimeout(() => this.end('shut down'), 0);
return;
@@ -231,7 +271,41 @@ class FakeRunner implements UnitRunner {
}
if (message.type !== 'run-unit') return;
this.fleet.unitsByWorker[this.index]!.push(message);
if (this.behaviour.hangOn?.includes(message.unitId) === true) return;
if (this.behaviour.hangOn?.includes(message.unitId) === true) {
const first = message.pairs[0];
for (const finishedRun of this.behaviour.serialRunsBeforeHang ?? []) {
const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat);
for (const [index, member] of finishedRun.record.members.entries()) {
const pair = message.pairs[index]!;
this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId });
this.events.onMessage({ type: 'serial-member', groupId, attempt: finishedRun.record, member });
}
this.events.onMessage({ type: 'serial-attempt', groupId, run: finishedRun });
}
const partial = this.behaviour.serialMembersBeforeHang ?? [];
if (partial.length > 0) {
const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat);
const attempt = { id: 'group-attempt-partial', index: this.behaviour.serialRunsBeforeHang?.length ?? 0, startedAt: '2026-01-01T00:00:00.000Z' };
for (const [index, member] of partial.entries()) {
const pair = message.pairs[index]!;
this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId });
this.events.onMessage({ type: 'serial-member', groupId, attempt, member });
}
}
const attempts = this.behaviour.attemptsBeforeHang ?? [];
const nextAttempt = this.behaviour.exitDuringNextAttemptOnInterrupt === true;
if ((attempts.length > 0 || nextAttempt) && first !== undefined) {
const pair = { testId: first.test.id, agent: first.agent, repeat: first.repeat };
this.events.onMessage({ type: 'pair-start', ...pair, title: first.test.id, file: first.test.file, serialId: undefined });
for (const attempt of attempts) {
this.events.onMessage({ type: 'attempt-start', ...pair, index: attempt.index });
this.events.onMessage({ type: 'attempt', ...pair, attempt });
}
if (nextAttempt) this.events.onMessage({ type: 'attempt-start', ...pair, index: attempts.length });
}
if (this.behaviour.exitInsteadOfHang === true) setTimeout(() => this.end('code 7, signal null'), 0);
return;
}
setTimeout(() => this.completeUnit(message), 0);
}
@@ -587,6 +661,63 @@ describe('scheduler gating and dispatch', () => {
});
});
/** Two members of one serial group in `tests/a.e2e.ts`. */
function serialPairs(target: ResolvedTarget): TestTargetPair[] {
return ['step 1', 'step 2'].map((title, declarationIndex) =>
makePair(makeTest('tests/a.e2e.ts', title, { declarationIndex, serialId: 'tests/a.e2e.ts::wizard' }), target),
);
}
/** One member's record in group attempt `attemptIndex`. */
function memberRecord(pairs: readonly TestTargetPair[], attemptIndex: number, memberIndex: number, status: 'passed' | 'failed'): SerialMemberRecord {
return {
id: `group-attempt-${attemptIndex}:member:${memberIndex}`,
index: memberIndex,
testId: pairs[memberIndex]!.test.id,
status,
startedAt: '2026-01-01T00:00:00.000Z',
durationMs: 1,
steps: [],
...(status === 'failed' ? { error: FAILED_ATTEMPT.error! } : {}),
secondaryErrors: [],
};
}
/** A finished group attempt whose members ended with `statuses`. */
function groupRun(pairs: readonly TestTargetPair[], index: number, statuses: readonly ('passed' | 'failed')[]): SerialAttemptRun {
const failed = statuses.includes('failed');
return {
reachedMembers: true,
record: {
id: `group-attempt-${index}`,
index,
status: failed ? 'failed' : 'passed',
startedAt: '2026-01-01T00:00:00.000Z',
durationMs: 5,
members: statuses.map((status, memberIndex) => memberRecord(pairs, index, memberIndex, status)),
artifacts: [],
...(failed ? { error: FAILED_ATTEMPT.error! } : {}),
secondaryErrors: [],
cleanup: 'complete',
},
};
}
/** Runs `pairs` on `fleet` and interrupts, then forces, while the unit hangs. */
async function runForcedInterrupt(pairs: TestTargetPair[], target: ResolvedTarget, fleet: FakeFleet): Promise<Collected> {
const interrupt = new AbortController();
const force = new AbortController();
const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)];
const collected = await run(
makeSelection([{ target, pairs }]),
makeCollection(['tests/a.e2e.ts'], pairs),
fleet,
{ workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 },
);
for (const timer of timers) clearTimeout(timer);
return collected;
}
describe('scheduler fault handling', () => {
it('synthesizes results and a run error when a worker dies mid-unit', async () => {
const target = makeTarget('web', 0);
@@ -745,6 +876,105 @@ describe('scheduler fault handling', () => {
expect(collected.runErrors).toEqual([]);
});
it.each([
{ finished: [FAILED_ATTEMPT], status: 'failed' },
{ finished: [FAILED_ATTEMPT, PASSED_RETRY], status: 'flaky' },
])('a forced interrupt keeps the attempts a test finished and their retry verdict ($status)', async ({ finished, status }) => {
const target = makeTarget('web', 0);
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], attemptsBeforeHang: finished });
const interrupt = new AbortController();
const force = new AbortController();
const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)];
const collected = await run(
makeSelection([{ target, pairs }]),
makeCollection(['tests/a.e2e.ts'], pairs),
fleet,
{ workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 },
);
for (const timer of timers) clearTimeout(timer);
expect(collected.results).toHaveLength(1);
expect(collected.results[0]!.status).toBe(status);
expect(collected.results[0]!.attempts).toEqual(finished);
});
it.each<{ statuses: ('passed' | 'failed')[][]; verdict: string; results: string[] }>([
{ statuses: [['passed', 'failed']], verdict: 'failed', results: ['passed', 'failed'] },
{ statuses: [['passed', 'failed'], ['passed', 'passed']], verdict: 'flaky', results: ['flaky', 'flaky'] },
])('a forced interrupt keeps the group attempts a serial group finished and their retry verdict ($verdict)', async ({ statuses, verdict, results }) => {
const target = makeTarget('web', 0);
const pairs = serialPairs(target);
const runs = statuses.map((members, index) => groupRun(pairs, index, members));
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: runs });
const collected = await runForcedInterrupt(pairs, target, fleet);
expect(collected.serialGroups).toHaveLength(1);
expect(collected.serialGroups[0]!.status).toBe(verdict);
expect(collected.serialGroups[0]!.attempts).toEqual(runs.map((entry) => entry.record));
expect(collected.results.map((result) => result.status)).toEqual(results);
});
it('a forced interrupt mid group attempt keeps the members that attempt finished', async () => {
const target = makeTarget('web', 0);
const pairs = serialPairs(target);
const finished = memberRecord(pairs, 0, 0, 'passed');
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialMembersBeforeHang: [finished] });
const collected = await runForcedInterrupt(pairs, target, fleet);
const group = collected.serialGroups[0]!;
expect(group.status).toBe('interrupted');
expect(group.attempts.map((attempt) => attempt.status)).toEqual(['interrupted']);
expect(group.attempts[0]!.members[0]).toEqual(finished);
expect(group.attempts[0]!.members[1]!.status).toBe('skipped');
expect(collected.results.map((result) => result.status)).toEqual(['passed', 'skipped']);
});
it('records a crash after a serial group finished its last attempt on that attempt', async () => {
const target = makeTarget('web', 0);
const pairs = serialPairs(target);
const passed = groupRun(pairs, 0, ['passed', 'passed']);
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: [passed], exitInsteadOfHang: true });
const collected = await run(makeSelection([{ target, pairs }]), makeCollection(['tests/a.e2e.ts'], pairs), fleet, { workers: 1 });
const group = collected.serialGroups[0]!;
expect(group.status).toBe('passed');
expect(group.attempts).toHaveLength(1);
expect(group.attempts[0]!.cleanup).toBe('forced');
expect(group.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
expect(collected.results.map((result) => result.status)).toEqual(['passed', 'passed']);
expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']);
});
it('a worker that exits on its own during the interrupt grace crashes the test it was running', async () => {
const target = makeTarget('web', 0);
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];
const fleet = new FakeFleet({
hangOn: ['file::web::tests/a.e2e.ts'],
attemptsBeforeHang: [FAILED_ATTEMPT],
exitDuringNextAttemptOnInterrupt: true,
});
const interrupt = new AbortController();
const timer = setTimeout(() => interrupt.abort(), 20);
const collected = await run(
makeSelection([{ target, pairs }]),
makeCollection(['tests/a.e2e.ts'], pairs),
fleet,
{ workers: 1, interruptSignal: interrupt.signal, interruptGraceMs: 30_000 },
);
clearTimeout(timer);
expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']);
expect(collected.results).toHaveLength(1);
expect(collected.results[0]!.status).toBe('failed');
expect(collected.results[0]!.attempts.map((attempt) => [attempt.index, attempt.error?.code])).toEqual([
[0, FAILED_ATTEMPT.error!.code],
[1, 'WORKER_CRASH'],
]);
});
it('kills a worker that ignores a forced interrupt once the force budget is spent', async () => {
const target = makeTarget('web', 0);
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];