diff --git a/.changeset/retry-crash-attempts.md b/.changeset/retry-crash-attempts.md new file mode 100644 index 000000000..8757ed80a --- /dev/null +++ b/.changeset/retry-crash-attempts.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +A worker that crashes during a retry no longer drops the attempts before it. Before, a test whose first attempt failed and whose retry crashed the worker showed a single attempt 0 with `WORKER_CRASH`, and the first attempt's error was lost. The report now keeps every finished attempt and adds the crash as the next one. A crash after the last attempt (in an `afterAll`) is recorded on that attempt instead of as a retry. For a serial group, the finished group attempts and the members that finished in the crashed attempt stay in the report, the member that was running fails with `WORKER_CRASH`, and a crash after a member finished fails the group attempt without blaming that member. A forced interrupt keeps the attempts that finished and the verdict they reach, `flaky` included. diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 736fd5e2a..428f0a0d5 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -119,7 +119,7 @@ These appear in terminal output and in the report as plain string codes. | `TRACE_WITHHELD` | A trace from a session that knows a secret value could not be rewritten, so it was deleted, or the engine returned an archive path outside the attempt's artifact directory, so nothing was registered. Recorded in the attempt's `secondaryErrors`; never fails the attempt. | infrastructure | 3 | | `BROWSER_INSTALL_FAILED` | Browser provisioning failed | infrastructure | 3 | | `WORKER_INIT_FAILED` | A worker process never became ready | infrastructure | 3 | -| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped, a serial group's finished members included, since their results wait for the whole group. An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 | +| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped. Attempts that finished before the exit stay in the report, a retry's earlier attempts and a serial group's finished members included, and the crash is recorded as the attempt after them, or on the last attempt when it came after it (an `afterAll`). An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 | | `WORKER_CRASH` | A worker exited during a specific test | infrastructure | 3 | | `WORKER_ENTRY_MISSING` | The worker entry module could not be resolved. Packaging fault. | infrastructure | 3 | | `NODE_UNSUPPORTED` | The runtime cannot run e2e's TypeScript loader: a Node.js outside `^22.22.3 \|\| >=24.8.0`, Bun (no `module.registerHooks`), or Deno. The CLI checks this before it starts and exits 2 with the same sentence; the loader checks it again where it registers its hooks, for a host that runs e2e's runner without the CLI | infrastructure | 3 | diff --git a/packages/e2e/src/run/execute.ts b/packages/e2e/src/run/execute.ts index 3183314ae..bd908989b 100644 --- a/packages/e2e/src/run/execute.ts +++ b/packages/e2e/src/run/execute.ts @@ -46,10 +46,11 @@ import type { ResultStatus, RunError, SerialGroupRecord, + SerialMemberRecord, } from './records.ts'; import { isFailedStatus } from './records.ts'; import { runWithRetries } from './retry.ts'; -import { runSerialUnit, type SerialHost, type SharedSerialSession } from './serial.ts'; +import { runSerialUnit, type SerialAttemptRun, type SerialAttemptStart, type SerialHost, type SharedSerialSession } from './serial.ts'; import { interruptedSkip, pairKey, pairResult, repeatSegment, unstartedResult } from './units.ts'; import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, redactsRecordings, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.ts'; import { isSecret } from '../secrets.ts'; @@ -64,6 +65,14 @@ import type { SetupFn } from '../types.ts'; export interface ExecutionEvents { onResult?(result: ResultRecord): void; onSerialGroup?(group: SerialGroupRecord): void; + /** An attempt of an ordinary or setup pair begins, its realm and `beforeAll` hooks included. */ + onAttemptStart?(pair: TestTargetPair, attemptIndex: number): void; + /** One finished attempt of an ordinary or setup pair, ahead of the pair's result. */ + onAttempt?(pair: TestTargetPair, attempt: AttemptRecord): void; + /** One member that finished in a serial group attempt, ahead of the group. */ + onSerialMember?(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void; + /** One finished serial group attempt, ahead of the group. */ + onSerialAttempt?(groupId: string, run: SerialAttemptRun): void; /** Fires before a runnable pair starts; a serial unit announces each member as it begins. */ onPairStart?(pair: TestTargetPair): void; /** Live step progress of one running attempt, for reporters. */ @@ -222,6 +231,16 @@ export class TargetExecutor implements SerialHost { this.options.events?.onSerialGroup?.(group); } + /** Streams one member finished in a serial group attempt (SerialHost). */ + serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void { + this.options.events?.onSerialMember?.(groupId, attempt, member); + } + + /** Streams one finished serial group attempt (SerialHost). */ + serialAttemptFinished(groupId: string, run: SerialAttemptRun): void { + this.options.events?.onSerialAttempt?.(groupId, run); + } + /** Builds one engine operation context. */ private op(attemptId: string, timeoutMs: number, signal: AbortSignal): OperationContext { return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' }; @@ -423,6 +442,7 @@ export class TargetExecutor implements SerialHost { pair.options.retries + 1, this.interruptSignal, async (attemptIndex) => { + this.options.events?.onAttemptStart?.(pair, attemptIndex); if (realm === null) realm = await this.realms.create(file); const registered = findRegistered(realm, pair.test); if (registered === undefined) { @@ -437,6 +457,7 @@ export class TargetExecutor implements SerialHost { kind: 'ordinary', }); attempts.push(attempt); + this.options.events?.onAttempt?.(pair, attempt); if (isFailedStatus(attempt.status)) { // A failed realm is never reused, but afterAll still runs for every // scope whose beforeAll started in it. A body that skipped itself @@ -501,6 +522,7 @@ export class TargetExecutor implements SerialHost { pair.options.retries + 1, this.interruptSignal, async (attemptIndex) => { + this.options.events?.onAttemptStart?.(pair, attemptIndex); const realm = attemptIndex === 0 && freshRegistration !== undefined ? this.realms.adopt(freshRegistration, file) @@ -520,23 +542,24 @@ export class TargetExecutor implements SerialHost { kind: 'setup', staging, }); + const missing = attempt.status === 'passed' ? staging.missing() : []; + if (missing.length > 0) { + attempt.status = 'failed'; + attempt.error = serializeError( + new E2EError( + 'test', + 'SESSION_CONTRACT', + `setup must save each declared session exactly once; missing: [${missing.join(', ')}]`, + ), + { phase: 'body' }, + ); + } attempts.push(attempt); + // Sent before teardown and the session saves, as in runOrdinaryPair: + // a crash in either then lands on this attempt instead of erasing it. + this.options.events?.onAttempt?.(pair, attempt); await this.realms.leave(realm); - if (attempt.status === 'passed') { - const missing = staging.missing(); - if (missing.length > 0) { - attempt.status = 'failed'; - attempt.error = serializeError( - new E2EError( - 'test', - 'SESSION_CONTRACT', - `setup must save each declared session exactly once; missing: [${missing.join(', ')}]`, - ), - { phase: 'body' }, - ); - return attempt; - } for (const [name, saved] of staging.entries()) { await this.options.sessionStore.save(name, this.sessionIdentity, saved); } diff --git a/packages/e2e/src/run/retry.ts b/packages/e2e/src/run/retry.ts index b363ee1f8..1aaa891c0 100644 --- a/packages/e2e/src/run/retry.ts +++ b/packages/e2e/src/run/retry.ts @@ -30,19 +30,38 @@ export async function runWithRetries( interruptSignal: AbortSignal, runOnce: (attemptIndex: number) => Promise, ): Promise { - let finalStatus: ResultStatus = 'failed'; - let attemptCount = 0; + const attempts: RetryAttempt[] = []; for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) { if (interruptSignal.aborted) break; const attempt = await runOnce(attemptIndex); if (attempt === undefined) break; - attemptCount += 1; - if (attempt.status === 'passed') return attemptCount > 1 ? 'flaky' : 'passed'; - if (attempt.status === 'interrupted') return attemptCount > 1 ? finalStatus : 'interrupted'; + attempts.push(attempt); // A body that skipped itself has decided; a retry would only ask again. - if (attempt.status === 'skipped') return 'skipped'; - finalStatus = attempt.status; + if (attempt.status === 'passed' || attempt.status === 'interrupted' || attempt.status === 'skipped') break; if (!isRetryEligible(attempt)) break; } - return finalStatus; + return retryVerdict(attempts); +} + +/** + * The verdict a sequence of finished attempts reaches: a pass is `flaky` + * after a failure, an interrupt during a retry keeps the verdict before it, + * and a self-skip decides. Also the verdict of attempts a worker finished + * before the run's interrupt stopped it, so both read alike. + */ +export function retryVerdict(attempts: readonly RetryAttempt[]): ResultStatus { + let verdict: ResultStatus = 'failed'; + for (const [index, attempt] of attempts.entries()) { + switch (attempt.status) { + case 'passed': + return index > 0 ? 'flaky' : 'passed'; + case 'interrupted': + return index > 0 ? verdict : 'interrupted'; + case 'skipped': + return 'skipped'; + default: + verdict = attempt.status; + } + } + return verdict; } diff --git a/packages/e2e/src/run/scheduler.ts b/packages/e2e/src/run/scheduler.ts index f2f607362..e09451978 100644 --- a/packages/e2e/src/run/scheduler.ts +++ b/packages/e2e/src/run/scheduler.ts @@ -14,7 +14,9 @@ import type { AiTraceSnapshot } from '../internal/ai-trace.ts'; import type { DebugSnapshot } from '../internal/debug.ts'; import { InfrastructureError, serializeError } from '../internal/errors.ts'; import { timestamp, uuidv7 } from '../internal/ids.ts'; -import type { ResultRecord, RunError, SerialGroupRecord } from './records.ts'; +import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord } from './records.ts'; +import { retryVerdict } from './retry.ts'; +import { SerialGroupProgress, serialGroupId } from './serial.ts'; import type { StepProgress } from './steps.ts'; import type { SpawnUnitRunner, UnitRunner } from './unit-runner.ts'; import { @@ -114,6 +116,24 @@ interface TargetState { failed: boolean; } +/** The attempts of one pair heard ahead of its result. */ +interface PairAttempts { + readonly finished: AttemptRecord[]; + /** The index of the attempt that started and has not finished: a crash now is that attempt's. */ + running: number | undefined; +} + +/** The result of a pair from the attempts it finished, with the verdict they reach. */ +function finishedResult(pair: TestTargetPair, attempts: AttemptRecord[]): ResultRecord { + const last = attempts.at(-1); + return pairResult(pair, { + status: retryVerdict(attempts), + selected: true, + attempts, + ...(last?.status === 'skipped' ? { skip: last.skip } : {}), + }); +} + /** * One worker and the unit it is executing. Owns its own lifecycle so the * scheduler never has to keep a separate record in sync with the transport. @@ -128,8 +148,10 @@ class SchedulerWorker { readonly reported = new Set(); /** The pair (`pairKey`) executing now, when one is. */ inFlightPair: string | undefined; - /** Pairs (`pairKey`) that started in `unit`; a serial member reports only once its whole group is done. */ - readonly started = new Set(); + /** Attempts of `unit`'s pairs heard ahead of the pair's result, by `pairKey`: what a crash keeps. */ + readonly attempts = new Map(); + /** What was heard of `unit`'s serial groups still running, by group report id: what a crash keeps. */ + readonly serialGroups = new Map(); sawFailure = false; becameReady = false; /** Told to tear down at once by a forced interrupt; its exit is then the one asked for. */ @@ -159,7 +181,8 @@ class SchedulerWorker { this.state = 'busy'; this.unit = unit; this.reported.clear(); - this.started.clear(); + this.attempts.clear(); + this.serialGroups.clear(); this.inFlightPair = undefined; this.sawFailure = false; const pairs: WirePair[] = unit.pairs.map((pair) => ({ @@ -205,6 +228,26 @@ class SchedulerWorker { this.killTimer.unref(); } + /** The attempts heard of one of `unit`'s pairs, started on first word of them. */ + pairAttempts(key: string): PairAttempts { + let attempts = this.attempts.get(key); + if (attempts === undefined) { + attempts = { finished: [], running: undefined }; + this.attempts.set(key, attempts); + } + return attempts; + } + + /** The progress of one of `unit`'s serial groups, started on first word of it. */ + serialGroup(groupId: string): SerialGroupProgress { + let progress = this.serialGroups.get(groupId); + if (progress === undefined) { + progress = new SerialGroupProgress(); + this.serialGroups.set(groupId, progress); + } + return progress; + } + clearKillTimer(): void { if (this.killTimer === undefined) return; clearTimeout(this.killTimer); @@ -584,7 +627,11 @@ class Scheduler { } case 'pair-start': { worker.inFlightPair = pairKey(message.testId, message.agent, message.repeat); - worker.started.add(worker.inFlightPair); + if (message.serialId !== undefined) { + worker + .serialGroup(serialGroupId(message.serialId, worker.targetName, message.agent, message.repeat)) + .memberStarted(message.testId); + } const { type: _type, ...start } = message; this.options.events.onTestStart?.(start, worker.targetName); break; @@ -609,7 +656,9 @@ class Scheduler { case 'result': { const state = this.targetState(worker); const result = decodeResult(message.result, state.target); - worker.reported.add(pairKey(result.test.id, result.agent, result.repeat)); + const key = pairKey(result.test.id, result.agent, result.repeat); + worker.reported.add(key); + worker.attempts.delete(key); if (result.status !== 'passed' && result.status !== 'flaky' && result.status !== 'skipped') { worker.sawFailure = true; } @@ -618,9 +667,28 @@ class Scheduler { break; } case 'serial-group': { + worker.serialGroups.delete(message.group.id); this.options.events.onSerialGroup(message.group); break; } + case 'attempt-start': { + worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat)).running = message.index; + break; + } + case 'attempt': { + const attempts = worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat)); + attempts.finished.push(message.attempt); + attempts.running = undefined; + break; + } + case 'serial-member': { + worker.serialGroup(message.groupId).memberFinished(message.attempt, message.member); + break; + } + case 'serial-attempt': { + worker.serialGroup(message.groupId).attemptFinished(message.run); + break; + } case 'unit-done': { for (const runError of message.runErrors) this.options.events.onRunError(runError); if (message.debug !== undefined) this.options.events.onDebug?.(message.debug); @@ -724,18 +792,53 @@ class Scheduler { return this.signalledWave.has(worker); } - /** Emits records for a unit whose worker died before reporting it done. */ + /** + * Emits records for a unit whose worker died before reporting it done. + * Attempts the worker finished stand. The test in flight gets a + * `WORKER_CRASH` attempt after them when the crash landed during an + * attempt; one between attempts (an `afterAll` after the last) is recorded + * on the last attempt instead, whose cleanup it forced. A serial group in + * flight gets a failed group attempt after its finished ones. A worker the + * run's interrupt stopped charges no crash: a test keeps the attempts it + * finished and the verdict they reach, as a retry the interrupt cut short + * does, and a serial group's attempt in flight is recorded interrupted. + */ private synthesizeCrashResults( state: TargetState, worker: SchedulerWorker, unit: WorkUnit, ): void { - const interrupted = this.interrupting; + const interrupted = this.interrupting && (worker.terminated || worker.killed); + const crash = serializeError(new InfrastructureError('WORKER_CRASH', 'worker process exited during this test')); + const settledGroups = new Set(); for (const pair of unit.pairs) { const key = pairKey(pair.test.id, pair.agent, pair.repeat); if (worker.reported.has(key)) continue; + const { serialId } = pair.test; + if (serialId !== undefined) { + const groupId = serialGroupId(serialId, worker.targetName, pair.agent, pair.repeat); + if (settledGroups.has(groupId)) continue; + const members = unit.pairs.filter( + (member) => member.test.serialId === serialId && member.agent === pair.agent && member.repeat === pair.repeat, + ); + const progress = worker.serialGroups.get(groupId); + const settled = interrupted + ? progress?.interrupted(members, state.target, this.stopSkip ?? INTERRUPTED_BEFORE_START) + : progress?.crashed(members, state.target, crash); + if (settled !== undefined) { + settledGroups.add(groupId); + this.options.events.onSerialGroup(settled.group); + for (const result of settled.results) this.report(result); + continue; + } + } + const { finished, running } = worker.attempts.get(key) ?? { finished: [], running: undefined }; if (interrupted) { - this.report(unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START)); + this.report( + finished.length === 0 + ? unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START) + : finishedResult(pair, finished), + ); continue; } // A crashed setup never persisted its sessions; dependents must skip. @@ -748,29 +851,36 @@ class Scheduler { this.report( unstartedResult(pair, { cause: 'infrastructure-unavailable', - reason: worker.started.has(key) - ? 'worker process exited before this serial group finished' - : 'worker process exited before this test started', + reason: 'worker process exited before this test started', }), ); continue; } + const last = finished.at(-1); + if (last !== undefined && running === undefined) { + this.report( + finishedResult(pair, [ + ...finished.slice(0, -1), + { ...last, secondaryErrors: [...last.secondaryErrors, crash], cleanup: 'forced' }, + ]), + ); + continue; + } this.report( pairResult(pair, { status: 'failed', selected: true, attempts: [ + ...finished, { id: uuidv7(), - index: 0, + index: running ?? finished.length, status: 'failed', startedAt: timestamp(), durationMs: 0, steps: [], artifacts: [], - error: serializeError( - new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'), - ), + error: crash, secondaryErrors: [], cleanup: 'forced', }, diff --git a/packages/e2e/src/run/serial.ts b/packages/e2e/src/run/serial.ts index cf0dd559b..b7f9a977f 100644 --- a/packages/e2e/src/run/serial.ts +++ b/packages/e2e/src/run/serial.ts @@ -20,7 +20,7 @@ import type { StepRecord } from './steps.ts'; import { findRegistered, type FileRef, type Realm, RealmManager } from './realm.ts'; import type { AttemptRecord, ResultRecord, ResultStatus, SerialAttemptRecord, SerialGroupRecord, SerialMemberRecord, FailedStatus } from './records.ts'; import { isFailedStatus } from './records.ts'; -import { runWithRetries } from './retry.ts'; +import { isRetryEligible, retryVerdict, runWithRetries } from './retry.ts'; import { interruptedSkip, pairResult, repeatSegment } from './units.ts'; /** @@ -87,6 +87,34 @@ export interface SerialHost { * steps, duration, and error from the group it has already seen. */ emitSerialGroup(group: SerialGroupRecord): void; + /** + * Streams one member that finished in the group attempt running now, so a + * worker that dies before the group ends does not take it along. + */ + serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void; + /** Streams one finished group attempt, for the same reason. */ + serialAttemptFinished(groupId: string, run: SerialAttemptRun): void; +} + +/** The identity of a group attempt, known from its start. */ +export type SerialAttemptStart = Pick; + +/** One group attempt as it ended, and whether it got as far as its members. */ +export interface SerialAttemptRun { + readonly record: SerialAttemptRecord; + /** + * False when the attempt stopped before any member ran (its file would not + * load, its session would not launch). Such a retry keeps the members' + * verdicts from the attempt before it. + */ + readonly reachedMembers: boolean; +} + +/** The report id of one serial group variant: a group runs once per agent and per repeat. */ +export function serialGroupId(serialId: string, targetName: string, agent: string, repeat: number): string { + return canonicalDigest( + repeat === 0 ? { serialId, targetId: targetName, agent } : { serialId, targetId: targetName, agent, repeat }, + ); } /** Runs one serial unit to completion and emits every member result. */ @@ -95,46 +123,20 @@ export async function runSerialUnit( members: readonly TestTargetPair[], file: FileRef, ): Promise { - const first = members[0]!; - const serialId = first.test.serialId!; - // Every member of a unit runs as the same agent (selection guarantees it), - // so the group is one variant of the flow and is identified as such. - const agent = first.agent; - const { repeat } = first; - const groupRecordId = canonicalDigest( - repeat === 0 ? { serialId, targetId: host.target.name, agent } : { serialId, targetId: host.target.name, agent, repeat }, - ); - - const group: SerialGroupRecord = { - id: groupRecordId, - serialId, - declarationIndex: first.test.declarationIndex, - file: first.test.file, - titlePath: serialTitlePath(first.test), - targetId: host.target.name, - platform: host.target.platform, - agent, - repeat, - memberTestIds: members.map((member) => member.test.id), - status: 'failed', - attempts: [], - }; - - const memberFinalStatus = new Map(); + const group = newSerialGroup(members, host.target); + const runs: SerialAttemptRun[] = []; const finalStatus = await runWithRetries( - first.options.retries + 1, + members[0]!.options.retries + 1, host.interruptSignal, async (attemptIndex) => { - const { record: attempt, reachedMembers } = await runSerialAttempt(host, members, file, attemptIndex); - group.attempts.push(attempt); - // A retry the run interrupted, or one that never reached its members, - // keeps the verdict before it, so each member keeps what that attempt said. - const cutRetry = attemptIndex > 0 && (attempt.status === 'interrupted' || !reachedMembers); - if (!cutRetry) for (const member of attempt.members) memberFinalStatus.set(member.testId, member); + const run = await runSerialAttempt(host, group.id, members, file, attemptIndex); + runs.push(run); + group.attempts.push(run.record); + host.serialAttemptFinished(group.id, run); // A beforeAll failure is not retry-eligible: the // attempt stands as recorded and the retry loop stops here. - if (attempt.error?.code === 'HOOK_FAILED') return undefined; - return attempt; + if (run.record.error?.code === 'HOOK_FAILED') return undefined; + return run.record; }, ); @@ -146,8 +148,51 @@ export async function runSerialUnit( } host.emitSerialGroup(group); - for (const member of members) { - const memberRecord = memberFinalStatus.get(member.test.id); + for (const result of memberResults(group, members, runs)) host.emit(result); + return group; +} + +/** A group record with no attempts yet. */ +function newSerialGroup(members: readonly TestTargetPair[], target: ResolvedTarget): SerialGroupRecord { + const first = members[0]!; + const serialId = first.test.serialId!; + // Every member of a unit runs as the same agent (selection guarantees it), + // so the group is one variant of the flow and is identified as such. + const { agent, repeat } = first; + return { + id: serialGroupId(serialId, target.name, agent, repeat), + serialId, + declarationIndex: first.test.declarationIndex, + file: first.test.file, + titlePath: serialTitlePath(first.test), + targetId: target.name, + platform: target.platform, + agent, + repeat, + memberTestIds: members.map((member) => member.test.id), + status: 'failed', + attempts: [], + }; +} + +/** + * Each member's result once its group is decided. Members carry no attempts + * of their own; the verdict is the member's record in the last attempt that + * counts. A retry the run interrupted, or one that never reached its + * members, keeps the verdict before it. + */ +function memberResults( + group: SerialGroupRecord, + members: readonly TestTargetPair[], + runs: readonly SerialAttemptRun[], +): ResultRecord[] { + const verdicts = new Map(); + for (const [attemptIndex, run] of runs.entries()) { + const cutRetry = attemptIndex > 0 && (run.record.status === 'interrupted' || !run.reachedMembers); + if (!cutRetry) for (const member of run.record.members) verdicts.set(member.testId, member); + } + return members.map((member) => { + const memberRecord = verdicts.get(member.test.id); let status: ResultStatus; let skip: SkipInfo | undefined; // A member that skipped itself stays skipped whatever the group did: a @@ -163,28 +208,160 @@ export async function runSerialUnit( } else { status = memberRecord.status; } - host.emit( - pairResult(member, { - status, - selected: true, - ...(skip !== undefined ? { skip } : {}), - attempts: [], - serialGroupId: groupRecordId, - }), - ); + return pairResult(member, { + status, + selected: true, + ...(skip !== undefined ? { skip } : {}), + attempts: [], + serialGroupId: group.id, + }); + }); +} + +/** A serial group decided without its worker's word: the record and its members' results. */ +interface SettledSerialGroup { + readonly group: SerialGroupRecord; + readonly results: readonly ResultRecord[]; +} + +/** + * What the runner heard of one serial group a worker is running: the + * attempts it finished, and the members finished so far in the attempt + * running now. When the worker dies before reporting the group, this is + * what the report keeps. + */ +export class SerialGroupProgress { + private readonly finished: SerialAttemptRun[] = []; + private current: SerialAttemptStart | undefined; + private currentMembers: SerialMemberRecord[] = []; + /** The member whose body is running now, by test id, until its record arrives. */ + private running: string | undefined; + + /** A member of the group announced its start. */ + memberStarted(testId: string): void { + this.running = testId; } - return group; + /** A member finished in the attempt running now. */ + memberFinished(attempt: SerialAttemptStart, member: SerialMemberRecord): void { + this.current = attempt; + this.currentMembers.push(member); + if (this.running === member.testId) this.running = undefined; + } + + /** A group attempt finished; the next one starts empty. */ + attemptFinished(run: SerialAttemptRun): void { + this.finished.push(run); + this.current = undefined; + this.currentMembers = []; + this.running = undefined; + } + + /** + * The group after its worker died with `error`, or undefined when the + * runner heard nothing of it running. The finished attempts stand, and the + * attempt in flight fails: the member whose body was running fails with + * `error` and the ones after it skip. A crash between members (a hook, + * the session closing) fails the attempt and leaves its finished members + * as they were; one in a retry before any member ran fails the first, as a + * launch failure does. A crash after the last attempt, when no retry + * follows it, is recorded on that attempt. + */ + crashed(members: readonly TestTargetPair[], target: ResolvedTarget, error: SerializedError): SettledSerialGroup | undefined { + const last = this.finished.at(-1); + if (!this.inFlight) { + if (last === undefined) return undefined; + if (!retryFollows(last.record, this.finished.length, members)) { + const closed: SerialAttemptRun = { + ...last, + record: { ...last.record, secondaryErrors: [...last.record.secondaryErrors, error], cleanup: 'forced' }, + }; + return this.settled([...this.finished.slice(0, -1), closed], members, target); + } + const record = this.attemptInFlight('failed', error); + endBeforeMembers(record, members, error, NEVER_INTERRUPTED); + return this.settled([...this.finished, { record, reachedMembers: false }], members, target); + } + const record = this.attemptInFlight('failed', error); + const next = this.currentMembers.length; + const wasRunning = this.running !== undefined && this.running === members[next]?.test.id; + const skip: SkipInfo = wasRunning + ? predecessorFailed(next) + : { cause: 'infrastructure-unavailable', reason: 'worker process exited during this group attempt' }; + for (const [memberIndex, member] of members.entries()) { + if (memberIndex < next) continue; + record.members.push( + wasRunning && memberIndex === next + ? memberFailedWith(record, memberIndex, member.test.id, error) + : skippedMember(record.id, memberIndex, member.test.id, skip), + ); + } + return this.settled([...this.finished, { record, reachedMembers: true }], members, target); + } + + /** + * The group after the run's interrupt stopped its worker, or undefined + * when the runner heard nothing of it running. The finished attempts + * stand, and an attempt in flight is interrupted: the members it finished + * keep their records and the rest skip with `skip`. The verdict is the one + * those attempts reach, as for a retry the interrupt cut short. + */ + interrupted(members: readonly TestTargetPair[], target: ResolvedTarget, skip: SkipInfo): SettledSerialGroup | undefined { + if (!this.inFlight) return this.finished.length === 0 ? undefined : this.settled(this.finished, members, target); + const record = this.attemptInFlight('interrupted', undefined); + for (const [memberIndex, member] of members.entries()) { + if (memberIndex >= this.currentMembers.length) record.members.push(skippedMember(record.id, memberIndex, member.test.id, skip)); + } + return this.settled([...this.finished, { record, reachedMembers: true }], members, target); + } + + /** Whether a member of the attempt in flight started or finished. */ + private get inFlight(): boolean { + return this.currentMembers.length > 0 || this.running !== undefined; + } + + /** The record of the attempt in flight as it stood, with the members it finished. */ + private attemptInFlight(status: SerialAttemptRecord['status'], error: SerializedError | undefined): SerialAttemptRecord { + return { + id: this.current?.id ?? uuidv7(), + index: this.finished.length, + status, + startedAt: this.current?.startedAt ?? timestamp(), + durationMs: 0, + members: [...this.currentMembers], + artifacts: [], + ...(error === undefined ? {} : { error }), + secondaryErrors: [], + cleanup: 'forced', + }; + } + + private settled( + runs: readonly SerialAttemptRun[], + members: readonly TestTargetPair[], + target: ResolvedTarget, + ): SettledSerialGroup { + const group = newSerialGroup(members, target); + group.status = retryVerdict(runs.map((run) => run.record)); + group.attempts = runs.map((run) => run.record); + return { group, results: memberResults(group, members, runs) }; + } } -/** One group attempt as it ended, and whether it got as far as its members. */ -interface SerialAttemptRun { - readonly record: SerialAttemptRecord; - readonly reachedMembers: boolean; +/** + * Whether the group retries after `record`, its attempt number `attempts`: + * the retry policy allows it, and a hook failure never retries. + */ +function retryFollows(record: SerialAttemptRecord, attempts: number, members: readonly TestTargetPair[]): boolean { + return attempts < members[0]!.options.retries + 1 && isRetryEligible(record) && record.error?.code !== 'HOOK_FAILED'; } +/** An abort signal that never fires. */ +const NEVER_INTERRUPTED = new AbortController().signal; + async function runSerialAttempt( host: SerialHost, + groupId: string, members: readonly TestTargetPair[], file: FileRef, attemptIndex: number, @@ -222,6 +399,12 @@ async function runSerialAttempt( secondaryErrors: [], cleanup: 'complete', }; + const start: SerialAttemptStart = { id: attemptId, index: attemptIndex, startedAt }; + /** Records one member and streams it, so a worker that dies later in the attempt does not take it along. */ + const finishMember = (member: SerialMemberRecord): void => { + memberRecords.push(member); + host.serialMemberFinished(groupId, start, member); + }; let realm: Realm; try { @@ -258,12 +441,12 @@ async function runSerialAttempt( skipRemaining = interruptedMemberSkip(host.interruptSignal); } if (skipRemaining !== undefined) { - memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining)); + finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining)); continue; } const registered = findRegistered(realm, member.test); if (registered === undefined) { - memberRecords.push({ + finishMember({ id: memberId, index: memberIndex, testId: member.test.id, @@ -285,7 +468,7 @@ async function runSerialAttempt( hookFailure = await host.realms.enterScopes(realm, registered); if (hookFailure !== undefined) { skipRemaining = { cause: 'hook-failed', reason: hookFailure.message }; - memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining)); + finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining)); continue; } host.pairStarted(member); @@ -294,7 +477,7 @@ async function runSerialAttempt( shared, }); shared.priorSteps.push(...memberAttempt.steps); - memberRecords.push({ + finishMember({ id: memberId, index: memberIndex, testId: member.test.id, @@ -367,22 +550,27 @@ function endBeforeMembers( members.forEach((member, memberIndex) => { record.members.push( memberIndex === 0 - ? { - id: `${record.id}:member:0`, - index: 0, - testId: member.test.id, - status: 'failed', - startedAt: record.startedAt, - durationMs: 0, - steps: [], - error, - secondaryErrors: [], - } + ? memberFailedWith(record, 0, member.test.id, error) : skippedMember(record.id, memberIndex, member.test.id, predecessorFailed(0)), ); }); } +/** A member that failed with an error of its group attempt's, having run no step of its own. */ +function memberFailedWith(record: SerialAttemptRecord, index: number, testId: string, error: SerializedError): SerialMemberRecord { + return { + id: `${record.id}:member:${index}`, + index, + testId, + status: 'failed', + startedAt: record.startedAt, + durationMs: 0, + steps: [], + error, + secondaryErrors: [], + }; +} + function isFailedMember(member: SerialMemberRecord): member is SerialMemberRecord & { status: FailedStatus } { return isFailedStatus(member.status); } diff --git a/packages/e2e/src/run/worker/entry.ts b/packages/e2e/src/run/worker/entry.ts index 4f2c0acbc..8caab52a7 100644 --- a/packages/e2e/src/run/worker/entry.ts +++ b/packages/e2e/src/run/worker/entry.ts @@ -39,20 +39,23 @@ let outbox: Promise = Promise.resolve(); /** How long an exit waits for the outbox: a channel whose runner is gone may never acknowledge. */ const FLUSH_GRACE_MS = 2_000; -/** Queues one message; resolves once the channel has taken it (or is gone). */ +/** + * Hands one message to the channel at once; resolves once the channel has + * taken it and every message before it (or is gone). The channel keeps the + * order. Handing it over synchronously is what lets a `pair-start` reach the + * runner when the test body exits the process right after it. + */ function send(message: WorkerToMain): Promise { - outbox = outbox.then( - () => - new Promise((resolve) => { - try { - if (process.send === undefined || !process.connected) resolve(); - else process.send(message, undefined, undefined, () => resolve()); - } catch { - // channel already closed; nothing left to deliver - resolve(); - } - }), - ); + const taken = new Promise((resolve) => { + try { + if (process.send === undefined || !process.connected) resolve(); + else process.send(message, undefined, undefined, () => resolve()); + } catch { + // channel already closed; nothing left to deliver + resolve(); + } + }); + outbox = outbox.then(() => taken); return outbox; } diff --git a/packages/e2e/src/run/worker/protocol.ts b/packages/e2e/src/run/worker/protocol.ts index 0e678b081..add33c905 100644 --- a/packages/e2e/src/run/worker/protocol.ts +++ b/packages/e2e/src/run/worker/protocol.ts @@ -13,7 +13,8 @@ import type { CliOverrides, PortAssignments, ResolvedTarget } from '../../config import type { AiTraceSnapshot } from '../../internal/ai-trace.ts'; import type { DebugSnapshot } from '../../internal/debug.ts'; import type { SerializedError } from '../../internal/errors.ts'; -import type { ResultRecord, RunError, SerialGroupRecord, WireResultRecord } from '../records.ts'; +import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord, WireResultRecord } from '../records.ts'; +import type { SerialAttemptRun, SerialAttemptStart } from '../serial.ts'; import type { StepProgress } from '../steps.ts'; /** One runnable pair on the wire; the worker resolves the test function. */ @@ -190,6 +191,52 @@ export interface SerialGroupMessage { readonly group: SerialGroupRecord; } +/** + * An attempt of an ordinary or setup pair begins, its realm and `beforeAll` + * hooks included. With `attempt`, it tells a crash during an attempt from + * one between attempts (an `afterAll` after the last one): only the first + * is charged a new attempt, with this index. + */ +export interface AttemptStartMessage { + readonly type: 'attempt-start'; + readonly testId: string; + readonly agent: string; + readonly repeat: number; + readonly index: number; +} + +/** + * One finished attempt of an ordinary or setup pair. A pair's result waits + * for its last attempt, so each attempt also goes out as it ends: a worker + * that dies during a retry must not take the attempts before it along. The + * result still carries every attempt; the scheduler keeps these only until + * it arrives. + */ +export interface AttemptMessage { + readonly type: 'attempt'; + readonly testId: string; + readonly agent: string; + readonly repeat: number; + readonly attempt: AttemptRecord; +} + +/** One member that finished in the serial group attempt running now; see `AttemptMessage`. */ +export interface SerialMemberMessage { + readonly type: 'serial-member'; + /** The group's report id (`SerialGroupRecord.id`). */ + readonly groupId: string; + readonly attempt: SerialAttemptStart; + readonly member: SerialMemberRecord; +} + +/** One finished serial group attempt; see `AttemptMessage`. */ +export interface SerialAttemptMessage { + readonly type: 'serial-attempt'; + /** The group's report id (`SerialGroupRecord.id`). */ + readonly groupId: string; + readonly run: SerialAttemptRun; +} + export interface UnitDoneMessage { readonly type: 'unit-done'; readonly unitId: string; @@ -236,6 +283,10 @@ export type WorkerToMain = | NoticeMessage | ResultMessage | SerialGroupMessage + | AttemptStartMessage + | AttemptMessage + | SerialMemberMessage + | SerialAttemptMessage | UnitDoneMessage | ShutdownDoneMessage | FatalMessage diff --git a/packages/e2e/src/run/worker/session.ts b/packages/e2e/src/run/worker/session.ts index 17712a9d7..92879a112 100644 --- a/packages/e2e/src/run/worker/session.ts +++ b/packages/e2e/src/run/worker/session.ts @@ -123,6 +123,12 @@ export class TargetWorker { if (countsTowardFailureLimit(result.status)) this.countFailure(); }, onSerialGroup: (group) => this.host.emit({ type: 'serial-group', group }), + onAttemptStart: (pair, index) => + this.host.emit({ type: 'attempt-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, index }), + onAttempt: (pair, attempt) => + this.host.emit({ type: 'attempt', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, attempt }), + onSerialMember: (groupId, attempt, member) => this.host.emit({ type: 'serial-member', groupId, attempt, member }), + onSerialAttempt: (groupId, run) => this.host.emit({ type: 'serial-attempt', groupId, run }), onPairStart: (pair) => { this.inFlight = { testId: pair.test.id, agent: pair.agent, repeat: pair.repeat }; this.host.emit({ diff --git a/packages/e2e/tests/integration/parallel-workers.test.ts b/packages/e2e/tests/integration/parallel-workers.test.ts index eeb3e24e5..46cad81c5 100644 --- a/packages/e2e/tests/integration/parallel-workers.test.ts +++ b/packages/e2e/tests/integration/parallel-workers.test.ts @@ -270,15 +270,20 @@ test.describe('wizard', { serial: true }, () => { { 'tests/serial-crash.e2e.ts': file }, { appUrl: app.url, configSource: workerConfigSource(1) }, ); + const group = outcome.report.run.serialGroups[0]!; + expect(group.status).toBe('failed'); + expect(group.attempts[0]!.members.map((member) => [member.status, member.error?.code])).toEqual([ + ['passed', undefined], + ['failed', 'WORKER_CRASH'], + ['skipped', undefined], + ]); + expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed'); const crashed = resultByTitle(outcome, 'step 2 crashes the worker'); expect(crashed.status).toBe('failed'); - expect(crashed.attempts[0]?.error?.code).toBe('WORKER_CRASH'); - const finished = resultByTitle(outcome, 'step 1 passes'); - expect(finished.status).toBe('skipped'); - expect(finished.skip?.reason).toBe('worker process exited before this serial group finished'); + expect(crashed.serialGroupId).toBe(group.id); const unreached = resultByTitle(outcome, 'step 3 never starts'); expect(unreached.status).toBe('skipped'); - expect(unreached.skip?.reason).toBe('worker process exited before this test started'); + expect(unreached.skip?.cause).toBe('serial-predecessor-failed'); expect(outcome.exitCode).toBe(3); expect(outcome.report.run.summary.failed).toBe(1); assertValidReport(outcome.report); @@ -287,6 +292,292 @@ test.describe('wizard', { serial: true }, () => { 120_000, ); + it( + 'keeps the failed attempt before a retry that crashed the worker', + async () => { + const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-crash-')), 'failed-once'); + const file = `import { existsSync, writeFileSync } from 'node:fs'; +import { test } from 'e2e'; + +test('fails, then crashes on retry', { retries: 1 }, async () => { + if (!existsSync(${JSON.stringify(marker)})) { + writeFileSync(${JSON.stringify(marker)}, ''); + throw new Error('first attempt fails'); + } + process.exit(7); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/retry-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const result = resultByTitle(outcome, 'fails, then crashes on retry'); + expect(result.status).toBe('failed'); + expect(result.attempts.map((attempt) => [attempt.index, attempt.status, attempt.error?.message])).toEqual([ + [0, 'failed', 'first attempt fails'], + [1, 'failed', 'worker process exited during this test'], + ]); + expect(result.attempts[1]!.error?.code).toBe('WORKER_CRASH'); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'keeps the failed group attempt before a serial retry that crashed the worker', + async () => { + const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-crash-')), 'failed-once'); + const file = `import { existsSync, writeFileSync } from 'node:fs'; +import { test } from 'e2e'; + +test.describe('wizard', { serial: true, retries: 1 }, () => { + test('step 1 passes', async () => {}); + test('step 2 fails, then crashes on retry', async () => { + if (!existsSync(${JSON.stringify(marker)})) { + writeFileSync(${JSON.stringify(marker)}, ''); + throw new Error('first attempt fails'); + } + process.exit(7); + }); + test('step 3 never passes', async () => {}); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/serial-retry-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const group = outcome.report.run.serialGroups[0]!; + expect(group.status).toBe('failed'); + expect( + group.attempts.map((attempt) => [attempt.index, attempt.members.map((member) => [member.status, member.error?.code])]), + ).toEqual([ + [0, [['passed', undefined], ['failed', 'ERROR'], ['skipped', undefined]]], + [1, [['passed', undefined], ['failed', 'WORKER_CRASH'], ['skipped', undefined]]], + ]); + expect(group.attempts[0]!.members[1]!.error?.message).toBe('first attempt fails'); + expect(group.attempts[1]!.error?.code).toBe('WORKER_CRASH'); + expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed'); + const crashed = resultByTitle(outcome, 'step 2 fails, then crashes on retry'); + expect(crashed.status).toBe('failed'); + expect(crashed.serialGroupId).toBe(group.id); + expect(resultByTitle(outcome, 'step 3 never passes').skip?.cause).toBe('serial-predecessor-failed'); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'records a crash in afterAll after a failed attempt on that attempt, not as a retry', + async () => { + const file = `import { test } from 'e2e'; + +test.afterAll(() => { + process.exit(7); +}); + +test('always fails', { retries: 1 }, async () => { + throw new Error('fails every time'); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/teardown-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const result = resultByTitle(outcome, 'always fails'); + expect(result.status).toBe('failed'); + expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.message, attempt.cleanup])).toEqual([ + [0, 'fails every time', 'forced'], + ]); + expect(result.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'keeps a passed setup attempt when afterAll crashes the worker, and skips its dependents', + async () => { + const setupFile = `import { test } from 'e2e'; + +test.afterAll(() => { + process.exit(7); +}); + +test.setup('seed storage', { sessions: ['seeded'] }, async ({ app, session }) => { + await app.open('/storage'); + await session.save('seeded'); +}); +`; + const consumerFile = `import { test } from 'e2e'; + +test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => { + await app.open('/storage'); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/auth.setup.e2e.ts': setupFile, 'tests/consumer.e2e.ts': consumerFile }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const setup = resultByTitle(outcome, 'seed storage'); + expect(setup.status).toBe('passed'); + expect(setup.attempts.map((attempt) => [attempt.index, attempt.status, attempt.cleanup])).toEqual([ + [0, 'passed', 'forced'], + ]); + expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']); + expect(resultByTitle(outcome, 'starts with the seeded state').skip?.cause).toBe('setup-failed'); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'keeps the SESSION_CONTRACT failure of a setup attempt when afterAll crashes the worker', + async () => { + const setupFile = `import { test } from 'e2e'; + +test.afterAll(() => { + process.exit(7); +}); + +test.setup('never saves', { sessions: ['seeded'] }, async ({ app }) => { + await app.open('/storage'); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { + 'tests/auth.setup.e2e.ts': setupFile, + 'tests/consumer.e2e.ts': `import { test } from 'e2e'; + +test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => { + await app.open('/storage'); +}); +`, + }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const setup = resultByTitle(outcome, 'never saves'); + expect(setup.status).toBe('failed'); + expect(setup.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.cleanup])).toEqual([ + [0, 'SESSION_CONTRACT', 'forced'], + ]); + expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'charges a crash in a retry beforeAll to the retry, not to the attempt before it', + async () => { + const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-hook-crash-')), 'failed-once'); + const file = `import { existsSync, writeFileSync } from 'node:fs'; +import { test } from 'e2e'; + +test.beforeAll(() => { + if (existsSync(${JSON.stringify(marker)})) process.exit(7); +}); + +test('fails once', { retries: 1 }, async () => { + writeFileSync(${JSON.stringify(marker)}, ''); + throw new Error('first attempt fails'); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/retry-hook-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const result = resultByTitle(outcome, 'fails once'); + expect(result.status).toBe('failed'); + expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.secondaryErrors.length])).toEqual([ + [0, 'ERROR', 0], + [1, 'WORKER_CRASH', 0], + ]); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'fails the first member when a serial retry crashes the worker before any member runs', + async () => { + const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-hook-crash-')), 'failed-once'); + const file = `import { existsSync, writeFileSync } from 'node:fs'; +import { test } from 'e2e'; + +test.describe('wizard', { serial: true, retries: 1 }, () => { + test.beforeAll(() => { + if (existsSync(${JSON.stringify(marker)})) process.exit(7); + }); + test('step 1 passes', async () => {}); + test('step 2 fails once', async () => { + writeFileSync(${JSON.stringify(marker)}, ''); + throw new Error('first attempt fails'); + }); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/serial-retry-hook-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const group = outcome.report.run.serialGroups[0]!; + expect(group.status).toBe('failed'); + expect( + group.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.members.map((member) => [member.status, member.error?.code])]), + ).toEqual([ + [0, 'ERROR', [['passed', undefined], ['failed', 'ERROR']]], + [1, 'WORKER_CRASH', [['failed', 'WORKER_CRASH'], ['skipped', undefined]]], + ]); + // The retry never reached its members, so they keep attempt 0's verdicts. + expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed'); + expect(resultByTitle(outcome, 'step 2 fails once').status).toBe('failed'); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'charges a crash after the last serial member to the group attempt, not to a member', + async () => { + const file = `import { test } from 'e2e'; + +test.describe('wizard', { serial: true }, () => { + test.afterAll(() => { + process.exit(7); + }); + test('step 1 passes', async () => {}); + test('step 2 passes', async () => {}); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/serial-teardown-crash.e2e.ts': file }, + { appUrl: app.url, configSource: workerConfigSource(1) }, + ); + const group = outcome.report.run.serialGroups[0]!; + expect(group.status).toBe('failed'); + expect(group.attempts).toHaveLength(1); + expect(group.attempts[0]!.error?.code).toBe('WORKER_CRASH'); + expect(group.attempts[0]!.members.map((member) => member.status)).toEqual(['passed', 'passed']); + expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed'); + expect(resultByTitle(outcome, 'step 2 passes').status).toBe('passed'); + expect(outcome.exitCode).toBe(3); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + it( 'interrupts workers, reports 130, and lists the file the one worker never reached', async () => { diff --git a/packages/e2e/tests/unit/retry.test.ts b/packages/e2e/tests/unit/retry.test.ts index 666f5e395..2664b1075 100644 --- a/packages/e2e/tests/unit/retry.test.ts +++ b/packages/e2e/tests/unit/retry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { isRetryEligible, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts'; +import { isRetryEligible, retryVerdict, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts'; import type { SerializedError } from '../../src/internal/errors.ts'; function error(category: SerializedError['category']): SerializedError { @@ -126,3 +126,19 @@ describe('runWithRetries', () => { expect(status).toBe('failed'); }); }); + +describe('retryVerdict', () => { + const failed: RetryAttempt = { status: 'failed', error: error('test') }; + + it.each<[readonly RetryAttempt[], string]>([ + [[], 'failed'], + [[{ status: 'passed' }], 'passed'], + [[failed, { status: 'passed' }], 'flaky'], + [[failed, { status: 'timed-out' }], 'timed-out'], + [[{ status: 'interrupted' }], 'interrupted'], + [[{ status: 'timed-out' }, { status: 'interrupted' }], 'timed-out'], + [[failed, { status: 'skipped' }], 'skipped'], + ])('reads %j as %s', (attempts, verdict) => { + expect(retryVerdict(attempts)).toBe(verdict); + }); +}); diff --git a/packages/e2e/tests/unit/scheduler.test.ts b/packages/e2e/tests/unit/scheduler.test.ts index e2472f546..08d69483a 100644 --- a/packages/e2e/tests/unit/scheduler.test.ts +++ b/packages/e2e/tests/unit/scheduler.test.ts @@ -15,8 +15,9 @@ import type { import type { ResolvedTarget } from '../../src/config/resolve.ts'; import { defineEngine, type EngineHandle } from '../../src/engine/index.ts'; import { classifyError, serializeError } from '../../src/internal/errors.ts'; -import type { ResultRecord, RunError, SerialGroupRecord } from '../../src/run/records.ts'; +import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord } from '../../src/run/records.ts'; import { runUnits } from '../../src/run/scheduler.ts'; +import { serialGroupId, type SerialAttemptRun } from '../../src/run/serial.ts'; import { buildWorkPlans, type TargetWorkPlan } from '../../src/run/units.ts'; import type { SpawnUnitRunner, UnitRunner, UnitRunnerEvents } from '../../src/run/unit-runner.ts'; import type { MainToWorker, RunUnitMessage } from '../../src/run/worker/protocol.ts'; @@ -142,6 +143,16 @@ interface FakeBehaviour { readonly neverReady?: readonly string[]; /** Unit ids whose worker never finishes: a test that will not end on its own. */ readonly hangOn?: readonly string[]; + /** Attempts the unit's first pair finishes before the worker hangs: a test caught between attempts. */ + readonly attemptsBeforeHang?: readonly AttemptRecord[]; + /** The hung pair starts its next attempt after `attemptsBeforeHang`, and the worker exits on its own once interrupted. */ + readonly exitDuringNextAttemptOnInterrupt?: boolean; + /** Serial group attempts the unit's pairs (one group) finish before the worker hangs. */ + readonly serialRunsBeforeHang?: readonly SerialAttemptRun[]; + /** Members the group's next attempt finishes before the worker hangs. */ + readonly serialMembersBeforeHang?: readonly SerialMemberRecord[]; + /** Instead of hanging, the worker exits on its own once it sent what came before. */ + readonly exitInsteadOfHang?: boolean; /** Workers ignore `terminate` and have to be killed. */ readonly ignoreTerminate?: boolean; /** @@ -152,6 +163,31 @@ interface FakeBehaviour { readonly signalledInit?: { readonly workers: number; readonly signal: NodeJS.Signals }; } +const FAILED_ATTEMPT: AttemptRecord = { + id: 'attempt-0', + index: 0, + status: 'failed', + startedAt: '2026-01-01T00:00:00.000Z', + durationMs: 5, + steps: [], + artifacts: [], + error: serializeError(classifyError(new Error('first attempt fails'))), + secondaryErrors: [], + cleanup: 'complete', +}; + +const PASSED_RETRY: AttemptRecord = { + id: 'attempt-1', + index: 1, + status: 'passed', + startedAt: '2026-01-01T00:00:01.000Z', + durationMs: 5, + steps: [], + artifacts: [], + secondaryErrors: [], + cleanup: 'complete', +}; + class FakeFleet { readonly spawned: { targetName: string; workerSlot: number }[] = []; readonly unitsByWorker: RunUnitMessage[][] = []; @@ -221,6 +257,10 @@ class FakeRunner implements UnitRunner { if (this.exited) return; if (message.type !== 'run-unit') this.fleet.controlMessages.push(message.type); if (message.type === 'interrupt') this.fleet.interruptSkips.push(message.skip); + if (message.type === 'interrupt' && this.behaviour.exitDuringNextAttemptOnInterrupt === true) { + setTimeout(() => this.end('code 7, signal null'), 0); + return; + } if (message.type === 'shutdown') { setTimeout(() => this.end('shut down'), 0); return; @@ -231,7 +271,41 @@ class FakeRunner implements UnitRunner { } if (message.type !== 'run-unit') return; this.fleet.unitsByWorker[this.index]!.push(message); - if (this.behaviour.hangOn?.includes(message.unitId) === true) return; + if (this.behaviour.hangOn?.includes(message.unitId) === true) { + const first = message.pairs[0]; + for (const finishedRun of this.behaviour.serialRunsBeforeHang ?? []) { + const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat); + for (const [index, member] of finishedRun.record.members.entries()) { + const pair = message.pairs[index]!; + this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId }); + this.events.onMessage({ type: 'serial-member', groupId, attempt: finishedRun.record, member }); + } + this.events.onMessage({ type: 'serial-attempt', groupId, run: finishedRun }); + } + const partial = this.behaviour.serialMembersBeforeHang ?? []; + if (partial.length > 0) { + const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat); + const attempt = { id: 'group-attempt-partial', index: this.behaviour.serialRunsBeforeHang?.length ?? 0, startedAt: '2026-01-01T00:00:00.000Z' }; + for (const [index, member] of partial.entries()) { + const pair = message.pairs[index]!; + this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId }); + this.events.onMessage({ type: 'serial-member', groupId, attempt, member }); + } + } + const attempts = this.behaviour.attemptsBeforeHang ?? []; + const nextAttempt = this.behaviour.exitDuringNextAttemptOnInterrupt === true; + if ((attempts.length > 0 || nextAttempt) && first !== undefined) { + const pair = { testId: first.test.id, agent: first.agent, repeat: first.repeat }; + this.events.onMessage({ type: 'pair-start', ...pair, title: first.test.id, file: first.test.file, serialId: undefined }); + for (const attempt of attempts) { + this.events.onMessage({ type: 'attempt-start', ...pair, index: attempt.index }); + this.events.onMessage({ type: 'attempt', ...pair, attempt }); + } + if (nextAttempt) this.events.onMessage({ type: 'attempt-start', ...pair, index: attempts.length }); + } + if (this.behaviour.exitInsteadOfHang === true) setTimeout(() => this.end('code 7, signal null'), 0); + return; + } setTimeout(() => this.completeUnit(message), 0); } @@ -587,6 +661,63 @@ describe('scheduler gating and dispatch', () => { }); }); +/** Two members of one serial group in `tests/a.e2e.ts`. */ +function serialPairs(target: ResolvedTarget): TestTargetPair[] { + return ['step 1', 'step 2'].map((title, declarationIndex) => + makePair(makeTest('tests/a.e2e.ts', title, { declarationIndex, serialId: 'tests/a.e2e.ts::wizard' }), target), + ); +} + +/** One member's record in group attempt `attemptIndex`. */ +function memberRecord(pairs: readonly TestTargetPair[], attemptIndex: number, memberIndex: number, status: 'passed' | 'failed'): SerialMemberRecord { + return { + id: `group-attempt-${attemptIndex}:member:${memberIndex}`, + index: memberIndex, + testId: pairs[memberIndex]!.test.id, + status, + startedAt: '2026-01-01T00:00:00.000Z', + durationMs: 1, + steps: [], + ...(status === 'failed' ? { error: FAILED_ATTEMPT.error! } : {}), + secondaryErrors: [], + }; +} + +/** A finished group attempt whose members ended with `statuses`. */ +function groupRun(pairs: readonly TestTargetPair[], index: number, statuses: readonly ('passed' | 'failed')[]): SerialAttemptRun { + const failed = statuses.includes('failed'); + return { + reachedMembers: true, + record: { + id: `group-attempt-${index}`, + index, + status: failed ? 'failed' : 'passed', + startedAt: '2026-01-01T00:00:00.000Z', + durationMs: 5, + members: statuses.map((status, memberIndex) => memberRecord(pairs, index, memberIndex, status)), + artifacts: [], + ...(failed ? { error: FAILED_ATTEMPT.error! } : {}), + secondaryErrors: [], + cleanup: 'complete', + }, + }; +} + +/** Runs `pairs` on `fleet` and interrupts, then forces, while the unit hangs. */ +async function runForcedInterrupt(pairs: TestTargetPair[], target: ResolvedTarget, fleet: FakeFleet): Promise { + const interrupt = new AbortController(); + const force = new AbortController(); + const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)]; + const collected = await run( + makeSelection([{ target, pairs }]), + makeCollection(['tests/a.e2e.ts'], pairs), + fleet, + { workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 }, + ); + for (const timer of timers) clearTimeout(timer); + return collected; +} + describe('scheduler fault handling', () => { it('synthesizes results and a run error when a worker dies mid-unit', async () => { const target = makeTarget('web', 0); @@ -745,6 +876,105 @@ describe('scheduler fault handling', () => { expect(collected.runErrors).toEqual([]); }); + it.each([ + { finished: [FAILED_ATTEMPT], status: 'failed' }, + { finished: [FAILED_ATTEMPT, PASSED_RETRY], status: 'flaky' }, + ])('a forced interrupt keeps the attempts a test finished and their retry verdict ($status)', async ({ finished, status }) => { + const target = makeTarget('web', 0); + const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)]; + const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], attemptsBeforeHang: finished }); + const interrupt = new AbortController(); + const force = new AbortController(); + const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)]; + + const collected = await run( + makeSelection([{ target, pairs }]), + makeCollection(['tests/a.e2e.ts'], pairs), + fleet, + { workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 }, + ); + for (const timer of timers) clearTimeout(timer); + + expect(collected.results).toHaveLength(1); + expect(collected.results[0]!.status).toBe(status); + expect(collected.results[0]!.attempts).toEqual(finished); + }); + + it.each<{ statuses: ('passed' | 'failed')[][]; verdict: string; results: string[] }>([ + { statuses: [['passed', 'failed']], verdict: 'failed', results: ['passed', 'failed'] }, + { statuses: [['passed', 'failed'], ['passed', 'passed']], verdict: 'flaky', results: ['flaky', 'flaky'] }, + ])('a forced interrupt keeps the group attempts a serial group finished and their retry verdict ($verdict)', async ({ statuses, verdict, results }) => { + const target = makeTarget('web', 0); + const pairs = serialPairs(target); + const runs = statuses.map((members, index) => groupRun(pairs, index, members)); + const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: runs }); + const collected = await runForcedInterrupt(pairs, target, fleet); + + expect(collected.serialGroups).toHaveLength(1); + expect(collected.serialGroups[0]!.status).toBe(verdict); + expect(collected.serialGroups[0]!.attempts).toEqual(runs.map((entry) => entry.record)); + expect(collected.results.map((result) => result.status)).toEqual(results); + }); + + it('a forced interrupt mid group attempt keeps the members that attempt finished', async () => { + const target = makeTarget('web', 0); + const pairs = serialPairs(target); + const finished = memberRecord(pairs, 0, 0, 'passed'); + const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialMembersBeforeHang: [finished] }); + const collected = await runForcedInterrupt(pairs, target, fleet); + + const group = collected.serialGroups[0]!; + expect(group.status).toBe('interrupted'); + expect(group.attempts.map((attempt) => attempt.status)).toEqual(['interrupted']); + expect(group.attempts[0]!.members[0]).toEqual(finished); + expect(group.attempts[0]!.members[1]!.status).toBe('skipped'); + expect(collected.results.map((result) => result.status)).toEqual(['passed', 'skipped']); + }); + + it('records a crash after a serial group finished its last attempt on that attempt', async () => { + const target = makeTarget('web', 0); + const pairs = serialPairs(target); + const passed = groupRun(pairs, 0, ['passed', 'passed']); + const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: [passed], exitInsteadOfHang: true }); + const collected = await run(makeSelection([{ target, pairs }]), makeCollection(['tests/a.e2e.ts'], pairs), fleet, { workers: 1 }); + + const group = collected.serialGroups[0]!; + expect(group.status).toBe('passed'); + expect(group.attempts).toHaveLength(1); + expect(group.attempts[0]!.cleanup).toBe('forced'); + expect(group.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']); + expect(collected.results.map((result) => result.status)).toEqual(['passed', 'passed']); + expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']); + }); + + it('a worker that exits on its own during the interrupt grace crashes the test it was running', async () => { + const target = makeTarget('web', 0); + const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)]; + const fleet = new FakeFleet({ + hangOn: ['file::web::tests/a.e2e.ts'], + attemptsBeforeHang: [FAILED_ATTEMPT], + exitDuringNextAttemptOnInterrupt: true, + }); + const interrupt = new AbortController(); + const timer = setTimeout(() => interrupt.abort(), 20); + + const collected = await run( + makeSelection([{ target, pairs }]), + makeCollection(['tests/a.e2e.ts'], pairs), + fleet, + { workers: 1, interruptSignal: interrupt.signal, interruptGraceMs: 30_000 }, + ); + clearTimeout(timer); + + expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']); + expect(collected.results).toHaveLength(1); + expect(collected.results[0]!.status).toBe('failed'); + expect(collected.results[0]!.attempts.map((attempt) => [attempt.index, attempt.error?.code])).toEqual([ + [0, FAILED_ATTEMPT.error!.code], + [1, 'WORKER_CRASH'], + ]); + }); + it('kills a worker that ignores a forced interrupt once the force budget is spent', async () => { const target = makeTarget('web', 0); const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];