mirror of
https://github.com/tester-army/e2e.git
synced 2026-11-22 06:16:50 +08:00
fix(run): a worker crash during a retry keeps the attempts before it (#837)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"e2e": patch
|
||||
---
|
||||
|
||||
A worker that crashes during a retry no longer drops the attempts before it. Before, a test whose first attempt failed and whose retry crashed the worker showed a single attempt 0 with `WORKER_CRASH`, and the first attempt's error was lost. The report now keeps every finished attempt and adds the crash as the next one. A crash after the last attempt (in an `afterAll`) is recorded on that attempt instead of as a retry. For a serial group, the finished group attempts and the members that finished in the crashed attempt stay in the report, the member that was running fails with `WORKER_CRASH`, and a crash after a member finished fails the group attempt without blaming that member. A forced interrupt keeps the attempts that finished and the verdict they reach, `flaky` included.
|
||||
@@ -119,7 +119,7 @@ These appear in terminal output and in the report as plain string codes.
|
||||
| `TRACE_WITHHELD` | A trace from a session that knows a secret value could not be rewritten, so it was deleted, or the engine returned an archive path outside the attempt's artifact directory, so nothing was registered. Recorded in the attempt's `secondaryErrors`; never fails the attempt. | infrastructure | 3 |
|
||||
| `BROWSER_INSTALL_FAILED` | Browser provisioning failed | infrastructure | 3 |
|
||||
| `WORKER_INIT_FAILED` | A worker process never became ready | infrastructure | 3 |
|
||||
| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped, a serial group's finished members included, since their results wait for the whole group. An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 |
|
||||
| `WORKER_EXIT` | A worker exited unexpectedly while running a file: the test in flight fails with `WORKER_CRASH` and the file's other tests are skipped. Attempts that finished before the exit stay in the report, a retry's earlier attempts and a serial group's finished members included, and the crash is recorded as the attempt after them, or on the last attempt when it came after it (an `afterAll`). An unhandled rejection is charged to the test running when it surfaces, or recorded as `UNHANDLED_REJECTION` between tests; before any test has finished there it ends the worker, and the rejection itself is the run error, not this one | infrastructure | 3 |
|
||||
| `WORKER_CRASH` | A worker exited during a specific test | infrastructure | 3 |
|
||||
| `WORKER_ENTRY_MISSING` | The worker entry module could not be resolved. Packaging fault. | infrastructure | 3 |
|
||||
| `NODE_UNSUPPORTED` | The runtime cannot run e2e's TypeScript loader: a Node.js outside `^22.22.3 \|\| >=24.8.0`, Bun (no `module.registerHooks`), or Deno. The CLI checks this before it starts and exits 2 with the same sentence; the loader checks it again where it registers its hooks, for a host that runs e2e's runner without the CLI | infrastructure | 3 |
|
||||
|
||||
@@ -46,10 +46,11 @@ import type {
|
||||
ResultStatus,
|
||||
RunError,
|
||||
SerialGroupRecord,
|
||||
SerialMemberRecord,
|
||||
} from './records.ts';
|
||||
import { isFailedStatus } from './records.ts';
|
||||
import { runWithRetries } from './retry.ts';
|
||||
import { runSerialUnit, type SerialHost, type SharedSerialSession } from './serial.ts';
|
||||
import { runSerialUnit, type SerialAttemptRun, type SerialAttemptStart, type SerialHost, type SharedSerialSession } from './serial.ts';
|
||||
import { interruptedSkip, pairKey, pairResult, repeatSegment, unstartedResult } from './units.ts';
|
||||
import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, redactsRecordings, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.ts';
|
||||
import { isSecret } from '../secrets.ts';
|
||||
@@ -64,6 +65,14 @@ import type { SetupFn } from '../types.ts';
|
||||
export interface ExecutionEvents {
|
||||
onResult?(result: ResultRecord): void;
|
||||
onSerialGroup?(group: SerialGroupRecord): void;
|
||||
/** An attempt of an ordinary or setup pair begins, its realm and `beforeAll` hooks included. */
|
||||
onAttemptStart?(pair: TestTargetPair, attemptIndex: number): void;
|
||||
/** One finished attempt of an ordinary or setup pair, ahead of the pair's result. */
|
||||
onAttempt?(pair: TestTargetPair, attempt: AttemptRecord): void;
|
||||
/** One member that finished in a serial group attempt, ahead of the group. */
|
||||
onSerialMember?(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
|
||||
/** One finished serial group attempt, ahead of the group. */
|
||||
onSerialAttempt?(groupId: string, run: SerialAttemptRun): void;
|
||||
/** Fires before a runnable pair starts; a serial unit announces each member as it begins. */
|
||||
onPairStart?(pair: TestTargetPair): void;
|
||||
/** Live step progress of one running attempt, for reporters. */
|
||||
@@ -222,6 +231,16 @@ export class TargetExecutor implements SerialHost {
|
||||
this.options.events?.onSerialGroup?.(group);
|
||||
}
|
||||
|
||||
/** Streams one member finished in a serial group attempt (SerialHost). */
|
||||
serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void {
|
||||
this.options.events?.onSerialMember?.(groupId, attempt, member);
|
||||
}
|
||||
|
||||
/** Streams one finished serial group attempt (SerialHost). */
|
||||
serialAttemptFinished(groupId: string, run: SerialAttemptRun): void {
|
||||
this.options.events?.onSerialAttempt?.(groupId, run);
|
||||
}
|
||||
|
||||
/** Builds one engine operation context. */
|
||||
private op(attemptId: string, timeoutMs: number, signal: AbortSignal): OperationContext {
|
||||
return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' };
|
||||
@@ -423,6 +442,7 @@ export class TargetExecutor implements SerialHost {
|
||||
pair.options.retries + 1,
|
||||
this.interruptSignal,
|
||||
async (attemptIndex) => {
|
||||
this.options.events?.onAttemptStart?.(pair, attemptIndex);
|
||||
if (realm === null) realm = await this.realms.create(file);
|
||||
const registered = findRegistered(realm, pair.test);
|
||||
if (registered === undefined) {
|
||||
@@ -437,6 +457,7 @@ export class TargetExecutor implements SerialHost {
|
||||
kind: 'ordinary',
|
||||
});
|
||||
attempts.push(attempt);
|
||||
this.options.events?.onAttempt?.(pair, attempt);
|
||||
if (isFailedStatus(attempt.status)) {
|
||||
// A failed realm is never reused, but afterAll still runs for every
|
||||
// scope whose beforeAll started in it. A body that skipped itself
|
||||
@@ -501,6 +522,7 @@ export class TargetExecutor implements SerialHost {
|
||||
pair.options.retries + 1,
|
||||
this.interruptSignal,
|
||||
async (attemptIndex) => {
|
||||
this.options.events?.onAttemptStart?.(pair, attemptIndex);
|
||||
const realm =
|
||||
attemptIndex === 0 && freshRegistration !== undefined
|
||||
? this.realms.adopt(freshRegistration, file)
|
||||
@@ -520,23 +542,24 @@ export class TargetExecutor implements SerialHost {
|
||||
kind: 'setup',
|
||||
staging,
|
||||
});
|
||||
const missing = attempt.status === 'passed' ? staging.missing() : [];
|
||||
if (missing.length > 0) {
|
||||
attempt.status = 'failed';
|
||||
attempt.error = serializeError(
|
||||
new E2EError(
|
||||
'test',
|
||||
'SESSION_CONTRACT',
|
||||
`setup must save each declared session exactly once; missing: [${missing.join(', ')}]`,
|
||||
),
|
||||
{ phase: 'body' },
|
||||
);
|
||||
}
|
||||
attempts.push(attempt);
|
||||
// Sent before teardown and the session saves, as in runOrdinaryPair:
|
||||
// a crash in either then lands on this attempt instead of erasing it.
|
||||
this.options.events?.onAttempt?.(pair, attempt);
|
||||
await this.realms.leave(realm);
|
||||
|
||||
if (attempt.status === 'passed') {
|
||||
const missing = staging.missing();
|
||||
if (missing.length > 0) {
|
||||
attempt.status = 'failed';
|
||||
attempt.error = serializeError(
|
||||
new E2EError(
|
||||
'test',
|
||||
'SESSION_CONTRACT',
|
||||
`setup must save each declared session exactly once; missing: [${missing.join(', ')}]`,
|
||||
),
|
||||
{ phase: 'body' },
|
||||
);
|
||||
return attempt;
|
||||
}
|
||||
for (const [name, saved] of staging.entries()) {
|
||||
await this.options.sessionStore.save(name, this.sessionIdentity, saved);
|
||||
}
|
||||
|
||||
@@ -30,19 +30,38 @@ export async function runWithRetries(
|
||||
interruptSignal: AbortSignal,
|
||||
runOnce: (attemptIndex: number) => Promise<RetryAttempt | undefined>,
|
||||
): Promise<ResultStatus> {
|
||||
let finalStatus: ResultStatus = 'failed';
|
||||
let attemptCount = 0;
|
||||
const attempts: RetryAttempt[] = [];
|
||||
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
||||
if (interruptSignal.aborted) break;
|
||||
const attempt = await runOnce(attemptIndex);
|
||||
if (attempt === undefined) break;
|
||||
attemptCount += 1;
|
||||
if (attempt.status === 'passed') return attemptCount > 1 ? 'flaky' : 'passed';
|
||||
if (attempt.status === 'interrupted') return attemptCount > 1 ? finalStatus : 'interrupted';
|
||||
attempts.push(attempt);
|
||||
// A body that skipped itself has decided; a retry would only ask again.
|
||||
if (attempt.status === 'skipped') return 'skipped';
|
||||
finalStatus = attempt.status;
|
||||
if (attempt.status === 'passed' || attempt.status === 'interrupted' || attempt.status === 'skipped') break;
|
||||
if (!isRetryEligible(attempt)) break;
|
||||
}
|
||||
return finalStatus;
|
||||
return retryVerdict(attempts);
|
||||
}
|
||||
|
||||
/**
|
||||
* The verdict a sequence of finished attempts reaches: a pass is `flaky`
|
||||
* after a failure, an interrupt during a retry keeps the verdict before it,
|
||||
* and a self-skip decides. Also the verdict of attempts a worker finished
|
||||
* before the run's interrupt stopped it, so both read alike.
|
||||
*/
|
||||
export function retryVerdict(attempts: readonly RetryAttempt[]): ResultStatus {
|
||||
let verdict: ResultStatus = 'failed';
|
||||
for (const [index, attempt] of attempts.entries()) {
|
||||
switch (attempt.status) {
|
||||
case 'passed':
|
||||
return index > 0 ? 'flaky' : 'passed';
|
||||
case 'interrupted':
|
||||
return index > 0 ? verdict : 'interrupted';
|
||||
case 'skipped':
|
||||
return 'skipped';
|
||||
default:
|
||||
verdict = attempt.status;
|
||||
}
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,9 @@ import type { AiTraceSnapshot } from '../internal/ai-trace.ts';
|
||||
import type { DebugSnapshot } from '../internal/debug.ts';
|
||||
import { InfrastructureError, serializeError } from '../internal/errors.ts';
|
||||
import { timestamp, uuidv7 } from '../internal/ids.ts';
|
||||
import type { ResultRecord, RunError, SerialGroupRecord } from './records.ts';
|
||||
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord } from './records.ts';
|
||||
import { retryVerdict } from './retry.ts';
|
||||
import { SerialGroupProgress, serialGroupId } from './serial.ts';
|
||||
import type { StepProgress } from './steps.ts';
|
||||
import type { SpawnUnitRunner, UnitRunner } from './unit-runner.ts';
|
||||
import {
|
||||
@@ -114,6 +116,24 @@ interface TargetState {
|
||||
failed: boolean;
|
||||
}
|
||||
|
||||
/** The attempts of one pair heard ahead of its result. */
|
||||
interface PairAttempts {
|
||||
readonly finished: AttemptRecord[];
|
||||
/** The index of the attempt that started and has not finished: a crash now is that attempt's. */
|
||||
running: number | undefined;
|
||||
}
|
||||
|
||||
/** The result of a pair from the attempts it finished, with the verdict they reach. */
|
||||
function finishedResult(pair: TestTargetPair, attempts: AttemptRecord[]): ResultRecord {
|
||||
const last = attempts.at(-1);
|
||||
return pairResult(pair, {
|
||||
status: retryVerdict(attempts),
|
||||
selected: true,
|
||||
attempts,
|
||||
...(last?.status === 'skipped' ? { skip: last.skip } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One worker and the unit it is executing. Owns its own lifecycle so the
|
||||
* scheduler never has to keep a separate record in sync with the transport.
|
||||
@@ -128,8 +148,10 @@ class SchedulerWorker {
|
||||
readonly reported = new Set<string>();
|
||||
/** The pair (`pairKey`) executing now, when one is. */
|
||||
inFlightPair: string | undefined;
|
||||
/** Pairs (`pairKey`) that started in `unit`; a serial member reports only once its whole group is done. */
|
||||
readonly started = new Set<string>();
|
||||
/** Attempts of `unit`'s pairs heard ahead of the pair's result, by `pairKey`: what a crash keeps. */
|
||||
readonly attempts = new Map<string, PairAttempts>();
|
||||
/** What was heard of `unit`'s serial groups still running, by group report id: what a crash keeps. */
|
||||
readonly serialGroups = new Map<string, SerialGroupProgress>();
|
||||
sawFailure = false;
|
||||
becameReady = false;
|
||||
/** Told to tear down at once by a forced interrupt; its exit is then the one asked for. */
|
||||
@@ -159,7 +181,8 @@ class SchedulerWorker {
|
||||
this.state = 'busy';
|
||||
this.unit = unit;
|
||||
this.reported.clear();
|
||||
this.started.clear();
|
||||
this.attempts.clear();
|
||||
this.serialGroups.clear();
|
||||
this.inFlightPair = undefined;
|
||||
this.sawFailure = false;
|
||||
const pairs: WirePair[] = unit.pairs.map((pair) => ({
|
||||
@@ -205,6 +228,26 @@ class SchedulerWorker {
|
||||
this.killTimer.unref();
|
||||
}
|
||||
|
||||
/** The attempts heard of one of `unit`'s pairs, started on first word of them. */
|
||||
pairAttempts(key: string): PairAttempts {
|
||||
let attempts = this.attempts.get(key);
|
||||
if (attempts === undefined) {
|
||||
attempts = { finished: [], running: undefined };
|
||||
this.attempts.set(key, attempts);
|
||||
}
|
||||
return attempts;
|
||||
}
|
||||
|
||||
/** The progress of one of `unit`'s serial groups, started on first word of it. */
|
||||
serialGroup(groupId: string): SerialGroupProgress {
|
||||
let progress = this.serialGroups.get(groupId);
|
||||
if (progress === undefined) {
|
||||
progress = new SerialGroupProgress();
|
||||
this.serialGroups.set(groupId, progress);
|
||||
}
|
||||
return progress;
|
||||
}
|
||||
|
||||
clearKillTimer(): void {
|
||||
if (this.killTimer === undefined) return;
|
||||
clearTimeout(this.killTimer);
|
||||
@@ -584,7 +627,11 @@ class Scheduler {
|
||||
}
|
||||
case 'pair-start': {
|
||||
worker.inFlightPair = pairKey(message.testId, message.agent, message.repeat);
|
||||
worker.started.add(worker.inFlightPair);
|
||||
if (message.serialId !== undefined) {
|
||||
worker
|
||||
.serialGroup(serialGroupId(message.serialId, worker.targetName, message.agent, message.repeat))
|
||||
.memberStarted(message.testId);
|
||||
}
|
||||
const { type: _type, ...start } = message;
|
||||
this.options.events.onTestStart?.(start, worker.targetName);
|
||||
break;
|
||||
@@ -609,7 +656,9 @@ class Scheduler {
|
||||
case 'result': {
|
||||
const state = this.targetState(worker);
|
||||
const result = decodeResult(message.result, state.target);
|
||||
worker.reported.add(pairKey(result.test.id, result.agent, result.repeat));
|
||||
const key = pairKey(result.test.id, result.agent, result.repeat);
|
||||
worker.reported.add(key);
|
||||
worker.attempts.delete(key);
|
||||
if (result.status !== 'passed' && result.status !== 'flaky' && result.status !== 'skipped') {
|
||||
worker.sawFailure = true;
|
||||
}
|
||||
@@ -618,9 +667,28 @@ class Scheduler {
|
||||
break;
|
||||
}
|
||||
case 'serial-group': {
|
||||
worker.serialGroups.delete(message.group.id);
|
||||
this.options.events.onSerialGroup(message.group);
|
||||
break;
|
||||
}
|
||||
case 'attempt-start': {
|
||||
worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat)).running = message.index;
|
||||
break;
|
||||
}
|
||||
case 'attempt': {
|
||||
const attempts = worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat));
|
||||
attempts.finished.push(message.attempt);
|
||||
attempts.running = undefined;
|
||||
break;
|
||||
}
|
||||
case 'serial-member': {
|
||||
worker.serialGroup(message.groupId).memberFinished(message.attempt, message.member);
|
||||
break;
|
||||
}
|
||||
case 'serial-attempt': {
|
||||
worker.serialGroup(message.groupId).attemptFinished(message.run);
|
||||
break;
|
||||
}
|
||||
case 'unit-done': {
|
||||
for (const runError of message.runErrors) this.options.events.onRunError(runError);
|
||||
if (message.debug !== undefined) this.options.events.onDebug?.(message.debug);
|
||||
@@ -724,18 +792,53 @@ class Scheduler {
|
||||
return this.signalledWave.has(worker);
|
||||
}
|
||||
|
||||
/** Emits records for a unit whose worker died before reporting it done. */
|
||||
/**
|
||||
* Emits records for a unit whose worker died before reporting it done.
|
||||
* Attempts the worker finished stand. The test in flight gets a
|
||||
* `WORKER_CRASH` attempt after them when the crash landed during an
|
||||
* attempt; one between attempts (an `afterAll` after the last) is recorded
|
||||
* on the last attempt instead, whose cleanup it forced. A serial group in
|
||||
* flight gets a failed group attempt after its finished ones. A worker the
|
||||
* run's interrupt stopped charges no crash: a test keeps the attempts it
|
||||
* finished and the verdict they reach, as a retry the interrupt cut short
|
||||
* does, and a serial group's attempt in flight is recorded interrupted.
|
||||
*/
|
||||
private synthesizeCrashResults(
|
||||
state: TargetState,
|
||||
worker: SchedulerWorker,
|
||||
unit: WorkUnit,
|
||||
): void {
|
||||
const interrupted = this.interrupting;
|
||||
const interrupted = this.interrupting && (worker.terminated || worker.killed);
|
||||
const crash = serializeError(new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'));
|
||||
const settledGroups = new Set<string>();
|
||||
for (const pair of unit.pairs) {
|
||||
const key = pairKey(pair.test.id, pair.agent, pair.repeat);
|
||||
if (worker.reported.has(key)) continue;
|
||||
const { serialId } = pair.test;
|
||||
if (serialId !== undefined) {
|
||||
const groupId = serialGroupId(serialId, worker.targetName, pair.agent, pair.repeat);
|
||||
if (settledGroups.has(groupId)) continue;
|
||||
const members = unit.pairs.filter(
|
||||
(member) => member.test.serialId === serialId && member.agent === pair.agent && member.repeat === pair.repeat,
|
||||
);
|
||||
const progress = worker.serialGroups.get(groupId);
|
||||
const settled = interrupted
|
||||
? progress?.interrupted(members, state.target, this.stopSkip ?? INTERRUPTED_BEFORE_START)
|
||||
: progress?.crashed(members, state.target, crash);
|
||||
if (settled !== undefined) {
|
||||
settledGroups.add(groupId);
|
||||
this.options.events.onSerialGroup(settled.group);
|
||||
for (const result of settled.results) this.report(result);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const { finished, running } = worker.attempts.get(key) ?? { finished: [], running: undefined };
|
||||
if (interrupted) {
|
||||
this.report(unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START));
|
||||
this.report(
|
||||
finished.length === 0
|
||||
? unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START)
|
||||
: finishedResult(pair, finished),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// A crashed setup never persisted its sessions; dependents must skip.
|
||||
@@ -748,29 +851,36 @@ class Scheduler {
|
||||
this.report(
|
||||
unstartedResult(pair, {
|
||||
cause: 'infrastructure-unavailable',
|
||||
reason: worker.started.has(key)
|
||||
? 'worker process exited before this serial group finished'
|
||||
: 'worker process exited before this test started',
|
||||
reason: 'worker process exited before this test started',
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const last = finished.at(-1);
|
||||
if (last !== undefined && running === undefined) {
|
||||
this.report(
|
||||
finishedResult(pair, [
|
||||
...finished.slice(0, -1),
|
||||
{ ...last, secondaryErrors: [...last.secondaryErrors, crash], cleanup: 'forced' },
|
||||
]),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
this.report(
|
||||
pairResult(pair, {
|
||||
status: 'failed',
|
||||
selected: true,
|
||||
attempts: [
|
||||
...finished,
|
||||
{
|
||||
id: uuidv7(),
|
||||
index: 0,
|
||||
index: running ?? finished.length,
|
||||
status: 'failed',
|
||||
startedAt: timestamp(),
|
||||
durationMs: 0,
|
||||
steps: [],
|
||||
artifacts: [],
|
||||
error: serializeError(
|
||||
new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'),
|
||||
),
|
||||
error: crash,
|
||||
secondaryErrors: [],
|
||||
cleanup: 'forced',
|
||||
},
|
||||
|
||||
+255
-67
@@ -20,7 +20,7 @@ import type { StepRecord } from './steps.ts';
|
||||
import { findRegistered, type FileRef, type Realm, RealmManager } from './realm.ts';
|
||||
import type { AttemptRecord, ResultRecord, ResultStatus, SerialAttemptRecord, SerialGroupRecord, SerialMemberRecord, FailedStatus } from './records.ts';
|
||||
import { isFailedStatus } from './records.ts';
|
||||
import { runWithRetries } from './retry.ts';
|
||||
import { isRetryEligible, retryVerdict, runWithRetries } from './retry.ts';
|
||||
import { interruptedSkip, pairResult, repeatSegment } from './units.ts';
|
||||
|
||||
/**
|
||||
@@ -87,6 +87,34 @@ export interface SerialHost {
|
||||
* steps, duration, and error from the group it has already seen.
|
||||
*/
|
||||
emitSerialGroup(group: SerialGroupRecord): void;
|
||||
/**
|
||||
* Streams one member that finished in the group attempt running now, so a
|
||||
* worker that dies before the group ends does not take it along.
|
||||
*/
|
||||
serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
|
||||
/** Streams one finished group attempt, for the same reason. */
|
||||
serialAttemptFinished(groupId: string, run: SerialAttemptRun): void;
|
||||
}
|
||||
|
||||
/** The identity of a group attempt, known from its start. */
|
||||
export type SerialAttemptStart = Pick<SerialAttemptRecord, 'id' | 'index' | 'startedAt'>;
|
||||
|
||||
/** One group attempt as it ended, and whether it got as far as its members. */
|
||||
export interface SerialAttemptRun {
|
||||
readonly record: SerialAttemptRecord;
|
||||
/**
|
||||
* False when the attempt stopped before any member ran (its file would not
|
||||
* load, its session would not launch). Such a retry keeps the members'
|
||||
* verdicts from the attempt before it.
|
||||
*/
|
||||
readonly reachedMembers: boolean;
|
||||
}
|
||||
|
||||
/** The report id of one serial group variant: a group runs once per agent and per repeat. */
|
||||
export function serialGroupId(serialId: string, targetName: string, agent: string, repeat: number): string {
|
||||
return canonicalDigest(
|
||||
repeat === 0 ? { serialId, targetId: targetName, agent } : { serialId, targetId: targetName, agent, repeat },
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs one serial unit to completion and emits every member result. */
|
||||
@@ -95,46 +123,20 @@ export async function runSerialUnit(
|
||||
members: readonly TestTargetPair[],
|
||||
file: FileRef,
|
||||
): Promise<SerialGroupRecord> {
|
||||
const first = members[0]!;
|
||||
const serialId = first.test.serialId!;
|
||||
// Every member of a unit runs as the same agent (selection guarantees it),
|
||||
// so the group is one variant of the flow and is identified as such.
|
||||
const agent = first.agent;
|
||||
const { repeat } = first;
|
||||
const groupRecordId = canonicalDigest(
|
||||
repeat === 0 ? { serialId, targetId: host.target.name, agent } : { serialId, targetId: host.target.name, agent, repeat },
|
||||
);
|
||||
|
||||
const group: SerialGroupRecord = {
|
||||
id: groupRecordId,
|
||||
serialId,
|
||||
declarationIndex: first.test.declarationIndex,
|
||||
file: first.test.file,
|
||||
titlePath: serialTitlePath(first.test),
|
||||
targetId: host.target.name,
|
||||
platform: host.target.platform,
|
||||
agent,
|
||||
repeat,
|
||||
memberTestIds: members.map((member) => member.test.id),
|
||||
status: 'failed',
|
||||
attempts: [],
|
||||
};
|
||||
|
||||
const memberFinalStatus = new Map<string, SerialMemberRecord>();
|
||||
const group = newSerialGroup(members, host.target);
|
||||
const runs: SerialAttemptRun[] = [];
|
||||
const finalStatus = await runWithRetries(
|
||||
first.options.retries + 1,
|
||||
members[0]!.options.retries + 1,
|
||||
host.interruptSignal,
|
||||
async (attemptIndex) => {
|
||||
const { record: attempt, reachedMembers } = await runSerialAttempt(host, members, file, attemptIndex);
|
||||
group.attempts.push(attempt);
|
||||
// A retry the run interrupted, or one that never reached its members,
|
||||
// keeps the verdict before it, so each member keeps what that attempt said.
|
||||
const cutRetry = attemptIndex > 0 && (attempt.status === 'interrupted' || !reachedMembers);
|
||||
if (!cutRetry) for (const member of attempt.members) memberFinalStatus.set(member.testId, member);
|
||||
const run = await runSerialAttempt(host, group.id, members, file, attemptIndex);
|
||||
runs.push(run);
|
||||
group.attempts.push(run.record);
|
||||
host.serialAttemptFinished(group.id, run);
|
||||
// A beforeAll failure is not retry-eligible: the
|
||||
// attempt stands as recorded and the retry loop stops here.
|
||||
if (attempt.error?.code === 'HOOK_FAILED') return undefined;
|
||||
return attempt;
|
||||
if (run.record.error?.code === 'HOOK_FAILED') return undefined;
|
||||
return run.record;
|
||||
},
|
||||
);
|
||||
|
||||
@@ -146,8 +148,51 @@ export async function runSerialUnit(
|
||||
}
|
||||
|
||||
host.emitSerialGroup(group);
|
||||
for (const member of members) {
|
||||
const memberRecord = memberFinalStatus.get(member.test.id);
|
||||
for (const result of memberResults(group, members, runs)) host.emit(result);
|
||||
return group;
|
||||
}
|
||||
|
||||
/** A group record with no attempts yet. */
|
||||
function newSerialGroup(members: readonly TestTargetPair[], target: ResolvedTarget): SerialGroupRecord {
|
||||
const first = members[0]!;
|
||||
const serialId = first.test.serialId!;
|
||||
// Every member of a unit runs as the same agent (selection guarantees it),
|
||||
// so the group is one variant of the flow and is identified as such.
|
||||
const { agent, repeat } = first;
|
||||
return {
|
||||
id: serialGroupId(serialId, target.name, agent, repeat),
|
||||
serialId,
|
||||
declarationIndex: first.test.declarationIndex,
|
||||
file: first.test.file,
|
||||
titlePath: serialTitlePath(first.test),
|
||||
targetId: target.name,
|
||||
platform: target.platform,
|
||||
agent,
|
||||
repeat,
|
||||
memberTestIds: members.map((member) => member.test.id),
|
||||
status: 'failed',
|
||||
attempts: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Each member's result once its group is decided. Members carry no attempts
|
||||
* of their own; the verdict is the member's record in the last attempt that
|
||||
* counts. A retry the run interrupted, or one that never reached its
|
||||
* members, keeps the verdict before it.
|
||||
*/
|
||||
function memberResults(
|
||||
group: SerialGroupRecord,
|
||||
members: readonly TestTargetPair[],
|
||||
runs: readonly SerialAttemptRun[],
|
||||
): ResultRecord[] {
|
||||
const verdicts = new Map<string, SerialMemberRecord>();
|
||||
for (const [attemptIndex, run] of runs.entries()) {
|
||||
const cutRetry = attemptIndex > 0 && (run.record.status === 'interrupted' || !run.reachedMembers);
|
||||
if (!cutRetry) for (const member of run.record.members) verdicts.set(member.testId, member);
|
||||
}
|
||||
return members.map((member) => {
|
||||
const memberRecord = verdicts.get(member.test.id);
|
||||
let status: ResultStatus;
|
||||
let skip: SkipInfo | undefined;
|
||||
// A member that skipped itself stays skipped whatever the group did: a
|
||||
@@ -163,28 +208,160 @@ export async function runSerialUnit(
|
||||
} else {
|
||||
status = memberRecord.status;
|
||||
}
|
||||
host.emit(
|
||||
pairResult(member, {
|
||||
status,
|
||||
selected: true,
|
||||
...(skip !== undefined ? { skip } : {}),
|
||||
attempts: [],
|
||||
serialGroupId: groupRecordId,
|
||||
}),
|
||||
);
|
||||
return pairResult(member, {
|
||||
status,
|
||||
selected: true,
|
||||
...(skip !== undefined ? { skip } : {}),
|
||||
attempts: [],
|
||||
serialGroupId: group.id,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** A serial group decided without its worker's word: the record and its members' results. */
|
||||
interface SettledSerialGroup {
|
||||
readonly group: SerialGroupRecord;
|
||||
readonly results: readonly ResultRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* What the runner heard of one serial group a worker is running: the
|
||||
* attempts it finished, and the members finished so far in the attempt
|
||||
* running now. When the worker dies before reporting the group, this is
|
||||
* what the report keeps.
|
||||
*/
|
||||
export class SerialGroupProgress {
|
||||
private readonly finished: SerialAttemptRun[] = [];
|
||||
private current: SerialAttemptStart | undefined;
|
||||
private currentMembers: SerialMemberRecord[] = [];
|
||||
/** The member whose body is running now, by test id, until its record arrives. */
|
||||
private running: string | undefined;
|
||||
|
||||
/** A member of the group announced its start. */
|
||||
memberStarted(testId: string): void {
|
||||
this.running = testId;
|
||||
}
|
||||
|
||||
return group;
|
||||
/** A member finished in the attempt running now. */
|
||||
memberFinished(attempt: SerialAttemptStart, member: SerialMemberRecord): void {
|
||||
this.current = attempt;
|
||||
this.currentMembers.push(member);
|
||||
if (this.running === member.testId) this.running = undefined;
|
||||
}
|
||||
|
||||
/** A group attempt finished; the next one starts empty. */
|
||||
attemptFinished(run: SerialAttemptRun): void {
|
||||
this.finished.push(run);
|
||||
this.current = undefined;
|
||||
this.currentMembers = [];
|
||||
this.running = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The group after its worker died with `error`, or undefined when the
|
||||
* runner heard nothing of it running. The finished attempts stand, and the
|
||||
* attempt in flight fails: the member whose body was running fails with
|
||||
* `error` and the ones after it skip. A crash between members (a hook,
|
||||
* the session closing) fails the attempt and leaves its finished members
|
||||
* as they were; one in a retry before any member ran fails the first, as a
|
||||
* launch failure does. A crash after the last attempt, when no retry
|
||||
* follows it, is recorded on that attempt.
|
||||
*/
|
||||
crashed(members: readonly TestTargetPair[], target: ResolvedTarget, error: SerializedError): SettledSerialGroup | undefined {
|
||||
const last = this.finished.at(-1);
|
||||
if (!this.inFlight) {
|
||||
if (last === undefined) return undefined;
|
||||
if (!retryFollows(last.record, this.finished.length, members)) {
|
||||
const closed: SerialAttemptRun = {
|
||||
...last,
|
||||
record: { ...last.record, secondaryErrors: [...last.record.secondaryErrors, error], cleanup: 'forced' },
|
||||
};
|
||||
return this.settled([...this.finished.slice(0, -1), closed], members, target);
|
||||
}
|
||||
const record = this.attemptInFlight('failed', error);
|
||||
endBeforeMembers(record, members, error, NEVER_INTERRUPTED);
|
||||
return this.settled([...this.finished, { record, reachedMembers: false }], members, target);
|
||||
}
|
||||
const record = this.attemptInFlight('failed', error);
|
||||
const next = this.currentMembers.length;
|
||||
const wasRunning = this.running !== undefined && this.running === members[next]?.test.id;
|
||||
const skip: SkipInfo = wasRunning
|
||||
? predecessorFailed(next)
|
||||
: { cause: 'infrastructure-unavailable', reason: 'worker process exited during this group attempt' };
|
||||
for (const [memberIndex, member] of members.entries()) {
|
||||
if (memberIndex < next) continue;
|
||||
record.members.push(
|
||||
wasRunning && memberIndex === next
|
||||
? memberFailedWith(record, memberIndex, member.test.id, error)
|
||||
: skippedMember(record.id, memberIndex, member.test.id, skip),
|
||||
);
|
||||
}
|
||||
return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
|
||||
}
|
||||
|
||||
/**
|
||||
* The group after the run's interrupt stopped its worker, or undefined
|
||||
* when the runner heard nothing of it running. The finished attempts
|
||||
* stand, and an attempt in flight is interrupted: the members it finished
|
||||
* keep their records and the rest skip with `skip`. The verdict is the one
|
||||
* those attempts reach, as for a retry the interrupt cut short.
|
||||
*/
|
||||
interrupted(members: readonly TestTargetPair[], target: ResolvedTarget, skip: SkipInfo): SettledSerialGroup | undefined {
|
||||
if (!this.inFlight) return this.finished.length === 0 ? undefined : this.settled(this.finished, members, target);
|
||||
const record = this.attemptInFlight('interrupted', undefined);
|
||||
for (const [memberIndex, member] of members.entries()) {
|
||||
if (memberIndex >= this.currentMembers.length) record.members.push(skippedMember(record.id, memberIndex, member.test.id, skip));
|
||||
}
|
||||
return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
|
||||
}
|
||||
|
||||
/** Whether a member of the attempt in flight started or finished. */
|
||||
private get inFlight(): boolean {
|
||||
return this.currentMembers.length > 0 || this.running !== undefined;
|
||||
}
|
||||
|
||||
/** The record of the attempt in flight as it stood, with the members it finished. */
|
||||
private attemptInFlight(status: SerialAttemptRecord['status'], error: SerializedError | undefined): SerialAttemptRecord {
|
||||
return {
|
||||
id: this.current?.id ?? uuidv7(),
|
||||
index: this.finished.length,
|
||||
status,
|
||||
startedAt: this.current?.startedAt ?? timestamp(),
|
||||
durationMs: 0,
|
||||
members: [...this.currentMembers],
|
||||
artifacts: [],
|
||||
...(error === undefined ? {} : { error }),
|
||||
secondaryErrors: [],
|
||||
cleanup: 'forced',
|
||||
};
|
||||
}
|
||||
|
||||
private settled(
|
||||
runs: readonly SerialAttemptRun[],
|
||||
members: readonly TestTargetPair[],
|
||||
target: ResolvedTarget,
|
||||
): SettledSerialGroup {
|
||||
const group = newSerialGroup(members, target);
|
||||
group.status = retryVerdict(runs.map((run) => run.record));
|
||||
group.attempts = runs.map((run) => run.record);
|
||||
return { group, results: memberResults(group, members, runs) };
|
||||
}
|
||||
}
|
||||
|
||||
/** One group attempt as it ended, and whether it got as far as its members. */
|
||||
interface SerialAttemptRun {
|
||||
readonly record: SerialAttemptRecord;
|
||||
readonly reachedMembers: boolean;
|
||||
/**
|
||||
* Whether the group retries after `record`, its attempt number `attempts`:
|
||||
* the retry policy allows it, and a hook failure never retries.
|
||||
*/
|
||||
function retryFollows(record: SerialAttemptRecord, attempts: number, members: readonly TestTargetPair[]): boolean {
|
||||
return attempts < members[0]!.options.retries + 1 && isRetryEligible(record) && record.error?.code !== 'HOOK_FAILED';
|
||||
}
|
||||
|
||||
/** An abort signal that never fires. */
|
||||
const NEVER_INTERRUPTED = new AbortController().signal;
|
||||
|
||||
async function runSerialAttempt(
|
||||
host: SerialHost,
|
||||
groupId: string,
|
||||
members: readonly TestTargetPair[],
|
||||
file: FileRef,
|
||||
attemptIndex: number,
|
||||
@@ -222,6 +399,12 @@ async function runSerialAttempt(
|
||||
secondaryErrors: [],
|
||||
cleanup: 'complete',
|
||||
};
|
||||
const start: SerialAttemptStart = { id: attemptId, index: attemptIndex, startedAt };
|
||||
/** Records one member and streams it, so a worker that dies later in the attempt does not take it along. */
|
||||
const finishMember = (member: SerialMemberRecord): void => {
|
||||
memberRecords.push(member);
|
||||
host.serialMemberFinished(groupId, start, member);
|
||||
};
|
||||
|
||||
let realm: Realm;
|
||||
try {
|
||||
@@ -258,12 +441,12 @@ async function runSerialAttempt(
|
||||
skipRemaining = interruptedMemberSkip(host.interruptSignal);
|
||||
}
|
||||
if (skipRemaining !== undefined) {
|
||||
memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
|
||||
finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
|
||||
continue;
|
||||
}
|
||||
const registered = findRegistered(realm, member.test);
|
||||
if (registered === undefined) {
|
||||
memberRecords.push({
|
||||
finishMember({
|
||||
id: memberId,
|
||||
index: memberIndex,
|
||||
testId: member.test.id,
|
||||
@@ -285,7 +468,7 @@ async function runSerialAttempt(
|
||||
hookFailure = await host.realms.enterScopes(realm, registered);
|
||||
if (hookFailure !== undefined) {
|
||||
skipRemaining = { cause: 'hook-failed', reason: hookFailure.message };
|
||||
memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
|
||||
finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
|
||||
continue;
|
||||
}
|
||||
host.pairStarted(member);
|
||||
@@ -294,7 +477,7 @@ async function runSerialAttempt(
|
||||
shared,
|
||||
});
|
||||
shared.priorSteps.push(...memberAttempt.steps);
|
||||
memberRecords.push({
|
||||
finishMember({
|
||||
id: memberId,
|
||||
index: memberIndex,
|
||||
testId: member.test.id,
|
||||
@@ -367,22 +550,27 @@ function endBeforeMembers(
|
||||
members.forEach((member, memberIndex) => {
|
||||
record.members.push(
|
||||
memberIndex === 0
|
||||
? {
|
||||
id: `${record.id}:member:0`,
|
||||
index: 0,
|
||||
testId: member.test.id,
|
||||
status: 'failed',
|
||||
startedAt: record.startedAt,
|
||||
durationMs: 0,
|
||||
steps: [],
|
||||
error,
|
||||
secondaryErrors: [],
|
||||
}
|
||||
? memberFailedWith(record, 0, member.test.id, error)
|
||||
: skippedMember(record.id, memberIndex, member.test.id, predecessorFailed(0)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** A member that failed with an error of its group attempt's, having run no step of its own. */
|
||||
function memberFailedWith(record: SerialAttemptRecord, index: number, testId: string, error: SerializedError): SerialMemberRecord {
|
||||
return {
|
||||
id: `${record.id}:member:${index}`,
|
||||
index,
|
||||
testId,
|
||||
status: 'failed',
|
||||
startedAt: record.startedAt,
|
||||
durationMs: 0,
|
||||
steps: [],
|
||||
error,
|
||||
secondaryErrors: [],
|
||||
};
|
||||
}
|
||||
|
||||
function isFailedMember(member: SerialMemberRecord): member is SerialMemberRecord & { status: FailedStatus } {
|
||||
return isFailedStatus(member.status);
|
||||
}
|
||||
|
||||
@@ -39,20 +39,23 @@ let outbox: Promise<void> = Promise.resolve();
|
||||
/** How long an exit waits for the outbox: a channel whose runner is gone may never acknowledge. */
|
||||
const FLUSH_GRACE_MS = 2_000;
|
||||
|
||||
/** Queues one message; resolves once the channel has taken it (or is gone). */
|
||||
/**
|
||||
* Hands one message to the channel at once; resolves once the channel has
|
||||
* taken it and every message before it (or is gone). The channel keeps the
|
||||
* order. Handing it over synchronously is what lets a `pair-start` reach the
|
||||
* runner when the test body exits the process right after it.
|
||||
*/
|
||||
function send(message: WorkerToMain): Promise<void> {
|
||||
outbox = outbox.then(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
try {
|
||||
if (process.send === undefined || !process.connected) resolve();
|
||||
else process.send(message, undefined, undefined, () => resolve());
|
||||
} catch {
|
||||
// channel already closed; nothing left to deliver
|
||||
resolve();
|
||||
}
|
||||
}),
|
||||
);
|
||||
const taken = new Promise<void>((resolve) => {
|
||||
try {
|
||||
if (process.send === undefined || !process.connected) resolve();
|
||||
else process.send(message, undefined, undefined, () => resolve());
|
||||
} catch {
|
||||
// channel already closed; nothing left to deliver
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
outbox = outbox.then(() => taken);
|
||||
return outbox;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ import type { CliOverrides, PortAssignments, ResolvedTarget } from '../../config
|
||||
import type { AiTraceSnapshot } from '../../internal/ai-trace.ts';
|
||||
import type { DebugSnapshot } from '../../internal/debug.ts';
|
||||
import type { SerializedError } from '../../internal/errors.ts';
|
||||
import type { ResultRecord, RunError, SerialGroupRecord, WireResultRecord } from '../records.ts';
|
||||
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord, WireResultRecord } from '../records.ts';
|
||||
import type { SerialAttemptRun, SerialAttemptStart } from '../serial.ts';
|
||||
import type { StepProgress } from '../steps.ts';
|
||||
|
||||
/** One runnable pair on the wire; the worker resolves the test function. */
|
||||
@@ -190,6 +191,52 @@ export interface SerialGroupMessage {
|
||||
readonly group: SerialGroupRecord;
|
||||
}
|
||||
|
||||
/**
|
||||
* An attempt of an ordinary or setup pair begins, its realm and `beforeAll`
|
||||
* hooks included. With `attempt`, it tells a crash during an attempt from
|
||||
* one between attempts (an `afterAll` after the last one): only the first
|
||||
* is charged a new attempt, with this index.
|
||||
*/
|
||||
export interface AttemptStartMessage {
|
||||
readonly type: 'attempt-start';
|
||||
readonly testId: string;
|
||||
readonly agent: string;
|
||||
readonly repeat: number;
|
||||
readonly index: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One finished attempt of an ordinary or setup pair. A pair's result waits
|
||||
* for its last attempt, so each attempt also goes out as it ends: a worker
|
||||
* that dies during a retry must not take the attempts before it along. The
|
||||
* result still carries every attempt; the scheduler keeps these only until
|
||||
* it arrives.
|
||||
*/
|
||||
export interface AttemptMessage {
|
||||
readonly type: 'attempt';
|
||||
readonly testId: string;
|
||||
readonly agent: string;
|
||||
readonly repeat: number;
|
||||
readonly attempt: AttemptRecord;
|
||||
}
|
||||
|
||||
/** One member that finished in the serial group attempt running now; see `AttemptMessage`. */
|
||||
export interface SerialMemberMessage {
|
||||
readonly type: 'serial-member';
|
||||
/** The group's report id (`SerialGroupRecord.id`). */
|
||||
readonly groupId: string;
|
||||
readonly attempt: SerialAttemptStart;
|
||||
readonly member: SerialMemberRecord;
|
||||
}
|
||||
|
||||
/** One finished serial group attempt; see `AttemptMessage`. */
|
||||
export interface SerialAttemptMessage {
|
||||
readonly type: 'serial-attempt';
|
||||
/** The group's report id (`SerialGroupRecord.id`). */
|
||||
readonly groupId: string;
|
||||
readonly run: SerialAttemptRun;
|
||||
}
|
||||
|
||||
export interface UnitDoneMessage {
|
||||
readonly type: 'unit-done';
|
||||
readonly unitId: string;
|
||||
@@ -236,6 +283,10 @@ export type WorkerToMain =
|
||||
| NoticeMessage
|
||||
| ResultMessage
|
||||
| SerialGroupMessage
|
||||
| AttemptStartMessage
|
||||
| AttemptMessage
|
||||
| SerialMemberMessage
|
||||
| SerialAttemptMessage
|
||||
| UnitDoneMessage
|
||||
| ShutdownDoneMessage
|
||||
| FatalMessage
|
||||
|
||||
@@ -123,6 +123,12 @@ export class TargetWorker {
|
||||
if (countsTowardFailureLimit(result.status)) this.countFailure();
|
||||
},
|
||||
onSerialGroup: (group) => this.host.emit({ type: 'serial-group', group }),
|
||||
onAttemptStart: (pair, index) =>
|
||||
this.host.emit({ type: 'attempt-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, index }),
|
||||
onAttempt: (pair, attempt) =>
|
||||
this.host.emit({ type: 'attempt', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, attempt }),
|
||||
onSerialMember: (groupId, attempt, member) => this.host.emit({ type: 'serial-member', groupId, attempt, member }),
|
||||
onSerialAttempt: (groupId, run) => this.host.emit({ type: 'serial-attempt', groupId, run }),
|
||||
onPairStart: (pair) => {
|
||||
this.inFlight = { testId: pair.test.id, agent: pair.agent, repeat: pair.repeat };
|
||||
this.host.emit({
|
||||
|
||||
@@ -270,15 +270,20 @@ test.describe('wizard', { serial: true }, () => {
|
||||
{ 'tests/serial-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const group = outcome.report.run.serialGroups[0]!;
|
||||
expect(group.status).toBe('failed');
|
||||
expect(group.attempts[0]!.members.map((member) => [member.status, member.error?.code])).toEqual([
|
||||
['passed', undefined],
|
||||
['failed', 'WORKER_CRASH'],
|
||||
['skipped', undefined],
|
||||
]);
|
||||
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
|
||||
const crashed = resultByTitle(outcome, 'step 2 crashes the worker');
|
||||
expect(crashed.status).toBe('failed');
|
||||
expect(crashed.attempts[0]?.error?.code).toBe('WORKER_CRASH');
|
||||
const finished = resultByTitle(outcome, 'step 1 passes');
|
||||
expect(finished.status).toBe('skipped');
|
||||
expect(finished.skip?.reason).toBe('worker process exited before this serial group finished');
|
||||
expect(crashed.serialGroupId).toBe(group.id);
|
||||
const unreached = resultByTitle(outcome, 'step 3 never starts');
|
||||
expect(unreached.status).toBe('skipped');
|
||||
expect(unreached.skip?.reason).toBe('worker process exited before this test started');
|
||||
expect(unreached.skip?.cause).toBe('serial-predecessor-failed');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
expect(outcome.report.run.summary.failed).toBe(1);
|
||||
assertValidReport(outcome.report);
|
||||
@@ -287,6 +292,292 @@ test.describe('wizard', { serial: true }, () => {
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'keeps the failed attempt before a retry that crashed the worker',
|
||||
async () => {
|
||||
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-crash-')), 'failed-once');
|
||||
const file = `import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { test } from 'e2e';
|
||||
|
||||
test('fails, then crashes on retry', { retries: 1 }, async () => {
|
||||
if (!existsSync(${JSON.stringify(marker)})) {
|
||||
writeFileSync(${JSON.stringify(marker)}, '');
|
||||
throw new Error('first attempt fails');
|
||||
}
|
||||
process.exit(7);
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/retry-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const result = resultByTitle(outcome, 'fails, then crashes on retry');
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.attempts.map((attempt) => [attempt.index, attempt.status, attempt.error?.message])).toEqual([
|
||||
[0, 'failed', 'first attempt fails'],
|
||||
[1, 'failed', 'worker process exited during this test'],
|
||||
]);
|
||||
expect(result.attempts[1]!.error?.code).toBe('WORKER_CRASH');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'keeps the failed group attempt before a serial retry that crashed the worker',
|
||||
async () => {
|
||||
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-crash-')), 'failed-once');
|
||||
const file = `import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { test } from 'e2e';
|
||||
|
||||
test.describe('wizard', { serial: true, retries: 1 }, () => {
|
||||
test('step 1 passes', async () => {});
|
||||
test('step 2 fails, then crashes on retry', async () => {
|
||||
if (!existsSync(${JSON.stringify(marker)})) {
|
||||
writeFileSync(${JSON.stringify(marker)}, '');
|
||||
throw new Error('first attempt fails');
|
||||
}
|
||||
process.exit(7);
|
||||
});
|
||||
test('step 3 never passes', async () => {});
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/serial-retry-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const group = outcome.report.run.serialGroups[0]!;
|
||||
expect(group.status).toBe('failed');
|
||||
expect(
|
||||
group.attempts.map((attempt) => [attempt.index, attempt.members.map((member) => [member.status, member.error?.code])]),
|
||||
).toEqual([
|
||||
[0, [['passed', undefined], ['failed', 'ERROR'], ['skipped', undefined]]],
|
||||
[1, [['passed', undefined], ['failed', 'WORKER_CRASH'], ['skipped', undefined]]],
|
||||
]);
|
||||
expect(group.attempts[0]!.members[1]!.error?.message).toBe('first attempt fails');
|
||||
expect(group.attempts[1]!.error?.code).toBe('WORKER_CRASH');
|
||||
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
|
||||
const crashed = resultByTitle(outcome, 'step 2 fails, then crashes on retry');
|
||||
expect(crashed.status).toBe('failed');
|
||||
expect(crashed.serialGroupId).toBe(group.id);
|
||||
expect(resultByTitle(outcome, 'step 3 never passes').skip?.cause).toBe('serial-predecessor-failed');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'records a crash in afterAll after a failed attempt on that attempt, not as a retry',
|
||||
async () => {
|
||||
const file = `import { test } from 'e2e';
|
||||
|
||||
test.afterAll(() => {
|
||||
process.exit(7);
|
||||
});
|
||||
|
||||
test('always fails', { retries: 1 }, async () => {
|
||||
throw new Error('fails every time');
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/teardown-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const result = resultByTitle(outcome, 'always fails');
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.message, attempt.cleanup])).toEqual([
|
||||
[0, 'fails every time', 'forced'],
|
||||
]);
|
||||
expect(result.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'keeps a passed setup attempt when afterAll crashes the worker, and skips its dependents',
|
||||
async () => {
|
||||
const setupFile = `import { test } from 'e2e';
|
||||
|
||||
test.afterAll(() => {
|
||||
process.exit(7);
|
||||
});
|
||||
|
||||
test.setup('seed storage', { sessions: ['seeded'] }, async ({ app, session }) => {
|
||||
await app.open('/storage');
|
||||
await session.save('seeded');
|
||||
});
|
||||
`;
|
||||
const consumerFile = `import { test } from 'e2e';
|
||||
|
||||
test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => {
|
||||
await app.open('/storage');
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/auth.setup.e2e.ts': setupFile, 'tests/consumer.e2e.ts': consumerFile },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const setup = resultByTitle(outcome, 'seed storage');
|
||||
expect(setup.status).toBe('passed');
|
||||
expect(setup.attempts.map((attempt) => [attempt.index, attempt.status, attempt.cleanup])).toEqual([
|
||||
[0, 'passed', 'forced'],
|
||||
]);
|
||||
expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
|
||||
expect(resultByTitle(outcome, 'starts with the seeded state').skip?.cause).toBe('setup-failed');
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'keeps the SESSION_CONTRACT failure of a setup attempt when afterAll crashes the worker',
|
||||
async () => {
|
||||
const setupFile = `import { test } from 'e2e';
|
||||
|
||||
test.afterAll(() => {
|
||||
process.exit(7);
|
||||
});
|
||||
|
||||
test.setup('never saves', { sessions: ['seeded'] }, async ({ app }) => {
|
||||
await app.open('/storage');
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{
|
||||
'tests/auth.setup.e2e.ts': setupFile,
|
||||
'tests/consumer.e2e.ts': `import { test } from 'e2e';
|
||||
|
||||
test('starts with the seeded state', { session: 'seeded' }, async ({ app }) => {
|
||||
await app.open('/storage');
|
||||
});
|
||||
`,
|
||||
},
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const setup = resultByTitle(outcome, 'never saves');
|
||||
expect(setup.status).toBe('failed');
|
||||
expect(setup.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.cleanup])).toEqual([
|
||||
[0, 'SESSION_CONTRACT', 'forced'],
|
||||
]);
|
||||
expect(setup.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'charges a crash in a retry beforeAll to the retry, not to the attempt before it',
|
||||
async () => {
|
||||
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-retry-hook-crash-')), 'failed-once');
|
||||
const file = `import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { test } from 'e2e';
|
||||
|
||||
test.beforeAll(() => {
|
||||
if (existsSync(${JSON.stringify(marker)})) process.exit(7);
|
||||
});
|
||||
|
||||
test('fails once', { retries: 1 }, async () => {
|
||||
writeFileSync(${JSON.stringify(marker)}, '');
|
||||
throw new Error('first attempt fails');
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/retry-hook-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const result = resultByTitle(outcome, 'fails once');
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.secondaryErrors.length])).toEqual([
|
||||
[0, 'ERROR', 0],
|
||||
[1, 'WORKER_CRASH', 0],
|
||||
]);
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'fails the first member when a serial retry crashes the worker before any member runs',
|
||||
async () => {
|
||||
const marker = path.join(mkdtempSync(path.join(tmpdir(), 'e2e-serial-retry-hook-crash-')), 'failed-once');
|
||||
const file = `import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { test } from 'e2e';
|
||||
|
||||
test.describe('wizard', { serial: true, retries: 1 }, () => {
|
||||
test.beforeAll(() => {
|
||||
if (existsSync(${JSON.stringify(marker)})) process.exit(7);
|
||||
});
|
||||
test('step 1 passes', async () => {});
|
||||
test('step 2 fails once', async () => {
|
||||
writeFileSync(${JSON.stringify(marker)}, '');
|
||||
throw new Error('first attempt fails');
|
||||
});
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/serial-retry-hook-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const group = outcome.report.run.serialGroups[0]!;
|
||||
expect(group.status).toBe('failed');
|
||||
expect(
|
||||
group.attempts.map((attempt) => [attempt.index, attempt.error?.code, attempt.members.map((member) => [member.status, member.error?.code])]),
|
||||
).toEqual([
|
||||
[0, 'ERROR', [['passed', undefined], ['failed', 'ERROR']]],
|
||||
[1, 'WORKER_CRASH', [['failed', 'WORKER_CRASH'], ['skipped', undefined]]],
|
||||
]);
|
||||
// The retry never reached its members, so they keep attempt 0's verdicts.
|
||||
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
|
||||
expect(resultByTitle(outcome, 'step 2 fails once').status).toBe('failed');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'charges a crash after the last serial member to the group attempt, not to a member',
|
||||
async () => {
|
||||
const file = `import { test } from 'e2e';
|
||||
|
||||
test.describe('wizard', { serial: true }, () => {
|
||||
test.afterAll(() => {
|
||||
process.exit(7);
|
||||
});
|
||||
test('step 1 passes', async () => {});
|
||||
test('step 2 passes', async () => {});
|
||||
});
|
||||
`;
|
||||
const { outcome, project } = await runProjectWithConfigFile(
|
||||
{ 'tests/serial-teardown-crash.e2e.ts': file },
|
||||
{ appUrl: app.url, configSource: workerConfigSource(1) },
|
||||
);
|
||||
const group = outcome.report.run.serialGroups[0]!;
|
||||
expect(group.status).toBe('failed');
|
||||
expect(group.attempts).toHaveLength(1);
|
||||
expect(group.attempts[0]!.error?.code).toBe('WORKER_CRASH');
|
||||
expect(group.attempts[0]!.members.map((member) => member.status)).toEqual(['passed', 'passed']);
|
||||
expect(resultByTitle(outcome, 'step 1 passes').status).toBe('passed');
|
||||
expect(resultByTitle(outcome, 'step 2 passes').status).toBe('passed');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
assertValidReport(outcome.report);
|
||||
project.cleanup();
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it(
|
||||
'interrupts workers, reports 130, and lists the file the one worker never reached',
|
||||
async () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { isRetryEligible, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts';
|
||||
import { isRetryEligible, retryVerdict, runWithRetries, type RetryAttempt } from '../../src/run/retry.ts';
|
||||
import type { SerializedError } from '../../src/internal/errors.ts';
|
||||
|
||||
function error(category: SerializedError['category']): SerializedError {
|
||||
@@ -126,3 +126,19 @@ describe('runWithRetries', () => {
|
||||
expect(status).toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('retryVerdict', () => {
|
||||
const failed: RetryAttempt = { status: 'failed', error: error('test') };
|
||||
|
||||
it.each<[readonly RetryAttempt[], string]>([
|
||||
[[], 'failed'],
|
||||
[[{ status: 'passed' }], 'passed'],
|
||||
[[failed, { status: 'passed' }], 'flaky'],
|
||||
[[failed, { status: 'timed-out' }], 'timed-out'],
|
||||
[[{ status: 'interrupted' }], 'interrupted'],
|
||||
[[{ status: 'timed-out' }, { status: 'interrupted' }], 'timed-out'],
|
||||
[[failed, { status: 'skipped' }], 'skipped'],
|
||||
])('reads %j as %s', (attempts, verdict) => {
|
||||
expect(retryVerdict(attempts)).toBe(verdict);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,8 +15,9 @@ import type {
|
||||
import type { ResolvedTarget } from '../../src/config/resolve.ts';
|
||||
import { defineEngine, type EngineHandle } from '../../src/engine/index.ts';
|
||||
import { classifyError, serializeError } from '../../src/internal/errors.ts';
|
||||
import type { ResultRecord, RunError, SerialGroupRecord } from '../../src/run/records.ts';
|
||||
import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord } from '../../src/run/records.ts';
|
||||
import { runUnits } from '../../src/run/scheduler.ts';
|
||||
import { serialGroupId, type SerialAttemptRun } from '../../src/run/serial.ts';
|
||||
import { buildWorkPlans, type TargetWorkPlan } from '../../src/run/units.ts';
|
||||
import type { SpawnUnitRunner, UnitRunner, UnitRunnerEvents } from '../../src/run/unit-runner.ts';
|
||||
import type { MainToWorker, RunUnitMessage } from '../../src/run/worker/protocol.ts';
|
||||
@@ -142,6 +143,16 @@ interface FakeBehaviour {
|
||||
readonly neverReady?: readonly string[];
|
||||
/** Unit ids whose worker never finishes: a test that will not end on its own. */
|
||||
readonly hangOn?: readonly string[];
|
||||
/** Attempts the unit's first pair finishes before the worker hangs: a test caught between attempts. */
|
||||
readonly attemptsBeforeHang?: readonly AttemptRecord[];
|
||||
/** The hung pair starts its next attempt after `attemptsBeforeHang`, and the worker exits on its own once interrupted. */
|
||||
readonly exitDuringNextAttemptOnInterrupt?: boolean;
|
||||
/** Serial group attempts the unit's pairs (one group) finish before the worker hangs. */
|
||||
readonly serialRunsBeforeHang?: readonly SerialAttemptRun[];
|
||||
/** Members the group's next attempt finishes before the worker hangs. */
|
||||
readonly serialMembersBeforeHang?: readonly SerialMemberRecord[];
|
||||
/** Instead of hanging, the worker exits on its own once it sent what came before. */
|
||||
readonly exitInsteadOfHang?: boolean;
|
||||
/** Workers ignore `terminate` and have to be killed. */
|
||||
readonly ignoreTerminate?: boolean;
|
||||
/**
|
||||
@@ -152,6 +163,31 @@ interface FakeBehaviour {
|
||||
readonly signalledInit?: { readonly workers: number; readonly signal: NodeJS.Signals };
|
||||
}
|
||||
|
||||
const FAILED_ATTEMPT: AttemptRecord = {
|
||||
id: 'attempt-0',
|
||||
index: 0,
|
||||
status: 'failed',
|
||||
startedAt: '2026-01-01T00:00:00.000Z',
|
||||
durationMs: 5,
|
||||
steps: [],
|
||||
artifacts: [],
|
||||
error: serializeError(classifyError(new Error('first attempt fails'))),
|
||||
secondaryErrors: [],
|
||||
cleanup: 'complete',
|
||||
};
|
||||
|
||||
const PASSED_RETRY: AttemptRecord = {
|
||||
id: 'attempt-1',
|
||||
index: 1,
|
||||
status: 'passed',
|
||||
startedAt: '2026-01-01T00:00:01.000Z',
|
||||
durationMs: 5,
|
||||
steps: [],
|
||||
artifacts: [],
|
||||
secondaryErrors: [],
|
||||
cleanup: 'complete',
|
||||
};
|
||||
|
||||
class FakeFleet {
|
||||
readonly spawned: { targetName: string; workerSlot: number }[] = [];
|
||||
readonly unitsByWorker: RunUnitMessage[][] = [];
|
||||
@@ -221,6 +257,10 @@ class FakeRunner implements UnitRunner {
|
||||
if (this.exited) return;
|
||||
if (message.type !== 'run-unit') this.fleet.controlMessages.push(message.type);
|
||||
if (message.type === 'interrupt') this.fleet.interruptSkips.push(message.skip);
|
||||
if (message.type === 'interrupt' && this.behaviour.exitDuringNextAttemptOnInterrupt === true) {
|
||||
setTimeout(() => this.end('code 7, signal null'), 0);
|
||||
return;
|
||||
}
|
||||
if (message.type === 'shutdown') {
|
||||
setTimeout(() => this.end('shut down'), 0);
|
||||
return;
|
||||
@@ -231,7 +271,41 @@ class FakeRunner implements UnitRunner {
|
||||
}
|
||||
if (message.type !== 'run-unit') return;
|
||||
this.fleet.unitsByWorker[this.index]!.push(message);
|
||||
if (this.behaviour.hangOn?.includes(message.unitId) === true) return;
|
||||
if (this.behaviour.hangOn?.includes(message.unitId) === true) {
|
||||
const first = message.pairs[0];
|
||||
for (const finishedRun of this.behaviour.serialRunsBeforeHang ?? []) {
|
||||
const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat);
|
||||
for (const [index, member] of finishedRun.record.members.entries()) {
|
||||
const pair = message.pairs[index]!;
|
||||
this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId });
|
||||
this.events.onMessage({ type: 'serial-member', groupId, attempt: finishedRun.record, member });
|
||||
}
|
||||
this.events.onMessage({ type: 'serial-attempt', groupId, run: finishedRun });
|
||||
}
|
||||
const partial = this.behaviour.serialMembersBeforeHang ?? [];
|
||||
if (partial.length > 0) {
|
||||
const groupId = serialGroupId(first!.test.serialId!, this.targetName, first!.agent, first!.repeat);
|
||||
const attempt = { id: 'group-attempt-partial', index: this.behaviour.serialRunsBeforeHang?.length ?? 0, startedAt: '2026-01-01T00:00:00.000Z' };
|
||||
for (const [index, member] of partial.entries()) {
|
||||
const pair = message.pairs[index]!;
|
||||
this.events.onMessage({ type: 'pair-start', testId: pair.test.id, agent: pair.agent, repeat: pair.repeat, title: pair.test.id, file: pair.test.file, serialId: pair.test.serialId });
|
||||
this.events.onMessage({ type: 'serial-member', groupId, attempt, member });
|
||||
}
|
||||
}
|
||||
const attempts = this.behaviour.attemptsBeforeHang ?? [];
|
||||
const nextAttempt = this.behaviour.exitDuringNextAttemptOnInterrupt === true;
|
||||
if ((attempts.length > 0 || nextAttempt) && first !== undefined) {
|
||||
const pair = { testId: first.test.id, agent: first.agent, repeat: first.repeat };
|
||||
this.events.onMessage({ type: 'pair-start', ...pair, title: first.test.id, file: first.test.file, serialId: undefined });
|
||||
for (const attempt of attempts) {
|
||||
this.events.onMessage({ type: 'attempt-start', ...pair, index: attempt.index });
|
||||
this.events.onMessage({ type: 'attempt', ...pair, attempt });
|
||||
}
|
||||
if (nextAttempt) this.events.onMessage({ type: 'attempt-start', ...pair, index: attempts.length });
|
||||
}
|
||||
if (this.behaviour.exitInsteadOfHang === true) setTimeout(() => this.end('code 7, signal null'), 0);
|
||||
return;
|
||||
}
|
||||
setTimeout(() => this.completeUnit(message), 0);
|
||||
}
|
||||
|
||||
@@ -587,6 +661,63 @@ describe('scheduler gating and dispatch', () => {
|
||||
});
|
||||
});
|
||||
|
||||
/** Two members of one serial group in `tests/a.e2e.ts`. */
|
||||
function serialPairs(target: ResolvedTarget): TestTargetPair[] {
|
||||
return ['step 1', 'step 2'].map((title, declarationIndex) =>
|
||||
makePair(makeTest('tests/a.e2e.ts', title, { declarationIndex, serialId: 'tests/a.e2e.ts::wizard' }), target),
|
||||
);
|
||||
}
|
||||
|
||||
/** One member's record in group attempt `attemptIndex`. */
|
||||
function memberRecord(pairs: readonly TestTargetPair[], attemptIndex: number, memberIndex: number, status: 'passed' | 'failed'): SerialMemberRecord {
|
||||
return {
|
||||
id: `group-attempt-${attemptIndex}:member:${memberIndex}`,
|
||||
index: memberIndex,
|
||||
testId: pairs[memberIndex]!.test.id,
|
||||
status,
|
||||
startedAt: '2026-01-01T00:00:00.000Z',
|
||||
durationMs: 1,
|
||||
steps: [],
|
||||
...(status === 'failed' ? { error: FAILED_ATTEMPT.error! } : {}),
|
||||
secondaryErrors: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** A finished group attempt whose members ended with `statuses`. */
|
||||
function groupRun(pairs: readonly TestTargetPair[], index: number, statuses: readonly ('passed' | 'failed')[]): SerialAttemptRun {
|
||||
const failed = statuses.includes('failed');
|
||||
return {
|
||||
reachedMembers: true,
|
||||
record: {
|
||||
id: `group-attempt-${index}`,
|
||||
index,
|
||||
status: failed ? 'failed' : 'passed',
|
||||
startedAt: '2026-01-01T00:00:00.000Z',
|
||||
durationMs: 5,
|
||||
members: statuses.map((status, memberIndex) => memberRecord(pairs, index, memberIndex, status)),
|
||||
artifacts: [],
|
||||
...(failed ? { error: FAILED_ATTEMPT.error! } : {}),
|
||||
secondaryErrors: [],
|
||||
cleanup: 'complete',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs `pairs` on `fleet` and interrupts, then forces, while the unit hangs. */
|
||||
async function runForcedInterrupt(pairs: TestTargetPair[], target: ResolvedTarget, fleet: FakeFleet): Promise<Collected> {
|
||||
const interrupt = new AbortController();
|
||||
const force = new AbortController();
|
||||
const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)];
|
||||
const collected = await run(
|
||||
makeSelection([{ target, pairs }]),
|
||||
makeCollection(['tests/a.e2e.ts'], pairs),
|
||||
fleet,
|
||||
{ workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 },
|
||||
);
|
||||
for (const timer of timers) clearTimeout(timer);
|
||||
return collected;
|
||||
}
|
||||
|
||||
describe('scheduler fault handling', () => {
|
||||
it('synthesizes results and a run error when a worker dies mid-unit', async () => {
|
||||
const target = makeTarget('web', 0);
|
||||
@@ -745,6 +876,105 @@ describe('scheduler fault handling', () => {
|
||||
expect(collected.runErrors).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ finished: [FAILED_ATTEMPT], status: 'failed' },
|
||||
{ finished: [FAILED_ATTEMPT, PASSED_RETRY], status: 'flaky' },
|
||||
])('a forced interrupt keeps the attempts a test finished and their retry verdict ($status)', async ({ finished, status }) => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];
|
||||
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], attemptsBeforeHang: finished });
|
||||
const interrupt = new AbortController();
|
||||
const force = new AbortController();
|
||||
const timers = [setTimeout(() => interrupt.abort(), 20), setTimeout(() => force.abort(), 40)];
|
||||
|
||||
const collected = await run(
|
||||
makeSelection([{ target, pairs }]),
|
||||
makeCollection(['tests/a.e2e.ts'], pairs),
|
||||
fleet,
|
||||
{ workers: 1, interruptSignal: interrupt.signal, forceSignal: force.signal, interruptGraceMs: 30_000 },
|
||||
);
|
||||
for (const timer of timers) clearTimeout(timer);
|
||||
|
||||
expect(collected.results).toHaveLength(1);
|
||||
expect(collected.results[0]!.status).toBe(status);
|
||||
expect(collected.results[0]!.attempts).toEqual(finished);
|
||||
});
|
||||
|
||||
it.each<{ statuses: ('passed' | 'failed')[][]; verdict: string; results: string[] }>([
|
||||
{ statuses: [['passed', 'failed']], verdict: 'failed', results: ['passed', 'failed'] },
|
||||
{ statuses: [['passed', 'failed'], ['passed', 'passed']], verdict: 'flaky', results: ['flaky', 'flaky'] },
|
||||
])('a forced interrupt keeps the group attempts a serial group finished and their retry verdict ($verdict)', async ({ statuses, verdict, results }) => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = serialPairs(target);
|
||||
const runs = statuses.map((members, index) => groupRun(pairs, index, members));
|
||||
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: runs });
|
||||
const collected = await runForcedInterrupt(pairs, target, fleet);
|
||||
|
||||
expect(collected.serialGroups).toHaveLength(1);
|
||||
expect(collected.serialGroups[0]!.status).toBe(verdict);
|
||||
expect(collected.serialGroups[0]!.attempts).toEqual(runs.map((entry) => entry.record));
|
||||
expect(collected.results.map((result) => result.status)).toEqual(results);
|
||||
});
|
||||
|
||||
it('a forced interrupt mid group attempt keeps the members that attempt finished', async () => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = serialPairs(target);
|
||||
const finished = memberRecord(pairs, 0, 0, 'passed');
|
||||
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialMembersBeforeHang: [finished] });
|
||||
const collected = await runForcedInterrupt(pairs, target, fleet);
|
||||
|
||||
const group = collected.serialGroups[0]!;
|
||||
expect(group.status).toBe('interrupted');
|
||||
expect(group.attempts.map((attempt) => attempt.status)).toEqual(['interrupted']);
|
||||
expect(group.attempts[0]!.members[0]).toEqual(finished);
|
||||
expect(group.attempts[0]!.members[1]!.status).toBe('skipped');
|
||||
expect(collected.results.map((result) => result.status)).toEqual(['passed', 'skipped']);
|
||||
});
|
||||
|
||||
it('records a crash after a serial group finished its last attempt on that attempt', async () => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = serialPairs(target);
|
||||
const passed = groupRun(pairs, 0, ['passed', 'passed']);
|
||||
const fleet = new FakeFleet({ hangOn: ['file::web::tests/a.e2e.ts'], serialRunsBeforeHang: [passed], exitInsteadOfHang: true });
|
||||
const collected = await run(makeSelection([{ target, pairs }]), makeCollection(['tests/a.e2e.ts'], pairs), fleet, { workers: 1 });
|
||||
|
||||
const group = collected.serialGroups[0]!;
|
||||
expect(group.status).toBe('passed');
|
||||
expect(group.attempts).toHaveLength(1);
|
||||
expect(group.attempts[0]!.cleanup).toBe('forced');
|
||||
expect(group.attempts[0]!.secondaryErrors.map((error) => error.code)).toEqual(['WORKER_CRASH']);
|
||||
expect(collected.results.map((result) => result.status)).toEqual(['passed', 'passed']);
|
||||
expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']);
|
||||
});
|
||||
|
||||
it('a worker that exits on its own during the interrupt grace crashes the test it was running', async () => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];
|
||||
const fleet = new FakeFleet({
|
||||
hangOn: ['file::web::tests/a.e2e.ts'],
|
||||
attemptsBeforeHang: [FAILED_ATTEMPT],
|
||||
exitDuringNextAttemptOnInterrupt: true,
|
||||
});
|
||||
const interrupt = new AbortController();
|
||||
const timer = setTimeout(() => interrupt.abort(), 20);
|
||||
|
||||
const collected = await run(
|
||||
makeSelection([{ target, pairs }]),
|
||||
makeCollection(['tests/a.e2e.ts'], pairs),
|
||||
fleet,
|
||||
{ workers: 1, interruptSignal: interrupt.signal, interruptGraceMs: 30_000 },
|
||||
);
|
||||
clearTimeout(timer);
|
||||
|
||||
expect(collected.runErrors.map((runError) => runError.error.code)).toEqual(['WORKER_EXIT']);
|
||||
expect(collected.results).toHaveLength(1);
|
||||
expect(collected.results[0]!.status).toBe('failed');
|
||||
expect(collected.results[0]!.attempts.map((attempt) => [attempt.index, attempt.error?.code])).toEqual([
|
||||
[0, FAILED_ATTEMPT.error!.code],
|
||||
[1, 'WORKER_CRASH'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('kills a worker that ignores a forced interrupt once the force budget is spent', async () => {
|
||||
const target = makeTarget('web', 0);
|
||||
const pairs = [makePair(makeTest('tests/a.e2e.ts', 'a'), target)];
|
||||
|
||||
Reference in New Issue
Block a user