test: bind scenario seeds and exercise the library baton in CI (#243)

* test: bind scenario seeds and exercise the library baton in CI

* test: run named scenarios through a remote Docker executor

* test: bound and clean up the testbed image-load check

---------

Co-authored-by: dev-driver <dev-driver@openrig-build>
Co-authored-by: dev-qa <dev-qa@openrig-build>
This commit is contained in:
Mike Schwarz
2026-09-30 17:55:20 -07:00
committed by GitHub
co-authored by dev-driver dev-qa
parent afde814f5b
commit c1ff76635e
16 changed files with 456 additions and 57 deletions
+1
View File
@@ -4,4 +4,5 @@ ARG TESTBED_IMAGE
FROM ${TESTBED_IMAGE}
COPY runner.mjs /opt/openrig-testbed/runner.mjs
COPY scenarios/ /opt/openrig-testbed/scenarios/
COPY library/ /opt/openrig-testbed/library/
CMD ["node", "/opt/openrig-testbed/runner.mjs", "healthy"]
@@ -215,8 +215,8 @@ export interface RunScenarioFileOptions {
topologyKind?: "stub" | "real";
/** Base environment for the hermetic scaffold (HOME/PATH/TERM). */
baseEnv?: Record<string, string | undefined>;
/** Overrides forwarded to buildRealDeps (clock/sleep/appendRecord/defaults/normalizer). */
deps?: Partial<Pick<RealDepsOptions, "now" | "sleep" | "appendRecord" | "defaults" | "normalizer">>;
/** Overrides forwarded to buildRealDeps, including an explicit test fault controller. */
deps?: Partial<Pick<RealDepsOptions, "now" | "sleep" | "appendRecord" | "defaults" | "normalizer" | "seedRegression">>;
/**
* 51-04 opt-in: how the scenario-local daemon is stood up. ABSENT => host-mode,
* byte-identical to pre-51-04 (`defaultHostDaemon` = spawnScenarioDaemon with rigBin).
@@ -17,7 +17,9 @@
* down → rig down <rigName|rigId> --json --force
* daemon {op} → the ScenarioDaemon lifecycle (sigterm | restart)
*
* restore/emit/mutate/policy/seed_regression have NO shipped runtime binding at v1
* seed_regression calls an explicitly supplied test fault controller. With no
* controller it fails loud; a declaration alone never counts as a seeded test.
* restore/emit/mutate/policy have NO shipped runtime binding at v1
* (they ride 51-03 / A5 items 6-8) — the adapter FAILS LOUD with a named
* UnboundActionError rather than fabricating a call (same floor as the FLAG-1
* `proof` surface: unbound is never silently-skipped).
@@ -92,6 +94,10 @@ export interface RealDepsOptions {
/** The mission scope reads audit (D3) — threaded from the scenario's
* env.scope_mission into `rig scope audit --mission <name> --json`. */
scopeMission?: string;
/** Arm a named, scenario-local test fault (or explicitly record its healthy
* control). The paired harness owns injection timing and verifies the actual
* failing observation; this must never invoke a production `rig` command. */
seedRegression?: (regressionClass: string) => Promise<ActionResult>;
}
/** `rig up` is heavy (real tmux seat launch) — give it a generous ceiling. */
@@ -153,8 +159,17 @@ export function buildRealDeps(opts: RealDepsOptions): ScenarioRunnerDeps {
if (op === "restart") { await daemon.restart(); return { code: 0, stdout: "", stderr: "" }; }
return fail(`daemon: unknown op ${JSON.stringify(op)} (allowed: sigterm, restart)`);
}
case "seed_regression": {
const regressionClass = payload && typeof payload === "object"
? (payload as { class?: unknown }).class : undefined;
if (typeof regressionClass !== "string" || regressionClass.length === 0) {
return fail("seed_regression: a non-empty class is required");
}
if (!opts.seedRegression) throw new UnboundActionError(verb);
return opts.seedRegression(regressionClass);
}
default:
// restore / emit / mutate / policy / seed_regression — no shipped binding at v1.
// restore / emit / mutate / policy — no shipped binding at v1.
throw new UnboundActionError(verb);
}
};
@@ -33,6 +33,23 @@ function write(name: string, body: string): string {
}
describe("loadScenarioFile", () => {
it("threads the explicit seed controller through the pipeline without a live daemon", async () => {
const { runScenarioFile } = await import("./helpers/scenario-pipeline.js");
const p = write("seed.yaml", "scenario: seeded\ntopology: ./unused.yaml\nsteps:\n - seed_regression: {class: baton-drop}\n");
const seedRegression = vi.fn(async () => ({ code: 0, stdout: "armed", stderr: "" }));
const result = await runScenarioFile(p, {
rigBin: "/not-executed",
deps: { seedRegression },
daemon: async scaffold => ({
port: 9, baseUrl: "http://127.0.0.1:9", readEnv: scaffold.env,
sigterm: async () => {}, restart: async () => {},
stop: async () => scaffold.cleanup(),
}),
});
expect(result).toEqual({ scenario: "seeded", verdict: "PASS" });
expect(seedRegression).toHaveBeenCalledExactlyOnceWith("baton-drop");
});
it("parses a valid scenario and resolves topology relative to the scenario file", () => {
const p = write("s.yaml", [
"scenario: baton-survives",
@@ -97,6 +97,25 @@ describe("buildRealDeps runAction verb->rig map", () => {
expect(deps.defaults.withinMs).toBeGreaterThan(0);
expect(deps.defaults.pollIntervalMs).toBeGreaterThan(0);
});
it("binds a named seed to the caller's fault controller and preserves its failure", async () => {
const result = { code: 1, stdout: "", stderr: "fault could not be armed" };
const seedRegression = vi.fn(async () => result);
const runRig = vi.fn(async () => ok());
const deps = buildRealDeps({ daemon: fakeDaemon(), rigBin: "/bin/rig", topologyPath: "/t.yaml", runRig, seedRegression });
expect(await deps.runAction("seed_regression", { class: "baton-drop" })).toBe(result);
expect(seedRegression).toHaveBeenCalledExactlyOnceWith("baton-drop");
expect(runRig).not.toHaveBeenCalled();
});
it("rejects a malformed seed before invoking the fault controller", async () => {
const seedRegression = vi.fn(async () => ok());
const deps = buildRealDeps({ daemon: fakeDaemon(), rigBin: "/bin/rig", topologyPath: "/t.yaml", seedRegression });
for (const payload of [undefined, {}, { class: "" }, { class: 42 }]) {
expect((await deps.runAction("seed_regression", payload)).code).toBe(1);
}
expect(seedRegression).not.toHaveBeenCalled();
});
});
describe("D2 — the unbound-verb message states WHY and names the v1 path", () => {
+9 -2
View File
@@ -2,8 +2,9 @@
The first PR automation increment is described in [ci/README.md](ci/README.md).
It runs the existing daemon-restart baton fixture plus a fault control in the
installed-package testbed. Hosted green/red execution is required before claiming
that increment verified. The eleven authored scenarios below are a broader target;
installed-package testbed, now alongside the library's queue-baton scenario and
its explicit seed binding. Hosted green/red execution is required before claiming
either verified. The eleven authored scenarios below are a broader target;
they are not eleven admitted CI passes.
## Historical 51-03 seed contract — the ten (+ one)
@@ -42,6 +43,12 @@ scenario's `expect` legs MUST catch it. Acceptance per scenario = the PAIR: GREE
shipped tip + RED on the seeded run whose runner diff (expected vs last-observed) names
the class. A scenario that cannot be shown RED is not delivered.
For executable seeds, place the step before the affected action and supply the
pipeline's `deps.seedRegression` controller. It must arm a real local fault in the
seeded run and explicitly record that fault as disabled in the healthy run. The
paired harness verifies the injection receipt and the specific failed observation.
The remaining historical markers after their assertions are not executable proofs.
## Run status honesty (authored before the runner exists)
YAML authoring is cleared ahead of the 51-02 runner (dispatch authority). These files
are the runner's acceptance targets; they are authored to the locked format and DO NOT
+63 -9
View File
@@ -32,7 +32,7 @@ fault is a storage-state reset, not a replay of a specific production bug.
## Isolation and evidence
Use `bash scripts/run-pr-scenarios.sh` on a disposable GitHub Linux runner after
`npm ci`. It refuses ordinary local invocation. Runtime is inside unprivileged,
`npm ci`, or select a prepared remote executor as below. Runtime is inside unprivileged,
network-disabled containers, with no host mounts, a read-only root, private
writable `/tmp`, dropped capabilities and bounded memory/processes/time. Image
construction uses network to retrieve the pinned base, Node and dependencies;
@@ -47,16 +47,70 @@ injection receipt. The pure `scripts/pr-scenarios.test.mjs` controls validate re
admission only; they are
**not** a substitute for the three actual container runs.
## Remaining authored scenarios
## Library scenario increment
The eleven YAML scenarios in `../scenarios/` remain unchanged and **unadmitted**.
All currently reach `seed_regression`, which `scenario-real-deps.ts` rejects as
unbound. Several additionally require step-time `emit`, `policy`, `mutate`, or
`restore`, and `kill-daemon-mid-handoff` still has its documented setup/observable
gaps. The bounded daemon-restart fixture used here is not the original seat-resume
scenario's full contract. Do not count this increment as eleven passing scenarios,
strip their assertions, or convert unsupported actions into no-ops.
The same job also runs `../scenarios/queue-baton-survives-restart.yaml` in three
fresh containers: healthy, `baton-drop`, healthy again. It brings up the library's
two-seat stub rig and asserts the exact `dev-qa@dev-pair-stub` claim after restart.
The original fixture and its three controls remain unchanged.
`seed_regression` now calls an explicit fault controller supplied through the
pipeline. With no controller it still fails loudly. The library baton declares
its seed **before** restart, not after the assertions; the controller records the
healthy control or arms the stopped-DB mutation. Unknown classes, a missing seed,
an injection failure, a surviving fault, or an unrelated failing observation fail
the job. Assertions continue to use the shipped queue read, never a fake observer.
The other ten library scenarios are **unadmitted**. Several need step-time `emit`,
`policy`, `mutate`, or `restore`. Others have incomplete assertions or setup:
clean-lifecycle has no post-down residue assertion, ps-scope neither brings up its
second topology nor excludes extra rows, and the home/preseed and send/render
scenarios need input behavior from the stub. `kill-daemon-mid-handoff` still has
its documented setup/observable gaps. A callback binding does not fix these gaps.
Do not count this increment as eleven passing scenarios or native seat-resume
coverage. Container evidence for each selected case is required for admission.
The existing CLI `run-scenarios.mjs` still accepts paths only; `--container` is
refused. This job runs the helper *inside* the isolated image, so it does not depend
on the host-to-container staging adapter's unsupported per-seat-script path.
## Run one case before pushing
Any developer or agent can use the same script with a prepared SSH Docker executor;
there is no seat-owner or per-run approval requirement. Use a private checkout with
the normal Node 22/24 development dependencies. Source build/pack runs on the client;
Docker image construction and the scenario run at the selected daemon.
```sh
DOCKER_HOST=ssh://your-test-executor bash scripts/run-pr-scenarios.sh \
--remote --case library --mode healthy --out dist/scenario-check
```
Leave `DOCKER_CONTEXT` unset when selecting `DOCKER_HOST`, so a saved context cannot
override the explicitly selected executor. This does not change Docker configuration.
Use a fresh output directory for each retained run. Without `--mode` the selected
case runs healthy / lost-baton / healthy. Without `--case` both cases run. CI keeps
its existing six-run default. A single healthy run establishes only that leg; it is
not the paired seeded-regression proof.
The client can be macOS/arm64 while Docker is Linux/amd64: the build reads the
**server** OS/architecture and passes the same platform to the image builds and
containers. The Node deadline helper replaces GNU `timeout`, preserves the real
exit status, returns 124 on deadline and escalates TERM to KILL after 15 seconds.
All daemon work remains in the container. No local OpenRig daemon is touched.
Build contexts are uploaded by Docker, not mounted from the client. Logs, actual
exit-code files, container names/inspection, server platform, image identity and
manifests are written in the client's `--out` directory. The image-load check still
runs once before the selected scenario to detect a broken package/native install.
It has a 120-second deadline, a recorded unique container name, actual exit/log
and inspection receipts, and bounded named cleanup even on failure or timeout.
Scenario runtime is 2 CPUs, 2GiB memory with no swap, 256 PIDs, network-none, non-root
and a read-only root plus scratch tmpfs. A rootless executor must enforce its
configured cgroup limits; its aggregate budget is an executor setting, not a
claim made by successful source checks.
The script removes only its named containers. If SSH becomes unavailable, it
reports incomplete cleanup with the exact name for later reconciliation; no remote
cleanup can be guaranteed through a broken connection. It never prunes shared
images or other agents' containers.
+13 -3
View File
@@ -2,14 +2,24 @@ import assert from 'node:assert/strict';
export const RESULT_PREFIX = 'OPENRIG_SCENARIO_RESULT=';
export const SCENARIO = 'queue-baton-survives-restart';
export const CASES = {
fixture: { file: 'scenarios/scenario-02-baton.yaml', destination: 'dev-worker@scn-baton', failedStep: 3 },
library: { file: 'library/queue-baton-survives-restart.yaml', destination: 'dev-qa@dev-pair-stub', failedStep: 4, seed: 'baton-drop' },
};
// A failed start, bad command, timeout, or unrelated assertion is NOT a caught regression.
export function verifyRun(mode, exitCode, report) {
export function verifyRun(mode, exitCode, report, caseName = 'fixture') {
const selected = CASES[caseName];
assert.ok(selected, 'unknown scenario case');
assert.equal(report.caseName ?? 'fixture', caseName);
assert.equal(report.mode, mode);
assert.equal(report.result?.scenario, SCENARIO);
assert.equal(report.error, undefined);
assert.equal(report.records.length, 1);
assert.deepEqual(report.records[0], report.result);
if (selected.seed) {
assert.deepEqual(report.seed, { class: selected.seed, enabled: mode !== 'healthy' });
}
if (mode === 'healthy') {
assert.equal(exitCode, 0);
assert.equal(report.result.verdict, 'PASS');
@@ -21,13 +31,13 @@ export function verifyRun(mode, exitCode, report) {
assert.equal(report.fault?.before, 'in-progress');
assert.equal(report.fault?.after, 'pending');
assert.equal(report.result.verdict, 'FAIL');
assert.equal(report.result.failedStep, 3); // the queue read AFTER restart
assert.equal(report.result.failedStep, selected.failedStep); // queue read AFTER restart
const observation = report.result.observation;
assert.equal(observation?.surface, 'queue');
assert.ok(Array.isArray(observation.value));
const baton = observation.value.filter(row => row?.qitemId === 'baton-1');
assert.equal(baton.length, 1);
assert.equal(baton[0].destinationSession, 'dev-worker@scn-baton');
assert.equal(baton[0].destinationSession, selected.destination);
assert.equal(baton[0].state, 'pending');
}
}
+19 -5
View File
@@ -7,7 +7,7 @@ import { createRequire } from 'node:module';
import { networkInterfaces } from 'node:os';
import { join } from 'node:path';
import { defaultHostDaemon, runScenarioFile } from '../../daemon/test/helpers/scenario-pipeline.ts';
import { RESULT_PREFIX } from './result.mjs';
import { CASES, RESULT_PREFIX } from './result.mjs';
assert.equal(process.platform, 'linux', 'container execution only');
assert.notEqual(process.getuid(), 0, 'run as the unprivileged testbed user');
@@ -15,11 +15,14 @@ assert.ok(Object.values(networkInterfaces()).flat().every(address => address.int
'requires --network none');
const mode = process.argv[2];
assert.ok(['healthy', 'lost-baton'].includes(mode), 'expected healthy or lost-baton');
const caseName = process.argv[3] ?? 'fixture';
const selected = CASES[caseName];
assert.ok(selected, 'unknown scenario case');
const rigBin = realpathSync('/usr/local/bin/rig');
const scenario = '/opt/openrig-testbed/scenarios/scenario-02-baton.yaml';
const scenario = join('/opt/openrig-testbed', selected.file);
const records = [];
const report = {
mode, records, fault: null,
mode, caseName, records, fault: null,
scenarioSha256: createHash('sha256').update(readFileSync(scenario)).digest('hex'),
};
@@ -27,13 +30,23 @@ try {
report.result = await runScenarioFile(scenario, {
rigBin,
baseEnv: { PATH: '/usr/local/bin:/usr/bin:/bin', TERM: 'xterm-256color' },
deps: { appendRecord: record => records.push(record) },
deps: {
appendRecord: record => records.push(record),
seedRegression: async regressionClass => {
assert.equal(regressionClass, selected.seed, 'unsupported regression class');
assert.equal(report.seed, undefined, 'seed must be armed exactly once');
report.seed = { class: regressionClass, enabled: mode !== 'healthy' };
return { code: 0, stdout: JSON.stringify(report.seed), stderr: '' };
},
},
daemon: async (scaffold, options) => {
const daemon = await defaultHostDaemon(scaffold, options);
if (mode === 'healthy') return daemon;
return {
...daemon,
restart: async () => {
if (selected.seed) assert.deepEqual(report.seed, { class: selected.seed, enabled: true },
'the scenario must arm its fault before restart');
await daemon.sigterm();
await assert.rejects(fetch(`${daemon.baseUrl}/healthz`),
error => error.cause?.code === 'ECONNREFUSED', 'daemon must be stopped before fault injection');
@@ -42,8 +55,9 @@ try {
const Database = createRequire(rigBin)('better-sqlite3');
const db = new Database(join(scaffold.stateDir, 'scenario.db'), { fileMustExist: true });
try {
const before = db.prepare('SELECT state FROM queue_items WHERE qitem_id = ?').get('baton-1');
const before = db.prepare('SELECT state, destination_session FROM queue_items WHERE qitem_id = ?').get('baton-1');
assert.equal(before?.state, 'in-progress');
assert.equal(before.destination_session, selected.destination);
const { changes } = db.prepare("UPDATE queue_items SET state = 'pending' WHERE qitem_id = ? AND state = 'in-progress'").run('baton-1');
assert.equal(changes, 1);
report.fault = {
@@ -33,6 +33,9 @@ steps:
- qitemId: baton-1
state: in-progress
destinationSession: dev-qa@dev-pair-stub
# Arm the paired harness BEFORE the affected action. Healthy runs record a
# disabled fault; seeded runs reset this exact claim while the daemon is down.
- seed_regression: {class: baton-drop}
- daemon: { op: restart }
# the baton MUST survive the restart: same item, still in-progress, still owned
- expect:
@@ -46,5 +49,4 @@ steps:
# expect { surface: pane, seat: qa, contains: "restored" }. The stub-runner at the
# current tip emits only READY/EXITED; the `restore` behavior that prints the
# "restored" pane marker is A5 items 6-8. Lights up when they land.
- seed_regression: {class: baton-drop}
- down: {}
+34 -14
View File
@@ -13,10 +13,16 @@ STUB_ASSETS_LIST="${TESTBED_DIR}/stub-assets.list"
OUT_DIR="${1:-${REPO_ROOT}/dist/testbed-image}"
command -v docker >/dev/null 2>&1 || {
echo "[testbed] docker not found — run this build HOST-side (locus ruling), not in the VM seat" >&2
echo "[testbed] Docker client not found; select the prepared disposable executor" >&2
exit 3
}
# Resolve the daemon's platform BEFORE building locally. A remote amd64 daemon
# reached from an arm64 Mac needs amd64 Node; client uname is not the target.
mkdir -p "${OUT_DIR}"
TARGET_PLATFORM="$(node "${REPO_ROOT}/scripts/scenario-executor.mjs" platform "${OUT_DIR}/docker-server.json")"
TARGETARCH="${TARGET_PLATFORM#linux/}"
# --- identity from the tree: the image is built AT this git sha, so gitSha == openrigSha ---
GIT_SHA="$(git -C "${REPO_ROOT}" rev-parse HEAD)"
IMAGE_TAG="openrig-testbed:${GIT_SHA}"
@@ -31,7 +37,15 @@ NODE_VERSION="$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9.]*\).*/\1/p' "${TESTBED_
# --- assemble a clean build context: Dockerfile + entrypoint + the openrig pack + staged stub assets ---
CONTEXT="$(mktemp -d)"
trap 'rm -rf "${CONTEXT}"' EXIT
LOAD_CONTAINER=""
cleanup() {
if [ -n "${LOAD_CONTAINER}" ]; then
node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null ||
echo "[testbed] cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2
fi
rm -rf "${CONTEXT}"
}
trap cleanup EXIT
cp "${TESTBED_DIR}/Dockerfile" "${TESTBED_DIR}/entrypoint.sh" "${CONTEXT}/"
# OpenRig CLI from the TREE (never the npm registry). ASSEMBLE the publishable @openrig/cli first
@@ -57,18 +71,8 @@ STUB_FILES_JSON="$(node -e \
'const fs=require("fs");const l=fs.readFileSync(process.argv[1],"utf8").split("\n").map(s=>s.replace(/#.*/,"").trim()).filter(Boolean);process.stdout.write(JSON.stringify(l))' \
"${STUB_ASSETS_LIST}")"
# --- resolve the target arch HOST-side and pass it EXPLICITLY (builder-agnostic: correct on the
# legacy builder — which NEVER populates the automatic TARGETARCH build-arg — AND on BuildKit). Fail
# CLOSED on an unknown arch rather than letting the Dockerfile silently default to amd64, which fetches
# x64 Node into an arm64 image and dies with a Rosetta/ELF failure (exit 133).
case "$(uname -m)" in
x86_64|amd64) TARGETARCH=amd64 ;;
arm64|aarch64) TARGETARCH=arm64 ;;
*) echo "[testbed] cannot resolve a supported TARGETARCH from 'uname -m'=$(uname -m); refusing to build (a silent amd64 default installs wrong-arch Node = exit 133)" >&2; exit 4 ;;
esac
# --- build (host-side) ---
docker build \
docker build --platform "${TARGET_PLATFORM}" \
--build-arg BASE_IMAGE="${BASE_IMAGE}" \
--build-arg NODE_VERSION="${NODE_VERSION}" \
--build-arg OPENRIG_TARBALL=openrig.tgz \
@@ -92,7 +96,23 @@ echo "[testbed] effect proof: daemon LOAD inside the container (better-sqlite3 m
# kernel, confirm readiness by hitting /healthz DIRECTLY (deterministic — no fixed sleep), then daemon
# status; an EXIT trap stops the daemon so a failed assertion still tears down. A broken native install
# fails `rig daemon start` here → set -e → non-zero → the build verb fails BEFORE the A/B pin.
docker run --rm --network none --cap-drop ALL --security-opt no-new-privileges "${IMAGE_TAG}" bash -lc 'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status'
LOAD_CONTAINER="openrig-testbed-load-$(node -p 'require("node:crypto").randomUUID()')"
printf '%s\n' "${LOAD_CONTAINER}" > "${OUT_DIR}/image-load.container-name.txt"
load_status=0
node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" \
--platform "${TARGET_PLATFORM}" --network none --cap-drop ALL --security-opt no-new-privileges \
--cpus 2 --memory 2g --memory-swap 2g --pids-limit 256 "${IMAGE_TAG}" bash -lc \
'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status' \
> "${OUT_DIR}/image-load.log" 2>&1 || load_status=$?
printf '%s\n' "${load_status}" > "${OUT_DIR}/image-load.exit-code.txt"
cat "${OUT_DIR}/image-load.log" >&2
inspect_status=0
node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker inspect "${LOAD_CONTAINER}" \
> "${OUT_DIR}/image-load.container.json" || inspect_status=$?
[ "${load_status}" -eq 0 ] || exit "${load_status}"
[ "${inspect_status}" -eq 0 ] || exit "${inspect_status}"
node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null
LOAD_CONTAINER=""
# --- emit the reproducible manifest + census receipt via the tested node orchestrator ---
INPUTS="$(mktemp)"
+60 -5
View File
@@ -1,6 +1,7 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { readFileSync, writeFileSync, mkdirSync, copyFileSync, mkdtempSync, rmSync, existsSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
@@ -61,11 +62,13 @@ test("Q2 fix A: packs the ASSEMBLED @openrig/cli (has the `rig` bin), NEVER the
assert.doesNotMatch(text, /cd\s+"?\$\{REPO_ROOT\}"?\s*&&\s*npm pack/, "must NOT pack the monorepo root");
});
test("Q2 fix B: resolves the target arch HOST-side + passes it explicitly, fail-CLOSED (never a silent amd64 default -> exit 133)", () => {
test("resolves the target from the Docker server and passes both platform and arch", () => {
const text = readScript();
assert.match(text, /uname -m/, "must resolve the host arch (uname -m) so the legacy builder gets a real TARGETARCH");
assert.match(text, /--build-arg\s+TARGETARCH=/, "must pass TARGETARCH explicitly (builder-agnostic)");
assert.match(text, /uname -m[\s\S]*?exit\s+[1-9]/, "must fail-closed (non-zero exit) on an unresolvable arch");
assert.match(text, /scenario-executor\.mjs.*platform/);
assert.match(text, /--build-arg\s+TARGETARCH=/);
assert.match(text, /docker build --platform/);
assert.match(text, /docker run --name/);
assert.doesNotMatch(text, /case.*uname -m/);
});
test("Q2 fix B: the Dockerfile fails CLOSED on an empty TARGETARCH (no silent amd64 default)", () => {
@@ -109,3 +112,55 @@ test("Q2 rider (effect proof): the build verb LOADS the daemon inside the contai
assert.match(text, /rig daemon start --no-kernel/, "must LOAD the daemon (better-sqlite3 binds) via the operator-corrected start, not merely check rig exists");
assert.match(text, /\/healthz/, "must confirm readiness deterministically via /healthz (operator correction — no fixed sleep)");
});
test("image-load success, failure and deadline all retain status and remove only the named container", () => {
const root = mkdtempSync(join(process.cwd(), ".testbed-load-"));
try {
for (const p of ["scripts", "docker/testbed", "packages/cli", "bin"]) mkdirSync(join(root, p), { recursive: true });
copyFileSync(SCRIPT, join(root, "scripts/build-testbed-image.sh"));
copyFileSync(join(HERE, "scenario-executor.mjs"), join(root, "scripts/scenario-executor.mjs"));
writeFileSync(join(root, "scripts/build-package.sh"), "#!/bin/sh\nexit 0\n");
writeFileSync(join(root, "scripts/testbed-build-inputs.mjs"), "export const readBaseImage = () => ({ref:'fixture@sha256:abc'});\n");
writeFileSync(join(root, "scripts/testbed-emit-manifest.mjs"), "import fs from 'node:fs';fs.writeFileSync(process.argv[3]+'/manifest.json','{}');\n");
writeFileSync(join(root, "docker/testbed/Dockerfile"), "ARG NODE_VERSION=22.22.1\n");
for (const p of ["docker/testbed/entrypoint.sh", "docker/testbed/base-image", "docker/testbed/stub-assets.list"]) writeFileSync(join(root, p), "");
writeFileSync(join(root, "bin/git"), "#!/bin/sh\nprintf 'fake-source\\n'\n", { mode: 0o755 });
writeFileSync(join(root, "bin/npm"), "#!/bin/sh\ntouch fixture.tgz\nprintf 'fixture.tgz\\n'\n", { mode: 0o755 });
writeFileSync(join(root, "bin/mktemp"), '#!/bin/sh\nexec /usr/bin/mktemp "$@" "$HOME/tmp.XXXXXXXX"\n', { mode: 0o755 });
// Use the real deadline helper with a short test deadline. No Docker daemon,
// build, package install or network is involved; the marker models a remote
// container surviving the client process until an explicit named removal.
writeFileSync(join(root, "bin/node"), `#!${process.execPath}
import {spawnSync} from 'node:child_process';const args=process.argv.slice(2);
if(args[1]==='timeout')args[2]='0.4';
const p=spawnSync(${JSON.stringify(process.execPath)},args,{stdio:'inherit'});process.exit(p.status??1);
`, { mode: 0o755 });
writeFileSync(join(root, "bin/docker"), `#!${process.execPath}
import fs from 'node:fs';const args=process.argv.slice(2);const marker=process.env.FAKE_CONTAINER;
if(args[0]==='version')console.log(JSON.stringify({Os:'linux',Arch:'amd64'}));
else if(args[0]==='build'){}
else if(args[0]==='run'){
const at=args.indexOf('--name');if(at<0)process.exit(9);
fs.writeFileSync(marker,args[at+1]);
if(process.env.LOAD_CASE==='timeout')setInterval(()=>{},1000);
else process.exit(process.env.LOAD_CASE==='failure'?7:0);
}else if(args[0]==='inspect')console.log('[]');
else if(args[0]==='rm'){
if(fs.readFileSync(marker,'utf8')!==args.at(-1))process.exit(8);
fs.unlinkSync(marker);
}else process.exit(8);
`, { mode: 0o755 });
for (const [mode, status] of [["success", 0], ["failure", 7], ["timeout", 124]]) {
const out = join(root, mode), marker = join(root, "container");
const run = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], {
env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker },
encoding: "utf8", timeout: 8000,
});
assert.equal(run.status, status, `${mode}: ${run.stderr}`);
assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), status);
assert.match(readFileSync(join(out, "image-load.container-name.txt"), "utf8"), /^openrig-testbed-load-/);
assert.equal(existsSync(marker), false, `${mode}: container must be removed even after timeout`);
assert.equal(existsSync(join(out, "manifest.json")), status === 0);
}
} finally { rmSync(root, { recursive: true, force: true }); }
});
+18
View File
@@ -19,6 +19,24 @@ test('accepts a healthy run and a specifically observed post-restart lost baton'
verifyRun('healthy', 0, green);
verifyRun('lost-baton', 1, red);
});
test('library admission requires its own seed, failing step, and exact destination', () => {
const libraryGreen = { ...green, caseName: 'library', seed: { class: 'baton-drop', enabled: false } };
const result = {
...red.result, failedStep: 4,
observation: { surface: 'queue', value: [
{ qitemId: 'baton-1', state: 'pending', destinationSession: 'dev-qa@dev-pair-stub' },
] },
};
const libraryRed = { ...report('lost-baton', result, red.fault), caseName: 'library', seed: { class: 'baton-drop', enabled: true } };
verifyRun('healthy', 0, libraryGreen, 'library');
verifyRun('lost-baton', 1, libraryRed, 'library');
for (const seed of [undefined, { class: 'typo', enabled: true }, { class: 'baton-drop', enabled: false }]) {
assert.throws(() => verifyRun('lost-baton', 1, { ...libraryRed, seed }, 'library'));
}
assert.throws(() => verifyRun('healthy', 0, libraryGreen));
assert.throws(() => verifyRun('lost-baton', 1, { ...libraryRed, result: red.result, records: [red.result] }, 'library'));
assert.throws(() => verifyRun('healthy', 0, green, 'unknown'));
});
test('rejects a surviving mutant and a failure before the seeded boundary', () => {
assert.throws(() => verifyRun('lost-baton', 0, { ...green, mode: 'lost-baton' }));
assert.throws(() => verifyRun('lost-baton', 1, report('lost-baton', {
+46 -14
View File
@@ -1,12 +1,36 @@
#!/usr/bin/env bash
# Hosted Linux only. All daemon/seat work runs inside the existing testbed.
# All daemon/seat work runs inside the existing disposable testbed.
# Default: GitHub CI. --remote: explicitly selected SSH Docker executor.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"
test "${GITHUB_ACTIONS:-}" = true || { echo 'Use a disposable GitHub runner, not a working host' >&2; exit 2; }
test "$(uname -s)" = Linux
REMOTE=false
CASES=(fixture library)
MODES=(healthy lost-baton healthy)
OUT="$REPO_ROOT/dist/pr-scenarios"
while [ "$#" -gt 0 ]; do
case "$1" in
--remote) REMOTE=true; shift ;;
--case)
case "${2:-}" in fixture|library) CASES=("$2");; *) echo 'Expected --case fixture|library' >&2; exit 2;; esac
shift 2 ;;
--mode)
case "${2:-}" in healthy|lost-baton) MODES=("$2");; *) echo 'Expected --mode healthy|lost-baton' >&2; exit 2;; esac
shift 2 ;;
--out) test -n "${2:-}" || exit 2; OUT="$2"; shift 2 ;;
*) echo "Unknown argument: $1" >&2; exit 2 ;;
esac
done
if "$REMOTE"; then
case "${DOCKER_HOST:-}" in ssh://?*) ;; *) echo '--remote requires explicit DOCKER_HOST=ssh://...' >&2; exit 2;; esac
test -z "${DOCKER_CONTEXT:-}" || { echo 'Unset DOCKER_CONTEXT so it cannot override the selected DOCKER_HOST' >&2; exit 2; }
else
test "${GITHUB_ACTIONS:-}" = true || { echo 'Use GitHub CI or --remote with the prepared disposable SSH executor' >&2; exit 2; }
test "$(uname -s)" = Linux
fi
mkdir -p "$OUT"
OUT="$(cd "$OUT" && pwd -P)"
TARGET_PLATFORM="$(node scripts/scenario-executor.mjs platform "$OUT/docker-server.json")"
# build-testbed-image performs the stock package build, clean target install,
# and daemon-load effect proof, then records the image inputs. It never pushes.
@@ -17,41 +41,49 @@ IMAGE="openrig-pr-scenarios:$SHA"
CONTEXT="$(mktemp -d)"
CONTAINER=""
cleanup() {
if [ -n "$CONTAINER" ]; then docker rm -f "$CONTAINER" >/dev/null; fi
if [ -n "$CONTAINER" ]; then
node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2
fi
rm -rf "$CONTEXT"
}
trap cleanup EXIT
cp docker/testbed/Dockerfile.scenarios "$CONTEXT/Dockerfile"
cp -R packages/daemon/test/fixtures/scenarios "$CONTEXT/scenarios"
cp -R packages/test-system/scenarios "$CONTEXT/library"
# esbuild already ships in the lockfile through tsx. Bundle the existing helper
# closure (including YAML) so the container needs no source tree or dev install.
node_modules/.bin/esbuild packages/test-system/ci/run.mjs --bundle --platform=node \
--format=esm --banner:js='import { createRequire as nodeRequire } from "node:module"; const require = nodeRequire(import.meta.url);' \
--outfile="$CONTEXT/runner.mjs" --metafile="$OUT/runner-inputs.json"
docker build --network none --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT"
docker build --network none --platform "$TARGET_PLATFORM" --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT"
docker image inspect "$IMAGE" > "$OUT/image-inspect.json"
attempt=0
for mode in healthy lost-baton healthy; do
for scenario in "${CASES[@]}"; do
for mode in "${MODES[@]}"; do
attempt=$((attempt + 1))
status=0
LOG="$OUT/$attempt-$mode.log"
LOG="$OUT/$attempt-$scenario-$mode.log"
CONTAINER="openrig-pr-${SHA:0:12}-$attempt-$$"
printf '%s\n' "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container-name.txt"
# Fresh writable scratch only. No mounts, host networking, credentials or Docker
# socket; the container is non-root, resource bounded and removed even on failure.
timeout --signal=TERM --kill-after=15s 300s docker run --name "$CONTAINER" --network none \
node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --platform "$TARGET_PLATFORM" --network none \
--read-only --tmpfs /tmp:rw,exec,nosuid,nodev,size=512m,mode=1777 \
--cap-drop ALL --security-opt no-new-privileges --pids-limit 256 \
--memory 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" \
--memory 2g --memory-swap 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" "$scenario" \
> "$LOG" 2>&1 || status=$?
printf '%s\n' "$status" > "$OUT/$attempt-$scenario-$mode.exit-code.txt"
cat "$LOG"
docker rm -f "$CONTAINER" >/dev/null
node scripts/scenario-executor.mjs timeout 30 docker inspect "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container.json"
node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null
CONTAINER=""
node --input-type=module - "$mode" "$status" "$LOG" <<'JS'
node --input-type=module - "$mode" "$status" "$LOG" "$scenario" <<'JS'
import { readFileSync } from 'node:fs';
import { readReport, verifyRun } from './packages/test-system/ci/result.mjs';
const [mode, status, log] = process.argv.slice(2);
verifyRun(mode, Number(status), readReport(readFileSync(log, 'utf8')));
console.log(`${mode}: ${mode === 'healthy' ? 'healthy scenario passed' : 'seeded durability regression caught at the expected assertion'}`);
const [mode, status, log, scenario] = process.argv.slice(2);
verifyRun(mode, Number(status), readReport(readFileSync(log, 'utf8')), scenario);
console.log(`${scenario}/${mode}: ${mode === 'healthy' ? 'healthy scenario passed' : 'seeded durability regression caught at the expected assertion'}`);
JS
done
done
+75
View File
@@ -0,0 +1,75 @@
// Client-side helpers for the existing scenario scripts; never runs a daemon.
import { spawn, spawnSync } from 'node:child_process';
import { writeFileSync } from 'node:fs';
import { constants } from 'node:os';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
export function dockerPlatform(server) {
if (server?.Os !== 'linux' || !['amd64', 'arm64'].includes(server?.Arch)) {
throw new Error(`Unsupported Docker server: ${server?.Os}/${server?.Arch}`);
}
return `linux/${server.Arch}`;
}
export async function runWithDeadline(argv, { timeoutMs, killAfterMs = 15000 }) {
if (!argv.length || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || killAfterMs <= 0) {
throw new Error('Command and positive finite deadline required');
}
return new Promise(resolveResult => {
const child = spawn(argv[0], argv.slice(1), { stdio: 'inherit', detached: true });
let forced, escalation, finished = false;
const signalChild = signal => {
if (!child.pid) return;
try { process.kill(-child.pid, signal); } catch (error) {
if (error.code !== 'ESRCH') throw error;
}
};
const terminate = code => {
if (forced !== undefined) return;
forced = code;
signalChild('SIGTERM');
escalation = setTimeout(() => signalChild('SIGKILL'), killAfterMs);
};
const timeout = setTimeout(() => terminate(124), timeoutMs);
const interrupt = () => terminate(130);
const shutdown = () => terminate(143);
process.once('SIGINT', interrupt);
process.once('SIGTERM', shutdown);
const finish = code => {
if (finished) return;
finished = true;
if (forced !== undefined) signalChild('SIGKILL');
clearTimeout(timeout); clearTimeout(escalation);
process.removeListener('SIGINT', interrupt);
process.removeListener('SIGTERM', shutdown);
resolveResult(forced ?? code);
};
child.once('error', error => {
console.error(`Cannot run ${argv[0]}: ${error.message}`);
finish(127);
});
child.once('close', (code, signal) => finish(code ?? (128 + (constants.signals[signal] ?? 0))));
});
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
try {
const [operation, ...args] = process.argv.slice(2);
if (operation === 'platform') {
const version = spawnSync('docker', ['version', '--format', '{{json .Server}}'], { encoding: 'utf8', timeout: 30000 });
if (version.error || version.status !== 0) throw new Error(version.error?.message ?? version.stderr);
const server = JSON.parse(version.stdout);
const platform = dockerPlatform(server);
if (args[0]) writeFileSync(args[0], JSON.stringify({ platform, server }, null, 2) + '\n');
console.log(platform);
} else if (operation === 'timeout') {
process.exitCode = await runWithDeadline(args.slice(1), { timeoutMs: Number(args[0]) * 1000 });
} else {
throw new Error('Expected platform [receipt-path] or timeout <seconds> <command> [args...]');
}
} catch (error) {
console.error(error.message);
process.exitCode = 2;
}
}
+60
View File
@@ -0,0 +1,60 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { dockerPlatform, runWithDeadline } from './scenario-executor.mjs';
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
test('target follows the Docker server, not this client platform', () => {
assert.equal(dockerPlatform({ Os: 'linux', Arch: 'amd64' }), 'linux/amd64');
assert.equal(dockerPlatform({ Os: 'linux', Arch: 'arm64' }), 'linux/arm64');
for (const server of [undefined, {}, { Os: 'darwin', Arch: 'arm64' }, { Os: 'linux', Arch: 'mips' }]) {
assert.throws(() => dockerPlatform(server), /Unsupported Docker server/);
}
});
test('deadline wrapper preserves actual successful and nonzero exits', async () => {
assert.equal(await runWithDeadline([process.execPath, '-e', 'process.exit(0)'], { timeoutMs: 3000 }), 0);
assert.equal(await runWithDeadline([process.execPath, '-e', 'process.exit(7)'], { timeoutMs: 3000 }), 7);
});
test('deadline expires even when the child ignores TERM', async () => {
const start = Date.now();
const status = await runWithDeadline([process.execPath, '-e', "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"], { timeoutMs: 300, killAfterMs: 100 });
assert.equal(status, 124);
assert.ok(Date.now() - start < 3000);
});
test('a missing executable and an invalid deadline cannot report success', async () => {
assert.equal(await runWithDeadline(['/nonexistent/openrig-test-program'], { timeoutMs: 1000 }), 127);
await assert.rejects(runWithDeadline([process.execPath], { timeoutMs: 0 }), /positive/);
});
test('the platform CLI queries a fake remote server and writes its actual receipt', () => {
const root = mkdtempSync(join(process.cwd(), '.scenario-platform-'));
try {
const server = { Os: 'linux', Arch: 'amd64', Version: 'test-only' };
writeFileSync(join(root, 'docker'), `#!/bin/sh
printf '%s\\n' '${JSON.stringify(server)}'
`, { mode: 0o755 });
const receipt = join(root, 'server.json');
const run = spawnSync(process.execPath, [resolve('scripts/scenario-executor.mjs'), 'platform', receipt], {
env: { ...process.env, PATH: root + ':/usr/bin:/bin', DOCKER_HOST: 'ssh://fixture.invalid' }, encoding: 'utf8',
});
assert.equal(run.status, 0, run.stderr);
assert.equal(run.stdout.trim(), 'linux/amd64');
assert.deepEqual(JSON.parse(readFileSync(receipt, 'utf8')), { platform: 'linux/amd64', server });
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('ordinary invocation and ambiguous remote selection refuse before any build', () => {
const script = resolve('scripts/run-pr-scenarios.sh');
for (const [args, extra, message] of [
[[], {}, /Use GitHub CI or --remote/],
[['--remote'], {}, /requires explicit DOCKER_HOST/],
[['--remote'], { DOCKER_HOST: 'ssh://fixture.invalid', DOCKER_CONTEXT: 'other' }, /Unset DOCKER_CONTEXT/],
[['--remote', '--case', 'typo'], {}, /Expected --case/],
[['--remote', '--mode', 'typo'], {}, /Expected --mode/],
]) {
const env = { ...process.env, GITHUB_ACTIONS: '', DOCKER_HOST: '', DOCKER_CONTEXT: '', ...extra };
const run = spawnSync('/bin/bash', [script, ...args], { env, encoding: 'utf8', timeout: 3000 });
assert.equal(run.status, 2, run.stderr);
assert.match(run.stderr, message);
}
});