diff --git a/docker/testbed/Dockerfile.scenarios b/docker/testbed/Dockerfile.scenarios index 0b639e6e..9660d23e 100644 --- a/docker/testbed/Dockerfile.scenarios +++ b/docker/testbed/Dockerfile.scenarios @@ -4,4 +4,5 @@ ARG TESTBED_IMAGE FROM ${TESTBED_IMAGE} COPY runner.mjs /opt/openrig-testbed/runner.mjs COPY scenarios/ /opt/openrig-testbed/scenarios/ +COPY library/ /opt/openrig-testbed/library/ CMD ["node", "/opt/openrig-testbed/runner.mjs", "healthy"] diff --git a/packages/daemon/test/helpers/scenario-pipeline.ts b/packages/daemon/test/helpers/scenario-pipeline.ts index 40013485..a8a70af4 100644 --- a/packages/daemon/test/helpers/scenario-pipeline.ts +++ b/packages/daemon/test/helpers/scenario-pipeline.ts @@ -215,8 +215,8 @@ export interface RunScenarioFileOptions { topologyKind?: "stub" | "real"; /** Base environment for the hermetic scaffold (HOME/PATH/TERM). */ baseEnv?: Record; - /** Overrides forwarded to buildRealDeps (clock/sleep/appendRecord/defaults/normalizer). */ - deps?: Partial>; + /** Overrides forwarded to buildRealDeps, including an explicit test fault controller. */ + deps?: Partial>; /** * 51-04 opt-in: how the scenario-local daemon is stood up. ABSENT => host-mode, * byte-identical to pre-51-04 (`defaultHostDaemon` = spawnScenarioDaemon with rigBin). diff --git a/packages/daemon/test/helpers/scenario-real-deps.ts b/packages/daemon/test/helpers/scenario-real-deps.ts index a63d97ba..5b864132 100644 --- a/packages/daemon/test/helpers/scenario-real-deps.ts +++ b/packages/daemon/test/helpers/scenario-real-deps.ts @@ -17,7 +17,9 @@ * down → rig down --json --force * daemon {op} → the ScenarioDaemon lifecycle (sigterm | restart) * - * restore/emit/mutate/policy/seed_regression have NO shipped runtime binding at v1 + * seed_regression calls an explicitly supplied test fault controller. With no + * controller it fails loud; a declaration alone never counts as a seeded test. + * restore/emit/mutate/policy have NO shipped runtime binding at v1 * (they ride 51-03 / A5 items 6-8) — the adapter FAILS LOUD with a named * UnboundActionError rather than fabricating a call (same floor as the FLAG-1 * `proof` surface: unbound is never silently-skipped). @@ -92,6 +94,10 @@ export interface RealDepsOptions { /** The mission scope reads audit (D3) — threaded from the scenario's * env.scope_mission into `rig scope audit --mission --json`. */ scopeMission?: string; + /** Arm a named, scenario-local test fault (or explicitly record its healthy + * control). The paired harness owns injection timing and verifies the actual + * failing observation; this must never invoke a production `rig` command. */ + seedRegression?: (regressionClass: string) => Promise; } /** `rig up` is heavy (real tmux seat launch) — give it a generous ceiling. */ @@ -153,8 +159,17 @@ export function buildRealDeps(opts: RealDepsOptions): ScenarioRunnerDeps { if (op === "restart") { await daemon.restart(); return { code: 0, stdout: "", stderr: "" }; } return fail(`daemon: unknown op ${JSON.stringify(op)} (allowed: sigterm, restart)`); } + case "seed_regression": { + const regressionClass = payload && typeof payload === "object" + ? (payload as { class?: unknown }).class : undefined; + if (typeof regressionClass !== "string" || regressionClass.length === 0) { + return fail("seed_regression: a non-empty class is required"); + } + if (!opts.seedRegression) throw new UnboundActionError(verb); + return opts.seedRegression(regressionClass); + } default: - // restore / emit / mutate / policy / seed_regression — no shipped binding at v1. + // restore / emit / mutate / policy — no shipped binding at v1. throw new UnboundActionError(verb); } }; diff --git a/packages/daemon/test/scenario-pipeline.test.ts b/packages/daemon/test/scenario-pipeline.test.ts index 1e991c05..d50a49b0 100644 --- a/packages/daemon/test/scenario-pipeline.test.ts +++ b/packages/daemon/test/scenario-pipeline.test.ts @@ -33,6 +33,23 @@ function write(name: string, body: string): string { } describe("loadScenarioFile", () => { + it("threads the explicit seed controller through the pipeline without a live daemon", async () => { + const { runScenarioFile } = await import("./helpers/scenario-pipeline.js"); + const p = write("seed.yaml", "scenario: seeded\ntopology: ./unused.yaml\nsteps:\n - seed_regression: {class: baton-drop}\n"); + const seedRegression = vi.fn(async () => ({ code: 0, stdout: "armed", stderr: "" })); + const result = await runScenarioFile(p, { + rigBin: "/not-executed", + deps: { seedRegression }, + daemon: async scaffold => ({ + port: 9, baseUrl: "http://127.0.0.1:9", readEnv: scaffold.env, + sigterm: async () => {}, restart: async () => {}, + stop: async () => scaffold.cleanup(), + }), + }); + expect(result).toEqual({ scenario: "seeded", verdict: "PASS" }); + expect(seedRegression).toHaveBeenCalledExactlyOnceWith("baton-drop"); + }); + it("parses a valid scenario and resolves topology relative to the scenario file", () => { const p = write("s.yaml", [ "scenario: baton-survives", diff --git a/packages/daemon/test/scenario-real-deps.test.ts b/packages/daemon/test/scenario-real-deps.test.ts index 29b041a5..bbe9552d 100644 --- a/packages/daemon/test/scenario-real-deps.test.ts +++ b/packages/daemon/test/scenario-real-deps.test.ts @@ -97,6 +97,25 @@ describe("buildRealDeps runAction verb->rig map", () => { expect(deps.defaults.withinMs).toBeGreaterThan(0); expect(deps.defaults.pollIntervalMs).toBeGreaterThan(0); }); + + it("binds a named seed to the caller's fault controller and preserves its failure", async () => { + const result = { code: 1, stdout: "", stderr: "fault could not be armed" }; + const seedRegression = vi.fn(async () => result); + const runRig = vi.fn(async () => ok()); + const deps = buildRealDeps({ daemon: fakeDaemon(), rigBin: "/bin/rig", topologyPath: "/t.yaml", runRig, seedRegression }); + expect(await deps.runAction("seed_regression", { class: "baton-drop" })).toBe(result); + expect(seedRegression).toHaveBeenCalledExactlyOnceWith("baton-drop"); + expect(runRig).not.toHaveBeenCalled(); + }); + + it("rejects a malformed seed before invoking the fault controller", async () => { + const seedRegression = vi.fn(async () => ok()); + const deps = buildRealDeps({ daemon: fakeDaemon(), rigBin: "/bin/rig", topologyPath: "/t.yaml", seedRegression }); + for (const payload of [undefined, {}, { class: "" }, { class: 42 }]) { + expect((await deps.runAction("seed_regression", payload)).code).toBe(1); + } + expect(seedRegression).not.toHaveBeenCalled(); + }); }); describe("D2 — the unbound-verb message states WHY and names the v1 path", () => { diff --git a/packages/test-system/README.md b/packages/test-system/README.md index 7bd50666..c9cd7134 100644 --- a/packages/test-system/README.md +++ b/packages/test-system/README.md @@ -2,8 +2,9 @@ The first PR automation increment is described in [ci/README.md](ci/README.md). It runs the existing daemon-restart baton fixture plus a fault control in the -installed-package testbed. Hosted green/red execution is required before claiming -that increment verified. The eleven authored scenarios below are a broader target; +installed-package testbed, now alongside the library's queue-baton scenario and +its explicit seed binding. Hosted green/red execution is required before claiming +either verified. The eleven authored scenarios below are a broader target; they are not eleven admitted CI passes. ## Historical 51-03 seed contract — the ten (+ one) @@ -42,6 +43,12 @@ scenario's `expect` legs MUST catch it. Acceptance per scenario = the PAIR: GREE shipped tip + RED on the seeded run whose runner diff (expected vs last-observed) names the class. A scenario that cannot be shown RED is not delivered. +For executable seeds, place the step before the affected action and supply the +pipeline's `deps.seedRegression` controller. It must arm a real local fault in the +seeded run and explicitly record that fault as disabled in the healthy run. The +paired harness verifies the injection receipt and the specific failed observation. +The remaining historical markers after their assertions are not executable proofs. + ## Run status honesty (authored before the runner exists) YAML authoring is cleared ahead of the 51-02 runner (dispatch authority). These files are the runner's acceptance targets; they are authored to the locked format and DO NOT diff --git a/packages/test-system/ci/README.md b/packages/test-system/ci/README.md index ebc2e679..468138da 100644 --- a/packages/test-system/ci/README.md +++ b/packages/test-system/ci/README.md @@ -32,7 +32,7 @@ fault is a storage-state reset, not a replay of a specific production bug. ## Isolation and evidence Use `bash scripts/run-pr-scenarios.sh` on a disposable GitHub Linux runner after -`npm ci`. It refuses ordinary local invocation. Runtime is inside unprivileged, +`npm ci`, or select a prepared remote executor as below. Runtime is inside unprivileged, network-disabled containers, with no host mounts, a read-only root, private writable `/tmp`, dropped capabilities and bounded memory/processes/time. Image construction uses network to retrieve the pinned base, Node and dependencies; @@ -47,16 +47,70 @@ injection receipt. The pure `scripts/pr-scenarios.test.mjs` controls validate re admission only; they are **not** a substitute for the three actual container runs. -## Remaining authored scenarios +## Library scenario increment -The eleven YAML scenarios in `../scenarios/` remain unchanged and **unadmitted**. -All currently reach `seed_regression`, which `scenario-real-deps.ts` rejects as -unbound. Several additionally require step-time `emit`, `policy`, `mutate`, or -`restore`, and `kill-daemon-mid-handoff` still has its documented setup/observable -gaps. The bounded daemon-restart fixture used here is not the original seat-resume -scenario's full contract. Do not count this increment as eleven passing scenarios, -strip their assertions, or convert unsupported actions into no-ops. +The same job also runs `../scenarios/queue-baton-survives-restart.yaml` in three +fresh containers: healthy, `baton-drop`, healthy again. It brings up the library's +two-seat stub rig and asserts the exact `dev-qa@dev-pair-stub` claim after restart. +The original fixture and its three controls remain unchanged. + +`seed_regression` now calls an explicit fault controller supplied through the +pipeline. With no controller it still fails loudly. The library baton declares +its seed **before** restart, not after the assertions; the controller records the +healthy control or arms the stopped-DB mutation. Unknown classes, a missing seed, +an injection failure, a surviving fault, or an unrelated failing observation fail +the job. Assertions continue to use the shipped queue read, never a fake observer. + +The other ten library scenarios are **unadmitted**. Several need step-time `emit`, +`policy`, `mutate`, or `restore`. Others have incomplete assertions or setup: +clean-lifecycle has no post-down residue assertion, ps-scope neither brings up its +second topology nor excludes extra rows, and the home/preseed and send/render +scenarios need input behavior from the stub. `kill-daemon-mid-handoff` still has +its documented setup/observable gaps. A callback binding does not fix these gaps. +Do not count this increment as eleven passing scenarios or native seat-resume +coverage. Container evidence for each selected case is required for admission. The existing CLI `run-scenarios.mjs` still accepts paths only; `--container` is refused. This job runs the helper *inside* the isolated image, so it does not depend on the host-to-container staging adapter's unsupported per-seat-script path. + +## Run one case before pushing + +Any developer or agent can use the same script with a prepared SSH Docker executor; +there is no seat-owner or per-run approval requirement. Use a private checkout with +the normal Node 22/24 development dependencies. Source build/pack runs on the client; +Docker image construction and the scenario run at the selected daemon. + +```sh +DOCKER_HOST=ssh://your-test-executor bash scripts/run-pr-scenarios.sh \ + --remote --case library --mode healthy --out dist/scenario-check +``` + +Leave `DOCKER_CONTEXT` unset when selecting `DOCKER_HOST`, so a saved context cannot +override the explicitly selected executor. This does not change Docker configuration. +Use a fresh output directory for each retained run. Without `--mode` the selected +case runs healthy / lost-baton / healthy. Without `--case` both cases run. CI keeps +its existing six-run default. A single healthy run establishes only that leg; it is +not the paired seeded-regression proof. + +The client can be macOS/arm64 while Docker is Linux/amd64: the build reads the +**server** OS/architecture and passes the same platform to the image builds and +containers. The Node deadline helper replaces GNU `timeout`, preserves the real +exit status, returns 124 on deadline and escalates TERM to KILL after 15 seconds. +All daemon work remains in the container. No local OpenRig daemon is touched. + +Build contexts are uploaded by Docker, not mounted from the client. Logs, actual +exit-code files, container names/inspection, server platform, image identity and +manifests are written in the client's `--out` directory. The image-load check still +runs once before the selected scenario to detect a broken package/native install. +It has a 120-second deadline, a recorded unique container name, actual exit/log +and inspection receipts, and bounded named cleanup even on failure or timeout. +Scenario runtime is 2 CPUs, 2GiB memory with no swap, 256 PIDs, network-none, non-root +and a read-only root plus scratch tmpfs. A rootless executor must enforce its +configured cgroup limits; its aggregate budget is an executor setting, not a +claim made by successful source checks. + +The script removes only its named containers. If SSH becomes unavailable, it +reports incomplete cleanup with the exact name for later reconciliation; no remote +cleanup can be guaranteed through a broken connection. It never prunes shared +images or other agents' containers. diff --git a/packages/test-system/ci/result.mjs b/packages/test-system/ci/result.mjs index 5532a9cd..038c00d4 100644 --- a/packages/test-system/ci/result.mjs +++ b/packages/test-system/ci/result.mjs @@ -2,14 +2,24 @@ import assert from 'node:assert/strict'; export const RESULT_PREFIX = 'OPENRIG_SCENARIO_RESULT='; export const SCENARIO = 'queue-baton-survives-restart'; +export const CASES = { + fixture: { file: 'scenarios/scenario-02-baton.yaml', destination: 'dev-worker@scn-baton', failedStep: 3 }, + library: { file: 'library/queue-baton-survives-restart.yaml', destination: 'dev-qa@dev-pair-stub', failedStep: 4, seed: 'baton-drop' }, +}; // A failed start, bad command, timeout, or unrelated assertion is NOT a caught regression. -export function verifyRun(mode, exitCode, report) { +export function verifyRun(mode, exitCode, report, caseName = 'fixture') { + const selected = CASES[caseName]; + assert.ok(selected, 'unknown scenario case'); + assert.equal(report.caseName ?? 'fixture', caseName); assert.equal(report.mode, mode); assert.equal(report.result?.scenario, SCENARIO); assert.equal(report.error, undefined); assert.equal(report.records.length, 1); assert.deepEqual(report.records[0], report.result); + if (selected.seed) { + assert.deepEqual(report.seed, { class: selected.seed, enabled: mode !== 'healthy' }); + } if (mode === 'healthy') { assert.equal(exitCode, 0); assert.equal(report.result.verdict, 'PASS'); @@ -21,13 +31,13 @@ export function verifyRun(mode, exitCode, report) { assert.equal(report.fault?.before, 'in-progress'); assert.equal(report.fault?.after, 'pending'); assert.equal(report.result.verdict, 'FAIL'); - assert.equal(report.result.failedStep, 3); // the queue read AFTER restart + assert.equal(report.result.failedStep, selected.failedStep); // queue read AFTER restart const observation = report.result.observation; assert.equal(observation?.surface, 'queue'); assert.ok(Array.isArray(observation.value)); const baton = observation.value.filter(row => row?.qitemId === 'baton-1'); assert.equal(baton.length, 1); - assert.equal(baton[0].destinationSession, 'dev-worker@scn-baton'); + assert.equal(baton[0].destinationSession, selected.destination); assert.equal(baton[0].state, 'pending'); } } diff --git a/packages/test-system/ci/run.mjs b/packages/test-system/ci/run.mjs index bcda8a7d..ed3ec548 100644 --- a/packages/test-system/ci/run.mjs +++ b/packages/test-system/ci/run.mjs @@ -7,7 +7,7 @@ import { createRequire } from 'node:module'; import { networkInterfaces } from 'node:os'; import { join } from 'node:path'; import { defaultHostDaemon, runScenarioFile } from '../../daemon/test/helpers/scenario-pipeline.ts'; -import { RESULT_PREFIX } from './result.mjs'; +import { CASES, RESULT_PREFIX } from './result.mjs'; assert.equal(process.platform, 'linux', 'container execution only'); assert.notEqual(process.getuid(), 0, 'run as the unprivileged testbed user'); @@ -15,11 +15,14 @@ assert.ok(Object.values(networkInterfaces()).flat().every(address => address.int 'requires --network none'); const mode = process.argv[2]; assert.ok(['healthy', 'lost-baton'].includes(mode), 'expected healthy or lost-baton'); +const caseName = process.argv[3] ?? 'fixture'; +const selected = CASES[caseName]; +assert.ok(selected, 'unknown scenario case'); const rigBin = realpathSync('/usr/local/bin/rig'); -const scenario = '/opt/openrig-testbed/scenarios/scenario-02-baton.yaml'; +const scenario = join('/opt/openrig-testbed', selected.file); const records = []; const report = { - mode, records, fault: null, + mode, caseName, records, fault: null, scenarioSha256: createHash('sha256').update(readFileSync(scenario)).digest('hex'), }; @@ -27,13 +30,23 @@ try { report.result = await runScenarioFile(scenario, { rigBin, baseEnv: { PATH: '/usr/local/bin:/usr/bin:/bin', TERM: 'xterm-256color' }, - deps: { appendRecord: record => records.push(record) }, + deps: { + appendRecord: record => records.push(record), + seedRegression: async regressionClass => { + assert.equal(regressionClass, selected.seed, 'unsupported regression class'); + assert.equal(report.seed, undefined, 'seed must be armed exactly once'); + report.seed = { class: regressionClass, enabled: mode !== 'healthy' }; + return { code: 0, stdout: JSON.stringify(report.seed), stderr: '' }; + }, + }, daemon: async (scaffold, options) => { const daemon = await defaultHostDaemon(scaffold, options); if (mode === 'healthy') return daemon; return { ...daemon, restart: async () => { + if (selected.seed) assert.deepEqual(report.seed, { class: selected.seed, enabled: true }, + 'the scenario must arm its fault before restart'); await daemon.sigterm(); await assert.rejects(fetch(`${daemon.baseUrl}/healthz`), error => error.cause?.code === 'ECONNREFUSED', 'daemon must be stopped before fault injection'); @@ -42,8 +55,9 @@ try { const Database = createRequire(rigBin)('better-sqlite3'); const db = new Database(join(scaffold.stateDir, 'scenario.db'), { fileMustExist: true }); try { - const before = db.prepare('SELECT state FROM queue_items WHERE qitem_id = ?').get('baton-1'); + const before = db.prepare('SELECT state, destination_session FROM queue_items WHERE qitem_id = ?').get('baton-1'); assert.equal(before?.state, 'in-progress'); + assert.equal(before.destination_session, selected.destination); const { changes } = db.prepare("UPDATE queue_items SET state = 'pending' WHERE qitem_id = ? AND state = 'in-progress'").run('baton-1'); assert.equal(changes, 1); report.fault = { diff --git a/packages/test-system/scenarios/queue-baton-survives-restart.yaml b/packages/test-system/scenarios/queue-baton-survives-restart.yaml index f14c8f80..9ff78811 100644 --- a/packages/test-system/scenarios/queue-baton-survives-restart.yaml +++ b/packages/test-system/scenarios/queue-baton-survives-restart.yaml @@ -33,6 +33,9 @@ steps: - qitemId: baton-1 state: in-progress destinationSession: dev-qa@dev-pair-stub + # Arm the paired harness BEFORE the affected action. Healthy runs record a + # disabled fault; seeded runs reset this exact claim while the daemon is down. + - seed_regression: {class: baton-drop} - daemon: { op: restart } # the baton MUST survive the restart: same item, still in-progress, still owned - expect: @@ -46,5 +49,4 @@ steps: # expect { surface: pane, seat: qa, contains: "restored" }. The stub-runner at the # current tip emits only READY/EXITED; the `restore` behavior that prints the # "restored" pane marker is A5 items 6-8. Lights up when they land. - - seed_regression: {class: baton-drop} - down: {} diff --git a/scripts/build-testbed-image.sh b/scripts/build-testbed-image.sh index ee547a0f..10b3950b 100755 --- a/scripts/build-testbed-image.sh +++ b/scripts/build-testbed-image.sh @@ -13,10 +13,16 @@ STUB_ASSETS_LIST="${TESTBED_DIR}/stub-assets.list" OUT_DIR="${1:-${REPO_ROOT}/dist/testbed-image}" command -v docker >/dev/null 2>&1 || { - echo "[testbed] docker not found — run this build HOST-side (locus ruling), not in the VM seat" >&2 + echo "[testbed] Docker client not found; select the prepared disposable executor" >&2 exit 3 } +# Resolve the daemon's platform BEFORE building locally. A remote amd64 daemon +# reached from an arm64 Mac needs amd64 Node; client uname is not the target. +mkdir -p "${OUT_DIR}" +TARGET_PLATFORM="$(node "${REPO_ROOT}/scripts/scenario-executor.mjs" platform "${OUT_DIR}/docker-server.json")" +TARGETARCH="${TARGET_PLATFORM#linux/}" + # --- identity from the tree: the image is built AT this git sha, so gitSha == openrigSha --- GIT_SHA="$(git -C "${REPO_ROOT}" rev-parse HEAD)" IMAGE_TAG="openrig-testbed:${GIT_SHA}" @@ -31,7 +37,15 @@ NODE_VERSION="$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9.]*\).*/\1/p' "${TESTBED_ # --- assemble a clean build context: Dockerfile + entrypoint + the openrig pack + staged stub assets --- CONTEXT="$(mktemp -d)" -trap 'rm -rf "${CONTEXT}"' EXIT +LOAD_CONTAINER="" +cleanup() { + if [ -n "${LOAD_CONTAINER}" ]; then + node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null || + echo "[testbed] cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2 + fi + rm -rf "${CONTEXT}" +} +trap cleanup EXIT cp "${TESTBED_DIR}/Dockerfile" "${TESTBED_DIR}/entrypoint.sh" "${CONTEXT}/" # OpenRig CLI from the TREE (never the npm registry). ASSEMBLE the publishable @openrig/cli first @@ -57,18 +71,8 @@ STUB_FILES_JSON="$(node -e \ 'const fs=require("fs");const l=fs.readFileSync(process.argv[1],"utf8").split("\n").map(s=>s.replace(/#.*/,"").trim()).filter(Boolean);process.stdout.write(JSON.stringify(l))' \ "${STUB_ASSETS_LIST}")" -# --- resolve the target arch HOST-side and pass it EXPLICITLY (builder-agnostic: correct on the -# legacy builder — which NEVER populates the automatic TARGETARCH build-arg — AND on BuildKit). Fail -# CLOSED on an unknown arch rather than letting the Dockerfile silently default to amd64, which fetches -# x64 Node into an arm64 image and dies with a Rosetta/ELF failure (exit 133). -case "$(uname -m)" in - x86_64|amd64) TARGETARCH=amd64 ;; - arm64|aarch64) TARGETARCH=arm64 ;; - *) echo "[testbed] cannot resolve a supported TARGETARCH from 'uname -m'=$(uname -m); refusing to build (a silent amd64 default installs wrong-arch Node = exit 133)" >&2; exit 4 ;; -esac - # --- build (host-side) --- -docker build \ +docker build --platform "${TARGET_PLATFORM}" \ --build-arg BASE_IMAGE="${BASE_IMAGE}" \ --build-arg NODE_VERSION="${NODE_VERSION}" \ --build-arg OPENRIG_TARBALL=openrig.tgz \ @@ -92,7 +96,23 @@ echo "[testbed] effect proof: daemon LOAD inside the container (better-sqlite3 m # kernel, confirm readiness by hitting /healthz DIRECTLY (deterministic — no fixed sleep), then daemon # status; an EXIT trap stops the daemon so a failed assertion still tears down. A broken native install # fails `rig daemon start` here → set -e → non-zero → the build verb fails BEFORE the A/B pin. -docker run --rm --network none --cap-drop ALL --security-opt no-new-privileges "${IMAGE_TAG}" bash -lc 'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status' +LOAD_CONTAINER="openrig-testbed-load-$(node -p 'require("node:crypto").randomUUID()')" +printf '%s\n' "${LOAD_CONTAINER}" > "${OUT_DIR}/image-load.container-name.txt" +load_status=0 +node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" \ + --platform "${TARGET_PLATFORM}" --network none --cap-drop ALL --security-opt no-new-privileges \ + --cpus 2 --memory 2g --memory-swap 2g --pids-limit 256 "${IMAGE_TAG}" bash -lc \ + 'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status' \ + > "${OUT_DIR}/image-load.log" 2>&1 || load_status=$? +printf '%s\n' "${load_status}" > "${OUT_DIR}/image-load.exit-code.txt" +cat "${OUT_DIR}/image-load.log" >&2 +inspect_status=0 +node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker inspect "${LOAD_CONTAINER}" \ + > "${OUT_DIR}/image-load.container.json" || inspect_status=$? +[ "${load_status}" -eq 0 ] || exit "${load_status}" +[ "${inspect_status}" -eq 0 ] || exit "${inspect_status}" +node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null +LOAD_CONTAINER="" # --- emit the reproducible manifest + census receipt via the tested node orchestrator --- INPUTS="$(mktemp)" diff --git a/scripts/build-testbed-image.test.mjs b/scripts/build-testbed-image.test.mjs index 2cab63d8..e2ab92fe 100644 --- a/scripts/build-testbed-image.test.mjs +++ b/scripts/build-testbed-image.test.mjs @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { readFileSync, writeFileSync, mkdirSync, copyFileSync, mkdtempSync, rmSync, existsSync } from "node:fs"; +import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; @@ -61,11 +62,13 @@ test("Q2 fix A: packs the ASSEMBLED @openrig/cli (has the `rig` bin), NEVER the assert.doesNotMatch(text, /cd\s+"?\$\{REPO_ROOT\}"?\s*&&\s*npm pack/, "must NOT pack the monorepo root"); }); -test("Q2 fix B: resolves the target arch HOST-side + passes it explicitly, fail-CLOSED (never a silent amd64 default -> exit 133)", () => { +test("resolves the target from the Docker server and passes both platform and arch", () => { const text = readScript(); - assert.match(text, /uname -m/, "must resolve the host arch (uname -m) so the legacy builder gets a real TARGETARCH"); - assert.match(text, /--build-arg\s+TARGETARCH=/, "must pass TARGETARCH explicitly (builder-agnostic)"); - assert.match(text, /uname -m[\s\S]*?exit\s+[1-9]/, "must fail-closed (non-zero exit) on an unresolvable arch"); + assert.match(text, /scenario-executor\.mjs.*platform/); + assert.match(text, /--build-arg\s+TARGETARCH=/); + assert.match(text, /docker build --platform/); + assert.match(text, /docker run --name/); + assert.doesNotMatch(text, /case.*uname -m/); }); test("Q2 fix B: the Dockerfile fails CLOSED on an empty TARGETARCH (no silent amd64 default)", () => { @@ -109,3 +112,55 @@ test("Q2 rider (effect proof): the build verb LOADS the daemon inside the contai assert.match(text, /rig daemon start --no-kernel/, "must LOAD the daemon (better-sqlite3 binds) via the operator-corrected start, not merely check rig exists"); assert.match(text, /\/healthz/, "must confirm readiness deterministically via /healthz (operator correction — no fixed sleep)"); }); + +test("image-load success, failure and deadline all retain status and remove only the named container", () => { + const root = mkdtempSync(join(process.cwd(), ".testbed-load-")); + try { + for (const p of ["scripts", "docker/testbed", "packages/cli", "bin"]) mkdirSync(join(root, p), { recursive: true }); + copyFileSync(SCRIPT, join(root, "scripts/build-testbed-image.sh")); + copyFileSync(join(HERE, "scenario-executor.mjs"), join(root, "scripts/scenario-executor.mjs")); + writeFileSync(join(root, "scripts/build-package.sh"), "#!/bin/sh\nexit 0\n"); + writeFileSync(join(root, "scripts/testbed-build-inputs.mjs"), "export const readBaseImage = () => ({ref:'fixture@sha256:abc'});\n"); + writeFileSync(join(root, "scripts/testbed-emit-manifest.mjs"), "import fs from 'node:fs';fs.writeFileSync(process.argv[3]+'/manifest.json','{}');\n"); + writeFileSync(join(root, "docker/testbed/Dockerfile"), "ARG NODE_VERSION=22.22.1\n"); + for (const p of ["docker/testbed/entrypoint.sh", "docker/testbed/base-image", "docker/testbed/stub-assets.list"]) writeFileSync(join(root, p), ""); + writeFileSync(join(root, "bin/git"), "#!/bin/sh\nprintf 'fake-source\\n'\n", { mode: 0o755 }); + writeFileSync(join(root, "bin/npm"), "#!/bin/sh\ntouch fixture.tgz\nprintf 'fixture.tgz\\n'\n", { mode: 0o755 }); + writeFileSync(join(root, "bin/mktemp"), '#!/bin/sh\nexec /usr/bin/mktemp "$@" "$HOME/tmp.XXXXXXXX"\n', { mode: 0o755 }); + // Use the real deadline helper with a short test deadline. No Docker daemon, + // build, package install or network is involved; the marker models a remote + // container surviving the client process until an explicit named removal. + writeFileSync(join(root, "bin/node"), `#!${process.execPath} +import {spawnSync} from 'node:child_process';const args=process.argv.slice(2); +if(args[1]==='timeout')args[2]='0.4'; +const p=spawnSync(${JSON.stringify(process.execPath)},args,{stdio:'inherit'});process.exit(p.status??1); +`, { mode: 0o755 }); + writeFileSync(join(root, "bin/docker"), `#!${process.execPath} +import fs from 'node:fs';const args=process.argv.slice(2);const marker=process.env.FAKE_CONTAINER; +if(args[0]==='version')console.log(JSON.stringify({Os:'linux',Arch:'amd64'})); +else if(args[0]==='build'){} +else if(args[0]==='run'){ + const at=args.indexOf('--name');if(at<0)process.exit(9); + fs.writeFileSync(marker,args[at+1]); + if(process.env.LOAD_CASE==='timeout')setInterval(()=>{},1000); + else process.exit(process.env.LOAD_CASE==='failure'?7:0); +}else if(args[0]==='inspect')console.log('[]'); +else if(args[0]==='rm'){ + if(fs.readFileSync(marker,'utf8')!==args.at(-1))process.exit(8); + fs.unlinkSync(marker); +}else process.exit(8); +`, { mode: 0o755 }); + for (const [mode, status] of [["success", 0], ["failure", 7], ["timeout", 124]]) { + const out = join(root, mode), marker = join(root, "container"); + const run = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], { + env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker }, + encoding: "utf8", timeout: 8000, + }); + assert.equal(run.status, status, `${mode}: ${run.stderr}`); + assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), status); + assert.match(readFileSync(join(out, "image-load.container-name.txt"), "utf8"), /^openrig-testbed-load-/); + assert.equal(existsSync(marker), false, `${mode}: container must be removed even after timeout`); + assert.equal(existsSync(join(out, "manifest.json")), status === 0); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/scripts/pr-scenarios.test.mjs b/scripts/pr-scenarios.test.mjs index 88505de4..7679f200 100644 --- a/scripts/pr-scenarios.test.mjs +++ b/scripts/pr-scenarios.test.mjs @@ -19,6 +19,24 @@ test('accepts a healthy run and a specifically observed post-restart lost baton' verifyRun('healthy', 0, green); verifyRun('lost-baton', 1, red); }); +test('library admission requires its own seed, failing step, and exact destination', () => { + const libraryGreen = { ...green, caseName: 'library', seed: { class: 'baton-drop', enabled: false } }; + const result = { + ...red.result, failedStep: 4, + observation: { surface: 'queue', value: [ + { qitemId: 'baton-1', state: 'pending', destinationSession: 'dev-qa@dev-pair-stub' }, + ] }, + }; + const libraryRed = { ...report('lost-baton', result, red.fault), caseName: 'library', seed: { class: 'baton-drop', enabled: true } }; + verifyRun('healthy', 0, libraryGreen, 'library'); + verifyRun('lost-baton', 1, libraryRed, 'library'); + for (const seed of [undefined, { class: 'typo', enabled: true }, { class: 'baton-drop', enabled: false }]) { + assert.throws(() => verifyRun('lost-baton', 1, { ...libraryRed, seed }, 'library')); + } + assert.throws(() => verifyRun('healthy', 0, libraryGreen)); + assert.throws(() => verifyRun('lost-baton', 1, { ...libraryRed, result: red.result, records: [red.result] }, 'library')); + assert.throws(() => verifyRun('healthy', 0, green, 'unknown')); +}); test('rejects a surviving mutant and a failure before the seeded boundary', () => { assert.throws(() => verifyRun('lost-baton', 0, { ...green, mode: 'lost-baton' })); assert.throws(() => verifyRun('lost-baton', 1, report('lost-baton', { diff --git a/scripts/run-pr-scenarios.sh b/scripts/run-pr-scenarios.sh index 17e9c058..05667ebf 100644 --- a/scripts/run-pr-scenarios.sh +++ b/scripts/run-pr-scenarios.sh @@ -1,12 +1,36 @@ #!/usr/bin/env bash -# Hosted Linux only. All daemon/seat work runs inside the existing testbed. +# All daemon/seat work runs inside the existing disposable testbed. +# Default: GitHub CI. --remote: explicitly selected SSH Docker executor. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" -test "${GITHUB_ACTIONS:-}" = true || { echo 'Use a disposable GitHub runner, not a working host' >&2; exit 2; } -test "$(uname -s)" = Linux +REMOTE=false +CASES=(fixture library) +MODES=(healthy lost-baton healthy) OUT="$REPO_ROOT/dist/pr-scenarios" +while [ "$#" -gt 0 ]; do + case "$1" in + --remote) REMOTE=true; shift ;; + --case) + case "${2:-}" in fixture|library) CASES=("$2");; *) echo 'Expected --case fixture|library' >&2; exit 2;; esac + shift 2 ;; + --mode) + case "${2:-}" in healthy|lost-baton) MODES=("$2");; *) echo 'Expected --mode healthy|lost-baton' >&2; exit 2;; esac + shift 2 ;; + --out) test -n "${2:-}" || exit 2; OUT="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done +if "$REMOTE"; then + case "${DOCKER_HOST:-}" in ssh://?*) ;; *) echo '--remote requires explicit DOCKER_HOST=ssh://...' >&2; exit 2;; esac + test -z "${DOCKER_CONTEXT:-}" || { echo 'Unset DOCKER_CONTEXT so it cannot override the selected DOCKER_HOST' >&2; exit 2; } +else + test "${GITHUB_ACTIONS:-}" = true || { echo 'Use GitHub CI or --remote with the prepared disposable SSH executor' >&2; exit 2; } + test "$(uname -s)" = Linux +fi mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd -P)" +TARGET_PLATFORM="$(node scripts/scenario-executor.mjs platform "$OUT/docker-server.json")" # build-testbed-image performs the stock package build, clean target install, # and daemon-load effect proof, then records the image inputs. It never pushes. @@ -17,41 +41,49 @@ IMAGE="openrig-pr-scenarios:$SHA" CONTEXT="$(mktemp -d)" CONTAINER="" cleanup() { - if [ -n "$CONTAINER" ]; then docker rm -f "$CONTAINER" >/dev/null; fi + if [ -n "$CONTAINER" ]; then + node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2 + fi rm -rf "$CONTEXT" } trap cleanup EXIT cp docker/testbed/Dockerfile.scenarios "$CONTEXT/Dockerfile" cp -R packages/daemon/test/fixtures/scenarios "$CONTEXT/scenarios" +cp -R packages/test-system/scenarios "$CONTEXT/library" # esbuild already ships in the lockfile through tsx. Bundle the existing helper # closure (including YAML) so the container needs no source tree or dev install. node_modules/.bin/esbuild packages/test-system/ci/run.mjs --bundle --platform=node \ --format=esm --banner:js='import { createRequire as nodeRequire } from "node:module"; const require = nodeRequire(import.meta.url);' \ --outfile="$CONTEXT/runner.mjs" --metafile="$OUT/runner-inputs.json" -docker build --network none --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT" +docker build --network none --platform "$TARGET_PLATFORM" --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT" docker image inspect "$IMAGE" > "$OUT/image-inspect.json" attempt=0 -for mode in healthy lost-baton healthy; do +for scenario in "${CASES[@]}"; do +for mode in "${MODES[@]}"; do attempt=$((attempt + 1)) status=0 - LOG="$OUT/$attempt-$mode.log" + LOG="$OUT/$attempt-$scenario-$mode.log" CONTAINER="openrig-pr-${SHA:0:12}-$attempt-$$" + printf '%s\n' "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container-name.txt" # Fresh writable scratch only. No mounts, host networking, credentials or Docker # socket; the container is non-root, resource bounded and removed even on failure. - timeout --signal=TERM --kill-after=15s 300s docker run --name "$CONTAINER" --network none \ + node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --platform "$TARGET_PLATFORM" --network none \ --read-only --tmpfs /tmp:rw,exec,nosuid,nodev,size=512m,mode=1777 \ --cap-drop ALL --security-opt no-new-privileges --pids-limit 256 \ - --memory 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" \ + --memory 2g --memory-swap 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" "$scenario" \ > "$LOG" 2>&1 || status=$? + printf '%s\n' "$status" > "$OUT/$attempt-$scenario-$mode.exit-code.txt" cat "$LOG" - docker rm -f "$CONTAINER" >/dev/null + node scripts/scenario-executor.mjs timeout 30 docker inspect "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container.json" + node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null CONTAINER="" - node --input-type=module - "$mode" "$status" "$LOG" <<'JS' + node --input-type=module - "$mode" "$status" "$LOG" "$scenario" <<'JS' import { readFileSync } from 'node:fs'; import { readReport, verifyRun } from './packages/test-system/ci/result.mjs'; -const [mode, status, log] = process.argv.slice(2); -verifyRun(mode, Number(status), readReport(readFileSync(log, 'utf8'))); -console.log(`${mode}: ${mode === 'healthy' ? 'healthy scenario passed' : 'seeded durability regression caught at the expected assertion'}`); +const [mode, status, log, scenario] = process.argv.slice(2); +verifyRun(mode, Number(status), readReport(readFileSync(log, 'utf8')), scenario); +console.log(`${scenario}/${mode}: ${mode === 'healthy' ? 'healthy scenario passed' : 'seeded durability regression caught at the expected assertion'}`); JS done +done diff --git a/scripts/scenario-executor.mjs b/scripts/scenario-executor.mjs new file mode 100644 index 00000000..c410b8f5 --- /dev/null +++ b/scripts/scenario-executor.mjs @@ -0,0 +1,75 @@ +// Client-side helpers for the existing scenario scripts; never runs a daemon. +import { spawn, spawnSync } from 'node:child_process'; +import { writeFileSync } from 'node:fs'; +import { constants } from 'node:os'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export function dockerPlatform(server) { + if (server?.Os !== 'linux' || !['amd64', 'arm64'].includes(server?.Arch)) { + throw new Error(`Unsupported Docker server: ${server?.Os}/${server?.Arch}`); + } + return `linux/${server.Arch}`; +} + +export async function runWithDeadline(argv, { timeoutMs, killAfterMs = 15000 }) { + if (!argv.length || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || killAfterMs <= 0) { + throw new Error('Command and positive finite deadline required'); + } + return new Promise(resolveResult => { + const child = spawn(argv[0], argv.slice(1), { stdio: 'inherit', detached: true }); + let forced, escalation, finished = false; + const signalChild = signal => { + if (!child.pid) return; + try { process.kill(-child.pid, signal); } catch (error) { + if (error.code !== 'ESRCH') throw error; + } + }; + const terminate = code => { + if (forced !== undefined) return; + forced = code; + signalChild('SIGTERM'); + escalation = setTimeout(() => signalChild('SIGKILL'), killAfterMs); + }; + const timeout = setTimeout(() => terminate(124), timeoutMs); + const interrupt = () => terminate(130); + const shutdown = () => terminate(143); + process.once('SIGINT', interrupt); + process.once('SIGTERM', shutdown); + const finish = code => { + if (finished) return; + finished = true; + if (forced !== undefined) signalChild('SIGKILL'); + clearTimeout(timeout); clearTimeout(escalation); + process.removeListener('SIGINT', interrupt); + process.removeListener('SIGTERM', shutdown); + resolveResult(forced ?? code); + }; + child.once('error', error => { + console.error(`Cannot run ${argv[0]}: ${error.message}`); + finish(127); + }); + child.once('close', (code, signal) => finish(code ?? (128 + (constants.signals[signal] ?? 0)))); + }); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const [operation, ...args] = process.argv.slice(2); + if (operation === 'platform') { + const version = spawnSync('docker', ['version', '--format', '{{json .Server}}'], { encoding: 'utf8', timeout: 30000 }); + if (version.error || version.status !== 0) throw new Error(version.error?.message ?? version.stderr); + const server = JSON.parse(version.stdout); + const platform = dockerPlatform(server); + if (args[0]) writeFileSync(args[0], JSON.stringify({ platform, server }, null, 2) + '\n'); + console.log(platform); + } else if (operation === 'timeout') { + process.exitCode = await runWithDeadline(args.slice(1), { timeoutMs: Number(args[0]) * 1000 }); + } else { + throw new Error('Expected platform [receipt-path] or timeout [args...]'); + } + } catch (error) { + console.error(error.message); + process.exitCode = 2; + } +} diff --git a/scripts/scenario-executor.test.mjs b/scripts/scenario-executor.test.mjs new file mode 100644 index 00000000..869ed451 --- /dev/null +++ b/scripts/scenario-executor.test.mjs @@ -0,0 +1,60 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { dockerPlatform, runWithDeadline } from './scenario-executor.mjs'; +import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +test('target follows the Docker server, not this client platform', () => { + assert.equal(dockerPlatform({ Os: 'linux', Arch: 'amd64' }), 'linux/amd64'); + assert.equal(dockerPlatform({ Os: 'linux', Arch: 'arm64' }), 'linux/arm64'); + for (const server of [undefined, {}, { Os: 'darwin', Arch: 'arm64' }, { Os: 'linux', Arch: 'mips' }]) { + assert.throws(() => dockerPlatform(server), /Unsupported Docker server/); + } +}); +test('deadline wrapper preserves actual successful and nonzero exits', async () => { + assert.equal(await runWithDeadline([process.execPath, '-e', 'process.exit(0)'], { timeoutMs: 3000 }), 0); + assert.equal(await runWithDeadline([process.execPath, '-e', 'process.exit(7)'], { timeoutMs: 3000 }), 7); +}); +test('deadline expires even when the child ignores TERM', async () => { + const start = Date.now(); + const status = await runWithDeadline([process.execPath, '-e', "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"], { timeoutMs: 300, killAfterMs: 100 }); + assert.equal(status, 124); + assert.ok(Date.now() - start < 3000); +}); +test('a missing executable and an invalid deadline cannot report success', async () => { + assert.equal(await runWithDeadline(['/nonexistent/openrig-test-program'], { timeoutMs: 1000 }), 127); + await assert.rejects(runWithDeadline([process.execPath], { timeoutMs: 0 }), /positive/); +}); + +test('the platform CLI queries a fake remote server and writes its actual receipt', () => { + const root = mkdtempSync(join(process.cwd(), '.scenario-platform-')); + try { + const server = { Os: 'linux', Arch: 'amd64', Version: 'test-only' }; + writeFileSync(join(root, 'docker'), `#!/bin/sh +printf '%s\\n' '${JSON.stringify(server)}' +`, { mode: 0o755 }); + const receipt = join(root, 'server.json'); + const run = spawnSync(process.execPath, [resolve('scripts/scenario-executor.mjs'), 'platform', receipt], { + env: { ...process.env, PATH: root + ':/usr/bin:/bin', DOCKER_HOST: 'ssh://fixture.invalid' }, encoding: 'utf8', + }); + assert.equal(run.status, 0, run.stderr); + assert.equal(run.stdout.trim(), 'linux/amd64'); + assert.deepEqual(JSON.parse(readFileSync(receipt, 'utf8')), { platform: 'linux/amd64', server }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); +test('ordinary invocation and ambiguous remote selection refuse before any build', () => { + const script = resolve('scripts/run-pr-scenarios.sh'); + for (const [args, extra, message] of [ + [[], {}, /Use GitHub CI or --remote/], + [['--remote'], {}, /requires explicit DOCKER_HOST/], + [['--remote'], { DOCKER_HOST: 'ssh://fixture.invalid', DOCKER_CONTEXT: 'other' }, /Unset DOCKER_CONTEXT/], + [['--remote', '--case', 'typo'], {}, /Expected --case/], + [['--remote', '--mode', 'typo'], {}, /Expected --mode/], + ]) { + const env = { ...process.env, GITHUB_ACTIONS: '', DOCKER_HOST: '', DOCKER_CONTEXT: '', ...extra }; + const run = spawnSync('/bin/bash', [script, ...args], { env, encoding: 'utf8', timeout: 3000 }); + assert.equal(run.status, 2, run.stderr); + assert.match(run.stderr, message); + } +});