import assert from "node:assert/strict"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { AGENT_PROVIDERS, type AgentEvent } from "@openbot/contracts/ipc"; import { Effect } from "effect"; import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle"; import { getString, isRecord } from "../src/backend/protocol"; const OUT = resolve(import.meta.dirname, "../.openbot-build/codex-compaction-queue-e2e"); const root = await mkdtemp(join(tmpdir(), "openbot-compaction-queue-")); const userData = join(root, "user-data"); const home = join(root, "home"); const logPath = join(root, "protocol.jsonl"); const controlPath = join(root, "control.json"); const memorySaved = join(root, "memory-saved"); const fakeCodex = join(root, "codex"); const COMPACTION_TIMEOUT_MS = 1_000; process.env.HOME = home; process.env.CODEX_HOME = join(home, ".codex"); process.env.OPENBOT_CODEX_PATH = fakeCodex; process.env.OPENBOT_FAKE_CODEX_LOG = logPath; process.env.OPENBOT_FAKE_CODEX_CONTROL = controlPath; mkdirSync(home, { recursive: true }); const { AgentService } = await import("../src/backend/agent-service"); const { USAGE_CHANGE_DELAY_MS } = await import("../src/backend/agent/context-compaction"); const { AgentStore } = await import("../src/backend/agent-store"); const { runCauseEffect } = await import("../src/backend/effect-boundary"); const { MailboxStore } = await import("../src/backend/mailbox-store"); const { OpenBotDatabase } = await import("../src/backend/openbot-database"); const { StoredStateFailure } = await import("../src/backend/stored-state-effects"); const FAKE_CODEX = `#!/usr/bin/env node const fs = require("node:fs"); if (process.argv.includes("--version")) { process.stdout.write("codex-cli 0.160.1\\n"); process.exit(0); } const log = (entry) => fs.appendFileSync(process.env.OPENBOT_FAKE_CODEX_LOG, JSON.stringify(entry) + "\\n"); const control = () => JSON.parse(fs.readFileSync(process.env.OPENBOT_FAKE_CODEX_CONTROL, "utf8")); const memorySaved = ${JSON.stringify(memorySaved)}; const config = { config: { mcp_servers: {} }, layers: [{ name: { type: "user", file: "/fake/.codex/config.toml", profile: null }, version: "1", config: { mcp_servers: {} } }] }; let buffer = ""; let threads = 0; let turns = 0; const threadTurns = new Map(); const compacting = new Map(); process.stdout.on("error", (error) => { if (error.code === "EPIPE") process.exit(0); throw error; }); const write = (message) => process.stdout.write(JSON.stringify(message) + "\\n"); process.on("SIGTERM", () => { if (!control().usageOnStop) process.exit(0); for (let thread = 1; thread <= threads; thread++) { const tokenUsage = { total: { totalTokens: 95000 }, last: { totalTokens: 95000 }, modelContextWindow: 100000 }; write({ method: "thread/tokenUsage/updated", params: { threadId: "thread-" + thread, turnId: "turn-stop", tokenUsage } }); } log({ usageOnStop: threads }); process.stdout.write("", () => process.exit(0)); }); const answer = (message) => { const params = message.params || {}; switch (message.method) { case "initialize": return write({ id: message.id, result: {} }); case "account/read": return write({ id: message.id, result: { account: { type: "chatgpt", email: "codex@example.com" } } }); case "account/rateLimits/read": return write({ id: message.id, result: { rateLimits: { limitId: "codex", primary: null, secondary: null }, rateLimitsByLimitId: null } }); case "model/list": return write({ id: message.id, result: { data: [{ model: "gpt-5.5", displayName: "GPT-5.5" }] } }); case "plugin/list": return write({ id: message.id, result: { marketplaces: [] } }); case "config/read": return write({ id: message.id, result: config }); case "config/batchWrite": return write({ id: message.id, result: {} }); case "thread/start": return write({ id: message.id, result: { thread: { id: "thread-" + (++threads), turns: [] } } }); case "thread/resume": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } }); case "thread/unsubscribe": return write({ id: message.id, result: { status: "Unsubscribed" } }); case "thread/turns/list": case "thread/items/list": return write({ id: message.id, result: { data: [], nextCursor: null } }); case "thread/read": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } }); case "turn/start": { if (compacting.has(params.threadId)) { log({ refused: params.threadId }); return write({ id: message.id, error: { code: -32603, message: "failed to submit turn input: ActiveTurnNotSteerable { turn_kind: Compact }" } }); } const turnId = "turn-" + (++turns); const { editMemory } = control(); const memoryWasEdited = editMemory && fs.existsSync(memorySaved); const usedTokens = memoryWasEdited ? 16000 : 90000; write({ id: message.id, result: { turn: { id: turnId, status: "inProgress", items: [] } } }); write({ method: "turn/started", params: { threadId: params.threadId, turn: { id: turnId } } }); const count = (threadTurns.get(params.threadId) ?? 0) + 1; threadTurns.set(params.threadId, count); const used = control().growUsage ? 84000 + 6000 * (count - 1) : usedTokens; write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage: { total: { totalTokens: used }, last: { totalTokens: used }, modelContextWindow: 100000 } } }); const finish = () => { const item = { type: "agentMessage", id: "message-" + turnId, text: "Done.", phase: "final_answer" }; write({ method: "item/completed", params: { threadId: params.threadId, turnId, item } }); write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: turnId, status: "completed" } } }); }; if (editMemory && !memoryWasEdited) { const deadline = Date.now() + 10_000; const timer = setInterval(() => { if (fs.existsSync(memorySaved)) { clearInterval(timer); finish(); } else if (Date.now() >= deadline) { clearInterval(timer); log({ fixtureError: "Fake Codex timed out waiting for the test to save agent memory." }); } }, 20); } else setTimeout(finish, 50); return; } case "thread/compact/start": { const { compactionMs, ignoreInterrupt, compactionStatus, editMemory, estimate } = control(); const turnId = "compact-turn-" + (++turns); const item = { type: "contextCompaction", id: "compact-item-" + turnId }; write({ id: message.id, result: {} }); write({ method: "turn/started", params: { threadId: params.threadId, turn: { id: turnId } } }); write({ method: "item/started", params: { threadId: params.threadId, turnId, item } }); const finish = (status) => { if (compacting.get(params.threadId)?.turnId !== turnId) return; compacting.delete(params.threadId); clearTimeout(timer); log({ compactionEnded: turnId, threadId: params.threadId, status }); if (status === "completed" && editMemory) { const last = { totalTokens: 16000, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningOutputTokens: 0 }; const tokenUsage = { total: { totalTokens: 90000 }, last, modelContextWindow: 100000 }; write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage } }); } if (status === "completed" && estimate) { const last = { totalTokens: 6000, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningOutputTokens: 0 }; const tokenUsage = { total: { totalTokens: 90000 }, last, modelContextWindow: 100000 }; write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage } }); } if (status === "completed") write({ method: "item/completed", params: { threadId: params.threadId, turnId, item } }); write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: turnId, status } } }); }; const timer = setTimeout(() => finish(compactionStatus ?? "completed"), compactionMs); compacting.set(params.threadId, { turnId, finish, ignoreInterrupt }); return; } case "turn/interrupt": { write({ id: message.id, result: {} }); const compaction = compacting.get(params.threadId); if (compaction?.turnId === params.turnId) { if (!compaction.ignoreInterrupt) compaction.finish("interrupted"); return; } return write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: params.turnId, status: "interrupted" } } }); } default: return write({ id: message.id, error: { code: -32601, message: "Method not found: " + message.method } }); } }; process.stdin.setEncoding("utf8"); process.stdin.on("data", (chunk) => { buffer += chunk; let newline = buffer.indexOf("\\n"); while (newline >= 0) { const line = buffer.slice(0, newline); buffer = buffer.slice(newline + 1); if (line) { const message = JSON.parse(line); log({ method: message.method ?? null, id: message.id ?? null, params: message.params ?? null }); if (message.method && message.id !== undefined) answer(message); } newline = buffer.indexOf("\\n"); } }); `; interface ProtocolEntry { method: string | null; threadId: string | null; turnId: string | null; refused: string | null; compactionEnded: string | null; fixtureError: string | null; status: string | null; usageOnStop: number | null; } interface CaseReport { name: string; compactionMs: number; ignoreInterrupt: boolean; threadId: string; statusWhileCompacting: string; finalStatus: string; finalError: string | null; compactionEnd: string | null; interruptSent: boolean; refusedStarts: number; passed: boolean; } const errors: { code: string; agentId?: string }[] = []; const report: { compactionTimeoutMs: number; cases: CaseReport[]; passed: boolean; error?: string } = { compactionTimeoutMs: COMPACTION_TIMEOUT_MS, cases: [], passed: false, }; const unused = () => Effect.die(new Error("Not used by this check.")); const browser: AgentBrowserHost = { listTabs: () => [], toolCanUseTab: () => false, beginTakeover: unused, endTakeover: () => {}, resolveUploadTarget: unused, handleDynamicTool: unused, userControlTabFor: () => null, awaitUserControlRelease: unused, onChanged: () => () => {}, onControlChanged: () => () => {}, onDocumentChanged: () => () => {}, clearControls: () => {}, endControl: () => {}, close: () => Effect.void, clearAgentSession: () => Effect.void, shareTab: () => undefined, unshareThread: () => undefined, agentTab: () => null, }; function protocol(): ProtocolEntry[] { try { return readFileSync(logPath, "utf8") .split("\n") .filter(Boolean) .map((line) => protocolEntry(JSON.parse(line))); } catch { return []; } } function protocolEntry(value: unknown): ProtocolEntry { const entry = isRecord(value) ? value : {}; const params = isRecord(entry.params) ? entry.params : {}; return { method: getString(entry, "method"), threadId: getString(params, "threadId") ?? getString(entry, "threadId"), turnId: getString(params, "turnId"), refused: getString(entry, "refused"), compactionEnded: getString(entry, "compactionEnded"), fixtureError: getString(entry, "fixtureError"), status: getString(entry, "status"), usageOnStop: typeof entry.usageOnStop === "number" ? entry.usageOnStop : null, }; } async function waitFor(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise { const deadline = Date.now() + timeoutMs; for (;;) { const value = read(); if (value) return value; if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`); await new Promise((done) => setTimeout(done, 25)); } } await writeFile(fakeCodex, FAKE_CODEX, { mode: 0o755 }); const database = new OpenBotDatabase(userData); const store = new AgentStore(userData, home, database); const mailbox = new MailboxStore(userData, store.sharedRoot, store.database); const agents = new AgentService({ store, mailbox, browser, preferredProvider: "codex", offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "codex"), contextCompactionTimeoutMs: COMPACTION_TIMEOUT_MS, }); let stopped = false; agents.on("event", (event: AgentEvent) => { if (event.type === "error") errors.push({ code: event.code, ...(event.agentId ? { agentId: event.agentId } : {}) }); }); try { await runCauseEffect(agents.initialize()); const cases = [ { name: "compaction ends after a memory edit", compactionMs: 400, ignoreInterrupt: false, editMemory: true, end: "completed", interrupt: false, }, { name: "compaction runs past the start deadline", compactionMs: 3_000, ignoreInterrupt: false, end: "completed", interrupt: false, }, { name: "deadline stops the compaction", compactionMs: 60_000, ignoreInterrupt: false, end: "interrupted", interrupt: true, }, { name: "refused start waits for the compaction", compactionMs: 12_000, ignoreInterrupt: true, end: "completed", interrupt: true, }, ] as const; for (const [index, entry] of cases.entries()) { await writeFile(controlPath, JSON.stringify(entry)); const agent = await runCauseEffect( agents.createAgent({ name: `Compaction ${index + 1}`, description: "Checks the queue during a compaction.", avatarSeed: `compaction-${index}`, avatarHue: null, initialMessage: "First message.", }), ); if ("editMemory" in entry) { await waitFor("the first usage report", () => agents .getRuntimeSnapshot() .contextStates?.some((state) => state.agentId === agent.id && state.usage?.usedTokens === 90000), ); agents.createMemory({ agentId: agent.id, text: "A test memory." }); await writeFile(memorySaved, ""); } const threadId = await waitFor("the first turn to end", () => { const session = store.activeProviderSession(agent.id)?.externalSessionId; const entries = protocol(); const fixtureError = entries.find((line) => line.fixtureError)?.fixtureError; if (fixtureError) throw new Error(fixtureError); const started = entries.some((line) => line.method === "turn/start" && line.threadId === session); const busy = agents .listQueue(agent.id) .deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status)); return session && started && !busy ? session : null; }); const receipt = await runCauseEffect(agents.sendMessage({ agentId: agent.id, text: "Second message." })); const deliveryId = receipt.deliveries[0]?.id; assert.ok(deliveryId, `${entry.name}: no delivery`); const compactionTurn = await waitFor("the compaction to start", () => protocol().find((line) => line.method === "thread/compact/start" && line.threadId === threadId), ); assert.ok(compactionTurn); const statusWhileCompacting = mailbox.getDelivery(deliveryId)?.delivery.status ?? "missing"; if ("editMemory" in entry) { const next = await waitFor( "the next turn after compaction", () => protocol().filter( (line) => line.threadId === threadId && (line.method === "turn/start" || line.method === "thread/compact/start"), )[2], ); assert.equal( next.method, "turn/start", "memory edit: the queue started another compaction instead of the message", ); assert.ok( protocol().some((line) => line.method === "thread/resume" && line.threadId === threadId), "memory edit: the session was not marked for reload", ); } const finalStatus = await waitFor("the second message to end", () => { const status = mailbox.getDelivery(deliveryId)?.delivery.status; return status === "completed" || status === "failed" || status === "interrupted" ? status : null; }); const entries = protocol(); const threadEntries = entries.filter((line) => line.threadId === threadId || line.refused === threadId); const result: CaseReport = { name: entry.name, compactionMs: entry.compactionMs, ignoreInterrupt: entry.ignoreInterrupt, threadId, statusWhileCompacting, finalStatus, finalError: mailbox.getDelivery(deliveryId)?.delivery.error ?? null, compactionEnd: entries.find((line) => line.compactionEnded && line.threadId === threadId)?.status ?? null, interruptSent: threadEntries.some( (line) => line.method === "turn/interrupt" && line.turnId?.startsWith("compact-turn-"), ), refusedStarts: threadEntries.filter((line) => line.refused === threadId).length, passed: false, }; report.cases.push(result); assert.equal(result.statusWhileCompacting, "queued", `${entry.name}: the message did not wait`); assert.equal(result.finalStatus, "completed", `${entry.name}: ${result.finalError ?? result.finalStatus}`); assert.equal(result.compactionEnd, entry.end, `${entry.name}: the compaction did not end as expected`); assert.equal(result.interruptSent, entry.interrupt, `${entry.name}: turn/interrupt`); if (entry.ignoreInterrupt) assert.ok(result.refusedStarts > 0, "Codex refused no start"); else assert.equal(result.refusedStarts, 0, `${entry.name}: Codex refused a start`); result.passed = true; } await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false })); const updatePreview = store.updatePreview; let previewFailures = 0; store.updatePreview = (id, preview) => { if (preview !== "Done." || previewFailures > 0) return updatePreview(id, preview); previewFailures += 1; return Effect.fail(new StoredStateFailure({ cause: new Error("disk I/O error") })); }; const held = await runCauseEffect( agents.createAgent({ name: "Failed turn end", description: "Checks the queue after a turn end that failed.", avatarSeed: "compaction-failed-end", avatarHue: null, initialMessage: "First message.", }), ); await waitFor("the failed turn end", () => { const busy = agents .listQueue(held.id) .deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status)); return previewFailures === 1 && !busy; }); store.updatePreview = updatePreview; const heldReceipt = await runCauseEffect(agents.sendMessage({ agentId: held.id, text: "Second message." })); const heldDeliveryId = heldReceipt.deliveries[0]?.id; assert.ok(heldDeliveryId, "failed turn end: no delivery"); const heldStatus = () => mailbox.getDelivery(heldDeliveryId)?.delivery.status ?? "missing"; const failedTurnEnd = { previewFailures, nextMessageStatus: await waitFor( "the next message after the failed turn end", () => (["completed", "failed", "interrupted"].includes(heldStatus()) ? heldStatus() : null), 30_000, ).catch(() => heldStatus()), }; Object.assign(report, { failedTurnEnd }); assert.equal(failedTurnEnd.nextMessageStatus, "completed", "failed turn end: the compaction hold stayed"); await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false })); const owner = await runCauseEffect( agents.createAgent({ name: "Side thread", description: "Checks the context of a side thread.", avatarSeed: "compaction-thread", avatarHue: null, initialMessage: "First message.", }), ); const mainSession = await waitFor("the agent's first turn to end", () => { const session = store.activeProviderSession(owner.id)?.externalSessionId; const busy = agents.listQueue(owner.id).deliveries.some((delivery) => delivery.status !== "completed"); return session && !busy ? session : null; }); const mainContext = () => agents.getRuntimeSnapshot().contextStates?.find((state) => state.agentId === owner.id) ?? null; const mainCompactStarts = () => protocol().filter((line) => line.method === "thread/compact/start" && line.threadId === mainSession).length; const mainBefore = await waitFor( "the agent's own compaction", () => mainContext()?.compaction?.status === "completed" && mainContext(), ); const mainStartsBefore = mainCompactStarts(); const task = await runCauseEffect(agents.startTask({ agentId: owner.id, text: "Thread message." })); const taskSession = await waitFor( "the thread session", () => store.database.activeProviderSession(task.threadId, "codex")?.externalSessionId, ); const threadContext = () => agents.getRuntimeSnapshot().threadContextStates?.find((state) => state.threadId === task.threadId) ?? null; const compactions = () => protocol().filter((line) => line.compactionEnded && line.threadId === taskSession); const automatic = await waitFor("the thread's automatic compaction", () => { const state = threadContext(); const idle = agents.listTasks(owner.id).tasks.find((candidate) => candidate.id === task.id)?.state === "idle"; return idle && compactions().length === 1 && state?.compaction?.status === "completed" ? state : null; }); await runCauseEffect(agents.compactTaskContext({ agentId: owner.id, taskId: task.id })); const manual = await waitFor("the thread's manual compaction", () => { const state = threadContext(); return compactions().length === 2 && state?.compaction?.status === "completed" ? state : null; }); const mainAfter = mainContext(); const sideThread = { threadUsage: automatic.usage, automaticCompaction: automatic.compaction?.id ?? null, manualCompaction: manual.compaction?.id ?? null, threadCompactions: compactions().map((line) => line.status), mainCompactStarts: mainCompactStarts() - mainStartsBefore, mainContextUnchanged: JSON.stringify(mainAfter) === JSON.stringify(mainBefore), }; Object.assign(report, { sideThread }); assert.equal(sideThread.threadUsage?.usedTokens, 90_000, "side thread: no usage"); assert.notEqual(sideThread.manualCompaction, sideThread.automaticCompaction, "side thread: no new compaction"); assert.deepEqual(sideThread.threadCompactions, ["completed", "completed"], "side thread: compactions"); assert.equal(sideThread.mainCompactStarts, 0, "side thread: the agent's own chat compacted"); assert.ok(sideThread.mainContextUnchanged, "side thread: the agent's own context changed"); await writeFile(controlPath, JSON.stringify({ compactionMs: 5_000, ignoreInterrupt: false })); const deleted = await runCauseEffect(agents.startTask({ agentId: owner.id, text: "Thread message." })); const deletedSession = await waitFor( "the deleted thread's session", () => store.database.activeProviderSession(deleted.threadId, "codex")?.externalSessionId, ); await waitFor("the deleted thread's compaction", () => protocol().some((line) => line.method === "thread/compact/start" && line.threadId === deletedSession), ); await runCauseEffect(agents.deleteTask({ agentId: owner.id, taskId: deleted.id })); const ownReceipt = await runCauseEffect(agents.sendMessage({ agentId: owner.id, text: "Own chat message." })); const ownDeliveryId = ownReceipt.deliveries[0]?.id; assert.ok(ownDeliveryId, "deleted thread: no delivery"); const ownStatus = () => mailbox.getDelivery(ownDeliveryId)?.delivery.status ?? "missing"; const deletedThread = { ownChatStatus: await waitFor( "the own chat message after the delete", () => (["completed", "failed", "interrupted"].includes(ownStatus()) ? ownStatus() : null), 30_000, ).catch(() => ownStatus()), }; Object.assign(report, { deletedThread }); assert.equal(deletedThread.ownChatStatus, "completed", "deleted thread: the agent's own chat stayed held"); await writeFile( controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, compactionStatus: "failed", growUsage: true }), ); const failing = await runCauseEffect( agents.createAgent({ name: "Failed compactions", description: "Checks that failed compactions stop.", avatarSeed: "compaction-failing", avatarHue: null, initialMessage: "First message.", }), ); const failingIdle = () => !agents .listQueue(failing.id) .deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status)); const failingSession = await waitFor("the first turn of the failing chat", () => { const session = store.activeProviderSession(failing.id)?.externalSessionId; return session && failingIdle() ? session : null; }); const failedEnds = () => protocol().filter((line) => line.compactionEnded && line.threadId === failingSession && line.status === "failed"); for (const text of ["Second message.", "Third message.", "Fourth message."]) { await waitFor( "the failing chat to settle", () => failingIdle() && !agents .getRuntimeSnapshot() .contextStates?.some((state) => state.agentId === failing.id && state.compaction?.status === "running"), ); await runCauseEffect(agents.sendMessage({ agentId: failing.id, text })); await waitFor("the message to end", failingIdle); } const failedCompactions = { compactStarts: protocol().filter( (line) => line.method === "thread/compact/start" && line.threadId === failingSession, ).length, failedEnds: failedEnds().length, turns: protocol().filter((line) => line.method === "turn/start" && line.threadId === failingSession).length, stoppedNotices: errors.filter( (error) => error.code === "context_compaction_stopped" && error.agentId === failing.id, ).length, deliveries: agents.listQueue(failing.id).deliveries.map((delivery) => delivery.status), }; Object.assign(report, { failedCompactions }); assert.equal(failedCompactions.turns, 4, "failed compactions: not every message ran"); assert.equal(failedCompactions.compactStarts, 2, "failed compactions: OpenBot did not stop after 2 failures"); assert.equal(failedCompactions.failedEnds, 2, "failed compactions: the compactions did not fail"); assert.equal(failedCompactions.stoppedNotices, 1, "failed compactions: one notice"); await writeFile( controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, growUsage: true, estimate: true }), ); const full = await runCauseEffect( agents.createAgent({ name: "Full after compaction", description: "Checks that compactions that free no space stop.", avatarSeed: "compaction-full", avatarHue: null, initialMessage: "First message.", }), ); const fullIdle = () => !agents .listQueue(full.id) .deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status)); const fullSession = await waitFor("the first turn of the full chat", () => { const session = store.activeProviderSession(full.id)?.externalSessionId; return session && fullIdle() ? session : null; }); for (const text of ["Second message.", "Third message.", "Fourth message."]) { await waitFor( "the full chat to settle", () => fullIdle() && !agents .getRuntimeSnapshot() .contextStates?.some((state) => state.agentId === full.id && state.compaction?.status === "running"), ); await runCauseEffect(agents.sendMessage({ agentId: full.id, text })); await waitFor("the message to end", fullIdle); } const ineffectiveCompactions = { compactStarts: protocol().filter((line) => line.method === "thread/compact/start" && line.threadId === fullSession) .length, turns: protocol().filter((line) => line.method === "turn/start" && line.threadId === fullSession).length, notices: errors.filter((error) => error.code === "context_compaction_ineffective" && error.agentId === full.id) .length, deliveries: agents.listQueue(full.id).deliveries.map((delivery) => delivery.status), }; Object.assign(report, { ineffectiveCompactions }); assert.equal(ineffectiveCompactions.turns, 4, "ineffective compactions: not every message ran"); assert.equal(ineffectiveCompactions.compactStarts, 2, "ineffective compactions: OpenBot did not stop after 2"); assert.equal(ineffectiveCompactions.notices, 1, "ineffective compactions: one notice"); await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, usageOnStop: true })); const lateErrors: string[] = []; const collect = (error: unknown) => lateErrors.push(error instanceof Error ? error.message : String(error)); process.on("uncaughtException", collect); await runCauseEffect(agents.stop()); database.close(); stopped = true; await new Promise((done) => setTimeout(done, USAGE_CHANGE_DELAY_MS)); process.off("uncaughtException", collect); const usageOnStop = { reportedThreads: protocol().find((line) => line.usageOnStop !== null)?.usageOnStop ?? 0, lateErrors, }; Object.assign(report, { usageOnStop }); assert.ok(usageOnStop.reportedThreads > 0, "stop: Codex reported no usage"); assert.deepEqual(usageOnStop.lateErrors, [], "stop: a late usage report used the closed database"); report.passed = true; } catch (error) { report.error = error instanceof Error ? (error.stack ?? error.message) : String(error); process.exitCode = 1; } finally { if (!stopped) { await runCauseEffect(agents.stop()).catch(() => {}); database.close(); } mkdirSync(OUT, { recursive: true }); writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`); await rm(root, { recursive: true, force: true }); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); }