Files
openbot/scripts/codex-compaction-queue-e2e.ts
NB 6fbd8bfc20 Let the main chat and side threads of an agent use each other's browser tabs (#13433)
* Let every chat of an agent use the browser tabs of the agent

Main chats and side threads of an agent can now use all browser tabs of
that agent. list_tabs shows the chat that opened each tab and tells when
another chat uses a tab now. Takeover, submit_secret and list_logins use
the same rule. Channel and messaging threads still use only their own
tabs and the tabs shared with an @ tag.

* Keep channel tabs out of other chats and make the working hint reliable

The main chat and side threads use only the tabs that the main chat, a
side thread or no chat opened. Tabs of a channel or a messaging thread
stay out of these chats unless a person tags them with @.

otherChatWorking now also uses the last chat that used each tab in the
last 2 minutes, because control sessions end 1.2 s after a call.

A takeover from one chat no longer deletes the held takeover result of
another chat. list_tabs looks up each chat once per call.

The browser user-control E2E now covers the cross-chat access.

* Keep a held takeover result for each chat of a tab

Held takeover results used only the tab id as the key. When two chats of
one agent took over the same tab, the second result replaced the first.
The key is now the tab id and the thread id.
2026-11-20 13:51:50 +01:00

656 lines
30 KiB
TypeScript

import assert from "node:assert/strict";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { AGENT_PROVIDERS, type AgentEvent } from "@openbot/contracts/ipc";
import { Effect } from "effect";
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
import { getString, isRecord } from "../src/backend/protocol";
const OUT = resolve(import.meta.dirname, "../.openbot-build/codex-compaction-queue-e2e");
const root = await mkdtemp(join(tmpdir(), "openbot-compaction-queue-"));
const userData = join(root, "user-data");
const home = join(root, "home");
const logPath = join(root, "protocol.jsonl");
const controlPath = join(root, "control.json");
const memorySaved = join(root, "memory-saved");
const fakeCodex = join(root, "codex");
const COMPACTION_TIMEOUT_MS = 1_000;
process.env.HOME = home;
process.env.CODEX_HOME = join(home, ".codex");
process.env.OPENBOT_CODEX_PATH = fakeCodex;
process.env.OPENBOT_FAKE_CODEX_LOG = logPath;
process.env.OPENBOT_FAKE_CODEX_CONTROL = controlPath;
mkdirSync(home, { recursive: true });
const { AgentService } = await import("../src/backend/agent-service");
const { USAGE_CHANGE_DELAY_MS } = await import("../src/backend/agent/context-compaction");
const { AgentStore } = await import("../src/backend/agent-store");
const { runCauseEffect } = await import("../src/backend/effect-boundary");
const { MailboxStore } = await import("../src/backend/mailbox-store");
const { OpenBotDatabase } = await import("../src/backend/openbot-database");
const { StoredStateFailure } = await import("../src/backend/stored-state-effects");
const FAKE_CODEX = `#!/usr/bin/env node
const fs = require("node:fs");
if (process.argv.includes("--version")) {
process.stdout.write("codex-cli 0.160.1\\n");
process.exit(0);
}
const log = (entry) => fs.appendFileSync(process.env.OPENBOT_FAKE_CODEX_LOG, JSON.stringify(entry) + "\\n");
const control = () => JSON.parse(fs.readFileSync(process.env.OPENBOT_FAKE_CODEX_CONTROL, "utf8"));
const memorySaved = ${JSON.stringify(memorySaved)};
const config = { config: { mcp_servers: {} }, layers: [{ name: { type: "user", file: "/fake/.codex/config.toml", profile: null }, version: "1", config: { mcp_servers: {} } }] };
let buffer = "";
let threads = 0;
let turns = 0;
const threadTurns = new Map();
const compacting = new Map();
process.stdout.on("error", (error) => {
if (error.code === "EPIPE") process.exit(0);
throw error;
});
const write = (message) => process.stdout.write(JSON.stringify(message) + "\\n");
process.on("SIGTERM", () => {
if (!control().usageOnStop) process.exit(0);
for (let thread = 1; thread <= threads; thread++) {
const tokenUsage = { total: { totalTokens: 95000 }, last: { totalTokens: 95000 }, modelContextWindow: 100000 };
write({ method: "thread/tokenUsage/updated", params: { threadId: "thread-" + thread, turnId: "turn-stop", tokenUsage } });
}
log({ usageOnStop: threads });
process.stdout.write("", () => process.exit(0));
});
const answer = (message) => {
const params = message.params || {};
switch (message.method) {
case "initialize": return write({ id: message.id, result: {} });
case "account/read": return write({ id: message.id, result: { account: { type: "chatgpt", email: "codex@example.com" } } });
case "account/rateLimits/read": return write({ id: message.id, result: { rateLimits: { limitId: "codex", primary: null, secondary: null }, rateLimitsByLimitId: null } });
case "model/list": return write({ id: message.id, result: { data: [{ model: "gpt-5.5", displayName: "GPT-5.5" }] } });
case "plugin/list": return write({ id: message.id, result: { marketplaces: [] } });
case "config/read": return write({ id: message.id, result: config });
case "config/batchWrite": return write({ id: message.id, result: {} });
case "thread/start": return write({ id: message.id, result: { thread: { id: "thread-" + (++threads), turns: [] } } });
case "thread/resume": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } });
case "thread/unsubscribe": return write({ id: message.id, result: { status: "Unsubscribed" } });
case "thread/turns/list":
case "thread/items/list": return write({ id: message.id, result: { data: [], nextCursor: null } });
case "thread/read": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } });
case "turn/start": {
if (compacting.has(params.threadId)) {
log({ refused: params.threadId });
return write({ id: message.id, error: { code: -32603, message: "failed to submit turn input: ActiveTurnNotSteerable { turn_kind: Compact }" } });
}
const turnId = "turn-" + (++turns);
const { editMemory } = control();
const memoryWasEdited = editMemory && fs.existsSync(memorySaved);
const usedTokens = memoryWasEdited ? 16000 : 90000;
write({ id: message.id, result: { turn: { id: turnId, status: "inProgress", items: [] } } });
write({ method: "turn/started", params: { threadId: params.threadId, turn: { id: turnId } } });
const count = (threadTurns.get(params.threadId) ?? 0) + 1;
threadTurns.set(params.threadId, count);
const used = control().growUsage ? 84000 + 6000 * (count - 1) : usedTokens;
write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage: { total: { totalTokens: used }, last: { totalTokens: used }, modelContextWindow: 100000 } } });
const finish = () => {
const item = { type: "agentMessage", id: "message-" + turnId, text: "Done.", phase: "final_answer" };
write({ method: "item/completed", params: { threadId: params.threadId, turnId, item } });
write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: turnId, status: "completed" } } });
};
if (editMemory && !memoryWasEdited) {
const deadline = Date.now() + 10_000;
const timer = setInterval(() => {
if (fs.existsSync(memorySaved)) {
clearInterval(timer);
finish();
} else if (Date.now() >= deadline) {
clearInterval(timer);
log({ fixtureError: "Fake Codex timed out waiting for the test to save agent memory." });
}
}, 20);
} else setTimeout(finish, 50);
return;
}
case "thread/compact/start": {
const { compactionMs, ignoreInterrupt, compactionStatus, editMemory, estimate } = control();
const turnId = "compact-turn-" + (++turns);
const item = { type: "contextCompaction", id: "compact-item-" + turnId };
write({ id: message.id, result: {} });
write({ method: "turn/started", params: { threadId: params.threadId, turn: { id: turnId } } });
write({ method: "item/started", params: { threadId: params.threadId, turnId, item } });
const finish = (status) => {
if (compacting.get(params.threadId)?.turnId !== turnId) return;
compacting.delete(params.threadId);
clearTimeout(timer);
log({ compactionEnded: turnId, threadId: params.threadId, status });
if (status === "completed" && editMemory) {
const last = { totalTokens: 16000, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningOutputTokens: 0 };
const tokenUsage = { total: { totalTokens: 90000 }, last, modelContextWindow: 100000 };
write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage } });
}
if (status === "completed" && estimate) {
const last = { totalTokens: 6000, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningOutputTokens: 0 };
const tokenUsage = { total: { totalTokens: 90000 }, last, modelContextWindow: 100000 };
write({ method: "thread/tokenUsage/updated", params: { threadId: params.threadId, turnId, tokenUsage } });
}
if (status === "completed") write({ method: "item/completed", params: { threadId: params.threadId, turnId, item } });
write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: turnId, status } } });
};
const timer = setTimeout(() => finish(compactionStatus ?? "completed"), compactionMs);
compacting.set(params.threadId, { turnId, finish, ignoreInterrupt });
return;
}
case "turn/interrupt": {
write({ id: message.id, result: {} });
const compaction = compacting.get(params.threadId);
if (compaction?.turnId === params.turnId) {
if (!compaction.ignoreInterrupt) compaction.finish("interrupted");
return;
}
return write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: params.turnId, status: "interrupted" } } });
}
default:
return write({ id: message.id, error: { code: -32601, message: "Method not found: " + message.method } });
}
};
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => {
buffer += chunk;
let newline = buffer.indexOf("\\n");
while (newline >= 0) {
const line = buffer.slice(0, newline);
buffer = buffer.slice(newline + 1);
if (line) {
const message = JSON.parse(line);
log({ method: message.method ?? null, id: message.id ?? null, params: message.params ?? null });
if (message.method && message.id !== undefined) answer(message);
}
newline = buffer.indexOf("\\n");
}
});
`;
interface ProtocolEntry {
method: string | null;
threadId: string | null;
turnId: string | null;
refused: string | null;
compactionEnded: string | null;
fixtureError: string | null;
status: string | null;
usageOnStop: number | null;
}
interface CaseReport {
name: string;
compactionMs: number;
ignoreInterrupt: boolean;
threadId: string;
statusWhileCompacting: string;
finalStatus: string;
finalError: string | null;
compactionEnd: string | null;
interruptSent: boolean;
refusedStarts: number;
passed: boolean;
}
const errors: { code: string; agentId?: string }[] = [];
const report: { compactionTimeoutMs: number; cases: CaseReport[]; passed: boolean; error?: string } = {
compactionTimeoutMs: COMPACTION_TIMEOUT_MS,
cases: [],
passed: false,
};
const unused = () => Effect.die(new Error("Not used by this check."));
const browser: AgentBrowserHost = {
listTabs: () => [],
toolCanUseTab: () => false,
beginTakeover: unused,
endTakeover: () => {},
resolveUploadTarget: unused,
handleDynamicTool: unused,
userControlTabFor: () => null,
awaitUserControlRelease: unused,
onChanged: () => () => {},
onControlChanged: () => () => {},
onDocumentChanged: () => () => {},
clearControls: () => {},
endControl: () => {},
close: () => Effect.void,
clearAgentSession: () => Effect.void,
shareTab: () => undefined,
unshareThread: () => undefined,
agentTab: () => null,
};
function protocol(): ProtocolEntry[] {
try {
return readFileSync(logPath, "utf8")
.split("\n")
.filter(Boolean)
.map((line) => protocolEntry(JSON.parse(line)));
} catch {
return [];
}
}
function protocolEntry(value: unknown): ProtocolEntry {
const entry = isRecord(value) ? value : {};
const params = isRecord(entry.params) ? entry.params : {};
return {
method: getString(entry, "method"),
threadId: getString(params, "threadId") ?? getString(entry, "threadId"),
turnId: getString(params, "turnId"),
refused: getString(entry, "refused"),
compactionEnded: getString(entry, "compactionEnded"),
fixtureError: getString(entry, "fixtureError"),
status: getString(entry, "status"),
usageOnStop: typeof entry.usageOnStop === "number" ? entry.usageOnStop : null,
};
}
async function waitFor<T>(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise<T> {
const deadline = Date.now() + timeoutMs;
for (;;) {
const value = read();
if (value) return value;
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`);
await new Promise((done) => setTimeout(done, 25));
}
}
await writeFile(fakeCodex, FAKE_CODEX, { mode: 0o755 });
const database = new OpenBotDatabase(userData);
const store = new AgentStore(userData, home, database);
const mailbox = new MailboxStore(userData, store.sharedRoot, store.database);
const agents = new AgentService({
store,
mailbox,
browser,
preferredProvider: "codex",
offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "codex"),
contextCompactionTimeoutMs: COMPACTION_TIMEOUT_MS,
});
let stopped = false;
agents.on("event", (event: AgentEvent) => {
if (event.type === "error") errors.push({ code: event.code, ...(event.agentId ? { agentId: event.agentId } : {}) });
});
try {
await runCauseEffect(agents.initialize());
const cases = [
{
name: "compaction ends after a memory edit",
compactionMs: 400,
ignoreInterrupt: false,
editMemory: true,
end: "completed",
interrupt: false,
},
{
name: "compaction runs past the start deadline",
compactionMs: 3_000,
ignoreInterrupt: false,
end: "completed",
interrupt: false,
},
{
name: "deadline stops the compaction",
compactionMs: 60_000,
ignoreInterrupt: false,
end: "interrupted",
interrupt: true,
},
{
name: "refused start waits for the compaction",
compactionMs: 12_000,
ignoreInterrupt: true,
end: "completed",
interrupt: true,
},
] as const;
for (const [index, entry] of cases.entries()) {
await writeFile(controlPath, JSON.stringify(entry));
const agent = await runCauseEffect(
agents.createAgent({
name: `Compaction ${index + 1}`,
description: "Checks the queue during a compaction.",
avatarSeed: `compaction-${index}`,
avatarHue: null,
initialMessage: "First message.",
}),
);
if ("editMemory" in entry) {
await waitFor("the first usage report", () =>
agents
.getRuntimeSnapshot()
.contextStates?.some((state) => state.agentId === agent.id && state.usage?.usedTokens === 90000),
);
agents.createMemory({ agentId: agent.id, text: "A test memory." });
await writeFile(memorySaved, "");
}
const threadId = await waitFor("the first turn to end", () => {
const session = store.activeProviderSession(agent.id)?.externalSessionId;
const entries = protocol();
const fixtureError = entries.find((line) => line.fixtureError)?.fixtureError;
if (fixtureError) throw new Error(fixtureError);
const started = entries.some((line) => line.method === "turn/start" && line.threadId === session);
const busy = agents
.listQueue(agent.id)
.deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status));
return session && started && !busy ? session : null;
});
const receipt = await runCauseEffect(agents.sendMessage({ agentId: agent.id, text: "Second message." }));
const deliveryId = receipt.deliveries[0]?.id;
assert.ok(deliveryId, `${entry.name}: no delivery`);
const compactionTurn = await waitFor("the compaction to start", () =>
protocol().find((line) => line.method === "thread/compact/start" && line.threadId === threadId),
);
assert.ok(compactionTurn);
const statusWhileCompacting = mailbox.getDelivery(deliveryId)?.delivery.status ?? "missing";
if ("editMemory" in entry) {
const next = await waitFor(
"the next turn after compaction",
() =>
protocol().filter(
(line) =>
line.threadId === threadId && (line.method === "turn/start" || line.method === "thread/compact/start"),
)[2],
);
assert.equal(
next.method,
"turn/start",
"memory edit: the queue started another compaction instead of the message",
);
assert.ok(
protocol().some((line) => line.method === "thread/resume" && line.threadId === threadId),
"memory edit: the session was not marked for reload",
);
}
const finalStatus = await waitFor("the second message to end", () => {
const status = mailbox.getDelivery(deliveryId)?.delivery.status;
return status === "completed" || status === "failed" || status === "interrupted" ? status : null;
});
const entries = protocol();
const threadEntries = entries.filter((line) => line.threadId === threadId || line.refused === threadId);
const result: CaseReport = {
name: entry.name,
compactionMs: entry.compactionMs,
ignoreInterrupt: entry.ignoreInterrupt,
threadId,
statusWhileCompacting,
finalStatus,
finalError: mailbox.getDelivery(deliveryId)?.delivery.error ?? null,
compactionEnd: entries.find((line) => line.compactionEnded && line.threadId === threadId)?.status ?? null,
interruptSent: threadEntries.some(
(line) => line.method === "turn/interrupt" && line.turnId?.startsWith("compact-turn-"),
),
refusedStarts: threadEntries.filter((line) => line.refused === threadId).length,
passed: false,
};
report.cases.push(result);
assert.equal(result.statusWhileCompacting, "queued", `${entry.name}: the message did not wait`);
assert.equal(result.finalStatus, "completed", `${entry.name}: ${result.finalError ?? result.finalStatus}`);
assert.equal(result.compactionEnd, entry.end, `${entry.name}: the compaction did not end as expected`);
assert.equal(result.interruptSent, entry.interrupt, `${entry.name}: turn/interrupt`);
if (entry.ignoreInterrupt) assert.ok(result.refusedStarts > 0, "Codex refused no start");
else assert.equal(result.refusedStarts, 0, `${entry.name}: Codex refused a start`);
result.passed = true;
}
await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false }));
const updatePreview = store.updatePreview;
let previewFailures = 0;
store.updatePreview = (id, preview) => {
if (preview !== "Done." || previewFailures > 0) return updatePreview(id, preview);
previewFailures += 1;
return Effect.fail(new StoredStateFailure({ cause: new Error("disk I/O error") }));
};
const held = await runCauseEffect(
agents.createAgent({
name: "Failed turn end",
description: "Checks the queue after a turn end that failed.",
avatarSeed: "compaction-failed-end",
avatarHue: null,
initialMessage: "First message.",
}),
);
await waitFor("the failed turn end", () => {
const busy = agents
.listQueue(held.id)
.deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status));
return previewFailures === 1 && !busy;
});
store.updatePreview = updatePreview;
const heldReceipt = await runCauseEffect(agents.sendMessage({ agentId: held.id, text: "Second message." }));
const heldDeliveryId = heldReceipt.deliveries[0]?.id;
assert.ok(heldDeliveryId, "failed turn end: no delivery");
const heldStatus = () => mailbox.getDelivery(heldDeliveryId)?.delivery.status ?? "missing";
const failedTurnEnd = {
previewFailures,
nextMessageStatus: await waitFor(
"the next message after the failed turn end",
() => (["completed", "failed", "interrupted"].includes(heldStatus()) ? heldStatus() : null),
30_000,
).catch(() => heldStatus()),
};
Object.assign(report, { failedTurnEnd });
assert.equal(failedTurnEnd.nextMessageStatus, "completed", "failed turn end: the compaction hold stayed");
await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false }));
const owner = await runCauseEffect(
agents.createAgent({
name: "Side thread",
description: "Checks the context of a side thread.",
avatarSeed: "compaction-thread",
avatarHue: null,
initialMessage: "First message.",
}),
);
const mainSession = await waitFor("the agent's first turn to end", () => {
const session = store.activeProviderSession(owner.id)?.externalSessionId;
const busy = agents.listQueue(owner.id).deliveries.some((delivery) => delivery.status !== "completed");
return session && !busy ? session : null;
});
const mainContext = () =>
agents.getRuntimeSnapshot().contextStates?.find((state) => state.agentId === owner.id) ?? null;
const mainCompactStarts = () =>
protocol().filter((line) => line.method === "thread/compact/start" && line.threadId === mainSession).length;
const mainBefore = await waitFor(
"the agent's own compaction",
() => mainContext()?.compaction?.status === "completed" && mainContext(),
);
const mainStartsBefore = mainCompactStarts();
const task = await runCauseEffect(agents.startTask({ agentId: owner.id, text: "Thread message." }));
const taskSession = await waitFor(
"the thread session",
() => store.database.activeProviderSession(task.threadId, "codex")?.externalSessionId,
);
const threadContext = () =>
agents.getRuntimeSnapshot().threadContextStates?.find((state) => state.threadId === task.threadId) ?? null;
const compactions = () => protocol().filter((line) => line.compactionEnded && line.threadId === taskSession);
const automatic = await waitFor("the thread's automatic compaction", () => {
const state = threadContext();
const idle = agents.listTasks(owner.id).tasks.find((candidate) => candidate.id === task.id)?.state === "idle";
return idle && compactions().length === 1 && state?.compaction?.status === "completed" ? state : null;
});
await runCauseEffect(agents.compactTaskContext({ agentId: owner.id, taskId: task.id }));
const manual = await waitFor("the thread's manual compaction", () => {
const state = threadContext();
return compactions().length === 2 && state?.compaction?.status === "completed" ? state : null;
});
const mainAfter = mainContext();
const sideThread = {
threadUsage: automatic.usage,
automaticCompaction: automatic.compaction?.id ?? null,
manualCompaction: manual.compaction?.id ?? null,
threadCompactions: compactions().map((line) => line.status),
mainCompactStarts: mainCompactStarts() - mainStartsBefore,
mainContextUnchanged: JSON.stringify(mainAfter) === JSON.stringify(mainBefore),
};
Object.assign(report, { sideThread });
assert.equal(sideThread.threadUsage?.usedTokens, 90_000, "side thread: no usage");
assert.notEqual(sideThread.manualCompaction, sideThread.automaticCompaction, "side thread: no new compaction");
assert.deepEqual(sideThread.threadCompactions, ["completed", "completed"], "side thread: compactions");
assert.equal(sideThread.mainCompactStarts, 0, "side thread: the agent's own chat compacted");
assert.ok(sideThread.mainContextUnchanged, "side thread: the agent's own context changed");
await writeFile(controlPath, JSON.stringify({ compactionMs: 5_000, ignoreInterrupt: false }));
const deleted = await runCauseEffect(agents.startTask({ agentId: owner.id, text: "Thread message." }));
const deletedSession = await waitFor(
"the deleted thread's session",
() => store.database.activeProviderSession(deleted.threadId, "codex")?.externalSessionId,
);
await waitFor("the deleted thread's compaction", () =>
protocol().some((line) => line.method === "thread/compact/start" && line.threadId === deletedSession),
);
await runCauseEffect(agents.deleteTask({ agentId: owner.id, taskId: deleted.id }));
const ownReceipt = await runCauseEffect(agents.sendMessage({ agentId: owner.id, text: "Own chat message." }));
const ownDeliveryId = ownReceipt.deliveries[0]?.id;
assert.ok(ownDeliveryId, "deleted thread: no delivery");
const ownStatus = () => mailbox.getDelivery(ownDeliveryId)?.delivery.status ?? "missing";
const deletedThread = {
ownChatStatus: await waitFor(
"the own chat message after the delete",
() => (["completed", "failed", "interrupted"].includes(ownStatus()) ? ownStatus() : null),
30_000,
).catch(() => ownStatus()),
};
Object.assign(report, { deletedThread });
assert.equal(deletedThread.ownChatStatus, "completed", "deleted thread: the agent's own chat stayed held");
await writeFile(
controlPath,
JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, compactionStatus: "failed", growUsage: true }),
);
const failing = await runCauseEffect(
agents.createAgent({
name: "Failed compactions",
description: "Checks that failed compactions stop.",
avatarSeed: "compaction-failing",
avatarHue: null,
initialMessage: "First message.",
}),
);
const failingIdle = () =>
!agents
.listQueue(failing.id)
.deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status));
const failingSession = await waitFor("the first turn of the failing chat", () => {
const session = store.activeProviderSession(failing.id)?.externalSessionId;
return session && failingIdle() ? session : null;
});
const failedEnds = () =>
protocol().filter((line) => line.compactionEnded && line.threadId === failingSession && line.status === "failed");
for (const text of ["Second message.", "Third message.", "Fourth message."]) {
await waitFor(
"the failing chat to settle",
() =>
failingIdle() &&
!agents
.getRuntimeSnapshot()
.contextStates?.some((state) => state.agentId === failing.id && state.compaction?.status === "running"),
);
await runCauseEffect(agents.sendMessage({ agentId: failing.id, text }));
await waitFor("the message to end", failingIdle);
}
const failedCompactions = {
compactStarts: protocol().filter(
(line) => line.method === "thread/compact/start" && line.threadId === failingSession,
).length,
failedEnds: failedEnds().length,
turns: protocol().filter((line) => line.method === "turn/start" && line.threadId === failingSession).length,
stoppedNotices: errors.filter(
(error) => error.code === "context_compaction_stopped" && error.agentId === failing.id,
).length,
deliveries: agents.listQueue(failing.id).deliveries.map((delivery) => delivery.status),
};
Object.assign(report, { failedCompactions });
assert.equal(failedCompactions.turns, 4, "failed compactions: not every message ran");
assert.equal(failedCompactions.compactStarts, 2, "failed compactions: OpenBot did not stop after 2 failures");
assert.equal(failedCompactions.failedEnds, 2, "failed compactions: the compactions did not fail");
assert.equal(failedCompactions.stoppedNotices, 1, "failed compactions: one notice");
await writeFile(
controlPath,
JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, growUsage: true, estimate: true }),
);
const full = await runCauseEffect(
agents.createAgent({
name: "Full after compaction",
description: "Checks that compactions that free no space stop.",
avatarSeed: "compaction-full",
avatarHue: null,
initialMessage: "First message.",
}),
);
const fullIdle = () =>
!agents
.listQueue(full.id)
.deliveries.some((delivery) => ["queued", "starting", "running"].includes(delivery.status));
const fullSession = await waitFor("the first turn of the full chat", () => {
const session = store.activeProviderSession(full.id)?.externalSessionId;
return session && fullIdle() ? session : null;
});
for (const text of ["Second message.", "Third message.", "Fourth message."]) {
await waitFor(
"the full chat to settle",
() =>
fullIdle() &&
!agents
.getRuntimeSnapshot()
.contextStates?.some((state) => state.agentId === full.id && state.compaction?.status === "running"),
);
await runCauseEffect(agents.sendMessage({ agentId: full.id, text }));
await waitFor("the message to end", fullIdle);
}
const ineffectiveCompactions = {
compactStarts: protocol().filter((line) => line.method === "thread/compact/start" && line.threadId === fullSession)
.length,
turns: protocol().filter((line) => line.method === "turn/start" && line.threadId === fullSession).length,
notices: errors.filter((error) => error.code === "context_compaction_ineffective" && error.agentId === full.id)
.length,
deliveries: agents.listQueue(full.id).deliveries.map((delivery) => delivery.status),
};
Object.assign(report, { ineffectiveCompactions });
assert.equal(ineffectiveCompactions.turns, 4, "ineffective compactions: not every message ran");
assert.equal(ineffectiveCompactions.compactStarts, 2, "ineffective compactions: OpenBot did not stop after 2");
assert.equal(ineffectiveCompactions.notices, 1, "ineffective compactions: one notice");
await writeFile(controlPath, JSON.stringify({ compactionMs: 200, ignoreInterrupt: false, usageOnStop: true }));
const lateErrors: string[] = [];
const collect = (error: unknown) => lateErrors.push(error instanceof Error ? error.message : String(error));
process.on("uncaughtException", collect);
await runCauseEffect(agents.stop());
database.close();
stopped = true;
await new Promise((done) => setTimeout(done, USAGE_CHANGE_DELAY_MS));
process.off("uncaughtException", collect);
const usageOnStop = {
reportedThreads: protocol().find((line) => line.usageOnStop !== null)?.usageOnStop ?? 0,
lateErrors,
};
Object.assign(report, { usageOnStop });
assert.ok(usageOnStop.reportedThreads > 0, "stop: Codex reported no usage");
assert.deepEqual(usageOnStop.lateErrors, [], "stop: a late usage report used the closed database");
report.passed = true;
} catch (error) {
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
process.exitCode = 1;
} finally {
if (!stopped) {
await runCauseEffect(agents.stop()).catch(() => {});
database.close();
}
mkdirSync(OUT, { recursive: true });
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
await rm(root, { recursive: true, force: true });
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
}