Files
openbot/scripts/opencode-usage-metering-e2e.ts
NB 6fbd8bfc20 Let the main chat and side threads of an agent use each other's browser tabs (#13433)
* Let every chat of an agent use the browser tabs of the agent

Main chats and side threads of an agent can now use all browser tabs of
that agent. list_tabs shows the chat that opened each tab and tells when
another chat uses a tab now. Takeover, submit_secret and list_logins use
the same rule. Channel and messaging threads still use only their own
tabs and the tabs shared with an @ tag.

* Keep channel tabs out of other chats and make the working hint reliable

The main chat and side threads use only the tabs that the main chat, a
side thread or no chat opened. Tabs of a channel or a messaging thread
stay out of these chats unless a person tags them with @.

otherChatWorking now also uses the last chat that used each tab in the
last 2 minutes, because control sessions end 1.2 s after a call.

A takeover from one chat no longer deletes the held takeover result of
another chat. list_tabs looks up each chat once per call.

The browser user-control E2E now covers the cross-chat access.

* Keep a held takeover result for each chat of a tab

Held takeover results used only the tab id as the key. When two chats of
one agent took over the same tab, the second result replaced the first.
The key is now the tab id and the thread id.
2026-11-20 13:51:50 +01:00

337 lines
14 KiB
TypeScript

import assert from "node:assert/strict";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { AGENT_PROVIDERS, decodeUsageTokens, type UsageTokens } from "@openbot/contracts/ipc";
import { Effect } from "effect";
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-usage-metering-e2e");
const root = await mkdtemp(join(tmpdir(), "openbot-opencode-metering-"));
const userData = join(root, "user-data");
const home = join(root, "home");
const fakeOpenCode = join(root, "opencode");
const logPath = join(root, "opencode.jsonl");
process.env.HOME = home;
process.env.OPENBOT_OPENCODE_PATH = fakeOpenCode;
process.env.FAKE_OPENCODE_LOG = logPath;
mkdirSync(home, { recursive: true });
const { AgentService } = await import("../src/backend/agent-service");
const { AgentStore } = await import("../src/backend/agent-store");
const { runCauseEffect } = await import("../src/backend/effect-boundary");
const { MailboxStore } = await import("../src/backend/mailbox-store");
const { OpenBotDatabase } = await import("../src/backend/openbot-database");
const FAKE_OPENCODE = `#!/usr/bin/env node
const fs = require("node:fs");
const NL = String.fromCharCode(10);
if (process.argv.includes("--version")) { process.stdout.write("1.19.3" + NL); process.exit(0); }
const log = (entry) => fs.appendFileSync(process.env.FAKE_OPENCODE_LOG, JSON.stringify({ pid: process.pid, ...entry }) + NL);
log({ start: true });
const write = (message) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + NL);
const MODEL = "opencode/big-pickle";
const setup = (sessionId) => ({ sessionId, models: { availableModels: [{ modelId: MODEL, name: "Big Pickle" }], currentModelId: MODEL } });
let sessions = 0;
function handle(message) {
const params = message.params ?? {};
if (message.method) log({ method: message.method, sessionId: params.sessionId ?? null });
if (message.id === undefined) return;
if (message.method === "initialize")
return write({ id: message.id, result: { protocolVersion: 1, authMethods: [], agentCapabilities: { loadSession: true, sessionCapabilities: { list: {}, resume: {} } } } });
if (message.method === "session/new") return write({ id: message.id, result: setup("ses-" + process.pid + "-" + ++sessions) });
if (message.method === "session/list") return write({ id: message.id, result: { sessions: [] } });
if (message.method === "session/resume" || message.method === "session/load") return write({ id: message.id, result: setup(params.sessionId) });
if (message.method === "session/prompt") {
const text = params.prompt.map((block) => block.text ?? "").join("");
const match = /tokens=(\\d+),(\\d+),(\\d+),(\\d+),(\\d+)/.exec(text);
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done." } } } });
if (!match) return write({ id: message.id, result: { stopReason: "end_turn" } });
const [input, cached, written, output, thought] = match.slice(1).map(Number);
const cost = /cost=([\\d.]+)/.exec(text);
if (cost)
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "usage_update", used: input + cached + written + output + thought, size: 200000, cost: { amount: Number(cost[1]), currency: "USD" } } } });
return write({ id: message.id, result: { stopReason: "end_turn", usage: {
inputTokens: input, outputTokens: output, totalTokens: input + output + thought + cached + written,
...(thought > 0 ? { thoughtTokens: thought } : {}),
...(cached > 0 ? { cachedReadTokens: cached } : {}),
...(written > 0 ? { cachedWriteTokens: written } : {}),
} } });
}
write({ id: message.id, result: {} });
}
let buffer = "";
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => {
buffer += chunk;
let index;
while ((index = buffer.indexOf(NL)) >= 0) {
const line = buffer.slice(0, index);
buffer = buffer.slice(index + 1);
if (line.trim()) handle(JSON.parse(line));
}
});
setInterval(() => {}, 1000);
`;
interface Turn {
input: number;
cached: number;
written: number;
output: number;
thought: number;
sessionCost?: number;
}
interface CaseReport {
name: string;
turnId: string;
sent: Turn;
recorded: UsageTokens[];
expectedCost: number | null;
recordedCost: number | null;
passed: boolean;
}
const report: {
model?: string;
cases: CaseReport[];
analytics?: { processedTokens: number; turns: number; missingUsageTurns: number };
passed: boolean;
error?: string;
} = { cases: [], passed: false };
const unused = () => Effect.die(new Error("Not used by this check."));
const browser: AgentBrowserHost = {
listTabs: () => [],
toolCanUseTab: () => false,
beginTakeover: unused,
endTakeover: () => {},
resolveUploadTarget: unused,
handleDynamicTool: unused,
userControlTabFor: () => null,
awaitUserControlRelease: unused,
onChanged: () => () => {},
onControlChanged: () => () => {},
onDocumentChanged: () => () => {},
clearControls: () => {},
endControl: () => {},
close: () => Effect.void,
clearAgentSession: () => Effect.void,
shareTab: () => undefined,
unshareThread: () => undefined,
agentTab: () => null,
};
async function waitFor<T>(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise<T> {
const deadline = Date.now() + timeoutMs;
for (;;) {
const value = read();
if (value) return value;
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`);
await new Promise((done) => setTimeout(done, 25));
}
}
interface LogEntry {
pid: number;
start?: boolean;
method?: string;
sessionId?: string | null;
}
function protocol(): LogEntry[] {
let text = "";
try {
text = readFileSync(logPath, "utf8");
} catch {
return [];
}
return text
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
}
await writeFile(fakeOpenCode, FAKE_OPENCODE, { mode: 0o755 });
const database = new OpenBotDatabase(userData);
const store = new AgentStore(userData, home, database);
const mailbox = new MailboxStore(userData, store.sharedRoot, store.database);
const agents = new AgentService({
store,
mailbox,
browser,
preferredProvider: "opencode",
offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "opencode"),
});
let connection: DatabaseSync | null = null;
const sqlite = () => {
connection ??= new DatabaseSync(database.path);
return connection;
};
const closeReader = () => connection?.close();
function records(turnId: string): UsageTokens[] {
return sqlite()
.prepare("SELECT tokens_json FROM agent_usage_records WHERE turn_id = ? ORDER BY recorded_at, rowid")
.all(turnId)
.map((row) => decodeUsageTokens(JSON.parse(String(row.tokens_json))));
}
function costOf(turnId: string): number | null {
const value = sqlite()
.prepare("SELECT estimated_cost_usd FROM agent_usage_records WHERE turn_id = ?")
.get(turnId)?.estimated_cost_usd;
return typeof value === "number" ? value : null;
}
function turnsOf(threadId: string): string[] {
return sqlite()
.prepare(
`SELECT a.turn_id FROM agent_usage_activity a JOIN projection_provider_sessions s ON s.id = a.session_id
WHERE a.kind = 'turn' AND s.thread_id = ? ORDER BY a.occurred_at, a.rowid`,
)
.all(threadId)
.map((row) => String(row.turn_id));
}
const message = (turn: Turn) =>
`tokens=${turn.input},${turn.cached},${turn.written},${turn.output},${turn.thought}` +
(turn.sessionCost === undefined ? "" : ` cost=${turn.sessionCost}`);
async function check(
name: string,
threadId: string,
index: number,
sent: Turn,
expectedCost: number | null,
): Promise<void> {
const turnId = await waitFor(`${name}: usage`, () => {
const id = turnsOf(threadId)[index];
return id && records(id).length > 0 && id;
});
const recorded = records(turnId);
const recordedCost = costOf(turnId);
const result: CaseReport = { name, turnId, sent, recorded, expectedCost, recordedCost, passed: false };
report.cases.push(result);
assert.equal(recorded.length, 1, `${name}: one record for the turn`);
assert.deepEqual(
recorded[0],
{
uncachedInput: sent.input,
cachedInput: sent.cached,
cacheCreation: sent.written,
output: sent.output + sent.thought,
},
`${name}: the turn's own tokens, with thinking as output`,
);
if (expectedCost === null) assert.equal(recordedCost, null, `${name}: no cost`);
else
assert.ok(
recordedCost !== null && Math.abs(recordedCost - expectedCost) < 1e-9,
`${name}: cost ${recordedCost}, expected ${expectedCost}`,
);
result.passed = true;
}
async function send(agentId: string, turn: Turn, label: string): Promise<void> {
const receipt = await runCauseEffect(agents.sendMessage({ agentId, text: message(turn) }));
const deliveryId = receipt.deliveries[0]?.id;
assert.ok(deliveryId, `${label}: no delivery`);
const status = await waitFor(label, () => {
const value = mailbox.getDelivery(deliveryId)?.delivery.status;
return value === "completed" || value === "failed" || value === "interrupted" ? value : null;
});
assert.equal(status, "completed", `${label}: ${mailbox.getDelivery(deliveryId)?.delivery.error ?? status}`);
}
const idle = (agentId: string) =>
agents
.listQueue(agentId)
.deliveries.every((delivery) => !["queued", "starting", "running"].includes(delivery.status));
try {
await runCauseEffect(agents.initialize());
const first: Turn = { input: 12_000, cached: 0, written: 0, output: 800, thought: 0, sessionCost: 0.05 };
const agent = await runCauseEffect(
agents.createAgent({
name: "OpenCode metering",
description: "Checks OpenCode usage metering.",
avatarSeed: "opencode-metering",
avatarHue: null,
initialMessage: message(first),
}),
);
report.model = agent.model;
const threadId = await waitFor("the agent's own thread", () => store.list().find((a) => a.id === agent.id)?.threadId);
await check("first turn", threadId, 0, first, 0.05);
await waitFor("the first turn to end", () => idle(agent.id));
const second: Turn = { input: 1_000, cached: 9_000, written: 0, output: 200, thought: 0 };
await send(agent.id, second, "smaller second turn");
await check("smaller second turn", threadId, 1, second, null);
const third: Turn = { input: 15_000, cached: 6_000, written: 2_000, output: 900, thought: 400, sessionCost: 0.2 };
await send(agent.id, third, "larger third turn");
await check("larger third turn", threadId, 2, third, 0.15);
const session = store.activeProviderSession(agent.id)?.externalSessionId;
assert.ok(session, "the agent has a provider session");
const killedPid = protocol().find((entry) => entry.start)?.pid ?? 0;
assert.ok(killedPid, "the fake server logged its start");
process.kill(killedPid, "SIGTERM");
await waitFor("OpenCode to start again", () => protocol().find((entry) => entry.start && entry.pid !== killedPid));
await waitFor("the provider to be ready again", () => agents.getStatus().phase === "ready");
const fourth: Turn = { input: 500, cached: 6_500, written: 0, output: 100, thought: 50, sessionCost: 0.26 };
await send(agent.id, fourth, "turn after a restart");
await check("turn after a restart", threadId, 3, fourth, 0.06);
assert.equal(
store.activeProviderSession(agent.id)?.externalSessionId,
session,
"the turn after a restart resumes the session",
);
const sessionId = store.activeProviderSession(agent.id)?.id;
assert.ok(sessionId, "the agent has an active provider session");
sqlite()
.prepare("DELETE FROM agent_usage_checkpoints WHERE agent_id = ? AND counter_id = ?")
.run(agent.id, `${sessionId}:opencode:cost`);
const fifth: Turn = { input: 700, cached: 7_000, written: 0, output: 120, thought: 0, sessionCost: 0.3 };
await send(agent.id, fifth, "first cost of an older session");
await check("first cost of an older session", threadId, 4, fifth, null);
const sixth: Turn = { input: 300, cached: 7_500, written: 0, output: 80, thought: 0, sessionCost: 0.31 };
await send(agent.id, sixth, "next cost of an older session");
await check("next cost of an older session", threadId, 5, sixth, 0.01);
const today = new Date().toISOString().slice(0, 10);
const analytics = agents.getAnalytics({ agentId: agent.id, startDate: today, endDate: today, timeZone: "UTC" });
report.analytics = {
processedTokens: analytics.totals.processedTokens,
turns: analytics.totals.turns,
missingUsageTurns: analytics.totals.missingUsageTurns,
};
const sent = report.cases.reduce(
(sum, { sent: turn }) => sum + turn.input + turn.cached + turn.written + turn.output + turn.thought,
0,
);
assert.equal(analytics.totals.turns, 6, "Analytics count every turn.");
assert.equal(analytics.totals.missingUsageTurns, 0, "Every turn has usage.");
assert.equal(analytics.totals.processedTokens, sent, "Analytics count every turn's tokens once.");
report.passed = true;
} catch (error) {
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
process.exitCode = 1;
} finally {
await runCauseEffect(agents.stop()).catch(() => {});
closeReader();
database.close();
mkdirSync(OUT, { recursive: true });
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
await rm(root, { recursive: true, force: true });
process.stdout.write(`${report.passed ? "PASS" : "FAIL"} ${join(OUT, "report.json")}\n`);
if (report.error) process.stdout.write(`${report.error}\n`);
}