Files
openbot/scripts/opencode-usage-metering-e2e.ts
T
NBandClaude Opus 5.5 b18d30f3c4 Let a user tag browser tabs and threads in chat (#5021)
* Let a user tag browser tabs and threads in chat

The @ picker of the chat composer now has sections for agents, browser
tabs and threads. A tag becomes a chip in the message.

- A tab tag lists only the tabs of the agent of the chat. When the user
  sends it, the host shares that tab with the target thread of the same
  agent, so a side thread can use a tab of the main chat.
- A thread tag gives the agent one short line: the title, the task id,
  the state and the start of the result. The agent reads more with
  history_read or history_search and the new optional taskId.
- The stored text keeps the marker. The marker expands at delivery, so
  the agent gets the current title and state.
- A click on a chip opens the tab page or the thread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Share a tagged tab when its prompt is built

- Share the tab with the public thread that gets the prompt, not when
  the message is queued. A queued message can move or change.
- Use the strict name rule for tab and thread titles.
- A popup keeps the shares of its opener tab.
- A tag from a teammate shows as a name only.
- The picker lists only tabs that have an owner agent.
- Use browserDisplayUrl in the picker.
- Make the thread question in chat-tags-e2e exact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep TaggedThreadState private to delivery-content

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 19:43:21 +01:00

362 lines
16 KiB
TypeScript

// Checks that token usage through OpenCode is metered once per turn, with each count. Runs the real
// agent service on a temporary profile with a fake OpenCode ACP server that answers `session/prompt`
// as OpenCode 1.19 and 2.x do: `usage` holds the tokens of that prompt only, `inputTokens` is the
// uncached input, and `cachedReadTokens`, `cachedWriteTokens` and `thoughtTokens` are left out when
// they are 0. A message `tokens=<input>,<cached>,<written>,<output>,<thought>` sets the usage of its
// turn. Cases: a first turn with no cache, a smaller second turn with cache reads (the session counter
// before this change booked 0 for it and had no uncached input), a larger third turn with a cache
// write and thinking, and a turn after the provider process starts again and resumes the session.
// A message can also carry `cost=<total>`: the fake then sends a `usage_update` with that session
// cost before the result, as OpenCode 2.1 does. Each turn books the difference from the last total,
// also after a restart. A session from before costs were kept books its first total as the start.
// `bun scripts/opencode-usage-metering-e2e.ts`. Writes
// .openbot-build/opencode-usage-metering-e2e/report.json.
import assert from "node:assert/strict";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { AGENT_PROVIDERS, decodeUsageTokens, type UsageTokens } from "@openbot/contracts/ipc";
import { Effect } from "effect";
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-usage-metering-e2e");
const root = await mkdtemp(join(tmpdir(), "openbot-opencode-metering-"));
const userData = join(root, "user-data");
const home = join(root, "home");
const fakeOpenCode = join(root, "opencode");
const logPath = join(root, "opencode.jsonl");
// Before the service modules load: the stores read the home folder.
process.env.HOME = home;
process.env.OPENBOT_OPENCODE_PATH = fakeOpenCode;
process.env.FAKE_OPENCODE_LOG = logPath;
mkdirSync(home, { recursive: true });
const { AgentService } = await import("../src/backend/agent-service");
const { AgentStore } = await import("../src/backend/agent-store");
const { runCauseEffect } = await import("../src/backend/effect-boundary");
const { MailboxStore } = await import("../src/backend/mailbox-store");
const { OpenBotDatabase } = await import("../src/backend/openbot-database");
/** An OpenCode ACP server with the methods that the service calls. */
const FAKE_OPENCODE = `#!/usr/bin/env node
const fs = require("node:fs");
const NL = String.fromCharCode(10);
if (process.argv.includes("--version")) { process.stdout.write("1.19.3" + NL); process.exit(0); }
const log = (entry) => fs.appendFileSync(process.env.FAKE_OPENCODE_LOG, JSON.stringify({ pid: process.pid, ...entry }) + NL);
log({ start: true });
const write = (message) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + NL);
const MODEL = "opencode/big-pickle";
const setup = (sessionId) => ({ sessionId, models: { availableModels: [{ modelId: MODEL, name: "Big Pickle" }], currentModelId: MODEL } });
let sessions = 0;
function handle(message) {
const params = message.params ?? {};
if (message.method) log({ method: message.method, sessionId: params.sessionId ?? null });
if (message.id === undefined) return;
if (message.method === "initialize")
return write({ id: message.id, result: { protocolVersion: 1, authMethods: [], agentCapabilities: { loadSession: true, sessionCapabilities: { list: {}, resume: {} } } } });
if (message.method === "session/new") return write({ id: message.id, result: setup("ses-" + process.pid + "-" + ++sessions) });
if (message.method === "session/list") return write({ id: message.id, result: { sessions: [] } });
if (message.method === "session/resume" || message.method === "session/load") return write({ id: message.id, result: setup(params.sessionId) });
if (message.method === "session/prompt") {
const text = params.prompt.map((block) => block.text ?? "").join("");
const match = /tokens=(\\d+),(\\d+),(\\d+),(\\d+),(\\d+)/.exec(text);
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done." } } } });
if (!match) return write({ id: message.id, result: { stopReason: "end_turn" } });
const [input, cached, written, output, thought] = match.slice(1).map(Number);
const cost = /cost=([\\d.]+)/.exec(text);
if (cost)
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "usage_update", used: input + cached + written + output + thought, size: 200000, cost: { amount: Number(cost[1]), currency: "USD" } } } });
return write({ id: message.id, result: { stopReason: "end_turn", usage: {
inputTokens: input, outputTokens: output, totalTokens: input + output + thought + cached + written,
...(thought > 0 ? { thoughtTokens: thought } : {}),
...(cached > 0 ? { cachedReadTokens: cached } : {}),
...(written > 0 ? { cachedWriteTokens: written } : {}),
} } });
}
write({ id: message.id, result: {} });
}
let buffer = "";
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => {
buffer += chunk;
let index;
while ((index = buffer.indexOf(NL)) >= 0) {
const line = buffer.slice(0, index);
buffer = buffer.slice(index + 1);
if (line.trim()) handle(JSON.parse(line));
}
});
setInterval(() => {}, 1000);
`;
interface Turn {
input: number;
cached: number;
written: number;
output: number;
thought: number;
/** The session's total cost that OpenCode reports after this turn. */
sessionCost?: number;
}
interface CaseReport {
name: string;
turnId: string;
sent: Turn;
recorded: UsageTokens[];
expectedCost: number | null;
recordedCost: number | null;
passed: boolean;
}
const report: {
model?: string;
cases: CaseReport[];
analytics?: { processedTokens: number; turns: number; missingUsageTurns: number };
passed: boolean;
error?: string;
} = { cases: [], passed: false };
const unused = () => Effect.die(new Error("Not used by this check."));
const browser: AgentBrowserHost = {
listTabs: () => [],
beginTakeover: unused,
endTakeover: () => {},
resolveUploadTarget: unused,
handleDynamicTool: unused,
userControlTabFor: () => null,
awaitUserControlRelease: unused,
onChanged: () => () => {},
onControlChanged: () => () => {},
onDocumentChanged: () => () => {},
clearControls: () => {},
endControl: () => {},
close: () => Effect.void,
clearAgentSession: () => Effect.void,
shareTab: () => undefined,
unshareThread: () => undefined,
agentTab: () => null,
};
/** Waits for a state of the service or of its database. */
async function waitFor<T>(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise<T> {
const deadline = Date.now() + timeoutMs;
for (;;) {
const value = read();
if (value) return value;
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`);
await new Promise((done) => setTimeout(done, 25));
}
}
interface LogEntry {
pid: number;
start?: boolean;
method?: string;
sessionId?: string | null;
}
function protocol(): LogEntry[] {
let text = "";
try {
text = readFileSync(logPath, "utf8");
} catch {
return [];
}
return text
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
}
await writeFile(fakeOpenCode, FAKE_OPENCODE, { mode: 0o755 });
const database = new OpenBotDatabase(userData);
const store = new AgentStore(userData, home, database);
const mailbox = new MailboxStore(userData, store.sharedRoot, store.database);
const agents = new AgentService({
store,
mailbox,
browser,
preferredProvider: "opencode",
offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "opencode"),
});
let connection: DatabaseSync | null = null;
const sqlite = () => {
connection ??= new DatabaseSync(database.path);
return connection;
};
const closeReader = () => connection?.close();
/** The usage records of one provider turn. */
function records(turnId: string): UsageTokens[] {
return sqlite()
.prepare("SELECT tokens_json FROM agent_usage_records WHERE turn_id = ? ORDER BY recorded_at, rowid")
.all(turnId)
.map((row) => decodeUsageTokens(JSON.parse(String(row.tokens_json))));
}
/** The cost booked for one provider turn. */
function costOf(turnId: string): number | null {
const value = sqlite()
.prepare("SELECT estimated_cost_usd FROM agent_usage_records WHERE turn_id = ?")
.get(turnId)?.estimated_cost_usd;
return typeof value === "number" ? value : null;
}
/** The provider turns that ran in one public thread, oldest first. */
function turnsOf(threadId: string): string[] {
return sqlite()
.prepare(
`SELECT a.turn_id FROM agent_usage_activity a JOIN projection_provider_sessions s ON s.id = a.session_id
WHERE a.kind = 'turn' AND s.thread_id = ? ORDER BY a.occurred_at, a.rowid`,
)
.all(threadId)
.map((row) => String(row.turn_id));
}
const message = (turn: Turn) =>
`tokens=${turn.input},${turn.cached},${turn.written},${turn.output},${turn.thought}` +
(turn.sessionCost === undefined ? "" : ` cost=${turn.sessionCost}`);
/** Checks that the turn at `index` of the thread has one record with the tokens of `sent` and `expectedCost`. */
async function check(
name: string,
threadId: string,
index: number,
sent: Turn,
expectedCost: number | null,
): Promise<void> {
const turnId = await waitFor(`${name}: usage`, () => {
const id = turnsOf(threadId)[index];
return id && records(id).length > 0 && id;
});
const recorded = records(turnId);
const recordedCost = costOf(turnId);
const result: CaseReport = { name, turnId, sent, recorded, expectedCost, recordedCost, passed: false };
report.cases.push(result);
assert.equal(recorded.length, 1, `${name}: one record for the turn`);
assert.deepEqual(
recorded[0],
{
uncachedInput: sent.input,
cachedInput: sent.cached,
cacheCreation: sent.written,
output: sent.output + sent.thought,
},
`${name}: the turn's own tokens, with thinking as output`,
);
if (expectedCost === null) assert.equal(recordedCost, null, `${name}: no cost`);
else
assert.ok(
recordedCost !== null && Math.abs(recordedCost - expectedCost) < 1e-9,
`${name}: cost ${recordedCost}, expected ${expectedCost}`,
);
result.passed = true;
}
async function send(agentId: string, turn: Turn, label: string): Promise<void> {
const receipt = await runCauseEffect(agents.sendMessage({ agentId, text: message(turn) }));
const deliveryId = receipt.deliveries[0]?.id;
assert.ok(deliveryId, `${label}: no delivery`);
const status = await waitFor(label, () => {
const value = mailbox.getDelivery(deliveryId)?.delivery.status;
return value === "completed" || value === "failed" || value === "interrupted" ? value : null;
});
assert.equal(status, "completed", `${label}: ${mailbox.getDelivery(deliveryId)?.delivery.error ?? status}`);
}
const idle = (agentId: string) =>
agents
.listQueue(agentId)
.deliveries.every((delivery) => !["queued", "starting", "running"].includes(delivery.status));
try {
await runCauseEffect(agents.initialize());
const first: Turn = { input: 12_000, cached: 0, written: 0, output: 800, thought: 0, sessionCost: 0.05 };
const agent = await runCauseEffect(
agents.createAgent({
name: "OpenCode metering",
description: "Checks OpenCode usage metering.",
avatarSeed: "opencode-metering",
avatarHue: null,
initialMessage: message(first),
}),
);
report.model = agent.model;
const threadId = await waitFor("the agent's own thread", () => store.list().find((a) => a.id === agent.id)?.threadId);
await check("first turn", threadId, 0, first, 0.05);
await waitFor("the first turn to end", () => idle(agent.id));
const second: Turn = { input: 1_000, cached: 9_000, written: 0, output: 200, thought: 0 };
await send(agent.id, second, "smaller second turn");
// No `usage_update`: the turn has no cost, and the next total includes it.
await check("smaller second turn", threadId, 1, second, null);
const third: Turn = { input: 15_000, cached: 6_000, written: 2_000, output: 900, thought: 400, sessionCost: 0.2 };
await send(agent.id, third, "larger third turn");
await check("larger third turn", threadId, 2, third, 0.15);
// The provider process stops between turns. The runtime starts it again and resumes the session,
// and OpenCode measures the next turn on its own.
const session = store.activeProviderSession(agent.id)?.externalSessionId;
assert.ok(session, "the agent has a provider session");
const killedPid = protocol().find((entry) => entry.start)?.pid ?? 0;
assert.ok(killedPid, "the fake server logged its start");
process.kill(killedPid, "SIGTERM");
await waitFor("OpenCode to start again", () => protocol().find((entry) => entry.start && entry.pid !== killedPid));
await waitFor("the provider to be ready again", () => agents.getStatus().phase === "ready");
const fourth: Turn = { input: 500, cached: 6_500, written: 0, output: 100, thought: 50, sessionCost: 0.26 };
await send(agent.id, fourth, "turn after a restart");
await check("turn after a restart", threadId, 3, fourth, 0.06);
assert.equal(
store.activeProviderSession(agent.id)?.externalSessionId,
session,
"the turn after a restart resumes the session",
);
// A session from before costs were kept: it has records but no cost checkpoint. Its first total
// also counts the earlier turns, so that turn has no cost and the next one books the difference.
const sessionId = store.activeProviderSession(agent.id)?.id;
assert.ok(sessionId, "the agent has an active provider session");
sqlite()
.prepare("DELETE FROM agent_usage_checkpoints WHERE agent_id = ? AND counter_id = ?")
.run(agent.id, `${sessionId}:opencode:cost`);
const fifth: Turn = { input: 700, cached: 7_000, written: 0, output: 120, thought: 0, sessionCost: 0.3 };
await send(agent.id, fifth, "first cost of an older session");
await check("first cost of an older session", threadId, 4, fifth, null);
const sixth: Turn = { input: 300, cached: 7_500, written: 0, output: 80, thought: 0, sessionCost: 0.31 };
await send(agent.id, sixth, "next cost of an older session");
await check("next cost of an older session", threadId, 5, sixth, 0.01);
const today = new Date().toISOString().slice(0, 10);
const analytics = agents.getAnalytics({ agentId: agent.id, startDate: today, endDate: today, timeZone: "UTC" });
report.analytics = {
processedTokens: analytics.totals.processedTokens,
turns: analytics.totals.turns,
missingUsageTurns: analytics.totals.missingUsageTurns,
};
const sent = report.cases.reduce(
(sum, { sent: turn }) => sum + turn.input + turn.cached + turn.written + turn.output + turn.thought,
0,
);
assert.equal(analytics.totals.turns, 6, "Analytics count every turn.");
assert.equal(analytics.totals.missingUsageTurns, 0, "Every turn has usage.");
assert.equal(analytics.totals.processedTokens, sent, "Analytics count every turn's tokens once.");
report.passed = true;
} catch (error) {
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
process.exitCode = 1;
} finally {
await runCauseEffect(agents.stop()).catch(() => {});
closeReader();
database.close();
mkdirSync(OUT, { recursive: true });
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
await rm(root, { recursive: true, force: true });
process.stdout.write(`${report.passed ? "PASS" : "FAIL"} ${join(OUT, "report.json")}\n`);
if (report.error) process.stdout.write(`${report.error}\n`);
}