import assert from "node:assert/strict"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { AGENT_PROVIDERS, decodeUsageTokens, type UsageTokens } from "@openbot/contracts/ipc"; import { Effect } from "effect"; import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle"; const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-usage-metering-e2e"); const root = await mkdtemp(join(tmpdir(), "openbot-opencode-metering-")); const userData = join(root, "user-data"); const home = join(root, "home"); const fakeOpenCode = join(root, "opencode"); const logPath = join(root, "opencode.jsonl"); process.env.HOME = home; process.env.OPENBOT_OPENCODE_PATH = fakeOpenCode; process.env.FAKE_OPENCODE_LOG = logPath; mkdirSync(home, { recursive: true }); const { AgentService } = await import("../src/backend/agent-service"); const { AgentStore } = await import("../src/backend/agent-store"); const { runCauseEffect } = await import("../src/backend/effect-boundary"); const { MailboxStore } = await import("../src/backend/mailbox-store"); const { OpenBotDatabase } = await import("../src/backend/openbot-database"); const FAKE_OPENCODE = `#!/usr/bin/env node const fs = require("node:fs"); const NL = String.fromCharCode(10); if (process.argv.includes("--version")) { process.stdout.write("1.19.3" + NL); process.exit(0); } const log = (entry) => fs.appendFileSync(process.env.FAKE_OPENCODE_LOG, JSON.stringify({ pid: process.pid, ...entry }) + NL); log({ start: true }); const write = (message) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + NL); const MODEL = "opencode/big-pickle"; const setup = (sessionId) => ({ sessionId, models: { availableModels: [{ modelId: MODEL, name: "Big Pickle" }], currentModelId: MODEL } }); let sessions = 0; function handle(message) { const params = message.params ?? {}; if (message.method) log({ method: message.method, sessionId: params.sessionId ?? null }); if (message.id === undefined) return; if (message.method === "initialize") return write({ id: message.id, result: { protocolVersion: 1, authMethods: [], agentCapabilities: { loadSession: true, sessionCapabilities: { list: {}, resume: {} } } } }); if (message.method === "session/new") return write({ id: message.id, result: setup("ses-" + process.pid + "-" + ++sessions) }); if (message.method === "session/list") return write({ id: message.id, result: { sessions: [] } }); if (message.method === "session/resume" || message.method === "session/load") return write({ id: message.id, result: setup(params.sessionId) }); if (message.method === "session/prompt") { const text = params.prompt.map((block) => block.text ?? "").join(""); const match = /tokens=(\\d+),(\\d+),(\\d+),(\\d+),(\\d+)/.exec(text); write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done." } } } }); if (!match) return write({ id: message.id, result: { stopReason: "end_turn" } }); const [input, cached, written, output, thought] = match.slice(1).map(Number); const cost = /cost=([\\d.]+)/.exec(text); if (cost) write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "usage_update", used: input + cached + written + output + thought, size: 200000, cost: { amount: Number(cost[1]), currency: "USD" } } } }); return write({ id: message.id, result: { stopReason: "end_turn", usage: { inputTokens: input, outputTokens: output, totalTokens: input + output + thought + cached + written, ...(thought > 0 ? { thoughtTokens: thought } : {}), ...(cached > 0 ? { cachedReadTokens: cached } : {}), ...(written > 0 ? { cachedWriteTokens: written } : {}), } } }); } write({ id: message.id, result: {} }); } let buffer = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (chunk) => { buffer += chunk; let index; while ((index = buffer.indexOf(NL)) >= 0) { const line = buffer.slice(0, index); buffer = buffer.slice(index + 1); if (line.trim()) handle(JSON.parse(line)); } }); setInterval(() => {}, 1000); `; interface Turn { input: number; cached: number; written: number; output: number; thought: number; sessionCost?: number; } interface CaseReport { name: string; turnId: string; sent: Turn; recorded: UsageTokens[]; expectedCost: number | null; recordedCost: number | null; passed: boolean; } const report: { model?: string; cases: CaseReport[]; analytics?: { processedTokens: number; turns: number; missingUsageTurns: number }; passed: boolean; error?: string; } = { cases: [], passed: false }; const unused = () => Effect.die(new Error("Not used by this check.")); const browser: AgentBrowserHost = { listTabs: () => [], toolCanUseTab: () => false, beginTakeover: unused, endTakeover: () => {}, resolveUploadTarget: unused, handleDynamicTool: unused, userControlTabFor: () => null, awaitUserControlRelease: unused, onChanged: () => () => {}, onControlChanged: () => () => {}, onDocumentChanged: () => () => {}, clearControls: () => {}, endControl: () => {}, close: () => Effect.void, clearAgentSession: () => Effect.void, shareTab: () => undefined, unshareThread: () => undefined, agentTab: () => null, }; async function waitFor(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise { const deadline = Date.now() + timeoutMs; for (;;) { const value = read(); if (value) return value; if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`); await new Promise((done) => setTimeout(done, 25)); } } interface LogEntry { pid: number; start?: boolean; method?: string; sessionId?: string | null; } function protocol(): LogEntry[] { let text = ""; try { text = readFileSync(logPath, "utf8"); } catch { return []; } return text .split("\n") .filter(Boolean) .map((line) => JSON.parse(line)); } await writeFile(fakeOpenCode, FAKE_OPENCODE, { mode: 0o755 }); const database = new OpenBotDatabase(userData); const store = new AgentStore(userData, home, database); const mailbox = new MailboxStore(userData, store.sharedRoot, store.database); const agents = new AgentService({ store, mailbox, browser, preferredProvider: "opencode", offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "opencode"), }); let connection: DatabaseSync | null = null; const sqlite = () => { connection ??= new DatabaseSync(database.path); return connection; }; const closeReader = () => connection?.close(); function records(turnId: string): UsageTokens[] { return sqlite() .prepare("SELECT tokens_json FROM agent_usage_records WHERE turn_id = ? ORDER BY recorded_at, rowid") .all(turnId) .map((row) => decodeUsageTokens(JSON.parse(String(row.tokens_json)))); } function costOf(turnId: string): number | null { const value = sqlite() .prepare("SELECT estimated_cost_usd FROM agent_usage_records WHERE turn_id = ?") .get(turnId)?.estimated_cost_usd; return typeof value === "number" ? value : null; } function turnsOf(threadId: string): string[] { return sqlite() .prepare( `SELECT a.turn_id FROM agent_usage_activity a JOIN projection_provider_sessions s ON s.id = a.session_id WHERE a.kind = 'turn' AND s.thread_id = ? ORDER BY a.occurred_at, a.rowid`, ) .all(threadId) .map((row) => String(row.turn_id)); } const message = (turn: Turn) => `tokens=${turn.input},${turn.cached},${turn.written},${turn.output},${turn.thought}` + (turn.sessionCost === undefined ? "" : ` cost=${turn.sessionCost}`); async function check( name: string, threadId: string, index: number, sent: Turn, expectedCost: number | null, ): Promise { const turnId = await waitFor(`${name}: usage`, () => { const id = turnsOf(threadId)[index]; return id && records(id).length > 0 && id; }); const recorded = records(turnId); const recordedCost = costOf(turnId); const result: CaseReport = { name, turnId, sent, recorded, expectedCost, recordedCost, passed: false }; report.cases.push(result); assert.equal(recorded.length, 1, `${name}: one record for the turn`); assert.deepEqual( recorded[0], { uncachedInput: sent.input, cachedInput: sent.cached, cacheCreation: sent.written, output: sent.output + sent.thought, }, `${name}: the turn's own tokens, with thinking as output`, ); if (expectedCost === null) assert.equal(recordedCost, null, `${name}: no cost`); else assert.ok( recordedCost !== null && Math.abs(recordedCost - expectedCost) < 1e-9, `${name}: cost ${recordedCost}, expected ${expectedCost}`, ); result.passed = true; } async function send(agentId: string, turn: Turn, label: string): Promise { const receipt = await runCauseEffect(agents.sendMessage({ agentId, text: message(turn) })); const deliveryId = receipt.deliveries[0]?.id; assert.ok(deliveryId, `${label}: no delivery`); const status = await waitFor(label, () => { const value = mailbox.getDelivery(deliveryId)?.delivery.status; return value === "completed" || value === "failed" || value === "interrupted" ? value : null; }); assert.equal(status, "completed", `${label}: ${mailbox.getDelivery(deliveryId)?.delivery.error ?? status}`); } const idle = (agentId: string) => agents .listQueue(agentId) .deliveries.every((delivery) => !["queued", "starting", "running"].includes(delivery.status)); try { await runCauseEffect(agents.initialize()); const first: Turn = { input: 12_000, cached: 0, written: 0, output: 800, thought: 0, sessionCost: 0.05 }; const agent = await runCauseEffect( agents.createAgent({ name: "OpenCode metering", description: "Checks OpenCode usage metering.", avatarSeed: "opencode-metering", avatarHue: null, initialMessage: message(first), }), ); report.model = agent.model; const threadId = await waitFor("the agent's own thread", () => store.list().find((a) => a.id === agent.id)?.threadId); await check("first turn", threadId, 0, first, 0.05); await waitFor("the first turn to end", () => idle(agent.id)); const second: Turn = { input: 1_000, cached: 9_000, written: 0, output: 200, thought: 0 }; await send(agent.id, second, "smaller second turn"); await check("smaller second turn", threadId, 1, second, null); const third: Turn = { input: 15_000, cached: 6_000, written: 2_000, output: 900, thought: 400, sessionCost: 0.2 }; await send(agent.id, third, "larger third turn"); await check("larger third turn", threadId, 2, third, 0.15); const session = store.activeProviderSession(agent.id)?.externalSessionId; assert.ok(session, "the agent has a provider session"); const killedPid = protocol().find((entry) => entry.start)?.pid ?? 0; assert.ok(killedPid, "the fake server logged its start"); process.kill(killedPid, "SIGTERM"); await waitFor("OpenCode to start again", () => protocol().find((entry) => entry.start && entry.pid !== killedPid)); await waitFor("the provider to be ready again", () => agents.getStatus().phase === "ready"); const fourth: Turn = { input: 500, cached: 6_500, written: 0, output: 100, thought: 50, sessionCost: 0.26 }; await send(agent.id, fourth, "turn after a restart"); await check("turn after a restart", threadId, 3, fourth, 0.06); assert.equal( store.activeProviderSession(agent.id)?.externalSessionId, session, "the turn after a restart resumes the session", ); const sessionId = store.activeProviderSession(agent.id)?.id; assert.ok(sessionId, "the agent has an active provider session"); sqlite() .prepare("DELETE FROM agent_usage_checkpoints WHERE agent_id = ? AND counter_id = ?") .run(agent.id, `${sessionId}:opencode:cost`); const fifth: Turn = { input: 700, cached: 7_000, written: 0, output: 120, thought: 0, sessionCost: 0.3 }; await send(agent.id, fifth, "first cost of an older session"); await check("first cost of an older session", threadId, 4, fifth, null); const sixth: Turn = { input: 300, cached: 7_500, written: 0, output: 80, thought: 0, sessionCost: 0.31 }; await send(agent.id, sixth, "next cost of an older session"); await check("next cost of an older session", threadId, 5, sixth, 0.01); const today = new Date().toISOString().slice(0, 10); const analytics = agents.getAnalytics({ agentId: agent.id, startDate: today, endDate: today, timeZone: "UTC" }); report.analytics = { processedTokens: analytics.totals.processedTokens, turns: analytics.totals.turns, missingUsageTurns: analytics.totals.missingUsageTurns, }; const sent = report.cases.reduce( (sum, { sent: turn }) => sum + turn.input + turn.cached + turn.written + turn.output + turn.thought, 0, ); assert.equal(analytics.totals.turns, 6, "Analytics count every turn."); assert.equal(analytics.totals.missingUsageTurns, 0, "Every turn has usage."); assert.equal(analytics.totals.processedTokens, sent, "Analytics count every turn's tokens once."); report.passed = true; } catch (error) { report.error = error instanceof Error ? (error.stack ?? error.message) : String(error); process.exitCode = 1; } finally { await runCauseEffect(agents.stop()).catch(() => {}); closeReader(); database.close(); mkdirSync(OUT, { recursive: true }); writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`); await rm(root, { recursive: true, force: true }); process.stdout.write(`${report.passed ? "PASS" : "FAIL"} ${join(OUT, "report.json")}\n`); if (report.error) process.stdout.write(`${report.error}\n`); }