mirror of
https://github.com/nightly-labs/openbot.git
synced 2026-11-23 02:01:26 +08:00
* Let every chat of an agent use the browser tabs of the agent Main chats and side threads of an agent can now use all browser tabs of that agent. list_tabs shows the chat that opened each tab and tells when another chat uses a tab now. Takeover, submit_secret and list_logins use the same rule. Channel and messaging threads still use only their own tabs and the tabs shared with an @ tag. * Keep channel tabs out of other chats and make the working hint reliable The main chat and side threads use only the tabs that the main chat, a side thread or no chat opened. Tabs of a channel or a messaging thread stay out of these chats unless a person tags them with @. otherChatWorking now also uses the last chat that used each tab in the last 2 minutes, because control sessions end 1.2 s after a call. A takeover from one chat no longer deletes the held takeover result of another chat. list_tabs looks up each chat once per call. The browser user-control E2E now covers the cross-chat access. * Keep a held takeover result for each chat of a tab Held takeover results used only the tab id as the key. When two chats of one agent took over the same tab, the second result replaced the first. The key is now the tab id and the thread id.
337 lines
14 KiB
TypeScript
337 lines
14 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { AGENT_PROVIDERS, decodeUsageTokens, type UsageTokens } from "@openbot/contracts/ipc";
|
|
import { Effect } from "effect";
|
|
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
|
|
|
|
const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-usage-metering-e2e");
|
|
const root = await mkdtemp(join(tmpdir(), "openbot-opencode-metering-"));
|
|
const userData = join(root, "user-data");
|
|
const home = join(root, "home");
|
|
const fakeOpenCode = join(root, "opencode");
|
|
const logPath = join(root, "opencode.jsonl");
|
|
|
|
process.env.HOME = home;
|
|
process.env.OPENBOT_OPENCODE_PATH = fakeOpenCode;
|
|
process.env.FAKE_OPENCODE_LOG = logPath;
|
|
mkdirSync(home, { recursive: true });
|
|
|
|
const { AgentService } = await import("../src/backend/agent-service");
|
|
const { AgentStore } = await import("../src/backend/agent-store");
|
|
const { runCauseEffect } = await import("../src/backend/effect-boundary");
|
|
const { MailboxStore } = await import("../src/backend/mailbox-store");
|
|
const { OpenBotDatabase } = await import("../src/backend/openbot-database");
|
|
|
|
const FAKE_OPENCODE = `#!/usr/bin/env node
|
|
const fs = require("node:fs");
|
|
const NL = String.fromCharCode(10);
|
|
if (process.argv.includes("--version")) { process.stdout.write("1.19.3" + NL); process.exit(0); }
|
|
const log = (entry) => fs.appendFileSync(process.env.FAKE_OPENCODE_LOG, JSON.stringify({ pid: process.pid, ...entry }) + NL);
|
|
log({ start: true });
|
|
const write = (message) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + NL);
|
|
const MODEL = "opencode/big-pickle";
|
|
const setup = (sessionId) => ({ sessionId, models: { availableModels: [{ modelId: MODEL, name: "Big Pickle" }], currentModelId: MODEL } });
|
|
let sessions = 0;
|
|
function handle(message) {
|
|
const params = message.params ?? {};
|
|
if (message.method) log({ method: message.method, sessionId: params.sessionId ?? null });
|
|
if (message.id === undefined) return;
|
|
if (message.method === "initialize")
|
|
return write({ id: message.id, result: { protocolVersion: 1, authMethods: [], agentCapabilities: { loadSession: true, sessionCapabilities: { list: {}, resume: {} } } } });
|
|
if (message.method === "session/new") return write({ id: message.id, result: setup("ses-" + process.pid + "-" + ++sessions) });
|
|
if (message.method === "session/list") return write({ id: message.id, result: { sessions: [] } });
|
|
if (message.method === "session/resume" || message.method === "session/load") return write({ id: message.id, result: setup(params.sessionId) });
|
|
if (message.method === "session/prompt") {
|
|
const text = params.prompt.map((block) => block.text ?? "").join("");
|
|
const match = /tokens=(\\d+),(\\d+),(\\d+),(\\d+),(\\d+)/.exec(text);
|
|
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done." } } } });
|
|
if (!match) return write({ id: message.id, result: { stopReason: "end_turn" } });
|
|
const [input, cached, written, output, thought] = match.slice(1).map(Number);
|
|
const cost = /cost=([\\d.]+)/.exec(text);
|
|
if (cost)
|
|
write({ method: "session/update", params: { sessionId: params.sessionId, update: { sessionUpdate: "usage_update", used: input + cached + written + output + thought, size: 200000, cost: { amount: Number(cost[1]), currency: "USD" } } } });
|
|
return write({ id: message.id, result: { stopReason: "end_turn", usage: {
|
|
inputTokens: input, outputTokens: output, totalTokens: input + output + thought + cached + written,
|
|
...(thought > 0 ? { thoughtTokens: thought } : {}),
|
|
...(cached > 0 ? { cachedReadTokens: cached } : {}),
|
|
...(written > 0 ? { cachedWriteTokens: written } : {}),
|
|
} } });
|
|
}
|
|
write({ id: message.id, result: {} });
|
|
}
|
|
let buffer = "";
|
|
process.stdin.setEncoding("utf8");
|
|
process.stdin.on("data", (chunk) => {
|
|
buffer += chunk;
|
|
let index;
|
|
while ((index = buffer.indexOf(NL)) >= 0) {
|
|
const line = buffer.slice(0, index);
|
|
buffer = buffer.slice(index + 1);
|
|
if (line.trim()) handle(JSON.parse(line));
|
|
}
|
|
});
|
|
setInterval(() => {}, 1000);
|
|
`;
|
|
|
|
interface Turn {
|
|
input: number;
|
|
cached: number;
|
|
written: number;
|
|
output: number;
|
|
thought: number;
|
|
sessionCost?: number;
|
|
}
|
|
|
|
interface CaseReport {
|
|
name: string;
|
|
turnId: string;
|
|
sent: Turn;
|
|
recorded: UsageTokens[];
|
|
expectedCost: number | null;
|
|
recordedCost: number | null;
|
|
passed: boolean;
|
|
}
|
|
|
|
const report: {
|
|
model?: string;
|
|
cases: CaseReport[];
|
|
analytics?: { processedTokens: number; turns: number; missingUsageTurns: number };
|
|
passed: boolean;
|
|
error?: string;
|
|
} = { cases: [], passed: false };
|
|
|
|
const unused = () => Effect.die(new Error("Not used by this check."));
|
|
const browser: AgentBrowserHost = {
|
|
listTabs: () => [],
|
|
toolCanUseTab: () => false,
|
|
beginTakeover: unused,
|
|
endTakeover: () => {},
|
|
resolveUploadTarget: unused,
|
|
handleDynamicTool: unused,
|
|
userControlTabFor: () => null,
|
|
awaitUserControlRelease: unused,
|
|
onChanged: () => () => {},
|
|
onControlChanged: () => () => {},
|
|
onDocumentChanged: () => () => {},
|
|
clearControls: () => {},
|
|
endControl: () => {},
|
|
close: () => Effect.void,
|
|
clearAgentSession: () => Effect.void,
|
|
shareTab: () => undefined,
|
|
unshareThread: () => undefined,
|
|
agentTab: () => null,
|
|
};
|
|
|
|
async function waitFor<T>(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise<T> {
|
|
const deadline = Date.now() + timeoutMs;
|
|
for (;;) {
|
|
const value = read();
|
|
if (value) return value;
|
|
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`);
|
|
await new Promise((done) => setTimeout(done, 25));
|
|
}
|
|
}
|
|
|
|
interface LogEntry {
|
|
pid: number;
|
|
start?: boolean;
|
|
method?: string;
|
|
sessionId?: string | null;
|
|
}
|
|
|
|
function protocol(): LogEntry[] {
|
|
let text = "";
|
|
try {
|
|
text = readFileSync(logPath, "utf8");
|
|
} catch {
|
|
return [];
|
|
}
|
|
return text
|
|
.split("\n")
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
}
|
|
|
|
await writeFile(fakeOpenCode, FAKE_OPENCODE, { mode: 0o755 });
|
|
const database = new OpenBotDatabase(userData);
|
|
const store = new AgentStore(userData, home, database);
|
|
const mailbox = new MailboxStore(userData, store.sharedRoot, store.database);
|
|
const agents = new AgentService({
|
|
store,
|
|
mailbox,
|
|
browser,
|
|
preferredProvider: "opencode",
|
|
offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "opencode"),
|
|
});
|
|
let connection: DatabaseSync | null = null;
|
|
const sqlite = () => {
|
|
connection ??= new DatabaseSync(database.path);
|
|
return connection;
|
|
};
|
|
const closeReader = () => connection?.close();
|
|
|
|
function records(turnId: string): UsageTokens[] {
|
|
return sqlite()
|
|
.prepare("SELECT tokens_json FROM agent_usage_records WHERE turn_id = ? ORDER BY recorded_at, rowid")
|
|
.all(turnId)
|
|
.map((row) => decodeUsageTokens(JSON.parse(String(row.tokens_json))));
|
|
}
|
|
|
|
function costOf(turnId: string): number | null {
|
|
const value = sqlite()
|
|
.prepare("SELECT estimated_cost_usd FROM agent_usage_records WHERE turn_id = ?")
|
|
.get(turnId)?.estimated_cost_usd;
|
|
return typeof value === "number" ? value : null;
|
|
}
|
|
|
|
function turnsOf(threadId: string): string[] {
|
|
return sqlite()
|
|
.prepare(
|
|
`SELECT a.turn_id FROM agent_usage_activity a JOIN projection_provider_sessions s ON s.id = a.session_id
|
|
WHERE a.kind = 'turn' AND s.thread_id = ? ORDER BY a.occurred_at, a.rowid`,
|
|
)
|
|
.all(threadId)
|
|
.map((row) => String(row.turn_id));
|
|
}
|
|
|
|
const message = (turn: Turn) =>
|
|
`tokens=${turn.input},${turn.cached},${turn.written},${turn.output},${turn.thought}` +
|
|
(turn.sessionCost === undefined ? "" : ` cost=${turn.sessionCost}`);
|
|
|
|
async function check(
|
|
name: string,
|
|
threadId: string,
|
|
index: number,
|
|
sent: Turn,
|
|
expectedCost: number | null,
|
|
): Promise<void> {
|
|
const turnId = await waitFor(`${name}: usage`, () => {
|
|
const id = turnsOf(threadId)[index];
|
|
return id && records(id).length > 0 && id;
|
|
});
|
|
const recorded = records(turnId);
|
|
const recordedCost = costOf(turnId);
|
|
const result: CaseReport = { name, turnId, sent, recorded, expectedCost, recordedCost, passed: false };
|
|
report.cases.push(result);
|
|
assert.equal(recorded.length, 1, `${name}: one record for the turn`);
|
|
assert.deepEqual(
|
|
recorded[0],
|
|
{
|
|
uncachedInput: sent.input,
|
|
cachedInput: sent.cached,
|
|
cacheCreation: sent.written,
|
|
output: sent.output + sent.thought,
|
|
},
|
|
`${name}: the turn's own tokens, with thinking as output`,
|
|
);
|
|
if (expectedCost === null) assert.equal(recordedCost, null, `${name}: no cost`);
|
|
else
|
|
assert.ok(
|
|
recordedCost !== null && Math.abs(recordedCost - expectedCost) < 1e-9,
|
|
`${name}: cost ${recordedCost}, expected ${expectedCost}`,
|
|
);
|
|
result.passed = true;
|
|
}
|
|
|
|
async function send(agentId: string, turn: Turn, label: string): Promise<void> {
|
|
const receipt = await runCauseEffect(agents.sendMessage({ agentId, text: message(turn) }));
|
|
const deliveryId = receipt.deliveries[0]?.id;
|
|
assert.ok(deliveryId, `${label}: no delivery`);
|
|
const status = await waitFor(label, () => {
|
|
const value = mailbox.getDelivery(deliveryId)?.delivery.status;
|
|
return value === "completed" || value === "failed" || value === "interrupted" ? value : null;
|
|
});
|
|
assert.equal(status, "completed", `${label}: ${mailbox.getDelivery(deliveryId)?.delivery.error ?? status}`);
|
|
}
|
|
|
|
const idle = (agentId: string) =>
|
|
agents
|
|
.listQueue(agentId)
|
|
.deliveries.every((delivery) => !["queued", "starting", "running"].includes(delivery.status));
|
|
|
|
try {
|
|
await runCauseEffect(agents.initialize());
|
|
const first: Turn = { input: 12_000, cached: 0, written: 0, output: 800, thought: 0, sessionCost: 0.05 };
|
|
const agent = await runCauseEffect(
|
|
agents.createAgent({
|
|
name: "OpenCode metering",
|
|
description: "Checks OpenCode usage metering.",
|
|
avatarSeed: "opencode-metering",
|
|
avatarHue: null,
|
|
initialMessage: message(first),
|
|
}),
|
|
);
|
|
report.model = agent.model;
|
|
const threadId = await waitFor("the agent's own thread", () => store.list().find((a) => a.id === agent.id)?.threadId);
|
|
await check("first turn", threadId, 0, first, 0.05);
|
|
await waitFor("the first turn to end", () => idle(agent.id));
|
|
|
|
const second: Turn = { input: 1_000, cached: 9_000, written: 0, output: 200, thought: 0 };
|
|
await send(agent.id, second, "smaller second turn");
|
|
await check("smaller second turn", threadId, 1, second, null);
|
|
|
|
const third: Turn = { input: 15_000, cached: 6_000, written: 2_000, output: 900, thought: 400, sessionCost: 0.2 };
|
|
await send(agent.id, third, "larger third turn");
|
|
await check("larger third turn", threadId, 2, third, 0.15);
|
|
|
|
const session = store.activeProviderSession(agent.id)?.externalSessionId;
|
|
assert.ok(session, "the agent has a provider session");
|
|
const killedPid = protocol().find((entry) => entry.start)?.pid ?? 0;
|
|
assert.ok(killedPid, "the fake server logged its start");
|
|
process.kill(killedPid, "SIGTERM");
|
|
await waitFor("OpenCode to start again", () => protocol().find((entry) => entry.start && entry.pid !== killedPid));
|
|
await waitFor("the provider to be ready again", () => agents.getStatus().phase === "ready");
|
|
const fourth: Turn = { input: 500, cached: 6_500, written: 0, output: 100, thought: 50, sessionCost: 0.26 };
|
|
await send(agent.id, fourth, "turn after a restart");
|
|
await check("turn after a restart", threadId, 3, fourth, 0.06);
|
|
assert.equal(
|
|
store.activeProviderSession(agent.id)?.externalSessionId,
|
|
session,
|
|
"the turn after a restart resumes the session",
|
|
);
|
|
|
|
const sessionId = store.activeProviderSession(agent.id)?.id;
|
|
assert.ok(sessionId, "the agent has an active provider session");
|
|
sqlite()
|
|
.prepare("DELETE FROM agent_usage_checkpoints WHERE agent_id = ? AND counter_id = ?")
|
|
.run(agent.id, `${sessionId}:opencode:cost`);
|
|
const fifth: Turn = { input: 700, cached: 7_000, written: 0, output: 120, thought: 0, sessionCost: 0.3 };
|
|
await send(agent.id, fifth, "first cost of an older session");
|
|
await check("first cost of an older session", threadId, 4, fifth, null);
|
|
const sixth: Turn = { input: 300, cached: 7_500, written: 0, output: 80, thought: 0, sessionCost: 0.31 };
|
|
await send(agent.id, sixth, "next cost of an older session");
|
|
await check("next cost of an older session", threadId, 5, sixth, 0.01);
|
|
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
const analytics = agents.getAnalytics({ agentId: agent.id, startDate: today, endDate: today, timeZone: "UTC" });
|
|
report.analytics = {
|
|
processedTokens: analytics.totals.processedTokens,
|
|
turns: analytics.totals.turns,
|
|
missingUsageTurns: analytics.totals.missingUsageTurns,
|
|
};
|
|
const sent = report.cases.reduce(
|
|
(sum, { sent: turn }) => sum + turn.input + turn.cached + turn.written + turn.output + turn.thought,
|
|
0,
|
|
);
|
|
assert.equal(analytics.totals.turns, 6, "Analytics count every turn.");
|
|
assert.equal(analytics.totals.missingUsageTurns, 0, "Every turn has usage.");
|
|
assert.equal(analytics.totals.processedTokens, sent, "Analytics count every turn's tokens once.");
|
|
report.passed = true;
|
|
} catch (error) {
|
|
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
|
|
process.exitCode = 1;
|
|
} finally {
|
|
await runCauseEffect(agents.stop()).catch(() => {});
|
|
closeReader();
|
|
database.close();
|
|
mkdirSync(OUT, { recursive: true });
|
|
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
await rm(root, { recursive: true, force: true });
|
|
process.stdout.write(`${report.passed ? "PASS" : "FAIL"} ${join(OUT, "report.json")}\n`);
|
|
if (report.error) process.stdout.write(`${report.error}\n`);
|
|
}
|