Files
openbot/scripts/webhook-analytics-e2e.ts
NB 6fbd8bfc20 Let the main chat and side threads of an agent use each other's browser tabs (#13433)
* Let every chat of an agent use the browser tabs of the agent

Main chats and side threads of an agent can now use all browser tabs of
that agent. list_tabs shows the chat that opened each tab and tells when
another chat uses a tab now. Takeover, submit_secret and list_logins use
the same rule. Channel and messaging threads still use only their own
tabs and the tabs shared with an @ tag.

* Keep channel tabs out of other chats and make the working hint reliable

The main chat and side threads use only the tabs that the main chat, a
side thread or no chat opened. Tabs of a channel or a messaging thread
stay out of these chats unless a person tags them with @.

otherChatWorking now also uses the last chat that used each tab in the
last 2 minutes, because control sessions end 1.2 s after a call.

A takeover from one chat no longer deletes the held takeover result of
another chat. list_tabs looks up each chat once per call.

The browser user-control E2E now covers the cross-chat access.

* Keep a held takeover result for each chat of a tab

Held takeover results used only the tab id as the key. When two chats of
one agent took over the same tab, the second result replaced the first.
The key is now the tab id and the thread id.
2026-11-20 13:51:50 +01:00

473 lines
19 KiB
TypeScript

import assert from "node:assert/strict";
import { createHmac, randomUUID } from "node:crypto";
import { mkdirSync, writeFileSync } from "node:fs";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { AGENT_PROVIDERS } from "@openbot/contracts/ipc";
import { WEBHOOK_DELIVERY_BODY_BYTES_LIMIT } from "@openbot/contracts/signal-protocol/messages";
import { WEBHOOK_ROUTES_LIMIT } from "@openbot/contracts/signal-protocol/webhook-route";
import { Effect } from "effect";
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
import type { HostOpenPanelClient } from "../src/main/analytics";
const OUT = resolve(import.meta.dirname, "../.openbot-build/webhook-analytics-e2e");
const root = await mkdtemp(join(tmpdir(), "openbot-webhook-analytics-"));
const userData = join(root, "user-data");
const home = join(root, "home");
const fakeCodex = join(root, "codex");
process.env.HOME = home;
process.env.CODEX_HOME = join(home, ".codex");
process.env.OPENBOT_CODEX_PATH = fakeCodex;
mkdirSync(home, { recursive: true });
const { AgentService } = await import("../src/backend/agent-service");
const { AgentStore } = await import("../src/backend/agent-store");
const { runCauseEffect } = await import("../src/backend/effect-boundary");
const { MailboxStore } = await import("../src/backend/mailbox-store");
const { OpenBotDatabase } = await import("../src/backend/openbot-database");
const { HostAnalytics } = await import("../src/main/analytics");
const { agentWebhookRoutineWriter, HostEventsService } = await import("../src/main/host-events-service");
const { webhookFailureCode } = await import("../src/renderer/src/webhook-failure-code");
const RELAY_ORIGIN = "https://signal.example.test";
const ROUTINE_NAME = "Failed builds webhook";
const BODY_MARK = "secret-build-log-text";
const FAKE_CODEX = `#!/usr/bin/env node
if (process.argv.includes("--version")) {
process.stdout.write("codex-cli 0.160.1\\n");
process.exit(0);
}
const config = { config: { mcp_servers: {} }, layers: [{ name: { type: "user", file: "/fake/.codex/config.toml", profile: null }, version: "1", config: { mcp_servers: {} } }] };
let buffer = "";
let threads = 0;
let turns = 0;
process.stdout.on("error", (error) => {
if (error.code === "EPIPE") process.exit(0);
throw error;
});
const write = (message) => process.stdout.write(JSON.stringify(message) + "\\n");
const answer = (message) => {
const params = message.params || {};
switch (message.method) {
case "initialize": return write({ id: message.id, result: {} });
case "account/read": return write({ id: message.id, result: { account: { type: "chatgpt", email: "codex@example.com" } } });
case "account/rateLimits/read": return write({ id: message.id, result: { rateLimits: { limitId: "codex", primary: null, secondary: null }, rateLimitsByLimitId: null } });
case "model/list": return write({ id: message.id, result: { data: [{ model: "gpt-6-luna", displayName: "GPT-6 Luna" }] } });
case "plugin/list": return write({ id: message.id, result: { marketplaces: [] } });
case "config/read": return write({ id: message.id, result: config });
case "config/batchWrite": return write({ id: message.id, result: {} });
case "thread/start": return write({ id: message.id, result: { thread: { id: "thread-" + (++threads), turns: [] } } });
case "thread/resume": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } });
case "thread/unsubscribe": return write({ id: message.id, result: { status: "Unsubscribed" } });
case "thread/turns/list":
case "thread/items/list": return write({ id: message.id, result: { data: [], nextCursor: null } });
case "thread/read": return write({ id: message.id, result: { thread: { id: params.threadId, turns: [] } } });
case "turn/start": {
const turnId = "turn-" + (++turns);
write({ id: message.id, result: { turn: { id: turnId, status: "inProgress", items: [] } } });
write({ method: "turn/started", params: { threadId: params.threadId, turn: { id: turnId } } });
return setTimeout(() => {
const item = { type: "agentMessage", id: "message-" + turnId, text: "Done.", phase: "final_answer" };
write({ method: "item/completed", params: { threadId: params.threadId, turnId, item } });
write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: turnId, status: "completed" } } });
}, 20);
}
case "turn/interrupt": {
write({ id: message.id, result: {} });
return write({ method: "turn/completed", params: { threadId: params.threadId, turn: { id: params.turnId, status: "interrupted" } } });
}
default:
return write({ id: message.id, error: { code: -32601, message: "Method not found: " + message.method } });
}
};
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => {
buffer += chunk;
let newline = buffer.indexOf("\\n");
while (newline >= 0) {
const line = buffer.slice(0, newline);
buffer = buffer.slice(newline + 1);
if (line) {
const message = JSON.parse(line);
if (message.method && message.id !== undefined) answer(message);
}
newline = buffer.indexOf("\\n");
}
});
`;
type Properties = NonNullable<Parameters<HostOpenPanelClient["track"]>[1]>;
interface Sent {
name: string;
properties: Properties;
}
const report: {
statuses: Record<string, number>;
deliveries: Properties[];
routineRuns: Properties[];
cap?: { sentInWindow: number; skippedCount: unknown };
routeLimitCode?: string;
checks: { name: string; passed: boolean }[];
passed: boolean;
error?: string;
} = { statuses: {}, deliveries: [], routineRuns: [], checks: [], passed: false };
const sent: Sent[] = [];
const client: HostOpenPanelClient = {
setGlobalProperties: () => {},
identify: async () => {},
clear: () => {},
track: async (name, properties) => {
sent.push({ name, properties: { ...properties } });
},
};
const deliveries = () => sent.filter((event) => event.name === "system_webhook_delivery").map((e) => e.properties);
const routineRuns = () => sent.filter((event) => event.name === "system_routine_run").map((e) => e.properties);
const unused = () => Effect.die(new Error("Not used by this check."));
const browser: AgentBrowserHost = {
listTabs: () => [],
toolCanUseTab: () => false,
beginTakeover: unused,
endTakeover: () => {},
resolveUploadTarget: unused,
handleDynamicTool: unused,
userControlTabFor: () => null,
awaitUserControlRelease: unused,
onChanged: () => () => {},
onControlChanged: () => () => {},
onDocumentChanged: () => () => {},
clearControls: () => {},
endControl: () => {},
close: () => Effect.void,
clearAgentSession: () => Effect.void,
shareTab: () => undefined,
unshareThread: () => undefined,
agentTab: () => null,
};
async function waitFor<T>(label: string, read: () => T | null | undefined | false, timeoutMs = 20_000): Promise<T> {
const deadline = Date.now() + timeoutMs;
for (;;) {
const value = read();
if (value) return value;
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}.`);
await new Promise((done) => setTimeout(done, 25));
}
}
function pass(name: string, condition: boolean, detail?: unknown) {
report.checks.push({ name, passed: condition });
assert.ok(condition, detail === undefined ? name : `${name}: ${JSON.stringify(detail)}`);
}
const DELIVERY_KEYS = new Set([
"profileId",
"owner_kind",
"format",
"result",
"ignored_reason",
"status_code",
"body_bytes",
"handle_ms",
"delivery_age_ms",
"test",
"skipped_count",
]);
await writeFile(fakeCodex, FAKE_CODEX, { mode: 0o755 });
const database = new OpenBotDatabase(userData);
const store = new AgentStore(userData, home, database);
const mailbox = new MailboxStore(userData, store.sharedRoot, store.database);
let principal: string | null = "account-1";
const agents = new AgentService({
store,
mailbox,
browser,
preferredProvider: "codex",
offProviders: AGENT_PROVIDERS.filter((provider) => provider !== "codex"),
webhookRoutines: () => agentWebhookRoutineWriter(events),
});
let reportDelivery:
| ((report: Parameters<InstanceType<typeof HostAnalytics>["handleWebhookDelivery"]>[0]) => void)
| null = null;
const events = new HostEventsService({
routines: agents.routineRecords,
cipher: {
encrypt: (value) => Buffer.from(`sealed:${Buffer.from(value).toString("base64")}`),
decrypt: (value) => Buffer.from(value.toString().slice("sealed:".length), "base64").toString(),
},
relay: {
connected: () => true,
setEnabled: () => {},
setAccountActive: () => {},
refresh: () => {},
ensureHost: () => Effect.void,
registerRoute: (routeId) => Effect.succeed(`${RELAY_ORIGIN}/v1/webhooks/${routeId}`),
revokeRoute: () => Effect.void,
},
accountPrincipal: () => principal,
onDelivery: (delivery) => reportDelivery?.(delivery),
});
const analytics = new HostAnalytics(
{
enabled: true,
trackingEnabled: true,
appVersion: "0.0.0",
platform: "darwin",
resolveOwner: () => ({ id: "account-1", email: "owner@example.test" }),
resolveAgent: () => null,
resolveRoutineRun: (agentId, routineId, runId) => {
const run = agents.listRoutineRuns({ agentId, routineId }).find((item) => item.id === runId);
const routine = agents.routineRecords.get({ kind: "agent", id: agentId }, routineId);
if (!run || !routine) return null;
return {
runKind: run.kind,
runSource: run.source ?? null,
triggerType: routine.trigger.kind === "webhook" ? "webhook" : routine.trigger.schedule.kind,
};
},
},
() => client,
);
reportDelivery = (delivery) => analytics.handleWebhookDelivery(delivery);
agents.on("event", (event) => analytics.handleAgentEvent(event));
const realNow = Date.now.bind(Date);
let clockOffset = 0;
Date.now = () => realNow() + clockOffset;
try {
await runCauseEffect(agents.initialize());
const agent = await runCauseEffect(
agents.createAgent({
name: "Builds",
description: "Watches builds.",
avatarSeed: "builds",
avatarHue: null,
initialMessage: "Hello.",
}),
);
await waitFor("the first chat turn to end", () =>
agents
.listQueue(agent.id)
.deliveries.every((delivery) => !["queued", "starting", "running"].includes(delivery.status)),
);
const owner = { kind: "agent", id: agent.id } as const;
const saved = await Effect.runPromise(
events.saveRoutine({
owner,
name: ROUTINE_NAME,
instruction: "Read the failed build.",
active: true,
timezone: "UTC",
trigger: { kind: "webhook", eventType: "build.failed", filters: [] },
}),
);
const secret = saved.secret;
assert.ok(secret, "The save made no secret.");
const route = agents.routineRecords.routes.list().find((item) => item.routineId === saved.routine.id);
assert.ok(route, "The save made no route.");
const routeId = route.routeId;
const deliveryIds: string[] = [];
const sign = (key: string, body: Uint8Array, timestamp: string, deliveryId: string) =>
`sha256=${createHmac("sha256", key).update(`${timestamp}.${deliveryId}.`).update(body).digest("hex")}`;
const deliver = async (
label: string,
options: { key?: string; route?: string; body?: Uint8Array; type?: string } = {},
) => {
const body =
options.body ??
new TextEncoder().encode(JSON.stringify({ type: options.type ?? "build.failed", data: { log: BODY_MARK } }));
const timestamp = String(Math.floor(Date.now() / 1000));
const deliveryId = randomUUID();
deliveryIds.push(deliveryId);
const { status } = await Effect.runPromise(
events.receive({
routeId: options.route ?? routeId,
deliveryId,
timestamp,
signature: sign(options.key ?? secret, body, timestamp, deliveryId),
body,
}),
);
report.statuses[label] = status;
return status;
};
const lastDelivery = async (count: number) => {
await waitFor(`delivery event ${count}`, () => deliveries().length >= count);
return deliveries()[count - 1] ?? {};
};
pass("A signed request starts the routine", (await deliver("started")) === 202);
const started = await lastDelivery(1);
pass(
"The started request reports its owner, result, status and size",
started.owner_kind === "agent" &&
started.format === "openbot" &&
started.result === "started" &&
started.status_code === 202 &&
typeof started.body_bytes === "number" &&
started.body_bytes > 0 &&
started.test === false,
started,
);
pass(
"The started request reports whole-number timings",
Number.isInteger(started.handle_ms) && Number.isInteger(started.delivery_age_ms),
started,
);
const run = await waitFor("the finished webhook run event", () =>
routineRuns().find((event) => event.status !== "running"),
);
pass(
"The webhook run reports trigger_type webhook and run_source webhook",
run.status === "succeeded" && run.trigger_type === "webhook" && run.run_source === "webhook",
run,
);
pass("Another event type is ignored", (await deliver("ignored", { type: "build.passed" })) === 202);
const ignored = await lastDelivery(2);
pass(
"The ignored request reports its reason",
ignored.result === "ignored" && ignored.ignored_reason === "event-type" && ignored.status_code === 202,
ignored,
);
pass("A wrong secret is refused", (await deliver("badSignature", { key: "whsec_not-the-secret" })) === 401);
const badSignature = await lastDelivery(3);
pass(
"The refused signature reports its owner and no delivery age",
badSignature.result === "bad_signature" &&
badSignature.status_code === 401 &&
badSignature.owner_kind === "agent" &&
badSignature.delivery_age_ms === undefined,
badSignature,
);
pass("An unknown route is refused", (await deliver("unknownRoute", { route: randomUUID() })) === 404);
const unknown = await lastDelivery(4);
pass(
"The unknown route reports no owner",
unknown.result === "unknown_route" && unknown.status_code === 404 && unknown.owner_kind === undefined,
unknown,
);
pass(
"A body that is not an event is refused",
(await deliver("badBody", { body: new TextEncoder().encode(`not json ${BODY_MARK}`) })) === 400,
);
const badBody = await lastDelivery(5);
pass(
"The bad body reports a delivery age, as it was signed",
badBody.result === "bad_body" && badBody.status_code === 400 && Number.isInteger(badBody.delivery_age_ms),
badBody,
);
pass(
"A body over the limit is refused",
(await deliver("tooLarge", { body: new Uint8Array(WEBHOOK_DELIVERY_BODY_BYTES_LIMIT + 1) })) === 413,
);
const tooLarge = await lastDelivery(6);
pass(
"The large body reports too_large and its size",
tooLarge.result === "too_large" &&
tooLarge.status_code === 413 &&
tooLarge.body_bytes === WEBHOOK_DELIVERY_BODY_BYTES_LIMIT + 1,
tooLarge,
);
principal = null;
pass("A signed-out host is unavailable", (await deliver("signedOut")) === 503);
principal = "account-1";
const signedOut = await lastDelivery(7);
pass(
"The signed-out request reports signed_out",
signedOut.result === "signed_out" && signedOut.status_code === 503,
signedOut,
);
const test = await Effect.runPromise(events.sendTestEvent({ owner, id: saved.routine.id }));
pass("The test event starts the routine", test.status === "started");
const testEvent = await lastDelivery(8);
pass("The test event reports test: true", testEvent.test === true && testEvent.result === "started", testEvent);
await waitFor(
"the test run to finish",
() => routineRuns().filter((event) => event.status !== "running").length >= 2,
);
clockOffset += 61_000;
const windowStart = deliveries().length;
for (let index = 0; index < 65; index += 1) await deliver("capped", { route: randomUUID() });
await waitFor("the capped window", () => deliveries().length >= windowStart + 60);
const sentInWindow = deliveries().length - windowStart;
clockOffset += 61_000;
await deliver("afterCap", { route: randomUUID() });
const afterCap = await lastDelivery(windowStart + 61);
report.cap = { sentInWindow, skippedCount: afterCap.skipped_count };
pass("Only 60 delivery events go in one minute", sentInWindow === 60, sentInWindow);
pass("The next event counts the 5 that did not go", afterCap.skipped_count === 5, afterCap);
report.deliveries = deliveries();
report.routineRuns = routineRuns();
pass(
"Delivery events carry only the allowed keys",
report.deliveries.every((properties) => Object.keys(properties).every((key) => DELIVERY_KEYS.has(key))),
);
const body = JSON.stringify(sent);
const leaks = [routeId, secret, saved.routine.id, ROUTINE_NAME, BODY_MARK, ...deliveryIds].filter((value) =>
body.includes(value),
);
pass("No event has a route ID, delivery ID, secret, routine name or body text", leaks.length === 0, leaks);
for (let index = agents.routineRecords.routes.list().length; index < WEBHOOK_ROUTES_LIMIT; index += 1) {
await Effect.runPromise(
events.saveRoutine({
owner,
name: `Filler ${index}`,
instruction: "Never runs.",
active: false,
timezone: "UTC",
trigger: { kind: "webhook", eventType: null, filters: [] },
}),
);
}
const limitError = await runCauseEffect(
events.saveRoutine({
owner,
name: "Over the limit",
instruction: "Never runs.",
active: false,
timezone: "UTC",
trigger: { kind: "webhook", eventType: null, filters: [] },
}),
).then(
() => null,
(error: unknown) => (error instanceof Error ? error.message : String(error)),
);
assert.ok(limitError, "The save over the route limit did not fail.");
report.routeLimitCode = webhookFailureCode(
new Error(`Error invoking remote method 'events:save-routine': Error: ${limitError}`),
"save_failed",
);
pass("The route limit failure maps to webhook_route_limit", report.routeLimitCode === "webhook_route_limit");
pass(
"Another failure keeps its fallback code",
webhookFailureCode(new Error("Other."), "save_failed") === "save_failed",
);
report.passed = true;
} catch (error) {
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
process.exitCode = 1;
} finally {
Date.now = realNow;
await runCauseEffect(agents.stop()).catch(() => {});
database.close();
mkdirSync(OUT, { recursive: true });
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
await rm(root, { recursive: true, force: true });
process.stdout.write(`${report.passed ? "PASS" : "FAIL"} ${join(OUT, "report.json")}\n`);
if (report.error) process.stdout.write(`${report.error}\n`);
}