mirror of
https://github.com/nightly-labs/openbot.git
synced 2026-11-23 02:01:26 +08:00
* Let every chat of an agent use the browser tabs of the agent Main chats and side threads of an agent can now use all browser tabs of that agent. list_tabs shows the chat that opened each tab and tells when another chat uses a tab now. Takeover, submit_secret and list_logins use the same rule. Channel and messaging threads still use only their own tabs and the tabs shared with an @ tag. * Keep channel tabs out of other chats and make the working hint reliable The main chat and side threads use only the tabs that the main chat, a side thread or no chat opened. Tabs of a channel or a messaging thread stay out of these chats unless a person tags them with @. otherChatWorking now also uses the last chat that used each tab in the last 2 minutes, because control sessions end 1.2 s after a call. A takeover from one chat no longer deletes the held takeover result of another chat. list_tabs looks up each chat once per call. The browser user-control E2E now covers the cross-chat access. * Keep a held takeover result for each chat of a tab Held takeover results used only the tab id as the key. When two chats of one agent took over the same tab, the second result replaced the first. The key is now the tab id and the thread id.
436 lines
17 KiB
TypeScript
436 lines
17 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { randomBytes } from "node:crypto";
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import type { AgentEvent } from "@openbot/contracts/ipc";
|
|
import { isDynamicRecord } from "@openbot/contracts/runtime-values";
|
|
import { TEAM_API_ROUTES } from "@openbot/contracts/team-api-routes";
|
|
import {
|
|
TEAM_APP_VERSION_HEADER,
|
|
TEAM_CAPABILITIES_HEADER,
|
|
TEAM_PROTOCOL_V9,
|
|
TEAM_PROTOCOL_VERSION_HEADER,
|
|
teamCapabilitiesForProtocol,
|
|
} from "@openbot/contracts/team-protocol";
|
|
import {
|
|
COMPUTER_ACTIVITY_CAPABILITY,
|
|
COMPUTER_ACTIVITY_CODECS,
|
|
COMPUTER_ACTIVITY_ROUTES,
|
|
} from "@openbot/contracts/team-protocol/computer-activity-v1";
|
|
import { createOpenBotLogger } from "@openbot/logging";
|
|
import { encodeLiveActivityBytes, LIVE_ACTIVITY_SECRET_BYTES } from "@openbot/team-client/live-activity-seal";
|
|
import { plugin } from "bun";
|
|
import { Effect } from "effect";
|
|
import type { AgentBrowserHost } from "../src/backend/agent/turn-lifecycle";
|
|
import { AgentService } from "../src/backend/agent-service";
|
|
import { AgentStore } from "../src/backend/agent-store";
|
|
import { runCauseEffect } from "../src/backend/effect-boundary";
|
|
import { MailboxStore } from "../src/backend/mailbox-store";
|
|
import { OpenBotDatabase } from "../src/backend/openbot-database";
|
|
import { SidebarLayoutStore } from "../src/backend/sidebar-layout-store";
|
|
import type { NotificationRelayRequest } from "../src/main/push-notifications";
|
|
import type { TeamApiBrowser } from "../src/main/team-api/dependencies";
|
|
|
|
const OUT = resolve(import.meta.dirname, "../.openbot-build/computer-activity-e2e");
|
|
const APP_VERSION = "0.0.0-e2e";
|
|
const STEP_DEADLINE_MS = 30_000;
|
|
const AWAY_AFTER_MS = 2_000;
|
|
const OWNER = { username: "owner-user", password: "owner-password-123" };
|
|
const OWNER_PHONE = "owner-phone-session";
|
|
const WITNESS = "witness-member";
|
|
const WITNESS_PHONE = "witness-phone-session";
|
|
const logger = createOpenBotLogger("computer-activity-e2e");
|
|
|
|
plugin({
|
|
name: "computer-activity-electron-boundary",
|
|
setup(build) {
|
|
class Inert {}
|
|
build.module("electron", () => ({
|
|
loader: "object",
|
|
exports: {
|
|
app: { getPath: () => tmpdir(), getVersion: () => APP_VERSION, isPackaged: false, on: () => undefined },
|
|
BrowserWindow: class extends Inert {
|
|
static getAllWindows() {
|
|
return [];
|
|
}
|
|
},
|
|
MessageChannelMain: class extends Inert {},
|
|
Notification: class extends Inert {
|
|
static isSupported() {
|
|
return false;
|
|
}
|
|
},
|
|
clipboard: {},
|
|
desktopCapturer: {},
|
|
dialog: {},
|
|
ipcMain: { handle: () => undefined, removeHandler: () => undefined, on: () => undefined },
|
|
nativeImage: {},
|
|
net: {},
|
|
powerMonitor: { on: () => undefined },
|
|
safeStorage: { isEncryptionAvailable: () => false },
|
|
screen: { getAllDisplays: () => [] },
|
|
session: {},
|
|
shell: { openExternal: async () => undefined },
|
|
systemPreferences: {},
|
|
webContents: { getAllWebContents: () => [] },
|
|
},
|
|
}));
|
|
},
|
|
});
|
|
const { TeamApiServer } = await import("../src/main/team-api-server");
|
|
const { TeamStore } = await import("../src/main/team-store");
|
|
const { PushNotificationService } = await import("../src/main/push-notifications");
|
|
const { ComputerActivityReport } = await import("../src/main/computer-activity-report");
|
|
const { decodeComputerActivityReply } = await import("../src/main/remote-host-decoding");
|
|
|
|
interface Step {
|
|
name: string;
|
|
passed: boolean;
|
|
detail?: unknown;
|
|
error?: string;
|
|
}
|
|
|
|
const report: { passed: boolean; unverified: string[]; steps: Step[] } = {
|
|
passed: false,
|
|
unverified: [
|
|
"The Electron idle time (`powerMonitor.getSystemIdleTime`). The check gives the report a synthetic idle time.",
|
|
"Which joined servers `RemoteServerManager` tells: online, with the capability, and shown on this desktop. That needs a remote connection.",
|
|
"A request over WebRTC. The check sends over HTTP; both paths run the same route codec.",
|
|
],
|
|
steps: [],
|
|
};
|
|
|
|
async function step<T>(name: string, run: () => Promise<T>, summarize: (value: T) => unknown = (value) => value) {
|
|
process.stdout.write(`- ${name}\n`);
|
|
let deadline: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
const value = await Promise.race([
|
|
run(),
|
|
new Promise<never>((_, reject) => {
|
|
deadline = setTimeout(() => reject(new Error(`No result within ${STEP_DEADLINE_MS} ms.`)), STEP_DEADLINE_MS);
|
|
}),
|
|
]);
|
|
report.steps.push({ name, passed: true, detail: summarize(value) });
|
|
return value;
|
|
} catch (error) {
|
|
report.steps.push({ name, passed: false, error: error instanceof Error ? error.message : String(error) });
|
|
throw error;
|
|
} finally {
|
|
clearTimeout(deadline);
|
|
}
|
|
}
|
|
|
|
const root = await mkdtemp(join(tmpdir(), "openbot-computer-activity-"));
|
|
const userData = join(root, "user-data");
|
|
const unused = () => Effect.die(new Error("Not used by this check."));
|
|
const agentBrowser: AgentBrowserHost = {
|
|
listTabs: () => [],
|
|
toolCanUseTab: () => false,
|
|
beginTakeover: unused,
|
|
endTakeover: () => {},
|
|
userControlTabFor: () => null,
|
|
awaitUserControlRelease: unused,
|
|
resolveUploadTarget: unused,
|
|
handleDynamicTool: unused,
|
|
onChanged: () => () => {},
|
|
onControlChanged: () => () => {},
|
|
onDocumentChanged: () => () => {},
|
|
clearControls: () => {},
|
|
endControl: () => {},
|
|
close: () => Effect.void,
|
|
clearAgentSession: () => Effect.void,
|
|
shareTab: () => undefined,
|
|
unshareThread: () => undefined,
|
|
agentTab: () => null,
|
|
};
|
|
const teamBrowser: TeamApiBrowser = {
|
|
listTabs: () => [],
|
|
getControlState: () => ({ sessions: [] }),
|
|
open: unused,
|
|
activate: unused,
|
|
navigate: unused,
|
|
reload: unused,
|
|
stop: unused,
|
|
takeUserControl: unused,
|
|
startTeach: unused,
|
|
stopTeach: () => [],
|
|
releaseUserControl: () => {},
|
|
releaseUserControlHeldBy: () => {},
|
|
releaseUserControlRefused: () => {},
|
|
close: unused,
|
|
capturePreview: unused,
|
|
setVisible: unused,
|
|
getDisplayState: () => ({ tabs: [], activeTabId: null }),
|
|
loadUrl: unused,
|
|
isPrivate: () => false,
|
|
startView: unused,
|
|
dispatchViewInput: unused,
|
|
copyViewSelection: unused,
|
|
importSignIn: unused,
|
|
};
|
|
|
|
const database = new OpenBotDatabase(userData);
|
|
const agentStore = new AgentStore(userData, join(root, "home"), database);
|
|
const mailbox = new MailboxStore(userData, agentStore.sharedRoot, agentStore.database);
|
|
const sidebar = new SidebarLayoutStore(join(userData, "sidebar-layout.json"));
|
|
const team = new TeamStore(join(userData, "openbot-team-server-v2.json"));
|
|
let agents: AgentService | undefined;
|
|
let server: InstanceType<typeof TeamApiServer> | undefined;
|
|
let phones: InstanceType<typeof PushNotificationService> | undefined;
|
|
|
|
let hostNow = Date.now();
|
|
let desktopNow = hostNow;
|
|
let idleSeconds = 0;
|
|
const sent: string[] = [];
|
|
const sentWaiters = new Set<() => void>();
|
|
const pending = new Set<Promise<void>>();
|
|
const answers: boolean[] = [];
|
|
|
|
function nextSendTo(sessionId: string): Promise<number> {
|
|
const from = sent.length;
|
|
return new Promise((resolveSent) => {
|
|
const check = () => {
|
|
const index = sent.indexOf(sessionId, from);
|
|
if (index < 0) return;
|
|
sentWaiters.delete(check);
|
|
resolveSent(index);
|
|
};
|
|
sentWaiters.add(check);
|
|
check();
|
|
});
|
|
}
|
|
|
|
async function wire(
|
|
port: number,
|
|
input: { method: string; path: string; body?: unknown; token?: string },
|
|
): Promise<{ status: number; body: unknown }> {
|
|
const headers = new Headers({
|
|
Accept: "application/json",
|
|
[TEAM_PROTOCOL_VERSION_HEADER]: String(TEAM_PROTOCOL_V9),
|
|
[TEAM_APP_VERSION_HEADER]: APP_VERSION,
|
|
[TEAM_CAPABILITIES_HEADER]: teamCapabilitiesForProtocol(TEAM_PROTOCOL_V9).join(","),
|
|
});
|
|
if (input.body !== undefined) headers.set("Content-Type", "application/json");
|
|
if (input.token) headers.set("Authorization", `Bearer ${input.token}`);
|
|
const response = await fetch(`http://127.0.0.1:${port}${input.path}`, {
|
|
method: input.method,
|
|
headers,
|
|
body: input.body === undefined ? null : JSON.stringify(input.body),
|
|
});
|
|
const text = await response.text();
|
|
return { status: response.status, body: text ? JSON.parse(text) : null };
|
|
}
|
|
|
|
function registerPhone(sessionId: string, memberId: string): void {
|
|
assert.ok(phones);
|
|
phones.register(
|
|
sessionId,
|
|
{ memberId, protocol: TEAM_PROTOCOL_V9, capabilities: new Set() },
|
|
{
|
|
platform: "android",
|
|
value: {
|
|
serverId: "fixture-server",
|
|
token: `${sessionId}-token-`.padEnd(40, "x"),
|
|
secret: encodeLiveActivityBytes(randomBytes(LIVE_ACTIVITY_SECRET_BYTES)),
|
|
locale: "en",
|
|
away: true,
|
|
level: "all",
|
|
mutedUntil: null,
|
|
preview: false,
|
|
sound: "default",
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
function failedTurn(agentId: string, threadId: string, turnId: string): AgentEvent {
|
|
return { type: "turn-completed", agentId, threadId, turnId, status: "failed" };
|
|
}
|
|
|
|
try {
|
|
const setup = await step("Start the host with phone notifications, and sign in the desktop", async () => {
|
|
await runCauseEffect(sidebar.initialize());
|
|
await runCauseEffect(agentStore.initialize());
|
|
await runCauseEffect(mailbox.initialize());
|
|
await runCauseEffect(team.initialize());
|
|
const agent = await runCauseEffect(
|
|
agentStore.createAgent({
|
|
name: "Activity fixture",
|
|
description: "Synthetic agent for the computer activity check",
|
|
avatarSeed: "computer-activity",
|
|
avatarHue: null,
|
|
}),
|
|
);
|
|
agents = new AgentService({ store: agentStore, mailbox, browser: agentBrowser, sidebarLayout: sidebar });
|
|
await runCauseEffect(team.configure("E2E Host", OWNER.username, OWNER.password));
|
|
const ownerId = team.getOwnerMemberId();
|
|
assert.ok(ownerId, "The host has no owner.");
|
|
const service = agents;
|
|
phones = new PushNotificationService({
|
|
agents: service,
|
|
store: { list: () => [], replace: () => Effect.void },
|
|
send: (request: NotificationRelayRequest) =>
|
|
Effect.sync(() => {
|
|
sent.push(request.push.sessionId);
|
|
for (const wake of [...sentWaiters]) wake();
|
|
return "sent" as const;
|
|
}),
|
|
hiddenAgentIds: () => new Set(),
|
|
memberActive: () => true,
|
|
lastUseElsewhere: (memberId, sessionId) => server?.lastMemberUseElsewhere(memberId, sessionId) ?? null,
|
|
awayAfterMs: AWAY_AFTER_MS,
|
|
atHostComputer: () => false,
|
|
liveActivityShows: () => false,
|
|
randomBytes: (size) => new Uint8Array(randomBytes(size)),
|
|
now: () => hostNow,
|
|
logger,
|
|
});
|
|
server = new TeamApiServer({
|
|
store: team,
|
|
agents: service,
|
|
mailbox,
|
|
browser: teamBrowser,
|
|
sidebarLayout: sidebar,
|
|
appVersion: APP_VERSION,
|
|
pushNotifications: phones,
|
|
});
|
|
const port = await Effect.runPromise(server.start());
|
|
phones.start();
|
|
registerPhone(OWNER_PHONE, ownerId);
|
|
registerPhone(WITNESS_PHONE, WITNESS);
|
|
const login = await wire(port, { method: "POST", path: "/v1/auth/login", body: OWNER });
|
|
assert.equal(login.status, 200, `Login failed: ${JSON.stringify(login.body)}`);
|
|
assert.ok(isDynamicRecord(login.body) && typeof login.body.sessionToken === "string");
|
|
return { port, token: login.body.sessionToken, agentId: agent.id, threadId: agent.threadId ?? "activity-thread" };
|
|
});
|
|
const { port, token, agentId, threadId } = setup;
|
|
const event = (turnId: string) => failedTurn(agentId, threadId, turnId);
|
|
assert.ok(agents && phones && server);
|
|
const service = agents;
|
|
|
|
async function phonesFor(turnId: string): Promise<{ owner: boolean }> {
|
|
const from = sent.length;
|
|
const witness = nextSendTo(WITNESS_PHONE);
|
|
service.emit("event", event(turnId));
|
|
const index = await witness;
|
|
return { owner: sent.slice(from, index).includes(OWNER_PHONE) };
|
|
}
|
|
|
|
let targets = ["fixture-server"];
|
|
const activity = new ComputerActivityReport({
|
|
awayAfterMs: AWAY_AFTER_MS,
|
|
idleSeconds: () => idleSeconds,
|
|
targets: () => targets,
|
|
send: (_serverId, idle) =>
|
|
Effect.tryPromise(async () => {
|
|
const body = { idleSeconds: idle };
|
|
const codec = COMPUTER_ACTIVITY_CODECS.get(COMPUTER_ACTIVITY_ROUTES.report);
|
|
assert.ok(codec, "The route has no codec.");
|
|
codec.request(body);
|
|
const response = await wire(port, { method: "POST", path: COMPUTER_ACTIVITY_ROUTES.report, body, token });
|
|
assert.equal(response.status, 200, `Activity failed: ${JSON.stringify(response.body)}`);
|
|
const phonesKnown = decodeComputerActivityReply(codec.response(response.status, response.body));
|
|
answers.push(phonesKnown);
|
|
return phonesKnown;
|
|
}),
|
|
run: (operation) => {
|
|
const done = Effect.runPromise(operation);
|
|
pending.add(done);
|
|
void done.finally(() => pending.delete(done));
|
|
},
|
|
now: () => desktopNow,
|
|
});
|
|
async function checkActivity(): Promise<number> {
|
|
const before = answers.length;
|
|
activity.check();
|
|
await Promise.all([...pending]);
|
|
return answers.length - before;
|
|
}
|
|
|
|
await step("The host advertises computer-activity-v1 to a protocol 9 peer", async () => {
|
|
const compatibility = await wire(port, { method: "GET", path: TEAM_API_ROUTES.compatibility });
|
|
assert.ok(isDynamicRecord(compatibility.body) && Array.isArray(compatibility.body.capabilities));
|
|
assert.ok(compatibility.body.capabilities.includes(COMPUTER_ACTIVITY_CAPABILITY), "The capability is missing.");
|
|
return { advertised: true };
|
|
});
|
|
|
|
await step("Without input, the owner's phone gets the event", async () => {
|
|
const result = await phonesFor("turn-before-input");
|
|
assert.equal(result.owner, true, "The owner's phone got no notification.");
|
|
return result;
|
|
});
|
|
|
|
await step("Input on the computer keeps the owner's phone quiet, and the host knows the phone", async () => {
|
|
idleSeconds = 0;
|
|
const sends = await checkActivity();
|
|
assert.equal(sends, 1, "The desktop did not tell the host.");
|
|
assert.equal(answers.at(-1), true, "The host did not say that it notifies a phone of the member.");
|
|
const result = await phonesFor("turn-during-input");
|
|
assert.equal(result.owner, false, "The owner's phone got a notification while the computer had input.");
|
|
return { sends, ...result };
|
|
});
|
|
|
|
await step("The desktop sends again only after new input", async () => {
|
|
desktopNow += 1_000;
|
|
idleSeconds = 1;
|
|
const quiet = await checkActivity();
|
|
assert.equal(quiet, 0, "The desktop sent with no input since its last send.");
|
|
desktopNow += 500;
|
|
idleSeconds = 0;
|
|
const again = await checkActivity();
|
|
assert.equal(again, 1, "The desktop did not send after new input.");
|
|
return { withoutInput: quiet, afterInput: again };
|
|
});
|
|
|
|
await step("Activity is not use of a hosted server", async () => {
|
|
assert.equal(server?.lastClientUseAt() ?? null, null, "An activity report moved the client use time.");
|
|
return { lastClientUseAt: null };
|
|
});
|
|
|
|
await step("After the away time without input, the owner's phone gets the event again", async () => {
|
|
desktopNow += AWAY_AFTER_MS;
|
|
idleSeconds = AWAY_AFTER_MS / 1_000;
|
|
const sends = await checkActivity();
|
|
assert.equal(sends, 0, "The desktop sent while the computer had no input for the away time.");
|
|
hostNow += AWAY_AFTER_MS;
|
|
const result = await phonesFor("turn-after-away");
|
|
assert.equal(result.owner, true, "The owner's phone stayed quiet after the away time.");
|
|
return { sends, ...result };
|
|
});
|
|
|
|
await step("A host with no phone of the member is not told again for a while", async () => {
|
|
phones?.remove(OWNER_PHONE);
|
|
desktopNow += 1_000;
|
|
idleSeconds = 0;
|
|
const first = await checkActivity();
|
|
assert.equal(first, 1, "The desktop did not tell the host.");
|
|
assert.equal(answers.at(-1), false, "The host said that it notifies a phone of the member, which has none.");
|
|
desktopNow += 1_000;
|
|
const next = await checkActivity();
|
|
assert.equal(next, 0, "The desktop told a host with no phone of the member again at once.");
|
|
targets = [];
|
|
desktopNow += 1_000;
|
|
await checkActivity();
|
|
targets = ["fixture-server"];
|
|
desktopNow += 1_000;
|
|
const afterTargetChange = await checkActivity();
|
|
assert.equal(afterTargetChange, 0, "A target change ended the pause of a host with no phone of the member.");
|
|
return { first, next, afterTargetChange };
|
|
});
|
|
|
|
report.passed = true;
|
|
} finally {
|
|
if (phones) await Effect.runPromise(phones.dispose()).catch(() => undefined);
|
|
if (server) await Effect.runPromise(server.stop()).catch(() => undefined);
|
|
if (agents) await runCauseEffect(agents.stop()).catch(() => undefined);
|
|
database.close();
|
|
await rm(root, { recursive: true, force: true });
|
|
mkdirSync(OUT, { recursive: true });
|
|
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
process.stdout.write(`Report: ${join(OUT, "report.json")}\n`);
|
|
}
|
|
|
|
if (!report.passed) process.exitCode = 1;
|