Merge pull request #297 from lyingbug/feat/task-popup-attribution

feat(extension): preserve observed popups while enabling task control
This commit is contained in:
Zhang GH
2026-09-22 20:32:41 +08:00
committed by GitHub
12 changed files with 1027 additions and 12 deletions
@@ -0,0 +1,129 @@
import { afterEach, expect, it, vi } from "vitest";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { watchDaemonConnection } from "@/lib/daemon-connection-preference";
import { OVERLAY_AGENT_STATE, type OverlayAgentStateMessage } from "@/lib/overlay-bridge";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { isAgentControlledTab } from "@/session-manager/manager";
import { ToolDispatcher } from "@/tools/dispatcher";
vi.hoisted(() => {
Object.assign(globalThis, { defineBackground: (main: unknown) => main });
});
// Keep background's real overlay wiring and SessionManager; isolate unrelated
// startup services so no transport, storage or browser connection is opened.
vi.mock("@/browser-driver/chromium-cdp");
vi.mock("@/debug/archive");
vi.mock("@/debug/bridge");
vi.mock("@/debug/manager");
vi.mock("@/lib/audit");
vi.mock("@/lib/audit-bridge");
vi.mock("@/lib/connection-controller");
vi.mock("@/lib/daemon-connection-preference");
vi.mock("@/lib/heartbeat");
vi.mock("@/lib/instance-id");
vi.mock("@/lib/interaction-preferences");
vi.mock("@/lib/keepalive");
vi.mock("@/lib/recording/frame-coordinator");
vi.mock("@/lib/sessions-live-flag");
vi.mock("@/long-screenshot/background");
vi.mock("@/session-manager/event-handler");
vi.mock("@/session-manager/agent-window", () => ({
AGENT_WINDOW_HOME: "about:blank",
chromeAgentWindowApi: {
create: async () => ({ windowId: 10, initialTabIds: [10] }),
ensureActiveTab: async () => 10,
},
}));
vi.mock("@/tools/borrow-confirmation");
vi.mock("@/tools/dispatcher");
vi.mock("@/tools/record");
vi.mock("@/transport/handshake");
vi.mock("@/transport/remote-authorization");
vi.mock("@/transport/ws-transport");
import background from "@/entrypoints/background";
afterEach(() => {
vi.unstubAllGlobals();
vi.clearAllMocks();
});
function event<T extends unknown[] = []>() {
const listeners: Array<(...args: T) => void> = [];
return {
addListener: (listener: (...args: T) => void) => listeners.push(listener),
emit: (...args: T) => {
for (const listener of listeners) listener(...args);
},
};
}
async function fixture() {
const overlay = new OverlayController();
const onDetached = event<[number]>();
const sendMessage = vi.fn(async (_tabId: number, message: OverlayAgentStateMessage) => {
overlay.applyAgentControlMode(message.sessionId, message.mode);
});
vi.stubGlobal("chrome", {
tabs: {
sendMessage,
onDetached,
onActivated: event(),
onUpdated: event(),
onCreated: event(),
onRemoved: event(),
},
debugger: { onDetach: event() },
runtime: { onMessage: event(), onConnect: event() },
notifications: { onClicked: event(), onButtonClicked: event() },
});
vi.mocked(watchDaemonConnection).mockReturnValue({ ready: Promise.resolve(), dispose() {} });
vi.mocked(attachSessionsLiveFlag).mockReturnValue({
refresh: vi.fn(async () => {}),
syncFromManager: vi.fn(async () => {}),
});
vi.mocked(ChromiumCdp.prototype.releaseSessionTab).mockResolvedValue();
(background as unknown as () => void)();
const deps = vi.mocked(ToolDispatcher).mock.calls.at(-1)![0];
const task = await deps.sessions.start("one");
return { overlay, onDetached, sendMessage, deps, task };
}
it.each([false, true])("hides an observed tab after detachment, remote=%s", async (remote) => {
const f = await fixture();
f.task.remote = remote;
f.task.observedTabs = new Set([20]);
f.deps.onAgentTabClaimed?.(20, 10);
expect(shouldShowAgentControlOverlay(f.overlay.snapshot())).toBe(true);
f.onDetached.emit(20);
expect(isAgentControlledTab(f.task, 20)).toBe(false);
expect(f.sendMessage).toHaveBeenLastCalledWith(20, {
type: OVERLAY_AGENT_STATE,
sessionId: null,
mode: "hidden",
generation: 0,
});
expect(shouldShowAgentControlOverlay(f.overlay.snapshot())).toBe(false);
expect(f.deps.cdp?.releaseSessionTab).toHaveBeenCalledWith("one", 20);
});
it("still releases control and CDP when the content script cannot receive hidden", async () => {
const f = await fixture();
f.task.observedTabs = new Set([20]);
f.sendMessage.mockRejectedValueOnce(new Error("Receiving end does not exist"));
f.onDetached.emit(20);
await Promise.resolve();
expect(isAgentControlledTab(f.task, 20)).toBe(false);
expect(f.deps.cdp?.releaseSessionTab).toHaveBeenCalledWith("one", 20);
});
it("does not reset tabs without an observed claim", async () => {
const f = await fixture();
f.onDetached.emit(10);
f.onDetached.emit(30);
expect(f.sendMessage).not.toHaveBeenCalled();
expect(f.deps.cdp?.releaseSessionTab).not.toHaveBeenCalled();
expect(isAgentControlledTab(f.task, 10)).toBe(true);
});
+10 -2
View File
@@ -166,7 +166,7 @@ export default defineBackground(() => {
/**
* Authoritative overlay state for a specific tab. Agent Window tabs are
* free by default; only tabs explicitly claimed through session startup,
* `tab_create`, or `tab_borrow` receive the control overlay.
* `tab_create`, `tab_borrow`, or popup observation receive the control overlay.
*/
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
if (typeof tabId === "number" && typeof windowId === "number") {
@@ -252,7 +252,15 @@ export default defineBackground(() => {
if (!sessions.findByWindowId(tab.windowId)) return;
void pushOverlayStateForTab(tab.id, tab.windowId);
});
chrome.tabs.onDetached.addListener((tabId) => debug.releaseTab(tabId));
chrome.tabs.onDetached.addListener((tabId) => {
debug.releaseTab(tabId);
const releasedSessionIds = sessions.releaseObservedTab(tabId);
if (releasedSessionIds.length > 0) void pushOverlayStateForTab(tabId);
for (const sessionId of releasedSessionIds)
void cdp
.releaseSessionTab(sessionId, tabId)
.catch((error) => console.debug("[bsk] observed tab release failed", error));
});
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
debug.stopTab(tabId, "tab_closed");
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
@@ -0,0 +1,487 @@
import { afterEach, expect, it, vi } from "vitest";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { handleSessionStop } from "@/tools/session";
import { handleTabBorrow, type TabManagementDeps } from "@/tools/tabs";
import { createDisconnectCleanup } from "../disconnect-cleanup";
import { isAgentControlledTab, SessionManager } from "../manager";
import { withTaskPopups } from "../task-popups";
afterEach(() => {
vi.unstubAllGlobals();
vi.useRealTimers();
});
function deferred<T>() {
let resolve!: (v: T) => void;
const promise = new Promise<T>((r) => {
resolve = r;
});
return { promise, resolve };
}
function event<T extends (...args: never[]) => void>() {
const listeners = new Set<T>();
return {
listeners,
addListener: (fn: T) => listeners.add(fn),
removeListener: (fn: T) => listeners.delete(fn),
};
}
async function fixture(remote = true) {
vi.useFakeTimers();
let next = 10;
const removeWindow = vi.fn(async () => {});
const manager = new SessionManager({
remote: () => remote,
agentWindow: {
create: async () => ({ windowId: next++, initialTabIds: [] }),
remove: removeWindow,
ensureActiveTab: async (id) => id,
},
});
const task = await manager.start("one"),
other = await manager.start("two");
const tabs = new Map<number, chrome.tabs.Tab>([
[10, { id: 10, windowId: 10, active: true } as chrome.tabs.Tab],
[11, { id: 11, windowId: 11, active: true } as chrome.tabs.Tab],
]);
const targets =
event<(e: { sourceTabId: number; sourceFrameId: number; tabId: number }) => void>();
const removed = event<(id: number) => void>(),
detached = event<(id: number) => void>();
const opened = event<() => void>();
const api = {
get: vi.fn(async (id: number) => {
const t = tabs.get(id);
if (!t) throw new Error("missing tab");
return { ...t };
}),
query: vi.fn(async (q: { windowId?: number }) =>
[...tabs.values()].filter((t) => q.windowId === undefined || t.windowId === q.windowId),
),
update: vi.fn(async (id: number, props: object) => Object.assign(tabs.get(id)!, props)),
move: vi.fn(async (id: number, props: { windowId: number }) =>
Object.assign(tabs.get(id)!, props),
),
remove: vi.fn(async (id: number) => {
tabs.delete(id);
}),
create: vi.fn(),
onCreated: opened,
onRemoved: removed,
onDetached: detached,
};
vi.stubGlobal("chrome", { tabs: api, webNavigation: { onCreatedNavigationTarget: targets } });
const open = (sourceTabId: number, tabId: number, windowId = 10, sourceFrameId = 0) => {
tabs.set(tabId, { id: tabId, windowId } as chrome.tabs.Tab);
for (const fn of opened.listeners) fn();
for (const fn of targets.listeners) fn({ sourceTabId, sourceFrameId, tabId });
};
const deps = {
tabs: api,
windows: {
get: vi.fn(async (id: number) => ({ id })),
getLastFocused: vi.fn(),
create: vi.fn(),
remove: vi.fn(),
},
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
approveBorrow: vi.fn(async () => true),
} as unknown as TabManagementDeps;
return {
manager,
task,
other,
tabs,
api,
targets,
removed,
detached,
opened,
open,
deps,
removeWindow,
};
}
it.each([
false,
true,
])("controls same-window and nested targets without destructive ownership, remote=%s", async (remote) => {
const f = await fixture(remote);
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(10, 20);
f.open(20, 21);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs).toEqual(new Set([20, 21]));
expect(f.task.agentCreatedTabs).toEqual(new Set([10]));
expect(isAgentControlledTab(f.task, 20)).toBe(true);
expect(f.api.move).not.toHaveBeenCalled();
expect(f.targets.listeners.size + f.removed.listeners.size + f.detached.listeners.size).toBe(0);
});
it.each([
[false, false],
[true, false],
[false, true],
[true, true],
])("releases observed overlays after session cleanup, remote=%s disconnect=%s", async (remote, disconnect) => {
const f = await fixture(remote);
const overlay = new OverlayController();
const work = withTaskPopups(
f.manager,
{ session_id: "one" },
async (input) => {
input(10);
f.open(10, 20);
},
() => overlay.activateAgentSession("one"),
);
await vi.advanceTimersByTimeAsync(100);
await work;
expect(shouldShowAgentControlOverlay(overlay.snapshot())).toBe(true);
// Closing the home tab can activate the popup and push control again.
const remove = f.api.remove.getMockImplementation()!;
f.api.remove.mockImplementation(async (id) => {
await remove(id);
if (id === 10) overlay.activateAgentSession("one");
});
const reset = vi.mocked(f.deps.agentOverlayReset!.resetAgentOverlays);
reset.mockImplementation(async (_tabId, sessionId) => {
expect(f.manager.get(sessionId)).toBeNull();
overlay.resetAgentOverlays(sessionId);
});
const deps = { tabManagement: f.deps, tabsQuery: f.api };
if (disconnect) {
const cleanup = createDisconnectCleanup({ manager: f.manager, sessionStopDeps: deps });
expect((await cleanup()).failures).toEqual([]);
} else {
expect(await handleSessionStop(f.manager, { session_id: "one" }, deps)).toMatchObject({
window_released: true,
});
}
expect(reset).toHaveBeenCalledExactlyOnceWith(20, "one");
expect(shouldShowAgentControlOverlay(overlay.snapshot())).toBe(false);
expect(f.api.remove).toHaveBeenCalledWith(10);
expect(f.api.remove).not.toHaveBeenCalledWith(20);
expect(f.removeWindow).not.toHaveBeenCalledWith(10);
expect(f.tabs.has(20)).toBe(true);
});
it.each([
false,
true,
])("overlay message failure does not prevent stop, queryFails=%s", async (queryFails) => {
const f = await fixture();
f.task.observedTabs = new Set([20]);
f.tabs.set(20, { id: 20, windowId: 10 } as chrome.tabs.Tab);
if (queryFails) f.api.query.mockRejectedValueOnce(new Error("temporary query failure"));
// Exercise production fallback wiring, without the injected reset API.
const sendMessage = vi.fn().mockRejectedValue(new Error("Receiving end does not exist"));
Object.assign(f.api, { sendMessage });
const result = await handleSessionStop(
f.manager,
{ session_id: "one" },
{
tabManagement: { ...f.deps, agentOverlayReset: undefined },
tabsQuery: f.api,
},
);
expect(result).toMatchObject({ window_released: true });
expect(f.manager.get("one")).toBeNull();
expect(f.tabs.has(20)).toBe(true);
expect(sendMessage).toHaveBeenCalledWith(20, {
type: "bh-agent-overlay-reset",
sessionId: "one",
});
});
it("does not wait for an unresponsive overlay receiver after releasing the session", async () => {
const f = await fixture();
f.task.observedTabs = new Set([20]);
f.tabs.set(20, { id: 20, windowId: 10 } as chrome.tabs.Tab);
const response = deferred<void>();
vi.mocked(f.deps.agentOverlayReset!.resetAgentOverlays).mockReturnValue(response.promise);
let completed = false;
const stop = handleSessionStop(
f.manager,
{ session_id: "one" },
{
tabManagement: f.deps,
tabsQuery: f.api,
},
).then((result) => {
completed = true;
return result;
});
await vi.advanceTimersByTimeAsync(0);
const completedBeforeReply = completed;
response.resolve();
expect(await stop).toMatchObject({ window_released: true });
expect(completedBeforeReply).toBe(true);
expect(f.manager.get("one")).toBeNull();
});
it("does not monitor action preparation or long work without native input", async () => {
const f = await fixture();
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
f.open(10, 20);
await vi.advanceTimersByTimeAsync(30000);
f.open(10, 21);
input(10);
f.open(10, 22);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs).toEqual(new Set([22]));
});
it("does not extend the input window or add a tail for unrelated tab creation", async () => {
const f = await fixture();
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(11, 30, 500);
await vi.advanceTimersByTimeAsync(101);
f.open(10, 20);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs?.size ?? 0).toBe(0);
expect(f.opened.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("leaves cross-window popups unclaimed, borrowable and safe from the original task's stop", async () => {
const f = await fixture();
f.api.move.mockRejectedValueOnce(new Error("Window busy"));
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(10, 20, 500);
f.open(20, 21);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.api.move).not.toHaveBeenCalled();
expect(isAgentControlledTab(f.task, 20)).toBe(false);
expect(isAgentControlledTab(f.task, 21)).toBe(false);
f.api.move
.mockReset()
.mockImplementation(async (id, props) => Object.assign(f.tabs.get(id)!, props));
expect(await handleTabBorrow(f.manager, { session_id: "two", tab_id: 20 }, f.deps)).toMatchObject(
{ tab_id: 20 },
);
await handleSessionStop(
f.manager,
{ session_id: "one" },
{ tabManagement: f.deps, tabsQuery: f.api },
);
expect(f.api.remove).not.toHaveBeenCalledWith(20);
expect(f.tabs.get(20)?.windowId).toBe(11);
});
it("requires a main-frame source event and rejects another task's source", async () => {
const f = await fixture();
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(10, 20, 10, 4);
f.open(11, 21);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs?.size ?? 0).toBe(0);
});
it.each([
"removed",
"detached",
"revoked",
"moved",
])("invalidates a source that is %s during tracking", async (kind) => {
const f = await fixture();
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
if (kind === "removed" || kind === "detached") for (const fn of f[kind].listeners) fn(10);
if (kind === "revoked") f.task.agentCreatedTabs.delete(10);
if (kind === "moved") f.tabs.get(10)!.windowId = 500;
f.open(10, 20);
});
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs?.size ?? 0).toBe(0);
});
it("bounds candidate queries and prevents late claims after the RPC result", async () => {
const f = await fixture();
const gate = deferred<chrome.tabs.Tab>();
const get = f.api.get.getMockImplementation()!;
f.api.get.mockImplementation((id) => (id === 20 ? gate.promise : get(id)));
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(10, 20);
return 42;
});
await vi.advanceTimersByTimeAsync(600);
expect(await work).toBe(42);
gate.resolve({ id: 20, windowId: 10 } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(0);
expect(f.task.observedTabs?.size ?? 0).toBe(0);
expect(f.targets.listeners.size + f.removed.listeners.size + f.detached.listeners.size).toBe(0);
});
it("removes every listener immediately on abort, even before the tool settles", async () => {
const f = await fixture(),
action = deferred<void>(),
controller = new AbortController();
const work = withTaskPopups(
f.manager,
{ session_id: "one" },
async (input) => {
input(10);
await action.promise;
},
undefined,
controller.signal,
);
expect(f.targets.listeners.size).toBe(1);
controller.abort();
expect(f.targets.listeners.size + f.removed.listeners.size + f.detached.listeners.size).toBe(0);
f.open(10, 20);
action.resolve();
await work;
expect(isAgentControlledTab(f.task, 20)).toBe(false);
});
it.each([
"cancel",
"other claim",
"reservation",
"target detached",
])("revalidates after lookup: %s", async (change) => {
const f = await fixture(),
controller = new AbortController(),
gate = deferred<chrome.tabs.Tab>();
const get = f.api.get.getMockImplementation()!;
f.api.get.mockImplementation((id) => (id === 20 ? gate.promise : get(id)));
const work = withTaskPopups(
f.manager,
{ session_id: "one" },
async (input) => {
input(10);
f.open(10, 20);
},
undefined,
controller.signal,
);
await vi.advanceTimersByTimeAsync(0);
if (change === "cancel") controller.abort();
if (change === "other claim") f.other.agentCreatedTabs.add(20);
if (change === "reservation") f.manager.tryReserveBorrow(20, "two");
if (change === "target detached") for (const fn of f.detached.listeners) fn(20);
gate.resolve({ id: 20, windowId: 10 } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(100);
await work;
expect(f.task.observedTabs?.has(20) ?? false).toBe(false);
});
it("rejects duplicate borrowing of created and observed targets across sessions", async () => {
const f = await fixture();
f.tabs.set(20, { id: 20, windowId: 500 } as chrome.tabs.Tab);
f.task.agentCreatedTabs.add(20);
for (const session_id of ["one", "two"])
expect(await handleTabBorrow(f.manager, { session_id, tab_id: 20 }, f.deps)).toMatchObject({
code: "permission_denied",
});
f.task.agentCreatedTabs.delete(20);
f.task.observedTabs = new Set([20]);
expect(await handleTabBorrow(f.manager, { session_id: "two", tab_id: 20 }, f.deps)).toMatchObject(
{ code: "permission_denied" },
);
expect(f.api.move).not.toHaveBeenCalled();
expect(f.manager.releaseObservedTab(20)).toEqual(["one"]);
expect(isAgentControlledTab(f.task, 20)).toBe(false);
});
it("observes late events until 100ms after the last input without extending for events", async () => {
const f = await fixture();
let actionReturned = false;
let settled = false;
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
await vi.advanceTimersByTimeAsync(40);
input(10);
actionReturned = true;
return 42;
}).then((result) => {
settled = true;
return result;
});
await vi.advanceTimersByTimeAsync(0);
expect(actionReturned).toBe(true);
await vi.advanceTimersByTimeAsync(50);
f.open(10, 20);
f.open(11, 30, 500);
await vi.advanceTimersByTimeAsync(49);
expect(settled).toBe(false);
expect(f.task.observedTabs).toEqual(new Set([20]));
await vi.advanceTimersByTimeAsync(1);
expect(await work).toBe(42);
f.open(10, 21);
expect(isAgentControlledTab(f.task, 21)).toBe(false);
expect(f.targets.listeners.size + f.removed.listeners.size + f.detached.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("does not add a tail when no input was sent", async () => {
const f = await fixture();
expect(await withTaskPopups(f.manager, { session_id: "one" }, async () => 42)).toBe(42);
expect(vi.getTimerCount()).toBe(0);
});
it("cancels the remaining input interval after the action returns", async () => {
const f = await fixture();
const controller = new AbortController();
const work = withTaskPopups(
f.manager,
{ session_id: "one" },
async (input) => {
input(10);
return 42;
},
undefined,
controller.signal,
);
await vi.advanceTimersByTimeAsync(50);
controller.abort();
expect(await work).toBe(42);
f.open(10, 20);
expect(isAgentControlledTab(f.task, 20)).toBe(false);
expect(f.targets.listeners.size + f.removed.listeners.size + f.detached.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("preserves observed tabs and their window when the final tab query fails", async () => {
const f = await fixture();
const work = withTaskPopups(f.manager, { session_id: "one" }, async (input) => {
input(10);
f.open(10, 20);
});
await vi.advanceTimersByTimeAsync(100);
await work;
f.api.query.mockRejectedValueOnce(new Error("temporary Chrome query failure"));
const result = await handleSessionStop(
f.manager,
{ session_id: "one" },
{
tabManagement: f.deps,
tabsQuery: f.api,
},
);
expect(result).toMatchObject({ window_released: true });
expect(f.removeWindow).not.toHaveBeenCalled();
expect(f.api.remove).toHaveBeenCalledWith(10);
expect(f.api.remove).not.toHaveBeenCalledWith(20);
expect(f.tabs.has(20)).toBe(true);
expect(f.manager.get("one")).toBeNull();
expect(f.deps.agentOverlayReset!.resetAgentOverlays).toHaveBeenCalledWith(20, "one");
});
+28 -2
View File
@@ -14,12 +14,18 @@ export interface SessionContext {
* Tabs opened through Chrome UI never enter this set.
*/
agentCreatedTabs: Set<number>;
/** Observed same-window popups: controllable, but preserved by session stop. */
observedTabs?: Set<number>;
createdAtMs: number;
}
/** Whether this session has explicitly claimed control of `tabId`. */
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
return (
ctx.agentCreatedTabs.has(tabId) ||
ctx.borrowedTabs.has(tabId) ||
(ctx.observedTabs?.has(tabId) ?? false)
);
}
export interface BorrowedTab {
@@ -152,10 +158,28 @@ export class SessionManager {
this.invalidateTabRefs(tabId);
for (const ctx of this.sessions.values()) {
ctx.agentCreatedTabs.delete(tabId);
ctx.observedTabs?.delete(tabId);
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
}
}
/** All kinds of committed control, not just borrowed tabs. */
findControllingSession(tabId: number): string | null {
return this.list().find((ctx) => isAgentControlledTab(ctx, tabId))?.sessionId ?? null;
}
/** Observation does not follow a page moved out by the browser user. */
releaseObservedTab(tabId: number): string[] {
const released: string[] = [];
for (const ctx of this.sessions.values()) {
if (ctx.observedTabs?.delete(tabId)) {
ctx.refStore.invalidateTab(tabId);
released.push(ctx.sessionId);
}
}
return released;
}
/**
* Look up whether `tabId` is currently borrowed by some *other*
* session than the one calling. Used by M8 `tab_borrow` to refuse
@@ -183,7 +207,9 @@ export class SessionManager {
*/
tryReserveBorrow(tabId: number, sessionId: string): BorrowReservation | { borrowedBy: string } {
const borrowedBy =
this.borrowReservations.get(tabId) ?? this.findBorrowingSession(tabId, sessionId);
this.borrowReservations.get(tabId) ??
this.findControllingSession(tabId) ??
this.findBorrowingSession(tabId, sessionId);
if (borrowedBy) return { borrowedBy };
this.borrowReservations.set(tabId, sessionId);
let closed = false;
@@ -0,0 +1,145 @@
import { isAgentControlledTab, type SessionManager } from "./manager";
const INPUT_WINDOW_MS = 100;
const CANDIDATE_BUDGET_MS = 500;
/** Browser source relationships are not proof of creation by an agent.
* Only observe a short interval starting at actual click/key dispatch. Same-window
* targets may be controlled, but are never added to the destructive cleanup set.
* Cross-window or late targets retain the explicit borrow flow. */
export async function withTaskPopups<T>(
manager: SessionManager,
params: { session_id?: string; tab_id?: number },
run: (inputSent: (tabId: number) => void) => Promise<T>,
onClaimed?: (tabId: number, windowId: number) => void,
signal?: AbortSignal,
): Promise<T> {
const task = params.session_id ? manager.get(params.session_id) : null;
const targets = globalThis.chrome?.webNavigation?.onCreatedNavigationTarget;
if (!task || !targets || signal?.aborted) return run(() => {});
let active = true;
let listening = false;
let observationEnded = Promise.resolve();
let finishObservation = () => {};
let timer: ReturnType<typeof setTimeout> | undefined;
const invalid = new Set<number>();
const candidates = new Map<number, Promise<boolean>>();
const live = () =>
active &&
!signal?.aborted &&
manager.get(task.sessionId) === task &&
!manager.isWindowCloseExpected(task);
const validSource = async (id: number) => {
if (!live() || invalid.has(id) || (task.remote && !isAgentControlledTab(task, id)))
return false;
try {
const tab = await chrome.tabs.get(id);
return (
live() &&
!invalid.has(id) &&
tab.windowId === task.agentWindowId &&
(!task.remote || isAgentControlledTab(task, id))
);
} catch {
return false;
}
};
const created = ({
sourceTabId,
sourceFrameId,
tabId,
}: {
sourceTabId: number;
sourceFrameId: number;
tabId: number;
}) => {
const parent = candidates.get(sourceTabId);
if (!live() || !listening || sourceFrameId !== 0 || !parent || candidates.has(tabId)) return;
const work = Promise.resolve()
.then(async () => {
if (!(await parent) || !(await validSource(sourceTabId))) return false;
const tab = await chrome.tabs.get(tabId);
if (
!(await validSource(sourceTabId)) ||
invalid.has(tabId) ||
tab.windowId !== task.agentWindowId ||
manager.findBorrowingSession(tabId, task.sessionId) ||
manager.findControllingSession(tabId)
)
return false;
// Non-destructive authority: stop releases this tab and preserves its window.
(task.observedTabs ??= new Set()).add(tabId);
onClaimed?.(tabId, tab.windowId);
return true;
})
.catch((error) => {
console.warn("[bsk] popup observation failed", error);
return false;
});
candidates.set(tabId, work);
};
const stopListening = () => {
listening = false;
clearTimeout(timer);
targets.removeListener(created);
finishObservation();
};
const invalidate = (id: number) => {
invalid.add(id);
};
const abort = () => {
active = false;
stopListening();
chrome.tabs.onRemoved?.removeListener(invalidate);
chrome.tabs.onDetached?.removeListener(invalidate);
};
const inputSent = (tabId: number) => {
if (!live()) return;
if (params.tab_id !== undefined && tabId !== params.tab_id) return;
candidates.set(tabId, Promise.resolve(true));
if (!listening) {
observationEnded = new Promise<void>((resolve) => {
finishObservation = resolve;
});
targets.addListener(created);
}
listening = true;
clearTimeout(timer);
timer = setTimeout(stopListening, INPUT_WINDOW_MS);
};
chrome.tabs.onRemoved?.addListener(invalidate);
chrome.tabs.onDetached?.addListener(invalidate);
signal?.addEventListener("abort", abort, { once: true });
try {
return await run(inputSent);
} finally {
// Chrome may deliver navigation-target events after the input response.
// Finish the interval armed by the last input; only input can extend it.
await observationEnded;
stopListening();
await settleOrExpire(Promise.all(candidates.values()), signal);
active = false;
signal?.removeEventListener("abort", abort);
chrome.tabs.onRemoved?.removeListener(invalidate);
chrome.tabs.onDetached?.removeListener(invalidate);
}
}
async function settleOrExpire(work: Promise<unknown>, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) return;
let abort = () => {};
let timer: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([
work,
new Promise<void>((resolve) => {
abort = resolve;
signal?.addEventListener("abort", abort, { once: true });
timer = setTimeout(resolve, CANDIDATE_BUDGET_MS);
}),
]);
} finally {
clearTimeout(timer);
signal?.removeEventListener("abort", abort);
}
}
@@ -1363,3 +1363,129 @@ describe("background execution dispatch integration", () => {
}
});
});
it.each([
"tool.click",
"tool.press",
])("tracks popups through the real %s dispatcher route", async (method) => {
const sessions = new SessionManager({
remote: () => true,
agentWindow: {
create: async () => ({ windowId: 10, initialTabIds: [] }),
remove: vi.fn(),
ensureActiveTab: async () => 10,
},
});
const task = await sessions.start("popup");
const listeners = new Set<
(e: { sourceTabId: number; sourceFrameId: number; tabId: number }) => void
>();
const event = {
addListener: (fn: (e: { sourceTabId: number; sourceFrameId: number; tabId: number }) => void) =>
listeners.add(fn),
removeListener: (
fn: (e: { sourceTabId: number; sourceFrameId: number; tabId: number }) => void,
) => listeners.delete(fn),
};
vi.stubGlobal("chrome", {
tabs: {
get: async (id: number) => ({ id, windowId: 10 }),
onCreated: { addListener: vi.fn(), removeListener: vi.fn() },
},
webNavigation: { onCreatedNavigationTarget: event },
});
const f = fakeTransport();
const dispatcher = new ToolDispatcher({ transport: f.transport, sessions });
vi.spyOn(
dispatcher as unknown as {
invoke: (
req: RequestFrame,
signal: AbortSignal,
inputSent: (tabId: number) => void,
) => Promise<unknown>;
},
"invoke",
).mockImplementation(async (_req, _signal, inputSent) => {
inputSent(10);
for (const fn of listeners) fn({ sourceTabId: 10, sourceFrameId: 0, tabId: 20 });
return {};
});
dispatcher.start();
try {
f.deliver({ id: "open", method, params: { session_id: "popup", tab_id: 10 } });
await vi.waitFor(() => expect(f.sent.some((r) => "id" in r && r.id === "open")).toBe(true));
expect(task.observedTabs?.has(20)).toBe(true);
expect(task.agentCreatedTabs.has(20)).toBe(false);
expect(listeners.size).toBe(0);
} finally {
dispatcher.stop();
vi.unstubAllGlobals();
}
});
it("passes dispatcher cancellation to popup tracking before the tool settles", async () => {
const sessions = new SessionManager({
remote: () => true,
agentWindow: {
create: async () => ({ windowId: 10, initialTabIds: [] }),
remove: vi.fn(),
ensureActiveTab: async () => 10,
},
});
const task = await sessions.start("popup");
const listeners = new Set<
(e: { sourceTabId: number; sourceFrameId: number; tabId: number }) => void
>();
vi.stubGlobal("chrome", {
tabs: {
get: async (id: number) => ({ id, windowId: 10 }),
onCreated: { addListener: vi.fn(), removeListener: vi.fn() },
},
webNavigation: {
onCreatedNavigationTarget: {
addListener: (fn: never) => listeners.add(fn),
removeListener: (fn: never) => listeners.delete(fn),
},
},
});
const f = fakeTransport();
const dispatcher = new ToolDispatcher({ transport: f.transport, sessions });
let finish!: () => void;
const gate = new Promise<void>((resolve) => {
finish = resolve;
});
const invoke = vi
.spyOn(
dispatcher as unknown as {
invoke: (
req: RequestFrame,
signal: AbortSignal,
inputSent: (tabId: number) => void,
) => Promise<unknown>;
},
"invoke",
)
.mockImplementation(async (_req, _signal, inputSent) => {
inputSent(10);
await gate;
return {};
});
dispatcher.start();
try {
f.deliver({ id: "open", method: "tool.press", params: { session_id: "popup", tab_id: 10 } });
await vi.waitFor(() => expect(invoke).toHaveBeenCalled());
f.deliver({ id: "cancel", method: "cancel", params: { rpc_id: "open" } });
await vi.waitFor(() =>
expect(f.sent).toContainEqual({ id: "cancel", result: { cancelled: true } }),
);
expect(invoke.mock.calls[0][1].aborted).toBe(true);
expect(listeners.size).toBe(0);
expect(task.agentCreatedTabs.has(20)).toBe(false);
finish();
await vi.waitFor(() => expect(f.sent.some((r) => "id" in r && r.id === "open")).toBe(true));
} finally {
finish();
dispatcher.stop();
vi.unstubAllGlobals();
}
});
@@ -1840,3 +1840,46 @@ describe("handleSelect", () => {
expect(fake.sent.some((c) => c.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
"click",
"press",
])("arms popup observation at native %s dispatch, after preparation", async (kind) => {
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const task = await manager.start("popup");
task.refStore.set("e1", 1234, { tabId: 4 });
const order: string[] = [];
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
order.push("prepare");
return {};
},
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Input.dispatchMouseEvent": (p) => {
order.push((p as { type: string }).type);
return {};
},
"Input.dispatchKeyEvent": (p) => {
order.push((p as { type: string }).type);
return {};
},
});
const onInputSent = vi.fn((id: number) => {
expect(id).toBe(4);
order.push("arm");
});
const deps = { cdp: fake.cdp, tabsApi: fake.tabsApi, onInputSent };
const result =
kind === "click"
? await handleClick(manager, { session_id: "popup", ref: "e1" }, deps)
: await handlePress(manager, { session_id: "popup", key: "Enter" }, deps);
expect(result).toHaveProperty("tab_id", 4);
expect(onInputSent).toHaveBeenCalledTimes(2);
if (kind === "click")
expect(order).toEqual(["prepare", "mouseMoved", "arm", "mousePressed", "arm", "mouseReleased"]);
else {
expect(order.indexOf("arm")).toBeLessThan(order.indexOf("rawKeyDown"));
expect(order.at(-2)).toBe("arm");
expect(order.at(-1)).toBe("keyUp");
}
});
+20 -3
View File
@@ -4,6 +4,7 @@ import type { InteractionPreferenceStore } from "@/lib/interaction-preferences";
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import { ScreenshotExports } from "@/long-screenshot/exports";
import type { SessionManager } from "@/session-manager/manager";
import { withTaskPopups } from "@/session-manager/task-popups";
import type { Transport } from "@/transport/transport";
import type {
BlurParams,
@@ -156,6 +157,9 @@ export interface DispatcherDeps {
helpNotificationCopy?: () => { title: string; body: string };
}
/** Tools whose page input can make the page open another tab or window. */
const OPENS_TABS = new Set(["tool.click", "tool.press"]);
/**
* Routes RPC requests pushed by the daemon over the Transport to the
* appropriate tool implementation.
@@ -319,7 +323,15 @@ export class ToolDispatcher {
/* Evidence must not block the operation. */
}
throwIfDispatchAborted(ac.signal);
const result = await this.invoke(req, ac.signal);
const result = OPENS_TABS.has(req.method)
? await withTaskPopups(
this.sessions,
(req.params ?? {}) as { session_id?: string; tab_id?: number },
(inputSent) => this.invoke(req, ac.signal, inputSent),
this.onAgentTabClaimed,
ac.signal,
)
: await this.invoke(req, ac.signal);
this.debug?.after(debugTicket, isRpcError(result) ? result.message : undefined);
debugTicket = undefined;
if (isRpcError(result)) {
@@ -388,7 +400,11 @@ export class ToolDispatcher {
}
}
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
private async invoke(
req: RequestFrame,
signal: AbortSignal,
onInputSent?: (tabId: number) => void,
): Promise<unknown | RpcError> {
const sessionId = (req.params as { session_id?: string } | undefined)?.session_id;
// Also enforce this for gateways backed by a local-mode daemon, where the
// standalone server's early IPC rejection does not apply.
@@ -649,6 +665,7 @@ export class ToolDispatcher {
tabsApi: chromeTabsApi,
signal,
bypassOverlay,
onInputSent,
}
: undefined,
),
@@ -737,7 +754,7 @@ export class ToolDispatcher {
handlePress(
this.sessions,
req.params as PressParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, onInputSent } : undefined,
),
signal,
);
+6
View File
@@ -53,6 +53,8 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
export interface InteractionDeps {
/** Arm short popup observation immediately before native input dispatch. */
onInputSent?: (tabId: number) => void;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
/** Abort hook (full chain wired in M10.2). */
@@ -610,6 +612,7 @@ async function dispatchClickAtPoint(
if (error) return failure(error);
}
if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" });
deps.onInputSent?.(tabId);
markSent?.();
attempted = true;
releaseNeeded = true;
@@ -620,6 +623,7 @@ async function dispatchClickAtPoint(
clickCount: count,
modifiers,
});
deps.onInputSent?.(tabId);
await release();
releaseNeeded = false;
}
@@ -1537,6 +1541,7 @@ export async function handlePress(
try {
let cancelled = false;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
deps.onInputSent?.(target.tabId);
input.markSent();
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
type: "rawKeyDown",
@@ -1569,6 +1574,7 @@ export async function handlePress(
cancelled = true;
}
}
deps.onInputSent?.(target.tabId);
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
type: "keyUp",
key: descriptor.key,
+14 -4
View File
@@ -10,7 +10,7 @@ import { rpcError } from "./errors";
import { clearRecordingForSession } from "./record";
import type { CdpRunner, ChromeTabsApi } from "./shared";
import { isRpcError } from "./shared";
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
import { chromeAgentOverlayResetApi, returnBorrowedTab, type TabManagementDeps } from "./tabs";
/** Valid range for Agent Window dimensions in CSS pixels. */
export const WINDOW_SIZE_MIN = 100;
@@ -334,7 +334,7 @@ async function stopSession(
}
// Step 5: decide whether to release (keep) the window or close it.
let shouldRelease = false;
let shouldRelease = (ctx.observedTabs?.size ?? 0) > 0;
if (queryApi) {
try {
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
@@ -371,14 +371,24 @@ async function stopSession(
}
shouldRelease = userTabs.length > 0;
} catch {
// Query failed (e.g. window already gone) — conservatively close it.
shouldRelease = false;
// A failed query does not prove the window is gone. Preserve the
// release decision when observed tabs are known to require protection.
}
}
if (shouldRelease) {
// Keep the window + its user tabs; only drop the session binding.
const observedTabIds = Array.from(ctx.observedTabs ?? []);
await manager.stop(params.session_id, { dropOnly: true });
// Closing an agent tab may have reactivated an observed tab. Reset only
// after dropping the session so subsequent overlay refreshes stay hidden.
const overlayReset = deps.tabManagement?.agentOverlayReset ?? chromeAgentOverlayResetApi;
for (const tabId of observedTabIds) {
// Match tab_return: content-script replies must not delay session cleanup.
void overlayReset.resetAgentOverlays(tabId, ctx.sessionId).catch(() => {
// Restricted pages and unloaded content scripts cannot receive messages.
});
}
result.window_released = true;
} else {
// Window is empty (or we couldn't verify state) — close it.
+8
View File
@@ -597,6 +597,7 @@ export async function handleTabClose(
// Keep the tracking set accurate so session_stop won't try to close a
// tab that's already gone (design §3.1).
ctx.agentCreatedTabs.delete(params.tab_id);
ctx.observedTabs?.delete(params.tab_id);
} catch (err) {
return {
code: "protocol_error",
@@ -698,6 +699,13 @@ async function validateBorrowTarget(
if (typeof tab.id !== "number" || typeof tab.windowId !== "number") {
return { code: "not_found", message: `tab ${tabId} not found` };
}
const owner = manager.findControllingSession(tabId);
if (owner)
return rpcError(
"permission_denied",
"borrow_conflict",
`tab_borrow: tab ${tabId} is already controlled by session ${owner}`,
);
if (tab.windowId === ctx.agentWindowId) {
return {
code: "invalid_params",
+11 -1
View File
@@ -193,7 +193,9 @@ Pair only with a server you trust to operate your browser. A paired server can c
Remote content reads, screenshots, recording and page operations require a tab explicitly created or borrowed by the task. Listing tab titles and URLs remains available to select a tab to borrow. A user tab moved or opened inside an Agent Window does not by itself become authorized. Borrowing uses the existing browser-controlled confirmation preference; remote request flags cannot change that preference. After a borrowed tab is returned, remote content access ends. Returning a tab during remote recording cancels that recording before releasing the tab.
This also applies to tabs or windows opened by a page through `target="_blank"`, `window.open`, or an OAuth flow. An opener relationship does not grant control. If such a tab is already inside the Agent Window, it cannot be borrowed in place: the browser user must first move it to a regular browser window, then the Agent can use the ordinary borrow flow. Tabs explicitly created through `bsk tab create` are controlled immediately. Automatic popup authorization is outside this version's scope.
A page may open targets through `target="_blank"`, `window.open`, or a login flow. An opener relationship and window membership alone do not authorize them. During a short observation window armed immediately before native click/key dispatch, a main-frame navigation-target event from a currently controlled source may grant control of a new tab already in the same Agent Window. Nested same-window targets follow the same source checks. These observed tabs are kept separate from explicitly agent-created tabs: session stop releases their control and preserves them and their window. If the final tab query fails, known observed tabs still require preserving the window. An explicit `tab_close` remains a separate destructive tool action.
Cross-window popups are never automatically moved or claimed. They use the ordinary `tab_borrow` flow. Late or unattributed targets also retain that flow; an unowned target already inside the Agent Window must first be moved to a regular browser window before borrowing. Tabs explicitly created by `tab_create` remain agent-owned and are closed on stop. A tab already controlled by any session cannot acquire a second borrow claim.
Disconnecting cancels task work, returns borrowed tabs and closes task-created tabs. User-created tabs survive cleanup. Failed returns preserve the window and must be resolved before reconnecting. Reconnection starts new tasks; commands and sessions are never replayed. Failed remote authentication does not select a local connection automatically.
@@ -216,3 +218,11 @@ BSK_REMOTE_CHROME=/absolute/path/to/chrome-for-testing \
```
The remote server integration tests cover credential exchange, rotation retries, stable device routing, replacement connections, revocation, connection capacity, file-lock contention and native TLS. Unit tests additionally cover rate-limit saturation, unavailable extension storage, local recovery, renewal retry frequency and popup authorization states. TLS fixtures contain a test-only private key and must never be used for deployment.
### Popup observation lifetime
Only `tool.click` and `tool.press` arm observation, immediately before sending native input rather than during target lookup, scrolling, navigation, evaluation or other RPC preparation. Each input phase opens 100 ms of observation. Operation completion waits for the remainder of the last input interval; cancellation closes the listener immediately. Events cannot extend the interval. There is no browser-global `tabs.onCreated` tail. This remains a best-effort source-and-time policy, not proof of causality. Targets delivered outside that interval require explicit borrowing.
Only main-frame source events qualify. Source ownership and its actual window, target window, competing claims and borrow reservations are rechecked before granting non-destructive control. After the action and its observation interval finish, candidate processing has a 500 ms budget, so observation and candidate processing together can delay the tool result by at most 600 ms. Expiry or abort removes listeners and invalidates late continuations without granting ownership; no Chrome movement needs compensation because observation never moves tabs.
Closing a tab or moving it out of its window revokes observed control. On stop, only the session's explicitly created tabs are closed. Observed tabs are preserved even when they originated from a page script or user interaction during the short input window.