From ece29530dd59a7173275d633449ff4cbacbfb668 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 23 Sep 2026 10:26:53 +0800 Subject: [PATCH 1/4] fix(extension): prevent control overlay from swallowing clicks --- .../src/tools/__tests__/click.browser.test.ts | 166 ++++++++++++++ .../src/tools/__tests__/dispatcher.test.ts | 21 +- .../src/tools/__tests__/file-transfer.test.ts | 14 ++ .../src/tools/__tests__/interaction.test.ts | 204 ++++++------------ .../tools/__tests__/visual-screenshot.test.ts | 15 +- apps/extension/src/tools/interaction.ts | 108 +++++----- 6 files changed, 320 insertions(+), 208 deletions(-) diff --git a/apps/extension/src/tools/__tests__/click.browser.test.ts b/apps/extension/src/tools/__tests__/click.browser.test.ts index b0e7bea..4ee416a 100644 --- a/apps/extension/src/tools/__tests__/click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click.browser.test.ts @@ -1,8 +1,13 @@ // @vitest-environment node // Opt in with BSK_CLICK_CHROME; each test owns its browser, profile and HTTP server. +import { readFileSync } from "node:fs"; import { createServer } from "node:http"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; import { describe, expect, it, vi } from "vitest"; import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { ControlOverlay } from "@/content/ControlOverlay"; +import { CAPTURE_SUPPRESS, type CaptureSuppressSendToTab } from "@/lib/capture-suppress-bridge"; import { SessionManager } from "@/session-manager/manager"; import { prepareBackgroundExecution } from "../background-execution"; import { handleClick, handlePress } from "../interaction"; @@ -354,4 +359,165 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( await new Promise((resolve) => server.close(() => resolve())); } }, 90_000); + + it("clicks through the real control pill and fails if the closed shadow overlay still intercepts input", async () => { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + const css = readFileSync(new URL("../../content/overlay.css", import.meta.url), "utf8"); + await withChrome( + { executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + const local: Send = (method, params) => send(method, params, sessionId); + const evaluate = async (expression: string) => { + const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { + expression, + returnByValue: true, + awaitPromise: true, + }, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await local("Page.bringToFront"); + await local("Emulation.setDeviceMetricsOverride", { + width: 1280, + height: 757, + deviceScaleFactor: 1, + mobile: false, + }); + await evaluate(`document.body.innerHTML = ''; + window.clicks=[]; window.stops=0; + document.querySelector('#target').onclick=e=>clicks.push(e.isTrusted); + window.overlay=document.createElement('browser-skill-overlay'); + overlay.setAttribute('data-bsk-overlay',''); overlay.setAttribute('data-bsk-overlay-surface',''); + document.documentElement.append(overlay); + window.overlayRoot=overlay.attachShadow({mode:'closed'});`); + const manager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("overlay-click"); + const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; + const tabsApi = { get: async () => tab, query: async () => [tab] }; + const mouse: string[] = []; + const cdp: CdpRunner = { + send: async (_tabId, method, params) => { + if (method === "Input.dispatchMouseEvent") + mouse.push((params as { type: string }).type); + return (await local(method, params)) as never; + }, + }; + const sendToTab: CaptureSuppressSendToTab = async (_tabId, message) => { + // Transport adapter: use the production CSS suppression marker, as the content script does. + await evaluate( + `overlay.toggleAttribute('data-bsk-capture-hidden', ${message.phase === "begin"})`, + ); + return { type: CAPTURE_SUPPRESS, ok: true }; + }; + // A retained hover bypass leaves the pill interactive even though the host is transparent. + for (const automationBypass of [false, true]) { + const markup = renderToStaticMarkup( + createElement(ControlOverlay, { + visible: true, + interrupting: false, + automationBypass, + onInterrupt: () => {}, + }), + ); + await evaluate(`overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass}); + overlayRoot.innerHTML = ${JSON.stringify(``)} + ${JSON.stringify(markup)}; + overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]').addEventListener('click',()=>stops++);`); + for (const slot of ["control-overlay-pill", "control-overlay-stop-all"]) { + const point = await evaluate<{ x: number; y: number }>(`(() => { + const r=overlayRoot.querySelector('[data-slot="${slot}"]').getBoundingClientRect(); + const x=${slot === "control-overlay-pill" ? "r.x+25" : "r.x+r.width/2"}, y=r.y+r.height/2; + Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'}); + clicks=[]; stops=0; return {x,y}; + })()`); + expect( + await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`), + ).toBe(true); + const nativeClick = async () => { + for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) { + await local("Input.dispatchMouseEvent", { + type, + ...point, + button: "left", + clickCount: 1, + }); + } + }; + await nativeClick(); // Reproduce the interception before using the fixed click path. + expect(await evaluate("clicks")).toEqual([]); + await evaluate("stops=0"); + const { root } = await local<{ root: { nodeId: number } }>("DOM.getDocument"); + const { nodeId } = await local<{ nodeId: number }>("DOM.querySelector", { + nodeId: root.nodeId, + selector: "#target", + }); + const { node } = await local<{ node: { backendNodeId: number } }>("DOM.describeNode", { + nodeId, + }); + ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 }); + mouse.length = 0; + const result = await handleClick( + manager, + { + session_id: ctx.sessionId, + ...(automationBypass ? { ref: "e1" } : { selector: "#target" }), + }, + { cdp, tabsApi, sendToTab }, + ); + expect(result, JSON.stringify(result)).not.toHaveProperty("code"); + expect(await evaluate("clicks")).toEqual([true]); + expect(await evaluate("stops")).toBe(0); + expect(mouse).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]); + expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false); + expect( + await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`), + ).toBe(true); + if (slot === "control-overlay-stop-all") { + await nativeClick(); + expect(await evaluate("stops")).toBe(1); + expect(await evaluate("clicks")).toEqual([true]); + } + mouse.length = 0; + const blocked = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { + cdp, + tabsApi, + sendToTab: async () => { + throw new Error("Content script unavailable"); + }, + }, + ); + expect(blocked).toMatchObject({ + code: "cdp_failed", + data: { reason: "input_not_ready", effect_state: "none" }, + }); + expect(mouse).toEqual(["mouseMoved"]); + expect(await evaluate("clicks")).toEqual([true]); + } + } + }, + ); + }, 30_000); }); diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 4f37386..08a8ac0 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -341,7 +341,7 @@ describe("ToolDispatcher", () => { }); }); - it("bypasses and restores the control overlay for an upload trigger click", async () => { + it("hides and restores the complete control overlay for an upload trigger click", async () => { const sendMessage = vi.fn(async () => undefined); vi.stubGlobal("chrome", { tabs: { @@ -383,15 +383,8 @@ describe("ToolDispatcher", () => { } if (method === "Runtime.evaluate") { const expression = (params as { expression?: string }).expression ?? ""; - if (expression.includes("hitIndex")) { - return { result: { value: { hitIndex: 0 } } } as T; - } - if (expression.includes("overlayHostPresent")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true }, - }, - } as T; + if (expression.startsWith("!!document.elementFromPoint")) { + return { result: { value: false } } as T; } if (expression.includes("count:")) { return { result: { value: { count: 1, multiple: false } } } as T; @@ -433,12 +426,12 @@ describe("ToolDispatcher", () => { expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } }); expect(sendMessage).toHaveBeenNthCalledWith(1, 7, { - type: "bh-automation-bypass", - enabled: true, + type: "bsk/capture-suppress", + phase: "begin", }); expect(sendMessage).toHaveBeenNthCalledWith(2, 7, { - type: "bh-automation-bypass", - enabled: false, + type: "bsk/capture-suppress", + phase: "end", }); }); diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 9b2f2a1..1fe1ad9 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -64,6 +64,13 @@ function uploadCdp( let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).startsWith( + "!!document.elementFromPoint", + ) + ) + return { result: { value: false } }; if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") @@ -594,6 +601,13 @@ describe("file transfer tools", () => { let cdpEvent: Parameters>[0] | undefined; let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).startsWith( + "!!document.elementFromPoint", + ) + ) + return { result: { value: false } }; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 6dea22e..90dd8a1 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { withInputReady } from "../input-readiness"; @@ -34,6 +35,7 @@ function makeFakeCdp( rendered: () => boolean | Promise = () => true, ) { const sent: Array<{ tabId: number; method: string; params?: object }> = []; + const overlayHit = vi.fn(async () => ({ result: { value: false } })); const sendImpl = async (tabId: number, method: string, params?: object) => { sent.push({ tabId, method, params }); if ( @@ -42,6 +44,13 @@ function makeFakeCdp( ) return { result: { value: visibility() } }; if (method === "Page.captureScreenshot") return { data: (await rendered()) ? "pixel" : "" }; + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).startsWith( + "!!document.elementFromPoint", + ) + ) + return overlayHit(); const h = handlers[method]; if (!h && method === "Accessibility.getPartialAXTree") return { nodes: [] }; if (!h && method === "Page.getLayoutMetrics") { @@ -65,7 +74,7 @@ function makeFakeCdp( ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }; - return { cdp, tabsApi, sent }; + return { cdp, tabsApi, sent, overlayHit }; } describe("modifiersBitfield", () => { @@ -241,10 +250,25 @@ describe("handleClick", () => { expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); }); - it("enables overlay bypass before mouse events when overlay blocks the click point", async () => { + it.each([ + "success", + "dispatch-error", + "cancel", + "still-covered", + "suppression-failed", + "hit-test-error", + "invalid-hit-test", + "no-content-script", + ])("scopes complete overlay suppression and checks the click point: %s", async (mode) => { const order: string[] = []; - const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => { - order.push(enabled ? "bypass-on" : "bypass-off"); + const controller = new AbortController(); + const bypassOverlay = vi.fn(); + const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => { + order.push(message.phase); + if (mode === "suppression-failed" || mode === "no-content-script") { + throw new Error("No receiving content script"); + } + return { type: CAPTURE_SUPPRESS, ok: true }; }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); @@ -252,146 +276,52 @@ describe("handleClick", () => { const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; - } - if (expr.includes("hitIndex")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, - }, - }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => { - order.push("mouse"); + "Input.dispatchMouseEvent": (params) => { + const { type } = params as { type: string }; + order.push(type); + if (type === "mouseMoved" && mode === "cancel") controller.abort(); + if (type === "mousePressed" && mode === "dispatch-error") throw new Error("Input failed"); return {}; }, }); - const res = await handleClick( + fake.overlayHit.mockImplementation(async () => { + order.push("hit-test"); + if (mode === "hit-test-error") throw new Error("Renderer unavailable"); + if (mode === "invalid-hit-test") return { exceptionDetails: {} } as never; + return { result: { value: mode === "still-covered" || mode === "suppression-failed" } }; + }); + const result = await handleClick( sm, { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, + { ...fake, sendToTab, bypassOverlay, signal: controller.signal }, ); - if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - expect(order.indexOf("bypass-on")).toBeLessThan(order.indexOf("mouse")); - expect(order.lastIndexOf("bypass-off")).toBeGreaterThan(order.lastIndexOf("mouse")); - }); - - it("disables overlay bypass when mouse dispatch throws", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - let mouseCalls = 0; - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; - } - if (expr.includes("hitIndex")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, - }, - }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => { - mouseCalls += 1; - if (mouseCalls === 2) throw new Error("mousePressed failed"); - return {}; - }, - }); - const res = await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, - ); - expect(res).toMatchObject({ code: "cdp_failed" }); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - }); - - it("enables overlay bypass when the overlay host itself captures the point", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const host = document.createElement("browser-skill-overlay"); - host.setAttribute("data-bsk-overlay", ""); - host.style.cssText = - "position:fixed;inset:0;width:100vw;height:100vh;pointer-events:auto;display:block"; - host.attachShadow({ mode: "closed" }); - document.body.append(host); - Object.defineProperty(host, "getBoundingClientRect", { - value: () => ({ - x: 0, - y: 0, - width: 1280, - height: 720, - top: 0, - left: 0, - right: 1280, - bottom: 720, - }), - }); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - const value = new Function(`return (${expr})`)(); - return { result: { value } }; - }, - "Input.dispatchMouseEvent": () => ({}), - }); - try { - const res = await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, - ); - if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - } finally { - host.remove(); + expect(order[0]).toBe("begin"); + expect(bypassOverlay).not.toHaveBeenCalled(); // Do not change a retained hover bypass. + if (mode === "success" || mode === "no-content-script") { + expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + expect(order.slice(1, 5)).toEqual([ + "mouseMoved", + "hit-test", + "mousePressed", + "mouseReleased", + ]); + } else { + expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed"); + if (mode === "dispatch-error") { + expect(order).toContain("mouseReleased"); + } else { + expect(order).not.toContain("mousePressed"); + } + if (mode === "still-covered" || mode === "suppression-failed") { + expect(result).toMatchObject({ data: { reason: "input_not_ready", effect_state: "none" } }); + } + } + if (mode === "suppression-failed" || mode === "no-content-script") { + expect(sendToTab).toHaveBeenCalledTimes(1); + } else { + expect(order.at(-1)).toBe("end"); + expect(sendToTab.mock.calls.map((call) => call[1].phase)).toEqual(["begin", "end"]); } - }); - - it("skips overlay bypass when overlay does not block the click point", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: false, overlayHostConnected: false } } }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => ({}), - }); - await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, - ); - expect(bypassOverlay).not.toHaveBeenCalled(); }); it("leaves disabled click behavior to the browser without an AX preflight", async () => { diff --git a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts index 49f1238..7f8911e 100644 --- a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts +++ b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts @@ -524,7 +524,7 @@ async function pointFixture(child = false, oopif = false) { ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]); const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; const tabsApi = { get: async () => tab, query: async () => [tab] }; - const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {}); + const sendToTab = vi.fn(async (_tab: number, _message: { phase: string }) => ({})); const original = f.send.getMockImplementation()!; const input: Record[] = []; const hitPoints: number[][] = []; @@ -539,6 +539,11 @@ async function pointFixture(child = false, oopif = false) { f.send.mockImplementation(async (target, method, params = {}) => { if (method === "Runtime.evaluate" && params.expression === "document.visibilityState") return { result: { value: control.visibility } }; + if ( + method === "Runtime.evaluate" && + String(params.expression).startsWith("!!document.elementFromPoint") + ) + return { result: { value: false } }; if (method === "Runtime.evaluate" && params.awaitPromise) return { result: { value: true } }; if (method === "Emulation.setFocusEmulationEnabled") { control.focusCommands.push(params.enabled as boolean); @@ -586,7 +591,7 @@ async function pointFixture(child = false, oopif = false) { input, hitPoints, pointControl: control, - deps: { cdp: f.cdp, tabsApi, bypassOverlay }, + deps: { cdp: f.cdp, tabsApi, sendToTab }, shot, }; } @@ -608,9 +613,9 @@ it.each([ "visual_capture_stale", ); expect(f.input).toHaveLength(count); - expect(f.deps.bypassOverlay.mock.calls).toEqual([ - [4, true], - [4, false], + expect(f.deps.sendToTab.mock.calls.map(([tab, message]) => [tab, message.phase])).toEqual([ + [4, "begin"], + [4, "end"], ]); }); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 60819b9..fc0e0b8 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -15,6 +15,11 @@ import { isAbortError } from "./vom/capture-abort"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { + type CaptureSuppressSendToTab, + withExtensionOverlayHidden, +} from "@/lib/capture-suppress-bridge"; +import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { BlurParams, @@ -60,6 +65,8 @@ export interface InteractionDeps { /** Abort hook (full chain wired in M10.2). */ signal?: AbortSignal; defaultTimeoutMs?: number; + /** Hide the complete extension overlay for the bounded click operation. */ + sendToTab?: CaptureSuppressSendToTab; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; /** Keep hover hit-testing active for the caller's next observation/action. */ @@ -517,36 +524,12 @@ export async function clickResolvedTarget( if (clickCount < 1) { return { code: "invalid_params", message: "click_count must be greater than zero" }; } - const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); - let automationBypassEnabled = false; - if (overlayBlocking && deps.bypassOverlay) { - try { - await deps.bypassOverlay(target.tabId, true); - automationBypassEnabled = true; - } catch (err) { - console.debug("[bsk interaction] overlay bypass enable failed", err); - } - } - - try { - const error = await dispatchClickAtPoint( - target.tabId, - centre, - params, - deps, - undefined, - markSent, - ); - if (error) return error; - } finally { - if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { - try { - await deps.bypassOverlay(target.tabId, false); - } catch (err) { - console.debug("[bsk interaction] overlay bypass disable failed", err); - } - } - } + const error = await withExtensionOverlayHidden( + target.tabId, + () => dispatchClickAtPoint(target.tabId, centre, params, deps, undefined, markSent), + deps.sendToTab, + ); + if (error) return error; return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, @@ -612,6 +595,28 @@ async function dispatchClickAtPoint( if (error) return failure(error); } if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); + // A closed shadow root retargets its inner controls to the host here. + // Check the real hit after mouseMoved, including when suppression failed. + const hit = await deps.cdp.send<{ + result?: { value?: boolean }; + exceptionDetails?: unknown; + }>(tabId, "Runtime.evaluate", { + expression: `!!document.elementFromPoint(${point.x},${point.y})?.closest(${JSON.stringify(OVERLAY_HOST_SELECTOR)})`, + returnByValue: true, + }); + if (hit.exceptionDetails || hit.result?.value !== false) { + return failure( + rpcError( + "cdp_failed", + "input_not_ready", + hit.result?.value === true + ? "Extension overlay intercepts the click point" + : "Could not verify that the click point is clear of the extension overlay", + { effect_state: attempted ? "unknown" : "none", pointer_moved: moved }, + ), + ); + } + if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); deps.onInputSent?.(tabId); markSent?.(); attempted = true; @@ -683,40 +688,39 @@ async function clickVisualPoint( return changed(); return null; }; - let bypass = false; try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - if (deps.bypassOverlay) { - await deps.bypassOverlay(target.tabId, true); - bypass = true; - } - const invalid = await validate(); - if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } }; - const error = await dispatchClickAtPoint( + return await withExtensionOverlayHidden( target.tabId, - point, - params, - deps, async () => { - await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability. - return validate(); + const invalid = await validate(); + if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } }; + const error = await dispatchClickAtPoint( + target.tabId, + point, + params, + deps, + async () => { + await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability. + return validate(); + }, + markSent, + ); + if (error) return error; + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: capture.ref, + ...point, + }); }, - markSent, + deps.sendToTab, ); - if (error) return error; - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: capture.ref, - ...point, - }); } catch (error) { return { code: deps.signal?.aborted || isAbortError(error) ? "cancelled" : "cdp_failed", message: error instanceof Error ? error.message : String(error), data: { effect_state: "none" }, }; - } finally { - if (bypass) await deps.bypassOverlay!(target.tabId, false).catch(() => {}); } } From f4465361252df81dd2ee6ff020132ceaf7285e88 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 23 Sep 2026 11:18:23 +0800 Subject: [PATCH 2/4] fix(extension): scope click passthrough to control surfaces --- .../__tests__/input-passthrough.test.ts | 72 +++++++ .../src/content/input-passthrough.ts | 30 +++ apps/extension/src/content/overlay.css | 8 + apps/extension/src/entrypoints/content.ts | 16 +- .../src/lib/input-passthrough-bridge.ts | 34 ++++ .../src/tools/__tests__/click.browser.test.ts | 125 ++++++++++-- .../src/tools/__tests__/dispatcher.test.ts | 15 +- .../src/tools/__tests__/file-transfer.test.ts | 45 +++-- .../src/tools/__tests__/interaction.test.ts | 186 +++++++++++++----- .../tools/__tests__/visual-screenshot.test.ts | 65 ++++-- apps/extension/src/tools/click-overlay.ts | 120 +++++++++++ apps/extension/src/tools/interaction.ts | 56 ++---- 12 files changed, 629 insertions(+), 143 deletions(-) create mode 100644 apps/extension/src/content/__tests__/input-passthrough.test.ts create mode 100644 apps/extension/src/content/input-passthrough.ts create mode 100644 apps/extension/src/lib/input-passthrough-bridge.ts create mode 100644 apps/extension/src/tools/click-overlay.ts diff --git a/apps/extension/src/content/__tests__/input-passthrough.test.ts b/apps/extension/src/content/__tests__/input-passthrough.test.ts new file mode 100644 index 0000000..1cc7129 --- /dev/null +++ b/apps/extension/src/content/__tests__/input-passthrough.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_ATTR, + type InputPassthroughMessage, + isInputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; +import { createInputPassthroughController } from "../input-passthrough"; + +describe("click input passthrough", () => { + let host: HTMLElement; + beforeEach(() => { + host = document.createElement("div"); + }); + const message = (phase: "begin" | "end", id = "click-1"): InputPassthroughMessage => ({ + type: INPUT_PASSTHROUGH, + phase, + id, + }); + + it("applies and acknowledges synchronously without hiding the host or scheduling frames", () => { + const controller = createInputPassthroughController(() => host); + const raf = vi.spyOn(window, "requestAnimationFrame"); + const ack = vi.fn(() => expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true)); + expect(controller.handleMessage(message("begin"), ack)).toBe(false); + expect(ack).toHaveBeenCalledWith({ type: INPUT_PASSTHROUGH, ok: true }); + expect(raf).not.toHaveBeenCalled(); + expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(false); + raf.mockRestore(); + }); + + it("counts independent clicks and ignores duplicate or unmatched messages", () => { + const controller = createInputPassthroughController(() => host); + for (const id of ["one", "one", "two"]) controller.handleMessage(message("begin", id), vi.fn()); + expect(controller.pendingCount).toBe(2); + for (const id of ["missing", "one", "one"]) + controller.handleMessage(message("end", id), vi.fn()); + expect(controller.pendingCount).toBe(1); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + controller.handleMessage(message("end", "two"), vi.fn()); + expect(controller.pendingCount).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("reapplies pending clicks after host replacement and leaves screenshot state alone", () => { + let current: HTMLElement | null = null; + const controller = createInputPassthroughController(() => current); + controller.handleMessage(message("begin"), vi.fn()); + current = host; + host.setAttribute("data-bsk-capture-hidden", ""); + controller.onHostMounted(host); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + controller.handleMessage(message("end"), vi.fn()); + expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(true); + current = document.createElement("div"); + controller.onHostMounted(current); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("accepts only paired click messages with an operation id", () => { + expect(isInputPassthroughMessage(message("begin"))).toBe(true); + expect(isInputPassthroughMessage(message("end"))).toBe(true); + for (const invalid of [ + null, + {}, + { ...message("begin"), id: "" }, + { ...message("begin"), phase: "reset" }, + { ...message("begin"), type: "bsk/capture-suppress" }, + ]) + expect(isInputPassthroughMessage(invalid)).toBe(false); + }); +}); diff --git a/apps/extension/src/content/input-passthrough.ts b/apps/extension/src/content/input-passthrough.ts new file mode 100644 index 0000000..c47f0d3 --- /dev/null +++ b/apps/extension/src/content/input-passthrough.ts @@ -0,0 +1,30 @@ +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_ATTR, + type InputPassthroughAck, + type InputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; + +export function createInputPassthroughController(getHost: () => HTMLElement | null) { + const pending = new Set(); + const apply = (host: HTMLElement | null) => + host?.toggleAttribute(INPUT_PASSTHROUGH_ATTR, pending.size > 0); + + return { + get pendingCount() { + return pending.size; + }, + handleMessage( + message: InputPassthroughMessage, + sendResponse: (ack: InputPassthroughAck) => void, + ): false { + if (message.phase === "begin") pending.add(message.id); + else pending.delete(message.id); + apply(getHost()); + // Hit testing uses current styles; unlike screenshots it needs no compositor frame. + sendResponse({ type: INPUT_PASSTHROUGH, ok: true }); + return false; + }, + onHostMounted: apply, + }; +} diff --git a/apps/extension/src/content/overlay.css b/apps/extension/src/content/overlay.css index 5e8af1a..e1a5f95 100644 --- a/apps/extension/src/content/overlay.css +++ b/apps/extension/src/content/overlay.css @@ -15,6 +15,14 @@ pointer-events: auto !important; } +/* Click passthrough changes only control hit targets, not visibility or other UI. */ +:host([data-bsk-overlay-surface][data-bsk-input-passthrough]), +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-blocker"], +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"], +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"] * { + pointer-events: none !important; +} + /* Screenshot suppression: the host lives in the main DOM, so hiding it removes the whole shadow subtree from compositing and captured frames only contain page content. */ diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index b481e05..639acd9 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -8,6 +8,7 @@ import { ControlOverlay } from "@/content/ControlOverlay"; import { createCaptureSuppressController } from "@/content/capture-suppress"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import { createHelpRequestData } from "@/content/help-request"; +import { createInputPassthroughController } from "@/content/input-passthrough"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; @@ -28,6 +29,11 @@ import { isHelpCancelMessage, isHelpRequestMessage, } from "@/lib/help-bridge"; +import { + type InputPassthroughAck, + type InputPassthroughMessage, + isInputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id"; import { isOverlayAgentOverlayResetMessage, @@ -87,6 +93,7 @@ export default defineContentScript({ } const captureSuppress = createCaptureSuppressController(() => overlayHost); + const inputPassthrough = createInputPassthroughController(() => overlayHost); const ui = await createShadowRootUi(ctx, { name: "browser-skill-overlay", @@ -101,6 +108,7 @@ export default defineContentScript({ hostLossReported = false; // A host rebuilt mid-capture must stay hidden until `end` arrives. captureSuppress.onHostMounted(shadowHost); + inputPassthrough.onHostMounted(shadowHost); const app = document.createElement("div"); app.className = "bsk-overlay-root"; container.append(app); @@ -257,6 +265,7 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | CaptureSuppressMessage + | InputPassthroughMessage | RecordStartMessage | RecordStopMessage | RecordCancelMessage @@ -264,11 +273,16 @@ export default defineContentScript({ | OverlayAgentStateMessage | OverlayAutomationBypassMessage, _sender: chrome.runtime.MessageSender, - sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void, + sendResponse: ( + response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck | InputPassthroughAck, + ) => void, ) => { if (isCaptureSuppressMessage(message)) { return captureSuppress.handleMessage(message, sendResponse); } + if (isInputPassthroughMessage(message)) { + return inputPassthrough.handleMessage(message, sendResponse); + } if (isRecordStartMessage(message)) { activeRecordRequestId = message.requestId; diff --git a/apps/extension/src/lib/input-passthrough-bridge.ts b/apps/extension/src/lib/input-passthrough-bridge.ts new file mode 100644 index 0000000..90515b5 --- /dev/null +++ b/apps/extension/src/lib/input-passthrough-bridge.ts @@ -0,0 +1,34 @@ +/** Click-only hit-test suppression; independent of screenshot visibility and hover bypass. */ +export const INPUT_PASSTHROUGH = "bsk/input-passthrough"; +export const INPUT_PASSTHROUGH_ATTR = "data-bsk-input-passthrough"; + +export interface InputPassthroughMessage { + type: typeof INPUT_PASSTHROUGH; + phase: "begin" | "end"; + /** Identifies one click so cleanup after a lost ack cannot release another click. */ + id: string; +} + +export interface InputPassthroughAck { + type: typeof INPUT_PASSTHROUGH; + ok: true; +} + +export function isInputPassthroughMessage(message: unknown): message is InputPassthroughMessage { + if (!message || typeof message !== "object") return false; + const m = message as Record; + return ( + m.type === INPUT_PASSTHROUGH && + (m.phase === "begin" || m.phase === "end") && + typeof m.id === "string" && + m.id.length > 0 + ); +} + +export type InputPassthroughSendToTab = ( + tabId: number, + message: InputPassthroughMessage, +) => Promise; + +export const sendInputPassthrough: InputPassthroughSendToTab = (tabId, message) => + chrome.tabs.sendMessage(tabId, message); diff --git a/apps/extension/src/tools/__tests__/click.browser.test.ts b/apps/extension/src/tools/__tests__/click.browser.test.ts index 4ee416a..dbc1903 100644 --- a/apps/extension/src/tools/__tests__/click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click.browser.test.ts @@ -4,10 +4,11 @@ import { readFileSync } from "node:fs"; import { createServer } from "node:http"; import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; +import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ControlOverlay } from "@/content/ControlOverlay"; -import { CAPTURE_SUPPRESS, type CaptureSuppressSendToTab } from "@/lib/capture-suppress-bridge"; +import { INPUT_PASSTHROUGH, type InputPassthroughSendToTab } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { prepareBackgroundExecution } from "../background-execution"; import { handleClick, handlePress } from "../interaction"; @@ -360,7 +361,7 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( } }, 90_000); - it("clicks through the real control pill and fails if the closed shadow overlay still intercepts input", async () => { + it("preserves hover and visible controls while scoped passthrough clears closed shadow click targets", async () => { const { withChrome } = await import( new URL( "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", @@ -368,6 +369,22 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( ).href ); const css = readFileSync(new URL("../../content/overlay.css", import.meta.url), "utf8"); + // Run the actual content-side controller in the browser, including its lease bookkeeping. + const moduleUrl = (source: string) => + `data:text/javascript;base64,${Buffer.from( + ts.transpileModule(source, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ES2022 }, + }).outputText, + ).toString("base64")}`; + const bridgeUrl = moduleUrl( + readFileSync(new URL("../../lib/input-passthrough-bridge.ts", import.meta.url), "utf8"), + ); + const controllerUrl = moduleUrl( + readFileSync(new URL("../../content/input-passthrough.ts", import.meta.url), "utf8").replace( + '"@/lib/input-passthrough-bridge"', + JSON.stringify(bridgeUrl), + ), + ); await withChrome( { executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 }, async (send: Send) => { @@ -398,13 +415,15 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( deviceScaleFactor: 1, mobile: false, }); - await evaluate(`document.body.innerHTML = ''; - window.clicks=[]; window.stops=0; + await evaluate(`(async () => { document.body.innerHTML = ''; + window.clicks=[]; window.stops=0; window.pageEvents=[]; document.querySelector('#target').onclick=e=>clicks.push(e.isTrusted); + for (const type of ['mouseover','mousedown','mouseup','click']) document.querySelector('#target').addEventListener(type,e=>pageEvents.push({type,trusted:e.isTrusted,hover:getComputedStyle(e.currentTarget).backgroundColor==='rgb(1, 2, 3)'})); window.overlay=document.createElement('browser-skill-overlay'); overlay.setAttribute('data-bsk-overlay',''); overlay.setAttribute('data-bsk-overlay-surface',''); document.documentElement.append(overlay); - window.overlayRoot=overlay.attachShadow({mode:'closed'});`); + window.overlayRoot=overlay.attachShadow({mode:'closed'}); + window.passthroughController=(await import(${JSON.stringify(controllerUrl)})).createInputPassthroughController(()=>overlay); })()`); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [] }), @@ -418,17 +437,30 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( const mouse: string[] = []; const cdp: CdpRunner = { send: async (_tabId, method, params) => { - if (method === "Input.dispatchMouseEvent") + if (method === "Input.dispatchMouseEvent") { mouse.push((params as { type: string }).type); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + expect( + await evaluate( + "getComputedStyle(overlayRoot.querySelector('[data-slot=control-overlay-pill]')).opacity", + ), + ).toBe("1"); + expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false); + } return (await local(method, params)) as never; }, }; - const sendToTab: CaptureSuppressSendToTab = async (_tabId, message) => { - // Transport adapter: use the production CSS suppression marker, as the content script does. - await evaluate( - `overlay.toggleAttribute('data-bsk-capture-hidden', ${message.phase === "begin"})`, + const phases: string[] = []; + const sendInputPassthrough: InputPassthroughSendToTab = async (_tabId, message) => { + phases.push(message.phase); + return evaluate( + `new Promise(resolve=>passthroughController.handleMessage(${JSON.stringify(message)},resolve))`, ); - return { type: CAPTURE_SUPPRESS, ok: true }; + }; + const bypassOverlay = async (_tabId: number, enabled: boolean) => { + await evaluate(`window.bypassCount += ${enabled ? 1 : -1}; + overlay.toggleAttribute('data-bsk-overlay-blocking',bypassCount===0); + overlayRoot.querySelector('[data-slot="control-overlay-blocker"]').style.pointerEvents=bypassCount>0?'none':'auto';`); }; // A retained hover bypass leaves the pill interactive even though the host is transparent. for (const automationBypass of [false, true]) { @@ -440,9 +472,23 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( onInterrupt: () => {}, }), ); - await evaluate(`overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass}); + await evaluate(`window.bypassCount=${automationBypass ? 1 : 0}; overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass}); overlayRoot.innerHTML = ${JSON.stringify(``)} + ${JSON.stringify(markup)}; + // Static markup starts before the component's reveal effect; model its settled state. + for (const control of overlayRoot.querySelectorAll('[data-slot^="control-overlay"]')) control.style.opacity='1'; overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]').addEventListener('click',()=>stops++);`); + await evaluate( + "Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})", + ); + phases.length = 0; + const clear = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, + ); + expect(clear).not.toHaveProperty("code"); + expect(phases).toEqual([]); + expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0); for (const slot of ["control-overlay-pill", "control-overlay-stop-all"]) { const point = await evaluate<{ x: number; y: number }>(`(() => { const r=overlayRoot.querySelector('[data-slot="${slot}"]').getBoundingClientRect(); @@ -465,7 +511,9 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( }; await nativeClick(); // Reproduce the interception before using the fixed click path. expect(await evaluate("clicks")).toEqual([]); - await evaluate("stops=0"); + await local("Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }); + await evaluate("stops=0; pageEvents=[]"); + phases.length = 0; const { root } = await local<{ root: { nodeId: number } }>("DOM.getDocument"); const { nodeId } = await local<{ nodeId: number }>("DOM.querySelector", { nodeId: root.nodeId, @@ -482,10 +530,23 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( session_id: ctx.sessionId, ...(automationBypass ? { ref: "e1" } : { selector: "#target" }), }, - { cdp, tabsApi, sendToTab }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, ); expect(result, JSON.stringify(result)).not.toHaveProperty("code"); expect(await evaluate("clicks")).toEqual([true]); + expect(phases).toEqual(["begin", "end"]); + expect(await evaluate("pageEvents")).toEqual( + ["mouseover", "mousedown", "mouseup", "click"].map((type) => ({ + type, + trusted: true, + hover: true, + })), + ); + expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe( + false, + ); expect(await evaluate("stops")).toBe(0); expect(mouse).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]); expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false); @@ -504,7 +565,8 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( { cdp, tabsApi, - sendToTab: async () => { + bypassOverlay, + sendInputPassthrough: async () => { throw new Error("Content script unavailable"); }, }, @@ -513,9 +575,40 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( code: "cdp_failed", data: { reason: "input_not_ready", effect_state: "none" }, }); - expect(mouse).toEqual(["mouseMoved"]); + expect(mouse).toEqual([]); expect(await evaluate("clicks")).toEqual([true]); } + // Other extension controls stay hit-testable even while a click lease is active. + await evaluate( + `(() => { const extra=document.createElement('div'); extra.innerHTML=''; overlayRoot.append(extra); })()`, + ); + await evaluate( + "Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})", + ); + mouse.length = 0; + const protectedControl = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, + ); + expect(protectedControl).toHaveProperty("data.reason", "input_not_ready"); + expect(mouse).toEqual([]); + await sendInputPassthrough(4, { + type: INPUT_PASSTHROUGH, + phase: "begin", + id: "css-check", + }); + expect(await evaluate("overlayRoot.elementFromPoint(40,30).id")).toBe("other"); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + await evaluate(`(() => { const replacement=document.createElement('browser-skill-overlay'); + for(const attr of overlay.attributes) replacement.setAttribute(attr.name,attr.value); + const replacementRoot=replacement.attachShadow({mode:'closed'}); + replacementRoot.innerHTML=overlayRoot.innerHTML; + overlay.replaceWith(replacement); overlay=replacement; overlayRoot=replacementRoot; + passthroughController.onHostMounted(overlay); })()`); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(true); + await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "end", id: "css-check" }); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(false); } }, ); diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 08a8ac0..6e88dba 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -341,7 +341,7 @@ describe("ToolDispatcher", () => { }); }); - it("hides and restores the complete control overlay for an upload trigger click", async () => { + it("leaves overlay state alone for an unobstructed upload trigger click", async () => { const sendMessage = vi.fn(async () => undefined); vi.stubGlobal("chrome", { tabs: { @@ -383,8 +383,8 @@ describe("ToolDispatcher", () => { } if (method === "Runtime.evaluate") { const expression = (params as { expression?: string }).expression ?? ""; - if (expression.startsWith("!!document.elementFromPoint")) { - return { result: { value: false } } as T; + if (expression.includes('return "absent"')) { + return { result: { value: "clear" } } as T; } if (expression.includes("count:")) { return { result: { value: { count: 1, multiple: false } } } as T; @@ -425,14 +425,7 @@ describe("ToolDispatcher", () => { await vi.waitFor(() => expect(sent).toHaveLength(1)); expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } }); - expect(sendMessage).toHaveBeenNthCalledWith(1, 7, { - type: "bsk/capture-suppress", - phase: "begin", - }); - expect(sendMessage).toHaveBeenNthCalledWith(2, 7, { - type: "bsk/capture-suppress", - phase: "end", - }); + expect(sendMessage).not.toHaveBeenCalled(); }); it("detaches CDP state before stopping a session", async () => { diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 1fe1ad9..d2085a4 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; @@ -58,6 +59,7 @@ function uploadCdp( multiple?: boolean; chooser?: { frameId?: string; backendNodeId?: number; mode?: string }; pendingResolve?: boolean; + overlayHit?: () => string; } = {}, ) { const calls: Array<{ method: string; params?: object }> = []; @@ -66,11 +68,9 @@ function uploadCdp( calls.push({ method, params }); if ( method === "Runtime.evaluate" && - String((params as { expression?: string })?.expression).startsWith( - "!!document.elementFromPoint", - ) + String((params as { expression?: string })?.expression).includes('return "absent"') ) - return { result: { value: false } }; + return { result: { value: options.overlayHit?.() ?? "clear" } }; if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") @@ -168,11 +168,20 @@ function dropCdp( } describe("file transfer tools", () => { - it("captures the file input activated by the requested click and injects only staged paths", async () => { + it.each([ + false, + true, + ])("uploads staged paths through the requested click (covered=%s)", async (covered) => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); - const { cdp, calls } = uploadCdp(); + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + }); + const { cdp, calls } = uploadCdp({ + overlayHit: () => (covered && !passthrough ? "covered" : "clear"), + }); const result = await handleUpload( manager, @@ -184,8 +193,12 @@ describe("file transfer tools", () => { { transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" }, ], }, - { cdp, tabsApi: tabsApi() }, + { cdp, tabsApi: tabsApi(), sendInputPassthrough }, ); + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual( + covered ? ["begin", "end"] : [], + ); + expect(passthrough).toBe(false); expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); expect(calls[0]).toEqual({ @@ -560,7 +573,7 @@ describe("file transfer tools", () => { expect(detach).toHaveBeenCalledWith(4); }); - it("routes one exact-target download through a browser-relative capability", async () => { + it.each([false, true])("routes an exact-target download (covered=%s)", async (covered) => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); @@ -600,14 +613,16 @@ describe("file transfer tools", () => { }; let cdpEvent: Parameters>[0] | undefined; let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + }); const send = vi.fn(async (_tabId: number, method: string, params?: object) => { if ( method === "Runtime.evaluate" && - String((params as { expression?: string })?.expression).startsWith( - "!!document.elementFromPoint", - ) + String((params as { expression?: string })?.expression).includes('return "absent"') ) - return { result: { value: false } }; + return { result: { value: covered && !passthrough ? "covered" : "clear" } }; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; @@ -640,8 +655,12 @@ describe("file transfer tools", () => { const result = await handleDownload( manager, { session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" }, - { cdp, tabsApi: tabsApi(), downloads }, + { cdp, tabsApi: tabsApi(), downloads, sendInputPassthrough }, ); + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual( + covered ? ["begin", "end"] : [], + ); + expect(passthrough).toBe(false); expect(suggested).toEqual({ filename: "BrowserSkill/tr_1/result.zip", diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 90dd8a1..e27aa86 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge"; +import { INPUT_PASSTHROUGH, type InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { withInputReady } from "../input-readiness"; @@ -35,7 +35,7 @@ function makeFakeCdp( rendered: () => boolean | Promise = () => true, ) { const sent: Array<{ tabId: number; method: string; params?: object }> = []; - const overlayHit = vi.fn(async () => ({ result: { value: false } })); + const overlayHit = vi.fn(async () => ({ result: { value: "clear" } })); const sendImpl = async (tabId: number, method: string, params?: object) => { sent.push({ tabId, method, params }); if ( @@ -46,9 +46,7 @@ function makeFakeCdp( if (method === "Page.captureScreenshot") return { data: (await rendered()) ? "pixel" : "" }; if ( method === "Runtime.evaluate" && - String((params as { expression?: string })?.expression).startsWith( - "!!document.elementFromPoint", - ) + String((params as { expression?: string })?.expression).includes('return "absent"') ) return overlayHit(); const h = handlers[method]; @@ -251,24 +249,40 @@ describe("handleClick", () => { }); it.each([ - "success", - "dispatch-error", - "cancel", + "clear", + "absent", + "bypass-only", + "covered", + "press-fails", + "cancel-after-move", + "cancel-during-begin", "still-covered", - "suppression-failed", - "hit-test-error", - "invalid-hit-test", - "no-content-script", - ])("scopes complete overlay suppression and checks the click point: %s", async (mode) => { + "begin-fails", + "lost-ack", + "restore-fails", + "probe-throws", + "probe-invalid", + "verify-throws", + "late-covered", + "late-unknown", + "known-late-unknown", + ])("scopes click passthrough without changing a retained hover bypass: %s", async (mode) => { const order: string[] = []; const controller = new AbortController(); - const bypassOverlay = vi.fn(); - const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => { + let bypassCount = 1; // A hover owns this reference throughout the click helper. + const bypassOverlay = vi.fn(async (_tab: number, enabled: boolean) => { + order.push(enabled ? "bypass-begin" : "bypass-end"); + bypassCount += enabled ? 1 : -1; + }); + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { order.push(message.phase); - if (mode === "suppression-failed" || mode === "no-content-script") { - throw new Error("No receiving content script"); - } - return { type: CAPTURE_SUPPRESS, ok: true }; + if (message.phase === "begin" && mode === "begin-fails") throw new Error("No receiver"); + passthrough = message.phase === "begin"; + if (passthrough && mode === "cancel-during-begin") controller.abort(); + if (passthrough && mode === "lost-ack") throw new Error("Response lost after application"); + if (!passthrough && mode === "restore-fails") throw new Error("Tab closed"); + return { type: INPUT_PASSTHROUGH, ok: true }; }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); @@ -279,49 +293,123 @@ describe("handleClick", () => { "Input.dispatchMouseEvent": (params) => { const { type } = params as { type: string }; order.push(type); - if (type === "mouseMoved" && mode === "cancel") controller.abort(); - if (type === "mousePressed" && mode === "dispatch-error") throw new Error("Input failed"); + if (type === "mouseMoved" && mode === "cancel-after-move") controller.abort(); + if (type === "mousePressed" && mode === "press-fails") throw new Error("Input failed"); return {}; }, }); + const initiallyClear = [ + "clear", + "absent", + "probe-throws", + "probe-invalid", + "late-covered", + "late-unknown", + ].includes(mode); fake.overlayHit.mockImplementation(async () => { - order.push("hit-test"); - if (mode === "hit-test-error") throw new Error("Renderer unavailable"); - if (mode === "invalid-hit-test") return { exceptionDetails: {} } as never; - return { result: { value: mode === "still-covered" || mode === "suppression-failed" } }; + order.push("probe"); + if (mode === "probe-throws" || (mode === "verify-throws" && passthrough)) + throw new Error("Renderer unavailable"); + if (mode === "probe-invalid") return { exceptionDetails: {} } as never; + if (order.includes("mouseMoved") && ["late-unknown", "known-late-unknown"].includes(mode)) + return {} as never; + const hit = + mode === "absent" + ? "absent" + : mode === "late-covered" && order.includes("mouseMoved") + ? "covered" + : mode === "bypass-only" && bypassCount === 2 + ? "clear" + : initiallyClear || (passthrough && mode !== "still-covered") + ? "clear" + : "covered"; + return { result: { value: hit } }; }); const result = await handleClick( sm, { session_id: "aa11", ref: "@e3" }, - { ...fake, sendToTab, bypassOverlay, signal: controller.signal }, + { + ...fake, + sendInputPassthrough, + bypassOverlay, + signal: controller.signal, + }, ); - expect(order[0]).toBe("begin"); - expect(bypassOverlay).not.toHaveBeenCalled(); // Do not change a retained hover bypass. - if (mode === "success" || mode === "no-content-script") { - expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); - expect(order.slice(1, 5)).toEqual([ + const beforeMoveFailure = [ + "still-covered", + "begin-fails", + "verify-throws", + "cancel-during-begin", + ].includes(mode); + const noPress = + beforeMoveFailure || + ["cancel-after-move", "late-covered", "known-late-unknown"].includes(mode); + const mouse = order.filter((step) => step.startsWith("mouse")); + expect(mouse).toEqual( + beforeMoveFailure + ? [] + : noPress + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], + ); + if (noPress || mode === "press-fails") { + expect(result).toHaveProperty( + "code", + mode.startsWith("cancel-") ? "cancelled" : "cdp_failed", + ); + if ( + [ + "still-covered", + "begin-fails", + "verify-throws", + "late-covered", + "known-late-unknown", + ].includes(mode) + ) + expect(result).toMatchObject({ + data: { + reason: "input_not_ready", + effect_state: "none", + pointer_moved: ["late-covered", "known-late-unknown"].includes(mode), + }, + }); + } else expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + if (initiallyClear || mode === "bypass-only") { + expect(sendInputPassthrough).not.toHaveBeenCalled(); + } else { + const messages = sendInputPassthrough.mock.calls.map(([, message]) => message); + expect(messages).toEqual([ + { type: INPUT_PASSTHROUGH, phase: "begin", id: expect.any(String) }, + { type: INPUT_PASSTHROUGH, phase: "end", id: messages[0].id }, + ]); + if (mouse.length) expect(order.indexOf("begin")).toBeLessThan(order.indexOf("mouseMoved")); + if (mouse.includes("mouseReleased")) + expect(order.indexOf("end")).toBeGreaterThan(order.indexOf("mouseReleased")); + expect(order.slice(-2)).toEqual(["end", "bypass-end"]); + } + if (mode === "clear" || mode === "absent") + expect(order).toEqual(["probe", "mouseMoved", "probe", "mousePressed", "mouseReleased"]); + if (mode === "covered") + expect(order).toEqual([ + "probe", + "bypass-begin", + "probe", + "begin", + "probe", "mouseMoved", - "hit-test", + "probe", "mousePressed", "mouseReleased", + "end", + "bypass-end", + ]); + expect(bypassCount).toBe(1); + if (initiallyClear) expect(bypassOverlay).not.toHaveBeenCalled(); + else + expect(bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], ]); - } else { - expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed"); - if (mode === "dispatch-error") { - expect(order).toContain("mouseReleased"); - } else { - expect(order).not.toContain("mousePressed"); - } - if (mode === "still-covered" || mode === "suppression-failed") { - expect(result).toMatchObject({ data: { reason: "input_not_ready", effect_state: "none" } }); - } - } - if (mode === "suppression-failed" || mode === "no-content-script") { - expect(sendToTab).toHaveBeenCalledTimes(1); - } else { - expect(order.at(-1)).toBe("end"); - expect(sendToTab.mock.calls.map((call) => call[1].phase)).toEqual(["begin", "end"]); - } }); it("leaves disabled click behavior to the browser without an AX preflight", async () => { diff --git a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts index 7f8911e..76d6950 100644 --- a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts +++ b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts @@ -524,26 +524,29 @@ async function pointFixture(child = false, oopif = false) { ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]); const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; const tabsApi = { get: async () => tab, query: async () => [tab] }; - const sendToTab = vi.fn(async (_tab: number, _message: { phase: string }) => ({})); + const sendInputPassthrough = vi.fn(async (_tab: number, _message: { phase: string }) => ({})); + const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {}); const original = f.send.getMockImplementation()!; const input: Record[] = []; const hitPoints: number[][] = []; const control = { hit: true, + overlayCovered: false, onMove: () => {}, failPress: false, visibility: "visible", onFocus: () => {}, focusCommands: [] as boolean[], }; + sendInputPassthrough.mockImplementation(async (_tab, message) => { + control.overlayCovered = message.phase !== "begin"; + return {}; + }); f.send.mockImplementation(async (target, method, params = {}) => { if (method === "Runtime.evaluate" && params.expression === "document.visibilityState") return { result: { value: control.visibility } }; - if ( - method === "Runtime.evaluate" && - String(params.expression).startsWith("!!document.elementFromPoint") - ) - return { result: { value: false } }; + if (method === "Runtime.evaluate" && String(params.expression).includes('return "absent"')) + return { result: { value: control.overlayCovered ? "covered" : "clear" } }; if (method === "Runtime.evaluate" && params.awaitPromise) return { result: { value: true } }; if (method === "Emulation.setFocusEmulationEnabled") { control.focusCommands.push(params.enabled as boolean); @@ -555,7 +558,7 @@ async function pointFixture(child = false, oopif = false) { String(params.functionDeclaration).includes("elementFromPoint") ) { hitPoints.push((params.arguments as { value: number }[]).map((a) => a.value)); - return { result: { value: control.hit } }; + return { result: { value: control.hit && !control.overlayCovered } }; } if (method === "Input.dispatchMouseEvent") { input.push(params); @@ -591,7 +594,7 @@ async function pointFixture(child = false, oopif = false) { input, hitPoints, pointControl: control, - deps: { cdp: f.cdp, tabsApi, sendToTab }, + deps: { cdp: f.cdp, tabsApi, sendInputPassthrough, bypassOverlay }, shot, }; } @@ -613,12 +616,52 @@ it.each([ "visual_capture_stale", ); expect(f.input).toHaveLength(count); - expect(f.deps.sendToTab.mock.calls.map(([tab, message]) => [tab, message.phase])).toEqual([ - [4, "begin"], - [4, "end"], + expect(f.deps.sendInputPassthrough).not.toHaveBeenCalled(); + expect(f.deps.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], ]); }); +it.each([ + "success", + "press-fails", + "cancel", + "still-covered", + "appears-during-bypass", +])("scopes visual click passthrough: %s", async (mode) => { + const f = await pointFixture(); + const abort = new AbortController(); + f.pointControl.overlayCovered = mode !== "appears-during-bypass"; + if (mode === "appears-during-bypass") + f.deps.bypassOverlay.mockImplementation(async (_tab, enabled) => { + if (enabled) f.pointControl.overlayCovered = true; + }); + f.pointControl.failPress = mode === "press-fails"; + if (mode === "cancel") f.pointControl.onMove = () => abort.abort(); + if (mode === "still-covered") f.deps.sendInputPassthrough.mockImplementation(async () => ({})); + const result = await handleClick(f.manager, f.params, { ...f.deps, signal: abort.signal }); + if (mode === "success" || mode === "appears-during-bypass") + expect(result).not.toHaveProperty("code"); + else expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed"); + expect(f.input.map((e) => e.type)).toEqual( + mode === "still-covered" + ? [] + : mode === "cancel" + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], + ); + expect(f.deps.sendInputPassthrough.mock.calls.map(([, message]) => message.phase)).toEqual([ + "begin", + "end", + ]); + expect(f.deps.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + if (mode === "still-covered") expect(result).toHaveProperty("data.effect_state", "none"); +}); + it.each([ "success", "geometry", diff --git a/apps/extension/src/tools/click-overlay.ts b/apps/extension/src/tools/click-overlay.ts new file mode 100644 index 0000000..78d6909 --- /dev/null +++ b/apps/extension/src/tools/click-overlay.ts @@ -0,0 +1,120 @@ +import { + INPUT_PASSTHROUGH, + type InputPassthroughSendToTab, + sendInputPassthrough, +} from "@/lib/input-passthrough-bridge"; +import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge"; +import type { RpcError } from "@/transport/types"; +import { rpcError } from "./errors"; +import type { CdpRunner } from "./shared"; + +export interface ClickOverlayDeps { + cdp: CdpRunner; + signal?: AbortSignal; + /** Acquire/release only this operation's automation bypass reference. */ + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + sendInputPassthrough?: InputPassthroughSendToTab; +} + +type OverlayHit = "absent" | "clear" | "covered" | "unknown"; + +/** Prepare before mouseMoved, then check again before every press. */ +export async function withClickOverlay( + tabId: number, + point: { x: number; y: number }, + deps: ClickOverlayDeps, + click: (beforePress: () => Promise) => Promise, + alwaysBypass = false, +): Promise { + const send = deps.sendInputPassthrough ?? sendInputPassthrough; + let bypass = false; + let passthroughId: string | undefined; + const notReady = () => + rpcError( + "cdp_failed", + "input_not_ready", + "Could not clear the extension overlay from the click point", + { + effect_state: "none", + pointer_moved: false, + }, + ); + const cancelled = (): RpcError => ({ + code: "cancelled", + message: "click aborted", + data: { effect_state: "none", pointer_moved: false }, + }); + const probe = async (): Promise => { + try { + const selector = JSON.stringify(OVERLAY_HOST_SELECTOR); + const hit = await deps.cdp.send<{ + result?: { value?: unknown }; + exceptionDetails?: unknown; + }>(tabId, "Runtime.evaluate", { + expression: `(() => { + const host = document.querySelector(${selector}); + if (!host) return "absent"; + return document.elementFromPoint(${point.x},${point.y})?.closest(${selector}) ? "covered" : "clear"; + })()`, + returnByValue: true, + }); + const value = hit.result?.value; + if (!hit.exceptionDetails && (value === "absent" || value === "clear" || value === "covered")) + return value; + console.debug("[bsk click] overlay hit-test returned no result", hit.exceptionDetails); + } catch (error) { + console.debug("[bsk click] overlay hit-test failed", error); + } + return "unknown"; + }; + try { + if (deps.signal?.aborted) return cancelled(); + let hit = await probe(); + let covered = hit === "covered"; + if (deps.signal?.aborted) return cancelled(); + if ((alwaysBypass || covered) && deps.bypassOverlay) { + try { + await deps.bypassOverlay(tabId, true); + bypass = true; + } catch (error) { + console.debug("[bsk click] overlay bypass enable failed", error); + } + // Enabling bypass can also render newly arrived control UI. + hit = await probe(); + covered ||= hit === "covered"; + } + if (deps.signal?.aborted) return cancelled(); + if (hit === "covered") { + passthroughId = crypto.randomUUID(); + try { + await send(tabId, { type: INPUT_PASSTHROUGH, phase: "begin", id: passthroughId }); + } catch (error) { + console.debug("[bsk click] overlay passthrough enable failed", error); + } + hit = await probe(); + } + if (deps.signal?.aborted) return cancelled(); + if (hit === "covered" || (covered && hit === "unknown")) return notReady(); + return await click(async () => { + const current = await probe(); + // Unknown probes preserve ordinary-page behavior, but cannot clear a known obstruction. + return current === "covered" || (covered && current === "unknown") ? notReady() : null; + }); + } finally { + // Release our lease even if begin's acknowledgement was lost. + if (passthroughId) { + try { + await send(tabId, { type: INPUT_PASSTHROUGH, phase: "end", id: passthroughId }); + } catch (error) { + console.debug("[bsk click] overlay passthrough restore failed", error); + } + } + if (bypass) { + try { + await deps.bypassOverlay!(tabId, false); + } catch (error) { + console.debug("[bsk click] overlay bypass restore failed", error); + } + } + } +} diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index fc0e0b8..ab14e8e 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -15,11 +15,6 @@ import { isAbortError } from "./vom/capture-abort"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; -import { - type CaptureSuppressSendToTab, - withExtensionOverlayHidden, -} from "@/lib/capture-suppress-bridge"; -import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { BlurParams, @@ -40,6 +35,7 @@ import type { SelectParams, SelectResult, } from "@/transport/types"; +import { type ClickOverlayDeps, withClickOverlay } from "./click-overlay"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; @@ -57,18 +53,11 @@ import { } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; -export interface InteractionDeps { +export interface InteractionDeps extends ClickOverlayDeps { /** Arm short popup observation immediately before native input dispatch. */ onInputSent?: (tabId: number) => void; - cdp: CdpRunner; tabsApi: ChromeTabsApi; - /** Abort hook (full chain wired in M10.2). */ - signal?: AbortSignal; defaultTimeoutMs?: number; - /** Hide the complete extension overlay for the bounded click operation. */ - sendToTab?: CaptureSuppressSendToTab; - /** Temporarily disable overlay click blocker during CDP automation. */ - bypassOverlay?: (tabId: number, enabled: boolean) => Promise; /** Keep hover hit-testing active for the caller's next observation/action. */ keepOverlayBypassAfterHover?: boolean; } @@ -524,10 +513,8 @@ export async function clickResolvedTarget( if (clickCount < 1) { return { code: "invalid_params", message: "click_count must be greater than zero" }; } - const error = await withExtensionOverlayHidden( - target.tabId, - () => dispatchClickAtPoint(target.tabId, centre, params, deps, undefined, markSent), - deps.sendToTab, + const error = await withClickOverlay(target.tabId, centre, deps, (verifyOverlay) => + dispatchClickAtPoint(target.tabId, centre, params, deps, verifyOverlay, undefined, markSent), ); if (error) return error; @@ -546,6 +533,7 @@ async function dispatchClickAtPoint( point: { x: number; y: number }, params: Pick, deps: InteractionDeps, + verifyOverlay: () => Promise, beforePress?: () => Promise, markSent?: () => void, ): Promise { @@ -564,7 +552,7 @@ async function dispatchClickAtPoint( modifiers, }); const failure = (error: RpcError): RpcError => - beforePress + beforePress || error.data?.reason === "input_not_ready" ? { ...error, data: { @@ -595,27 +583,8 @@ async function dispatchClickAtPoint( if (error) return failure(error); } if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); - // A closed shadow root retargets its inner controls to the host here. - // Check the real hit after mouseMoved, including when suppression failed. - const hit = await deps.cdp.send<{ - result?: { value?: boolean }; - exceptionDetails?: unknown; - }>(tabId, "Runtime.evaluate", { - expression: `!!document.elementFromPoint(${point.x},${point.y})?.closest(${JSON.stringify(OVERLAY_HOST_SELECTOR)})`, - returnByValue: true, - }); - if (hit.exceptionDetails || hit.result?.value !== false) { - return failure( - rpcError( - "cdp_failed", - "input_not_ready", - hit.result?.value === true - ? "Extension overlay intercepts the click point" - : "Could not verify that the click point is clear of the extension overlay", - { effect_state: attempted ? "unknown" : "none", pointer_moved: moved }, - ), - ); - } + const blocked = await verifyOverlay(); + if (blocked) return failure(blocked); if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); deps.onInputSent?.(tabId); markSent?.(); @@ -690,9 +659,11 @@ async function clickVisualPoint( }; try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - return await withExtensionOverlayHidden( + return await withClickOverlay( target.tabId, - async () => { + point, + deps, + async (verifyOverlay) => { const invalid = await validate(); if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } }; const error = await dispatchClickAtPoint( @@ -700,6 +671,7 @@ async function clickVisualPoint( point, params, deps, + verifyOverlay, async () => { await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability. return validate(); @@ -713,7 +685,7 @@ async function clickVisualPoint( ...point, }); }, - deps.sendToTab, + true, // Visual target validation also needs the page blocker bypassed. ); } catch (error) { return { From d426a40c2be63e16e518363297e73c239d28e1a4 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 23 Sep 2026 11:51:44 +0800 Subject: [PATCH 3/4] fix(extension): expire orphaned click passthrough leases --- .../__tests__/input-passthrough.test.ts | 62 ++++++- .../src/content/input-passthrough.ts | 32 +++- apps/extension/src/content/overlay.css | 3 +- .../__tests__/content-passthrough.test.ts | 151 ++++++++++++++++++ apps/extension/src/entrypoints/content.ts | 7 + .../src/lib/input-passthrough-bridge.ts | 2 + .../src/tools/__tests__/click.browser.test.ts | 38 ++++- .../src/tools/__tests__/interaction.test.ts | 75 ++++++++- apps/extension/src/tools/click-overlay.ts | 30 +++- 9 files changed, 388 insertions(+), 12 deletions(-) create mode 100644 apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts diff --git a/apps/extension/src/content/__tests__/input-passthrough.test.ts b/apps/extension/src/content/__tests__/input-passthrough.test.ts index 1cc7129..ab8e9a5 100644 --- a/apps/extension/src/content/__tests__/input-passthrough.test.ts +++ b/apps/extension/src/content/__tests__/input-passthrough.test.ts @@ -1,7 +1,8 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_ATTR, + INPUT_PASSTHROUGH_TTL_MS, type InputPassthroughMessage, isInputPassthroughMessage, } from "@/lib/input-passthrough-bridge"; @@ -10,8 +11,13 @@ import { createInputPassthroughController } from "../input-passthrough"; describe("click input passthrough", () => { let host: HTMLElement; beforeEach(() => { + vi.useFakeTimers(); host = document.createElement("div"); }); + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); const message = (phase: "begin" | "end", id = "click-1"): InputPassthroughMessage => ({ type: INPUT_PASSTHROUGH, phase, @@ -69,4 +75,58 @@ describe("click input passthrough", () => { ]) expect(isInputPassthroughMessage(invalid)).toBe(false); }); + + it("expires a lost end without extending the lease on duplicate begin", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin"), vi.fn()); + vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1); + controller.handleMessage(message("begin"), vi.fn()); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + vi.advanceTimersByTime(1); + expect(controller.pendingCount).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); + + it("expires only the old click and ignores its late end while another click is active", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin", "old"), vi.fn()); + vi.advanceTimersByTime(1000); + controller.handleMessage(message("begin", "new"), vi.fn()); + vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1000); + expect(controller.pendingCount).toBe(1); + controller.handleMessage(message("end", "old"), vi.fn()); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + vi.advanceTimersByTime(1000); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("cancels timers on end and reset, and does not restore old leases on remount", () => { + let current = host; + const controller = createInputPassthroughController(() => current); + controller.handleMessage(message("begin", "one"), vi.fn()); + controller.handleMessage(message("begin", "two"), vi.fn()); + controller.handleMessage(message("end", "one"), vi.fn()); + expect(vi.getTimerCount()).toBe(1); + controller.reset(); + expect(vi.getTimerCount()).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + current = document.createElement("div"); + controller.onHostMounted(current); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + controller.handleMessage(message("begin", "new-session"), vi.fn()); + controller.handleMessage(message("end", "two"), vi.fn()); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + }); + + it("does not revive an expired lease when a frozen page remounts before timers run", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin"), vi.fn()); + vi.setSystemTime(Date.now() + INPUT_PASSTHROUGH_TTL_MS); + const replacement = document.createElement("div"); + controller.onHostMounted(replacement); + expect(controller.pendingCount).toBe(0); + expect(replacement.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); }); diff --git a/apps/extension/src/content/input-passthrough.ts b/apps/extension/src/content/input-passthrough.ts index c47f0d3..ec835f9 100644 --- a/apps/extension/src/content/input-passthrough.ts +++ b/apps/extension/src/content/input-passthrough.ts @@ -1,14 +1,25 @@ import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_ATTR, + INPUT_PASSTHROUGH_TTL_MS, type InputPassthroughAck, type InputPassthroughMessage, } from "@/lib/input-passthrough-bridge"; export function createInputPassthroughController(getHost: () => HTMLElement | null) { - const pending = new Set(); - const apply = (host: HTMLElement | null) => + const pending = new Map; expiresAt: number }>(); + const release = (id: string) => { + const lease = pending.get(id); + if (lease) clearTimeout(lease.timer); + pending.delete(id); + }; + const apply = (host: HTMLElement | null) => { + // A suspended page can resume/remount before its overdue timers run. + for (const [id, lease] of pending) { + if (lease.expiresAt <= Date.now()) release(id); + } host?.toggleAttribute(INPUT_PASSTHROUGH_ATTR, pending.size > 0); + }; return { get pendingCount() { @@ -18,13 +29,26 @@ export function createInputPassthroughController(getHost: () => HTMLElement | nu message: InputPassthroughMessage, sendResponse: (ack: InputPassthroughAck) => void, ): false { - if (message.phase === "begin") pending.add(message.id); - else pending.delete(message.id); + if (message.phase === "begin") { + if (!pending.has(message.id)) { + pending.set(message.id, { + expiresAt: Date.now() + INPUT_PASSTHROUGH_TTL_MS, + timer: setTimeout(() => { + release(message.id); + apply(getHost()); + }, INPUT_PASSTHROUGH_TTL_MS), + }); + } + } else release(message.id); apply(getHost()); // Hit testing uses current styles; unlike screenshots it needs no compositor frame. sendResponse({ type: INPUT_PASSTHROUGH, ok: true }); return false; }, + reset() { + for (const id of pending.keys()) release(id); + apply(getHost()); + }, onHostMounted: apply, }; } diff --git a/apps/extension/src/content/overlay.css b/apps/extension/src/content/overlay.css index e1a5f95..03e4d0f 100644 --- a/apps/extension/src/content/overlay.css +++ b/apps/extension/src/content/overlay.css @@ -15,7 +15,8 @@ pointer-events: auto !important; } -/* Click passthrough changes only control hit targets, not visibility or other UI. */ +/* Click passthrough changes only control hit targets, not visibility or other UI. + Keep this after the equally specific host blocking rule above. */ :host([data-bsk-overlay-surface][data-bsk-input-passthrough]), :host([data-bsk-input-passthrough]) [data-slot="control-overlay-blocker"], :host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"], diff --git a/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts b/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts new file mode 100644 index 0000000..e5513a2 --- /dev/null +++ b/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts @@ -0,0 +1,151 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_ATTR } from "@/lib/input-passthrough-bridge"; +import { + OVERLAY_AGENT_OVERLAY_RESET, + OVERLAY_AGENT_STATE, + type OverlayMode, +} from "@/lib/overlay-bridge"; +import { RECORD_START, RECORD_STOP } from "@/lib/record-bridge"; + +vi.hoisted(() => { + Object.assign(globalThis, { defineContentScript: (definition: unknown) => definition }); +}); +// Exercise the real content entrypoint/controller wiring without mounting React components. +vi.mock("react-dom/client", () => ({ + default: { createRoot: () => ({ render() {}, unmount() {} }) }, +})); +vi.mock("@/lib/instance-id", () => ({ + getControlHintsHidden: async () => false, + STORAGE_KEYS: { CONTROL_HINTS_HIDDEN: "control_hints_hidden" }, +})); + +import content from "../content"; + +let dispose: (() => void) | undefined; +afterEach(() => { + dispose?.(); + dispose = undefined; + document.querySelectorAll("browser-skill-overlay").forEach((host) => host.remove()); + vi.clearAllTimers(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +async function fixture() { + vi.useFakeTimers(); + let receive: (message: unknown, sender: object, ack: () => void) => void; + vi.stubGlobal("chrome", { + runtime: { + sendMessage: vi.fn(async () => { + throw new Error("Background unavailable"); + }), + onMessage: { + addListener: (listener: typeof receive) => { + receive = listener; + }, + removeListener: vi.fn(), + }, + }, + storage: { onChanged: { addListener: vi.fn(), removeListener: vi.fn() } }, + }); + let host: HTMLElement; + let mount: () => void; + vi.stubGlobal( + "createShadowRootUi", + async ( + _ctx: unknown, + options: { + onMount(container: HTMLElement, shadow: ShadowRoot, host: HTMLElement): unknown; + onRemove(root: unknown): void; + }, + ) => { + let root: unknown; + mount = () => { + if (host) { + options.onRemove(root); + host.remove(); + } + host = document.createElement("browser-skill-overlay"); + const shadow = host.attachShadow({ mode: "closed" }); + const container = document.createElement("div"); + shadow.append(container); + document.documentElement.append(host); + root = options.onMount(container, shadow, host); + }; + return { mount }; + }, + ); + await (content.main as (ctx: { onInvalidated(fn: () => void): void }) => Promise)({ + onInvalidated: (fn) => { + dispose = fn; + }, + }); + const send = (message: unknown) => receive(message, {}, vi.fn()); + const state = (sessionId = "one", mode: OverlayMode = "control") => + send({ type: OVERLAY_AGENT_STATE, sessionId, mode, generation: 0 }); + const begin = (id = "click-one") => send({ type: INPUT_PASSTHROUGH, phase: "begin", id }); + const active = () => host.hasAttribute(INPUT_PASSTHROUGH_ATTR); + state(); + return { send, state, begin, active, remount: () => mount() }; +} + +it.each([ + "paused", + "hidden", + "interrupting", +] as const)("clears leases when control becomes %s", async (mode) => { + const f = await fixture(); + f.begin(); + expect(f.active()).toBe(true); + f.state("one", mode); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); + f.state(); + f.remount(); + expect(f.active()).toBe(false); +}); + +it("preserves same-session control updates and remounts until the click ends", async () => { + const f = await fixture(); + f.begin(); + f.state(); + f.remount(); + expect(f.active()).toBe(true); + f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" }); + expect(f.active()).toBe(false); +}); + +it("clears old session leases without letting stale reset/end clear the new session", async () => { + const f = await fixture(); + f.begin(); + f.state("two"); + expect(f.active()).toBe(false); + f.begin("click-two"); + f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "one" }); + f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" }); + expect(f.active()).toBe(true); + f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "two" }); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); + f.remount(); + expect(f.active()).toBe(false); +}); + +it("clears leases when recording starts or finishes", async () => { + const f = await fixture(); + f.begin(); + f.send({ type: RECORD_START, requestId: "record" }); + expect(f.active()).toBe(false); + f.begin("late-click"); + f.send({ type: RECORD_STOP, requestId: "record" }); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); + +it("removes passthrough and timers when the content context is invalidated", async () => { + const f = await fixture(); + f.begin(); + dispose?.(); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 639acd9..82bd71f 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -174,6 +174,8 @@ export default defineContentScript({ function renderReactOverlays(): void { const overlayState = overlays.snapshot(); const controlOverlayVisible = shouldShowAgentControlOverlay(overlayState); + // Leaving control (including help/record UI) must not retain click leases. + if (!controlOverlayVisible) inputPassthrough.reset(); const interactiveOverlayVisible = overlayState.borrowRequests.length > 0 || overlayState.activeHelp !== null || @@ -226,12 +228,16 @@ export default defineContentScript({ } function applyOverlayState(state: OverlayAgentStateMessage): void { + if (overlays.snapshot().activeSessionId !== state.sessionId) inputPassthrough.reset(); activeAgentState = state; overlays.applyAgentControlMode(state.sessionId, state.mode); renderAll(); } function resetAgentOverlayState(sessionId: string) { + const activeSessionId = overlays.snapshot().activeSessionId; + if (activeSessionId && activeSessionId !== sessionId) return; + inputPassthrough.reset(); const previousHelp = overlays.resetAgentOverlays(sessionId); if (previousHelp) { void sendHelpFinish(previousHelp.id, "cancelled"); @@ -528,6 +534,7 @@ export default defineContentScript({ hostObserver.observe(document.documentElement, { childList: true, subtree: false }); ctx.onInvalidated(() => { + inputPassthrough.reset(); hostObserver.disconnect(); chrome.runtime.onMessage.removeListener(onMessage); chrome.storage.onChanged.removeListener(onStorageChange); diff --git a/apps/extension/src/lib/input-passthrough-bridge.ts b/apps/extension/src/lib/input-passthrough-bridge.ts index 90515b5..944bc3c 100644 --- a/apps/extension/src/lib/input-passthrough-bridge.ts +++ b/apps/extension/src/lib/input-passthrough-bridge.ts @@ -1,6 +1,8 @@ /** Click-only hit-test suppression; independent of screenshot visibility and hover bypass. */ export const INPUT_PASSTHROUGH = "bsk/input-passthrough"; export const INPUT_PASSTHROUGH_ATTR = "data-bsk-input-passthrough"; +/** Bound orphaned click leases even if the background never sends end. */ +export const INPUT_PASSTHROUGH_TTL_MS = 5_000; export interface InputPassthroughMessage { type: typeof INPUT_PASSTHROUGH; diff --git a/apps/extension/src/tools/__tests__/click.browser.test.ts b/apps/extension/src/tools/__tests__/click.browser.test.ts index dbc1903..68eddb7 100644 --- a/apps/extension/src/tools/__tests__/click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click.browser.test.ts @@ -8,7 +8,11 @@ import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ControlOverlay } from "@/content/ControlOverlay"; -import { INPUT_PASSTHROUGH, type InputPassthroughSendToTab } from "@/lib/input-passthrough-bridge"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughSendToTab, +} from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { prepareBackgroundExecution } from "../background-execution"; import { handleClick, handlePress } from "../interaction"; @@ -610,6 +614,38 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "end", id: "css-check" }); expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(false); } + // Losing end must restore the user's Stop button without another background message. + const stopPoint = await evaluate<{ x: number; y: number }>(`(() => { + const stop=overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]'); + stop.addEventListener('click',()=>stops++); + const r=stop.getBoundingClientRect(), x=r.x+r.width/2, y=r.y+r.height/2; + Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'}); + stops=0; clicks=[]; return {x,y}; + })()`); + await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "begin", id: "lost-end" }); + expect(await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y}).id`)).toBe( + "target", + ); + await expect + .poll(() => evaluate("overlay.hasAttribute('data-bsk-input-passthrough')"), { + timeout: INPUT_PASSTHROUGH_TTL_MS + 3_000, + }) + .toBe(false); + expect(await evaluate("passthroughController.pendingCount")).toBe(0); + expect( + await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y})===overlay`), + ).toBe(true); + for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) { + await local("Input.dispatchMouseEvent", { + type, + ...stopPoint, + button: "left", + clickCount: 1, + }); + } + expect(await evaluate("stops")).toBe(1); + expect(await evaluate("clicks")).toEqual([]); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); }, ); }, 30_000); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index e27aa86..f80a7d4 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from "vitest"; -import { INPUT_PASSTHROUGH, type InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { withInputReady } from "../input-readiness"; @@ -412,6 +416,75 @@ describe("handleClick", () => { ]); }); + it.each([ + "begin", + "move", + "press", + "release", + "clear-slow", + ])("handles lease expiry during %s without retrying input", async (phase) => { + let now = Date.now(); + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("expiry"); + ctx.refStore.set("e1", 100, { tabId: 4 }); + const mouse: string[] = []; + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + if (passthrough && phase === "begin") now += INPUT_PASSTHROUGH_TTL_MS; + }); + const bypassOverlay = vi.fn(async () => {}); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[0, 0, 40, 0, 40, 20, 0, 20]] }), + "Input.dispatchMouseEvent": (params) => { + const type = (params as { type: string }).type; + mouse.push(type); + if ( + (phase === "move" && type === "mouseMoved") || + (["press", "clear-slow"].includes(phase) && type === "mousePressed") || + (phase === "release" && type === "mouseReleased") + ) + now += INPUT_PASSTHROUGH_TTL_MS; + return {}; + }, + }); + // Another concurrent lease may keep the point clear after ours has expired. + fake.overlayHit.mockImplementation(async () => ({ + result: { value: passthrough || phase === "clear-slow" ? "clear" : "covered" }, + })); + const result = await handleClick( + manager, + { session_id: "expiry", ref: "e1" }, + { ...fake, bypassOverlay, sendInputPassthrough }, + ); + if (phase === "clear-slow") { + expect(result).not.toHaveProperty("code"); + expect(sendInputPassthrough).not.toHaveBeenCalled(); + } else { + expect(result).toMatchObject({ + data: { + reason: ["begin", "move"].includes(phase) ? "input_not_ready" : "input_outcome_unknown", + effect_state: ["begin", "move"].includes(phase) ? "none" : "unknown", + }, + }); + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(["begin", "end"]); + expect(bypassOverlay.mock.calls).toHaveLength(2); + } + expect(mouse).toEqual( + phase === "begin" + ? [] + : phase === "move" + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], + ); + } finally { + clock.mockRestore(); + } + }); + it("leaves disabled click behavior to the browser without an AX preflight", async () => { const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await manager.start("aa11"); diff --git a/apps/extension/src/tools/click-overlay.ts b/apps/extension/src/tools/click-overlay.ts index 78d6909..01afbe6 100644 --- a/apps/extension/src/tools/click-overlay.ts +++ b/apps/extension/src/tools/click-overlay.ts @@ -1,12 +1,13 @@ import { INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, type InputPassthroughSendToTab, sendInputPassthrough, } from "@/lib/input-passthrough-bridge"; import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge"; import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; -import type { CdpRunner } from "./shared"; +import { type CdpRunner, isRpcError } from "./shared"; export interface ClickOverlayDeps { cdp: CdpRunner; @@ -29,6 +30,8 @@ export async function withClickOverlay( const send = deps.sendInputPassthrough ?? sendInputPassthrough; let bypass = false; let passthroughId: string | undefined; + let passthroughExpiresAt = Infinity; + const expired = () => Date.now() >= passthroughExpiresAt; const notReady = () => rpcError( "cdp_failed", @@ -86,6 +89,8 @@ export async function withClickOverlay( if (deps.signal?.aborted) return cancelled(); if (hit === "covered") { passthroughId = crypto.randomUUID(); + // Start before sending, so our deadline cannot outlive the content-side lease. + passthroughExpiresAt = Date.now() + INPUT_PASSTHROUGH_TTL_MS; try { await send(tabId, { type: INPUT_PASSTHROUGH, phase: "begin", id: passthroughId }); } catch (error) { @@ -94,12 +99,29 @@ export async function withClickOverlay( hit = await probe(); } if (deps.signal?.aborted) return cancelled(); - if (hit === "covered" || (covered && hit === "unknown")) return notReady(); - return await click(async () => { + if (expired() || hit === "covered" || (covered && hit === "unknown")) return notReady(); + const result = await click(async () => { + if (expired()) return notReady(); const current = await probe(); // Unknown probes preserve ordinary-page behavior, but cannot clear a known obstruction. - return current === "covered" || (covered && current === "unknown") ? notReady() : null; + return expired() || current === "covered" || (covered && current === "unknown") + ? notReady() + : null; }); + // The lease can expire while a slow press/release is in flight. Always release + // the mouse, but do not claim success (or retry) when delivery is uncertain. + if (expired() && !isRpcError(result)) { + return rpcError( + "cdp_failed", + "input_outcome_unknown", + "Overlay passthrough expired during click", + { + effect_state: "unknown", + pointer_moved: true, + }, + ); + } + return result; } finally { // Release our lease even if begin's acknowledgement was lost. if (passthroughId) { From 531a1fa91db0bfeedf64a2c155a4625055ed8889 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 23 Sep 2026 12:05:31 +0800 Subject: [PATCH 4/4] fix(extension): reserve time for click delivery before lease expiry --- .../src/tools/__tests__/interaction.test.ts | 44 ++++++++++++------- apps/extension/src/tools/click-overlay.ts | 9 ++-- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index f80a7d4..74e6ebb 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -417,12 +417,18 @@ describe("handleClick", () => { }); it.each([ - "begin", - "move", - "press", - "release", - "clear-slow", - ])("handles lease expiry during %s without retrying input", async (phase) => { + ["begin", INPUT_PASSTHROUGH_TTL_MS], + ["move", INPUT_PASSTHROUGH_TTL_MS], + ["press", INPUT_PASSTHROUGH_TTL_MS], + ["release", INPUT_PASSTHROUGH_TTL_MS], + ["clear-slow", INPUT_PASSTHROUGH_TTL_MS], + ["begin", 4_000], + ["move", 4_000], + ["probe", 4_000], + ["move", 3_999], + ["press", 4_500], + ["release", 4_500], + ] as const)("handles %s delayed by %i ms without retrying input", async (phase, elapsed) => { let now = Date.now(); const clock = vi.spyOn(Date, "now").mockImplementation(() => now); try { @@ -433,7 +439,7 @@ describe("handleClick", () => { let passthrough = false; const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { passthrough = message.phase === "begin"; - if (passthrough && phase === "begin") now += INPUT_PASSTHROUGH_TTL_MS; + if (passthrough && phase === "begin") now += elapsed; }); const bypassOverlay = vi.fn(async () => {}); const fake = makeFakeCdp({ @@ -447,36 +453,40 @@ describe("handleClick", () => { (["press", "clear-slow"].includes(phase) && type === "mousePressed") || (phase === "release" && type === "mouseReleased") ) - now += INPUT_PASSTHROUGH_TTL_MS; + now += elapsed; return {}; }, }); // Another concurrent lease may keep the point clear after ours has expired. - fake.overlayHit.mockImplementation(async () => ({ - result: { value: passthrough || phase === "clear-slow" ? "clear" : "covered" }, - })); + fake.overlayHit.mockImplementation(async () => { + if (phase === "probe" && mouse.length > 0) now += elapsed; + return { result: { value: passthrough || phase === "clear-slow" ? "clear" : "covered" } }; + }); const result = await handleClick( manager, { session_id: "expiry", ref: "e1" }, { ...fake, bypassOverlay, sendInputPassthrough }, ); - if (phase === "clear-slow") { + const blockedBeforePress = ["begin", "move", "probe"].includes(phase) && elapsed >= 4_000; + if (phase === "clear-slow" || (!blockedBeforePress && elapsed < INPUT_PASSTHROUGH_TTL_MS)) { expect(result).not.toHaveProperty("code"); - expect(sendInputPassthrough).not.toHaveBeenCalled(); } else { expect(result).toMatchObject({ data: { - reason: ["begin", "move"].includes(phase) ? "input_not_ready" : "input_outcome_unknown", - effect_state: ["begin", "move"].includes(phase) ? "none" : "unknown", + reason: blockedBeforePress ? "input_not_ready" : "input_outcome_unknown", + effect_state: blockedBeforePress ? "none" : "unknown", }, }); + } + if (phase === "clear-slow") expect(sendInputPassthrough).not.toHaveBeenCalled(); + else { expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(["begin", "end"]); expect(bypassOverlay.mock.calls).toHaveLength(2); } expect(mouse).toEqual( - phase === "begin" + blockedBeforePress && phase === "begin" ? [] - : phase === "move" + : blockedBeforePress ? ["mouseMoved"] : ["mouseMoved", "mousePressed", "mouseReleased"], ); diff --git a/apps/extension/src/tools/click-overlay.ts b/apps/extension/src/tools/click-overlay.ts index 01afbe6..7fa7d0b 100644 --- a/apps/extension/src/tools/click-overlay.ts +++ b/apps/extension/src/tools/click-overlay.ts @@ -32,6 +32,8 @@ export async function withClickOverlay( let passthroughId: string | undefined; let passthroughExpiresAt = Infinity; const expired = () => Date.now() >= passthroughExpiresAt; + // Reserve one second for input delivery; completed clicks still use the full lease. + const pressDeadlineReached = () => Date.now() >= passthroughExpiresAt - 1_000; const notReady = () => rpcError( "cdp_failed", @@ -99,12 +101,13 @@ export async function withClickOverlay( hit = await probe(); } if (deps.signal?.aborted) return cancelled(); - if (expired() || hit === "covered" || (covered && hit === "unknown")) return notReady(); + if (pressDeadlineReached() || hit === "covered" || (covered && hit === "unknown")) + return notReady(); const result = await click(async () => { - if (expired()) return notReady(); + if (pressDeadlineReached()) return notReady(); const current = await probe(); // Unknown probes preserve ordinary-page behavior, but cannot clear a known obstruction. - return expired() || current === "covered" || (covered && current === "unknown") + return pressDeadlineReached() || current === "covered" || (covered && current === "unknown") ? notReady() : null; });