diff --git a/apps/extension/src/content/__tests__/input-passthrough.test.ts b/apps/extension/src/content/__tests__/input-passthrough.test.ts new file mode 100644 index 0000000..ab8e9a5 --- /dev/null +++ b/apps/extension/src/content/__tests__/input-passthrough.test.ts @@ -0,0 +1,132 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_ATTR, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughMessage, + isInputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; +import { createInputPassthroughController } from "../input-passthrough"; + +describe("click input passthrough", () => { + let host: HTMLElement; + beforeEach(() => { + vi.useFakeTimers(); + host = document.createElement("div"); + }); + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); + const message = (phase: "begin" | "end", id = "click-1"): InputPassthroughMessage => ({ + type: INPUT_PASSTHROUGH, + phase, + id, + }); + + it("applies and acknowledges synchronously without hiding the host or scheduling frames", () => { + const controller = createInputPassthroughController(() => host); + const raf = vi.spyOn(window, "requestAnimationFrame"); + const ack = vi.fn(() => expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true)); + expect(controller.handleMessage(message("begin"), ack)).toBe(false); + expect(ack).toHaveBeenCalledWith({ type: INPUT_PASSTHROUGH, ok: true }); + expect(raf).not.toHaveBeenCalled(); + expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(false); + raf.mockRestore(); + }); + + it("counts independent clicks and ignores duplicate or unmatched messages", () => { + const controller = createInputPassthroughController(() => host); + for (const id of ["one", "one", "two"]) controller.handleMessage(message("begin", id), vi.fn()); + expect(controller.pendingCount).toBe(2); + for (const id of ["missing", "one", "one"]) + controller.handleMessage(message("end", id), vi.fn()); + expect(controller.pendingCount).toBe(1); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + controller.handleMessage(message("end", "two"), vi.fn()); + expect(controller.pendingCount).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("reapplies pending clicks after host replacement and leaves screenshot state alone", () => { + let current: HTMLElement | null = null; + const controller = createInputPassthroughController(() => current); + controller.handleMessage(message("begin"), vi.fn()); + current = host; + host.setAttribute("data-bsk-capture-hidden", ""); + controller.onHostMounted(host); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + controller.handleMessage(message("end"), vi.fn()); + expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(true); + current = document.createElement("div"); + controller.onHostMounted(current); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("accepts only paired click messages with an operation id", () => { + expect(isInputPassthroughMessage(message("begin"))).toBe(true); + expect(isInputPassthroughMessage(message("end"))).toBe(true); + for (const invalid of [ + null, + {}, + { ...message("begin"), id: "" }, + { ...message("begin"), phase: "reset" }, + { ...message("begin"), type: "bsk/capture-suppress" }, + ]) + expect(isInputPassthroughMessage(invalid)).toBe(false); + }); + + it("expires a lost end without extending the lease on duplicate begin", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin"), vi.fn()); + vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1); + controller.handleMessage(message("begin"), vi.fn()); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + vi.advanceTimersByTime(1); + expect(controller.pendingCount).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); + + it("expires only the old click and ignores its late end while another click is active", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin", "old"), vi.fn()); + vi.advanceTimersByTime(1000); + controller.handleMessage(message("begin", "new"), vi.fn()); + vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1000); + expect(controller.pendingCount).toBe(1); + controller.handleMessage(message("end", "old"), vi.fn()); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + vi.advanceTimersByTime(1000); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + }); + + it("cancels timers on end and reset, and does not restore old leases on remount", () => { + let current = host; + const controller = createInputPassthroughController(() => current); + controller.handleMessage(message("begin", "one"), vi.fn()); + controller.handleMessage(message("begin", "two"), vi.fn()); + controller.handleMessage(message("end", "one"), vi.fn()); + expect(vi.getTimerCount()).toBe(1); + controller.reset(); + expect(vi.getTimerCount()).toBe(0); + expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + current = document.createElement("div"); + controller.onHostMounted(current); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + controller.handleMessage(message("begin", "new-session"), vi.fn()); + controller.handleMessage(message("end", "two"), vi.fn()); + expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true); + }); + + it("does not revive an expired lease when a frozen page remounts before timers run", () => { + const controller = createInputPassthroughController(() => host); + controller.handleMessage(message("begin"), vi.fn()); + vi.setSystemTime(Date.now() + INPUT_PASSTHROUGH_TTL_MS); + const replacement = document.createElement("div"); + controller.onHostMounted(replacement); + expect(controller.pendingCount).toBe(0); + expect(replacement.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/apps/extension/src/content/input-passthrough.ts b/apps/extension/src/content/input-passthrough.ts new file mode 100644 index 0000000..ec835f9 --- /dev/null +++ b/apps/extension/src/content/input-passthrough.ts @@ -0,0 +1,54 @@ +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_ATTR, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughAck, + type InputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; + +export function createInputPassthroughController(getHost: () => HTMLElement | null) { + const pending = new Map; expiresAt: number }>(); + const release = (id: string) => { + const lease = pending.get(id); + if (lease) clearTimeout(lease.timer); + pending.delete(id); + }; + const apply = (host: HTMLElement | null) => { + // A suspended page can resume/remount before its overdue timers run. + for (const [id, lease] of pending) { + if (lease.expiresAt <= Date.now()) release(id); + } + host?.toggleAttribute(INPUT_PASSTHROUGH_ATTR, pending.size > 0); + }; + + return { + get pendingCount() { + return pending.size; + }, + handleMessage( + message: InputPassthroughMessage, + sendResponse: (ack: InputPassthroughAck) => void, + ): false { + if (message.phase === "begin") { + if (!pending.has(message.id)) { + pending.set(message.id, { + expiresAt: Date.now() + INPUT_PASSTHROUGH_TTL_MS, + timer: setTimeout(() => { + release(message.id); + apply(getHost()); + }, INPUT_PASSTHROUGH_TTL_MS), + }); + } + } else release(message.id); + apply(getHost()); + // Hit testing uses current styles; unlike screenshots it needs no compositor frame. + sendResponse({ type: INPUT_PASSTHROUGH, ok: true }); + return false; + }, + reset() { + for (const id of pending.keys()) release(id); + apply(getHost()); + }, + onHostMounted: apply, + }; +} diff --git a/apps/extension/src/content/overlay.css b/apps/extension/src/content/overlay.css index 5e8af1a..03e4d0f 100644 --- a/apps/extension/src/content/overlay.css +++ b/apps/extension/src/content/overlay.css @@ -15,6 +15,15 @@ pointer-events: auto !important; } +/* Click passthrough changes only control hit targets, not visibility or other UI. + Keep this after the equally specific host blocking rule above. */ +:host([data-bsk-overlay-surface][data-bsk-input-passthrough]), +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-blocker"], +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"], +:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"] * { + pointer-events: none !important; +} + /* Screenshot suppression: the host lives in the main DOM, so hiding it removes the whole shadow subtree from compositing and captured frames only contain page content. */ diff --git a/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts b/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts new file mode 100644 index 0000000..e5513a2 --- /dev/null +++ b/apps/extension/src/entrypoints/__tests__/content-passthrough.test.ts @@ -0,0 +1,151 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_ATTR } from "@/lib/input-passthrough-bridge"; +import { + OVERLAY_AGENT_OVERLAY_RESET, + OVERLAY_AGENT_STATE, + type OverlayMode, +} from "@/lib/overlay-bridge"; +import { RECORD_START, RECORD_STOP } from "@/lib/record-bridge"; + +vi.hoisted(() => { + Object.assign(globalThis, { defineContentScript: (definition: unknown) => definition }); +}); +// Exercise the real content entrypoint/controller wiring without mounting React components. +vi.mock("react-dom/client", () => ({ + default: { createRoot: () => ({ render() {}, unmount() {} }) }, +})); +vi.mock("@/lib/instance-id", () => ({ + getControlHintsHidden: async () => false, + STORAGE_KEYS: { CONTROL_HINTS_HIDDEN: "control_hints_hidden" }, +})); + +import content from "../content"; + +let dispose: (() => void) | undefined; +afterEach(() => { + dispose?.(); + dispose = undefined; + document.querySelectorAll("browser-skill-overlay").forEach((host) => host.remove()); + vi.clearAllTimers(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +async function fixture() { + vi.useFakeTimers(); + let receive: (message: unknown, sender: object, ack: () => void) => void; + vi.stubGlobal("chrome", { + runtime: { + sendMessage: vi.fn(async () => { + throw new Error("Background unavailable"); + }), + onMessage: { + addListener: (listener: typeof receive) => { + receive = listener; + }, + removeListener: vi.fn(), + }, + }, + storage: { onChanged: { addListener: vi.fn(), removeListener: vi.fn() } }, + }); + let host: HTMLElement; + let mount: () => void; + vi.stubGlobal( + "createShadowRootUi", + async ( + _ctx: unknown, + options: { + onMount(container: HTMLElement, shadow: ShadowRoot, host: HTMLElement): unknown; + onRemove(root: unknown): void; + }, + ) => { + let root: unknown; + mount = () => { + if (host) { + options.onRemove(root); + host.remove(); + } + host = document.createElement("browser-skill-overlay"); + const shadow = host.attachShadow({ mode: "closed" }); + const container = document.createElement("div"); + shadow.append(container); + document.documentElement.append(host); + root = options.onMount(container, shadow, host); + }; + return { mount }; + }, + ); + await (content.main as (ctx: { onInvalidated(fn: () => void): void }) => Promise)({ + onInvalidated: (fn) => { + dispose = fn; + }, + }); + const send = (message: unknown) => receive(message, {}, vi.fn()); + const state = (sessionId = "one", mode: OverlayMode = "control") => + send({ type: OVERLAY_AGENT_STATE, sessionId, mode, generation: 0 }); + const begin = (id = "click-one") => send({ type: INPUT_PASSTHROUGH, phase: "begin", id }); + const active = () => host.hasAttribute(INPUT_PASSTHROUGH_ATTR); + state(); + return { send, state, begin, active, remount: () => mount() }; +} + +it.each([ + "paused", + "hidden", + "interrupting", +] as const)("clears leases when control becomes %s", async (mode) => { + const f = await fixture(); + f.begin(); + expect(f.active()).toBe(true); + f.state("one", mode); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); + f.state(); + f.remount(); + expect(f.active()).toBe(false); +}); + +it("preserves same-session control updates and remounts until the click ends", async () => { + const f = await fixture(); + f.begin(); + f.state(); + f.remount(); + expect(f.active()).toBe(true); + f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" }); + expect(f.active()).toBe(false); +}); + +it("clears old session leases without letting stale reset/end clear the new session", async () => { + const f = await fixture(); + f.begin(); + f.state("two"); + expect(f.active()).toBe(false); + f.begin("click-two"); + f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "one" }); + f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" }); + expect(f.active()).toBe(true); + f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "two" }); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); + f.remount(); + expect(f.active()).toBe(false); +}); + +it("clears leases when recording starts or finishes", async () => { + const f = await fixture(); + f.begin(); + f.send({ type: RECORD_START, requestId: "record" }); + expect(f.active()).toBe(false); + f.begin("late-click"); + f.send({ type: RECORD_STOP, requestId: "record" }); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); + +it("removes passthrough and timers when the content context is invalidated", async () => { + const f = await fixture(); + f.begin(); + dispose?.(); + expect(f.active()).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index b481e05..82bd71f 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -8,6 +8,7 @@ import { ControlOverlay } from "@/content/ControlOverlay"; import { createCaptureSuppressController } from "@/content/capture-suppress"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import { createHelpRequestData } from "@/content/help-request"; +import { createInputPassthroughController } from "@/content/input-passthrough"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; @@ -28,6 +29,11 @@ import { isHelpCancelMessage, isHelpRequestMessage, } from "@/lib/help-bridge"; +import { + type InputPassthroughAck, + type InputPassthroughMessage, + isInputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id"; import { isOverlayAgentOverlayResetMessage, @@ -87,6 +93,7 @@ export default defineContentScript({ } const captureSuppress = createCaptureSuppressController(() => overlayHost); + const inputPassthrough = createInputPassthroughController(() => overlayHost); const ui = await createShadowRootUi(ctx, { name: "browser-skill-overlay", @@ -101,6 +108,7 @@ export default defineContentScript({ hostLossReported = false; // A host rebuilt mid-capture must stay hidden until `end` arrives. captureSuppress.onHostMounted(shadowHost); + inputPassthrough.onHostMounted(shadowHost); const app = document.createElement("div"); app.className = "bsk-overlay-root"; container.append(app); @@ -166,6 +174,8 @@ export default defineContentScript({ function renderReactOverlays(): void { const overlayState = overlays.snapshot(); const controlOverlayVisible = shouldShowAgentControlOverlay(overlayState); + // Leaving control (including help/record UI) must not retain click leases. + if (!controlOverlayVisible) inputPassthrough.reset(); const interactiveOverlayVisible = overlayState.borrowRequests.length > 0 || overlayState.activeHelp !== null || @@ -218,12 +228,16 @@ export default defineContentScript({ } function applyOverlayState(state: OverlayAgentStateMessage): void { + if (overlays.snapshot().activeSessionId !== state.sessionId) inputPassthrough.reset(); activeAgentState = state; overlays.applyAgentControlMode(state.sessionId, state.mode); renderAll(); } function resetAgentOverlayState(sessionId: string) { + const activeSessionId = overlays.snapshot().activeSessionId; + if (activeSessionId && activeSessionId !== sessionId) return; + inputPassthrough.reset(); const previousHelp = overlays.resetAgentOverlays(sessionId); if (previousHelp) { void sendHelpFinish(previousHelp.id, "cancelled"); @@ -257,6 +271,7 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | CaptureSuppressMessage + | InputPassthroughMessage | RecordStartMessage | RecordStopMessage | RecordCancelMessage @@ -264,11 +279,16 @@ export default defineContentScript({ | OverlayAgentStateMessage | OverlayAutomationBypassMessage, _sender: chrome.runtime.MessageSender, - sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void, + sendResponse: ( + response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck | InputPassthroughAck, + ) => void, ) => { if (isCaptureSuppressMessage(message)) { return captureSuppress.handleMessage(message, sendResponse); } + if (isInputPassthroughMessage(message)) { + return inputPassthrough.handleMessage(message, sendResponse); + } if (isRecordStartMessage(message)) { activeRecordRequestId = message.requestId; @@ -514,6 +534,7 @@ export default defineContentScript({ hostObserver.observe(document.documentElement, { childList: true, subtree: false }); ctx.onInvalidated(() => { + inputPassthrough.reset(); hostObserver.disconnect(); chrome.runtime.onMessage.removeListener(onMessage); chrome.storage.onChanged.removeListener(onStorageChange); diff --git a/apps/extension/src/lib/input-passthrough-bridge.ts b/apps/extension/src/lib/input-passthrough-bridge.ts new file mode 100644 index 0000000..944bc3c --- /dev/null +++ b/apps/extension/src/lib/input-passthrough-bridge.ts @@ -0,0 +1,36 @@ +/** Click-only hit-test suppression; independent of screenshot visibility and hover bypass. */ +export const INPUT_PASSTHROUGH = "bsk/input-passthrough"; +export const INPUT_PASSTHROUGH_ATTR = "data-bsk-input-passthrough"; +/** Bound orphaned click leases even if the background never sends end. */ +export const INPUT_PASSTHROUGH_TTL_MS = 5_000; + +export interface InputPassthroughMessage { + type: typeof INPUT_PASSTHROUGH; + phase: "begin" | "end"; + /** Identifies one click so cleanup after a lost ack cannot release another click. */ + id: string; +} + +export interface InputPassthroughAck { + type: typeof INPUT_PASSTHROUGH; + ok: true; +} + +export function isInputPassthroughMessage(message: unknown): message is InputPassthroughMessage { + if (!message || typeof message !== "object") return false; + const m = message as Record; + return ( + m.type === INPUT_PASSTHROUGH && + (m.phase === "begin" || m.phase === "end") && + typeof m.id === "string" && + m.id.length > 0 + ); +} + +export type InputPassthroughSendToTab = ( + tabId: number, + message: InputPassthroughMessage, +) => Promise; + +export const sendInputPassthrough: InputPassthroughSendToTab = (tabId, message) => + chrome.tabs.sendMessage(tabId, message); diff --git a/apps/extension/src/tools/__tests__/click.browser.test.ts b/apps/extension/src/tools/__tests__/click.browser.test.ts index b0e7bea..68eddb7 100644 --- a/apps/extension/src/tools/__tests__/click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click.browser.test.ts @@ -1,8 +1,18 @@ // @vitest-environment node // Opt in with BSK_CLICK_CHROME; each test owns its browser, profile and HTTP server. +import { readFileSync } from "node:fs"; import { createServer } from "node:http"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { ControlOverlay } from "@/content/ControlOverlay"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughSendToTab, +} from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { prepareBackgroundExecution } from "../background-execution"; import { handleClick, handlePress } from "../interaction"; @@ -354,4 +364,289 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", ( await new Promise((resolve) => server.close(() => resolve())); } }, 90_000); + + it("preserves hover and visible controls while scoped passthrough clears closed shadow click targets", async () => { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + const css = readFileSync(new URL("../../content/overlay.css", import.meta.url), "utf8"); + // Run the actual content-side controller in the browser, including its lease bookkeeping. + const moduleUrl = (source: string) => + `data:text/javascript;base64,${Buffer.from( + ts.transpileModule(source, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ES2022 }, + }).outputText, + ).toString("base64")}`; + const bridgeUrl = moduleUrl( + readFileSync(new URL("../../lib/input-passthrough-bridge.ts", import.meta.url), "utf8"), + ); + const controllerUrl = moduleUrl( + readFileSync(new URL("../../content/input-passthrough.ts", import.meta.url), "utf8").replace( + '"@/lib/input-passthrough-bridge"', + JSON.stringify(bridgeUrl), + ), + ); + await withChrome( + { executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + const local: Send = (method, params) => send(method, params, sessionId); + const evaluate = async (expression: string) => { + const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { + expression, + returnByValue: true, + awaitPromise: true, + }, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await local("Page.bringToFront"); + await local("Emulation.setDeviceMetricsOverride", { + width: 1280, + height: 757, + deviceScaleFactor: 1, + mobile: false, + }); + await evaluate(`(async () => { document.body.innerHTML = ''; + window.clicks=[]; window.stops=0; window.pageEvents=[]; + document.querySelector('#target').onclick=e=>clicks.push(e.isTrusted); + for (const type of ['mouseover','mousedown','mouseup','click']) document.querySelector('#target').addEventListener(type,e=>pageEvents.push({type,trusted:e.isTrusted,hover:getComputedStyle(e.currentTarget).backgroundColor==='rgb(1, 2, 3)'})); + window.overlay=document.createElement('browser-skill-overlay'); + overlay.setAttribute('data-bsk-overlay',''); overlay.setAttribute('data-bsk-overlay-surface',''); + document.documentElement.append(overlay); + window.overlayRoot=overlay.attachShadow({mode:'closed'}); + window.passthroughController=(await import(${JSON.stringify(controllerUrl)})).createInputPassthroughController(()=>overlay); })()`); + const manager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("overlay-click"); + const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; + const tabsApi = { get: async () => tab, query: async () => [tab] }; + const mouse: string[] = []; + const cdp: CdpRunner = { + send: async (_tabId, method, params) => { + if (method === "Input.dispatchMouseEvent") { + mouse.push((params as { type: string }).type); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + expect( + await evaluate( + "getComputedStyle(overlayRoot.querySelector('[data-slot=control-overlay-pill]')).opacity", + ), + ).toBe("1"); + expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false); + } + return (await local(method, params)) as never; + }, + }; + const phases: string[] = []; + const sendInputPassthrough: InputPassthroughSendToTab = async (_tabId, message) => { + phases.push(message.phase); + return evaluate( + `new Promise(resolve=>passthroughController.handleMessage(${JSON.stringify(message)},resolve))`, + ); + }; + const bypassOverlay = async (_tabId: number, enabled: boolean) => { + await evaluate(`window.bypassCount += ${enabled ? 1 : -1}; + overlay.toggleAttribute('data-bsk-overlay-blocking',bypassCount===0); + overlayRoot.querySelector('[data-slot="control-overlay-blocker"]').style.pointerEvents=bypassCount>0?'none':'auto';`); + }; + // A retained hover bypass leaves the pill interactive even though the host is transparent. + for (const automationBypass of [false, true]) { + const markup = renderToStaticMarkup( + createElement(ControlOverlay, { + visible: true, + interrupting: false, + automationBypass, + onInterrupt: () => {}, + }), + ); + await evaluate(`window.bypassCount=${automationBypass ? 1 : 0}; overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass}); + overlayRoot.innerHTML = ${JSON.stringify(``)} + ${JSON.stringify(markup)}; + // Static markup starts before the component's reveal effect; model its settled state. + for (const control of overlayRoot.querySelectorAll('[data-slot^="control-overlay"]')) control.style.opacity='1'; + overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]').addEventListener('click',()=>stops++);`); + await evaluate( + "Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})", + ); + phases.length = 0; + const clear = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, + ); + expect(clear).not.toHaveProperty("code"); + expect(phases).toEqual([]); + expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0); + for (const slot of ["control-overlay-pill", "control-overlay-stop-all"]) { + const point = await evaluate<{ x: number; y: number }>(`(() => { + const r=overlayRoot.querySelector('[data-slot="${slot}"]').getBoundingClientRect(); + const x=${slot === "control-overlay-pill" ? "r.x+25" : "r.x+r.width/2"}, y=r.y+r.height/2; + Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'}); + clicks=[]; stops=0; return {x,y}; + })()`); + expect( + await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`), + ).toBe(true); + const nativeClick = async () => { + for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) { + await local("Input.dispatchMouseEvent", { + type, + ...point, + button: "left", + clickCount: 1, + }); + } + }; + await nativeClick(); // Reproduce the interception before using the fixed click path. + expect(await evaluate("clicks")).toEqual([]); + await local("Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }); + await evaluate("stops=0; pageEvents=[]"); + phases.length = 0; + const { root } = await local<{ root: { nodeId: number } }>("DOM.getDocument"); + const { nodeId } = await local<{ nodeId: number }>("DOM.querySelector", { + nodeId: root.nodeId, + selector: "#target", + }); + const { node } = await local<{ node: { backendNodeId: number } }>("DOM.describeNode", { + nodeId, + }); + ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 }); + mouse.length = 0; + const result = await handleClick( + manager, + { + session_id: ctx.sessionId, + ...(automationBypass ? { ref: "e1" } : { selector: "#target" }), + }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, + ); + expect(result, JSON.stringify(result)).not.toHaveProperty("code"); + expect(await evaluate("clicks")).toEqual([true]); + expect(phases).toEqual(["begin", "end"]); + expect(await evaluate("pageEvents")).toEqual( + ["mouseover", "mousedown", "mouseup", "click"].map((type) => ({ + type, + trusted: true, + hover: true, + })), + ); + expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe( + false, + ); + expect(await evaluate("stops")).toBe(0); + expect(mouse).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]); + expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false); + expect( + await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`), + ).toBe(true); + if (slot === "control-overlay-stop-all") { + await nativeClick(); + expect(await evaluate("stops")).toBe(1); + expect(await evaluate("clicks")).toEqual([true]); + } + mouse.length = 0; + const blocked = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { + cdp, + tabsApi, + bypassOverlay, + sendInputPassthrough: async () => { + throw new Error("Content script unavailable"); + }, + }, + ); + expect(blocked).toMatchObject({ + code: "cdp_failed", + data: { reason: "input_not_ready", effect_state: "none" }, + }); + expect(mouse).toEqual([]); + expect(await evaluate("clicks")).toEqual([true]); + } + // Other extension controls stay hit-testable even while a click lease is active. + await evaluate( + `(() => { const extra=document.createElement('div'); extra.innerHTML=''; overlayRoot.append(extra); })()`, + ); + await evaluate( + "Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})", + ); + mouse.length = 0; + const protectedControl = await handleClick( + manager, + { session_id: ctx.sessionId, selector: "#target" }, + { cdp, tabsApi, sendInputPassthrough, bypassOverlay }, + ); + expect(protectedControl).toHaveProperty("data.reason", "input_not_ready"); + expect(mouse).toEqual([]); + await sendInputPassthrough(4, { + type: INPUT_PASSTHROUGH, + phase: "begin", + id: "css-check", + }); + expect(await evaluate("overlayRoot.elementFromPoint(40,30).id")).toBe("other"); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + await evaluate(`(() => { const replacement=document.createElement('browser-skill-overlay'); + for(const attr of overlay.attributes) replacement.setAttribute(attr.name,attr.value); + const replacementRoot=replacement.attachShadow({mode:'closed'}); + replacementRoot.innerHTML=overlayRoot.innerHTML; + overlay.replaceWith(replacement); overlay=replacement; overlayRoot=replacementRoot; + passthroughController.onHostMounted(overlay); })()`); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(true); + await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "end", id: "css-check" }); + expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(false); + } + // Losing end must restore the user's Stop button without another background message. + const stopPoint = await evaluate<{ x: number; y: number }>(`(() => { + const stop=overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]'); + stop.addEventListener('click',()=>stops++); + const r=stop.getBoundingClientRect(), x=r.x+r.width/2, y=r.y+r.height/2; + Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'}); + stops=0; clicks=[]; return {x,y}; + })()`); + await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "begin", id: "lost-end" }); + expect(await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y}).id`)).toBe( + "target", + ); + await expect + .poll(() => evaluate("overlay.hasAttribute('data-bsk-input-passthrough')"), { + timeout: INPUT_PASSTHROUGH_TTL_MS + 3_000, + }) + .toBe(false); + expect(await evaluate("passthroughController.pendingCount")).toBe(0); + expect( + await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y})===overlay`), + ).toBe(true); + for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) { + await local("Input.dispatchMouseEvent", { + type, + ...stopPoint, + button: "left", + clickCount: 1, + }); + } + expect(await evaluate("stops")).toBe(1); + expect(await evaluate("clicks")).toEqual([]); + expect(await evaluate("getComputedStyle(overlay).display")).toBe("block"); + }, + ); + }, 30_000); }); diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 4f37386..6e88dba 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -341,7 +341,7 @@ describe("ToolDispatcher", () => { }); }); - it("bypasses and restores the control overlay for an upload trigger click", async () => { + it("leaves overlay state alone for an unobstructed upload trigger click", async () => { const sendMessage = vi.fn(async () => undefined); vi.stubGlobal("chrome", { tabs: { @@ -383,15 +383,8 @@ describe("ToolDispatcher", () => { } if (method === "Runtime.evaluate") { const expression = (params as { expression?: string }).expression ?? ""; - if (expression.includes("hitIndex")) { - return { result: { value: { hitIndex: 0 } } } as T; - } - if (expression.includes("overlayHostPresent")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true }, - }, - } as T; + if (expression.includes('return "absent"')) { + return { result: { value: "clear" } } as T; } if (expression.includes("count:")) { return { result: { value: { count: 1, multiple: false } } } as T; @@ -432,14 +425,7 @@ describe("ToolDispatcher", () => { await vi.waitFor(() => expect(sent).toHaveLength(1)); expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } }); - expect(sendMessage).toHaveBeenNthCalledWith(1, 7, { - type: "bh-automation-bypass", - enabled: true, - }); - expect(sendMessage).toHaveBeenNthCalledWith(2, 7, { - type: "bh-automation-bypass", - enabled: false, - }); + expect(sendMessage).not.toHaveBeenCalled(); }); it("detaches CDP state before stopping a session", async () => { diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 9b2f2a1..d2085a4 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; @@ -58,12 +59,18 @@ function uploadCdp( multiple?: boolean; chooser?: { frameId?: string; backendNodeId?: number; mode?: string }; pendingResolve?: boolean; + overlayHit?: () => string; } = {}, ) { const calls: Array<{ method: string; params?: object }> = []; let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).includes('return "absent"') + ) + return { result: { value: options.overlayHit?.() ?? "clear" } }; if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") @@ -161,11 +168,20 @@ function dropCdp( } describe("file transfer tools", () => { - it("captures the file input activated by the requested click and injects only staged paths", async () => { + it.each([ + false, + true, + ])("uploads staged paths through the requested click (covered=%s)", async (covered) => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); - const { cdp, calls } = uploadCdp(); + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + }); + const { cdp, calls } = uploadCdp({ + overlayHit: () => (covered && !passthrough ? "covered" : "clear"), + }); const result = await handleUpload( manager, @@ -177,8 +193,12 @@ describe("file transfer tools", () => { { transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" }, ], }, - { cdp, tabsApi: tabsApi() }, + { cdp, tabsApi: tabsApi(), sendInputPassthrough }, ); + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual( + covered ? ["begin", "end"] : [], + ); + expect(passthrough).toBe(false); expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); expect(calls[0]).toEqual({ @@ -553,7 +573,7 @@ describe("file transfer tools", () => { expect(detach).toHaveBeenCalledWith(4); }); - it("routes one exact-target download through a browser-relative capability", async () => { + it.each([false, true])("routes an exact-target download (covered=%s)", async (covered) => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); @@ -593,7 +613,16 @@ describe("file transfer tools", () => { }; let cdpEvent: Parameters>[0] | undefined; let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + }); const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).includes('return "absent"') + ) + return { result: { value: covered && !passthrough ? "covered" : "clear" } }; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; @@ -626,8 +655,12 @@ describe("file transfer tools", () => { const result = await handleDownload( manager, { session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" }, - { cdp, tabsApi: tabsApi(), downloads }, + { cdp, tabsApi: tabsApi(), downloads, sendInputPassthrough }, ); + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual( + covered ? ["begin", "end"] : [], + ); + expect(passthrough).toBe(false); expect(suggested).toEqual({ filename: "BrowserSkill/tr_1/result.zip", diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 6dea22e..74e6ebb 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1,4 +1,9 @@ import { describe, expect, it, vi } from "vitest"; +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughMessage, +} from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { withInputReady } from "../input-readiness"; @@ -34,6 +39,7 @@ function makeFakeCdp( rendered: () => boolean | Promise = () => true, ) { const sent: Array<{ tabId: number; method: string; params?: object }> = []; + const overlayHit = vi.fn(async () => ({ result: { value: "clear" } })); const sendImpl = async (tabId: number, method: string, params?: object) => { sent.push({ tabId, method, params }); if ( @@ -42,6 +48,11 @@ function makeFakeCdp( ) return { result: { value: visibility() } }; if (method === "Page.captureScreenshot") return { data: (await rendered()) ? "pixel" : "" }; + if ( + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression).includes('return "absent"') + ) + return overlayHit(); const h = handlers[method]; if (!h && method === "Accessibility.getPartialAXTree") return { nodes: [] }; if (!h && method === "Page.getLayoutMetrics") { @@ -65,7 +76,7 @@ function makeFakeCdp( ), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }; - return { cdp, tabsApi, sent }; + return { cdp, tabsApi, sent, overlayHit }; } describe("modifiersBitfield", () => { @@ -241,10 +252,41 @@ describe("handleClick", () => { expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); }); - it("enables overlay bypass before mouse events when overlay blocks the click point", async () => { + it.each([ + "clear", + "absent", + "bypass-only", + "covered", + "press-fails", + "cancel-after-move", + "cancel-during-begin", + "still-covered", + "begin-fails", + "lost-ack", + "restore-fails", + "probe-throws", + "probe-invalid", + "verify-throws", + "late-covered", + "late-unknown", + "known-late-unknown", + ])("scopes click passthrough without changing a retained hover bypass: %s", async (mode) => { const order: string[] = []; - const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => { - order.push(enabled ? "bypass-on" : "bypass-off"); + const controller = new AbortController(); + let bypassCount = 1; // A hover owns this reference throughout the click helper. + const bypassOverlay = vi.fn(async (_tab: number, enabled: boolean) => { + order.push(enabled ? "bypass-begin" : "bypass-end"); + bypassCount += enabled ? 1 : -1; + }); + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + order.push(message.phase); + if (message.phase === "begin" && mode === "begin-fails") throw new Error("No receiver"); + passthrough = message.phase === "begin"; + if (passthrough && mode === "cancel-during-begin") controller.abort(); + if (passthrough && mode === "lost-ack") throw new Error("Response lost after application"); + if (!passthrough && mode === "restore-fails") throw new Error("Tab closed"); + return { type: INPUT_PASSTHROUGH, ok: true }; }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); @@ -252,146 +294,205 @@ describe("handleClick", () => { const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; - } - if (expr.includes("hitIndex")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, - }, - }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => { - order.push("mouse"); + "Input.dispatchMouseEvent": (params) => { + const { type } = params as { type: string }; + order.push(type); + if (type === "mouseMoved" && mode === "cancel-after-move") controller.abort(); + if (type === "mousePressed" && mode === "press-fails") throw new Error("Input failed"); return {}; }, }); - const res = await handleClick( + const initiallyClear = [ + "clear", + "absent", + "probe-throws", + "probe-invalid", + "late-covered", + "late-unknown", + ].includes(mode); + fake.overlayHit.mockImplementation(async () => { + order.push("probe"); + if (mode === "probe-throws" || (mode === "verify-throws" && passthrough)) + throw new Error("Renderer unavailable"); + if (mode === "probe-invalid") return { exceptionDetails: {} } as never; + if (order.includes("mouseMoved") && ["late-unknown", "known-late-unknown"].includes(mode)) + return {} as never; + const hit = + mode === "absent" + ? "absent" + : mode === "late-covered" && order.includes("mouseMoved") + ? "covered" + : mode === "bypass-only" && bypassCount === 2 + ? "clear" + : initiallyClear || (passthrough && mode !== "still-covered") + ? "clear" + : "covered"; + return { result: { value: hit } }; + }); + const result = await handleClick( sm, { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, + { + ...fake, + sendInputPassthrough, + bypassOverlay, + signal: controller.signal, + }, ); - if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - expect(order.indexOf("bypass-on")).toBeLessThan(order.indexOf("mouse")); - expect(order.lastIndexOf("bypass-off")).toBeGreaterThan(order.lastIndexOf("mouse")); - }); - - it("disables overlay bypass when mouse dispatch throws", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - let mouseCalls = 0; - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; - } - if (expr.includes("hitIndex")) { - return { - result: { - value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, - }, - }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => { - mouseCalls += 1; - if (mouseCalls === 2) throw new Error("mousePressed failed"); - return {}; - }, - }); - const res = await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, + const beforeMoveFailure = [ + "still-covered", + "begin-fails", + "verify-throws", + "cancel-during-begin", + ].includes(mode); + const noPress = + beforeMoveFailure || + ["cancel-after-move", "late-covered", "known-late-unknown"].includes(mode); + const mouse = order.filter((step) => step.startsWith("mouse")); + expect(mouse).toEqual( + beforeMoveFailure + ? [] + : noPress + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], ); - expect(res).toMatchObject({ code: "cdp_failed" }); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - }); - - it("enables overlay bypass when the overlay host itself captures the point", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const host = document.createElement("browser-skill-overlay"); - host.setAttribute("data-bsk-overlay", ""); - host.style.cssText = - "position:fixed;inset:0;width:100vw;height:100vh;pointer-events:auto;display:block"; - host.attachShadow({ mode: "closed" }); - document.body.append(host); - Object.defineProperty(host, "getBoundingClientRect", { - value: () => ({ - x: 0, - y: 0, - width: 1280, - height: 720, - top: 0, - left: 0, - right: 1280, - bottom: 720, - }), - }); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - const value = new Function(`return (${expr})`)(); - return { result: { value } }; - }, - "Input.dispatchMouseEvent": () => ({}), - }); - try { - const res = await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, + if (noPress || mode === "press-fails") { + expect(result).toHaveProperty( + "code", + mode.startsWith("cancel-") ? "cancelled" : "cdp_failed", ); - if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(bypassOverlay).toHaveBeenCalledWith(4, true); - expect(bypassOverlay).toHaveBeenCalledWith(4, false); - } finally { - host.remove(); + if ( + [ + "still-covered", + "begin-fails", + "verify-throws", + "late-covered", + "known-late-unknown", + ].includes(mode) + ) + expect(result).toMatchObject({ + data: { + reason: "input_not_ready", + effect_state: "none", + pointer_moved: ["late-covered", "known-late-unknown"].includes(mode), + }, + }); + } else expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + if (initiallyClear || mode === "bypass-only") { + expect(sendInputPassthrough).not.toHaveBeenCalled(); + } else { + const messages = sendInputPassthrough.mock.calls.map(([, message]) => message); + expect(messages).toEqual([ + { type: INPUT_PASSTHROUGH, phase: "begin", id: expect.any(String) }, + { type: INPUT_PASSTHROUGH, phase: "end", id: messages[0].id }, + ]); + if (mouse.length) expect(order.indexOf("begin")).toBeLessThan(order.indexOf("mouseMoved")); + if (mouse.includes("mouseReleased")) + expect(order.indexOf("end")).toBeGreaterThan(order.indexOf("mouseReleased")); + expect(order.slice(-2)).toEqual(["end", "bypass-end"]); } + if (mode === "clear" || mode === "absent") + expect(order).toEqual(["probe", "mouseMoved", "probe", "mousePressed", "mouseReleased"]); + if (mode === "covered") + expect(order).toEqual([ + "probe", + "bypass-begin", + "probe", + "begin", + "probe", + "mouseMoved", + "probe", + "mousePressed", + "mouseReleased", + "end", + "bypass-end", + ]); + expect(bypassCount).toBe(1); + if (initiallyClear) expect(bypassOverlay).not.toHaveBeenCalled(); + else + expect(bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); }); - it("skips overlay bypass when overlay does not block the click point", async () => { - const bypassOverlay = vi.fn().mockResolvedValue(undefined); - const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await sm.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - "Runtime.evaluate": (params: unknown) => { - const expr = String((params as { expression?: string })?.expression ?? ""); - if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { - return { result: { value: { overlayHostPresent: false, overlayHostConnected: false } } }; - } - throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); - }, - "Input.dispatchMouseEvent": () => ({}), - }); - await handleClick( - sm, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, - ); - expect(bypassOverlay).not.toHaveBeenCalled(); + it.each([ + ["begin", INPUT_PASSTHROUGH_TTL_MS], + ["move", INPUT_PASSTHROUGH_TTL_MS], + ["press", INPUT_PASSTHROUGH_TTL_MS], + ["release", INPUT_PASSTHROUGH_TTL_MS], + ["clear-slow", INPUT_PASSTHROUGH_TTL_MS], + ["begin", 4_000], + ["move", 4_000], + ["probe", 4_000], + ["move", 3_999], + ["press", 4_500], + ["release", 4_500], + ] as const)("handles %s delayed by %i ms without retrying input", async (phase, elapsed) => { + let now = Date.now(); + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("expiry"); + ctx.refStore.set("e1", 100, { tabId: 4 }); + const mouse: string[] = []; + let passthrough = false; + const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => { + passthrough = message.phase === "begin"; + if (passthrough && phase === "begin") now += elapsed; + }); + const bypassOverlay = vi.fn(async () => {}); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[0, 0, 40, 0, 40, 20, 0, 20]] }), + "Input.dispatchMouseEvent": (params) => { + const type = (params as { type: string }).type; + mouse.push(type); + if ( + (phase === "move" && type === "mouseMoved") || + (["press", "clear-slow"].includes(phase) && type === "mousePressed") || + (phase === "release" && type === "mouseReleased") + ) + now += elapsed; + return {}; + }, + }); + // Another concurrent lease may keep the point clear after ours has expired. + fake.overlayHit.mockImplementation(async () => { + if (phase === "probe" && mouse.length > 0) now += elapsed; + return { result: { value: passthrough || phase === "clear-slow" ? "clear" : "covered" } }; + }); + const result = await handleClick( + manager, + { session_id: "expiry", ref: "e1" }, + { ...fake, bypassOverlay, sendInputPassthrough }, + ); + const blockedBeforePress = ["begin", "move", "probe"].includes(phase) && elapsed >= 4_000; + if (phase === "clear-slow" || (!blockedBeforePress && elapsed < INPUT_PASSTHROUGH_TTL_MS)) { + expect(result).not.toHaveProperty("code"); + } else { + expect(result).toMatchObject({ + data: { + reason: blockedBeforePress ? "input_not_ready" : "input_outcome_unknown", + effect_state: blockedBeforePress ? "none" : "unknown", + }, + }); + } + if (phase === "clear-slow") expect(sendInputPassthrough).not.toHaveBeenCalled(); + else { + expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(["begin", "end"]); + expect(bypassOverlay.mock.calls).toHaveLength(2); + } + expect(mouse).toEqual( + blockedBeforePress && phase === "begin" + ? [] + : blockedBeforePress + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], + ); + } finally { + clock.mockRestore(); + } }); it("leaves disabled click behavior to the browser without an AX preflight", async () => { diff --git a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts index 49f1238..76d6950 100644 --- a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts +++ b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts @@ -524,21 +524,29 @@ async function pointFixture(child = false, oopif = false) { ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]); const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; const tabsApi = { get: async () => tab, query: async () => [tab] }; + const sendInputPassthrough = vi.fn(async (_tab: number, _message: { phase: string }) => ({})); const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {}); const original = f.send.getMockImplementation()!; const input: Record[] = []; const hitPoints: number[][] = []; const control = { hit: true, + overlayCovered: false, onMove: () => {}, failPress: false, visibility: "visible", onFocus: () => {}, focusCommands: [] as boolean[], }; + sendInputPassthrough.mockImplementation(async (_tab, message) => { + control.overlayCovered = message.phase !== "begin"; + return {}; + }); f.send.mockImplementation(async (target, method, params = {}) => { if (method === "Runtime.evaluate" && params.expression === "document.visibilityState") return { result: { value: control.visibility } }; + if (method === "Runtime.evaluate" && String(params.expression).includes('return "absent"')) + return { result: { value: control.overlayCovered ? "covered" : "clear" } }; if (method === "Runtime.evaluate" && params.awaitPromise) return { result: { value: true } }; if (method === "Emulation.setFocusEmulationEnabled") { control.focusCommands.push(params.enabled as boolean); @@ -550,7 +558,7 @@ async function pointFixture(child = false, oopif = false) { String(params.functionDeclaration).includes("elementFromPoint") ) { hitPoints.push((params.arguments as { value: number }[]).map((a) => a.value)); - return { result: { value: control.hit } }; + return { result: { value: control.hit && !control.overlayCovered } }; } if (method === "Input.dispatchMouseEvent") { input.push(params); @@ -586,7 +594,7 @@ async function pointFixture(child = false, oopif = false) { input, hitPoints, pointControl: control, - deps: { cdp: f.cdp, tabsApi, bypassOverlay }, + deps: { cdp: f.cdp, tabsApi, sendInputPassthrough, bypassOverlay }, shot, }; } @@ -608,12 +616,52 @@ it.each([ "visual_capture_stale", ); expect(f.input).toHaveLength(count); + expect(f.deps.sendInputPassthrough).not.toHaveBeenCalled(); expect(f.deps.bypassOverlay.mock.calls).toEqual([ [4, true], [4, false], ]); }); +it.each([ + "success", + "press-fails", + "cancel", + "still-covered", + "appears-during-bypass", +])("scopes visual click passthrough: %s", async (mode) => { + const f = await pointFixture(); + const abort = new AbortController(); + f.pointControl.overlayCovered = mode !== "appears-during-bypass"; + if (mode === "appears-during-bypass") + f.deps.bypassOverlay.mockImplementation(async (_tab, enabled) => { + if (enabled) f.pointControl.overlayCovered = true; + }); + f.pointControl.failPress = mode === "press-fails"; + if (mode === "cancel") f.pointControl.onMove = () => abort.abort(); + if (mode === "still-covered") f.deps.sendInputPassthrough.mockImplementation(async () => ({})); + const result = await handleClick(f.manager, f.params, { ...f.deps, signal: abort.signal }); + if (mode === "success" || mode === "appears-during-bypass") + expect(result).not.toHaveProperty("code"); + else expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed"); + expect(f.input.map((e) => e.type)).toEqual( + mode === "still-covered" + ? [] + : mode === "cancel" + ? ["mouseMoved"] + : ["mouseMoved", "mousePressed", "mouseReleased"], + ); + expect(f.deps.sendInputPassthrough.mock.calls.map(([, message]) => message.phase)).toEqual([ + "begin", + "end", + ]); + expect(f.deps.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + if (mode === "still-covered") expect(result).toHaveProperty("data.effect_state", "none"); +}); + it.each([ "success", "geometry", diff --git a/apps/extension/src/tools/click-overlay.ts b/apps/extension/src/tools/click-overlay.ts new file mode 100644 index 0000000..7fa7d0b --- /dev/null +++ b/apps/extension/src/tools/click-overlay.ts @@ -0,0 +1,145 @@ +import { + INPUT_PASSTHROUGH, + INPUT_PASSTHROUGH_TTL_MS, + type InputPassthroughSendToTab, + sendInputPassthrough, +} from "@/lib/input-passthrough-bridge"; +import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge"; +import type { RpcError } from "@/transport/types"; +import { rpcError } from "./errors"; +import { type CdpRunner, isRpcError } from "./shared"; + +export interface ClickOverlayDeps { + cdp: CdpRunner; + signal?: AbortSignal; + /** Acquire/release only this operation's automation bypass reference. */ + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + sendInputPassthrough?: InputPassthroughSendToTab; +} + +type OverlayHit = "absent" | "clear" | "covered" | "unknown"; + +/** Prepare before mouseMoved, then check again before every press. */ +export async function withClickOverlay( + tabId: number, + point: { x: number; y: number }, + deps: ClickOverlayDeps, + click: (beforePress: () => Promise) => Promise, + alwaysBypass = false, +): Promise { + const send = deps.sendInputPassthrough ?? sendInputPassthrough; + let bypass = false; + let passthroughId: string | undefined; + let passthroughExpiresAt = Infinity; + const expired = () => Date.now() >= passthroughExpiresAt; + // Reserve one second for input delivery; completed clicks still use the full lease. + const pressDeadlineReached = () => Date.now() >= passthroughExpiresAt - 1_000; + const notReady = () => + rpcError( + "cdp_failed", + "input_not_ready", + "Could not clear the extension overlay from the click point", + { + effect_state: "none", + pointer_moved: false, + }, + ); + const cancelled = (): RpcError => ({ + code: "cancelled", + message: "click aborted", + data: { effect_state: "none", pointer_moved: false }, + }); + const probe = async (): Promise => { + try { + const selector = JSON.stringify(OVERLAY_HOST_SELECTOR); + const hit = await deps.cdp.send<{ + result?: { value?: unknown }; + exceptionDetails?: unknown; + }>(tabId, "Runtime.evaluate", { + expression: `(() => { + const host = document.querySelector(${selector}); + if (!host) return "absent"; + return document.elementFromPoint(${point.x},${point.y})?.closest(${selector}) ? "covered" : "clear"; + })()`, + returnByValue: true, + }); + const value = hit.result?.value; + if (!hit.exceptionDetails && (value === "absent" || value === "clear" || value === "covered")) + return value; + console.debug("[bsk click] overlay hit-test returned no result", hit.exceptionDetails); + } catch (error) { + console.debug("[bsk click] overlay hit-test failed", error); + } + return "unknown"; + }; + try { + if (deps.signal?.aborted) return cancelled(); + let hit = await probe(); + let covered = hit === "covered"; + if (deps.signal?.aborted) return cancelled(); + if ((alwaysBypass || covered) && deps.bypassOverlay) { + try { + await deps.bypassOverlay(tabId, true); + bypass = true; + } catch (error) { + console.debug("[bsk click] overlay bypass enable failed", error); + } + // Enabling bypass can also render newly arrived control UI. + hit = await probe(); + covered ||= hit === "covered"; + } + if (deps.signal?.aborted) return cancelled(); + if (hit === "covered") { + passthroughId = crypto.randomUUID(); + // Start before sending, so our deadline cannot outlive the content-side lease. + passthroughExpiresAt = Date.now() + INPUT_PASSTHROUGH_TTL_MS; + try { + await send(tabId, { type: INPUT_PASSTHROUGH, phase: "begin", id: passthroughId }); + } catch (error) { + console.debug("[bsk click] overlay passthrough enable failed", error); + } + hit = await probe(); + } + if (deps.signal?.aborted) return cancelled(); + if (pressDeadlineReached() || hit === "covered" || (covered && hit === "unknown")) + return notReady(); + const result = await click(async () => { + if (pressDeadlineReached()) return notReady(); + const current = await probe(); + // Unknown probes preserve ordinary-page behavior, but cannot clear a known obstruction. + return pressDeadlineReached() || current === "covered" || (covered && current === "unknown") + ? notReady() + : null; + }); + // The lease can expire while a slow press/release is in flight. Always release + // the mouse, but do not claim success (or retry) when delivery is uncertain. + if (expired() && !isRpcError(result)) { + return rpcError( + "cdp_failed", + "input_outcome_unknown", + "Overlay passthrough expired during click", + { + effect_state: "unknown", + pointer_moved: true, + }, + ); + } + return result; + } finally { + // Release our lease even if begin's acknowledgement was lost. + if (passthroughId) { + try { + await send(tabId, { type: INPUT_PASSTHROUGH, phase: "end", id: passthroughId }); + } catch (error) { + console.debug("[bsk click] overlay passthrough restore failed", error); + } + } + if (bypass) { + try { + await deps.bypassOverlay!(tabId, false); + } catch (error) { + console.debug("[bsk click] overlay bypass restore failed", error); + } + } + } +} diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 60819b9..ab14e8e 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -35,6 +35,7 @@ import type { SelectParams, SelectResult, } from "@/transport/types"; +import { type ClickOverlayDeps, withClickOverlay } from "./click-overlay"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; @@ -52,16 +53,11 @@ import { } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; -export interface InteractionDeps { +export interface InteractionDeps extends ClickOverlayDeps { /** Arm short popup observation immediately before native input dispatch. */ onInputSent?: (tabId: number) => void; - cdp: CdpRunner; tabsApi: ChromeTabsApi; - /** Abort hook (full chain wired in M10.2). */ - signal?: AbortSignal; defaultTimeoutMs?: number; - /** Temporarily disable overlay click blocker during CDP automation. */ - bypassOverlay?: (tabId: number, enabled: boolean) => Promise; /** Keep hover hit-testing active for the caller's next observation/action. */ keepOverlayBypassAfterHover?: boolean; } @@ -517,36 +513,10 @@ export async function clickResolvedTarget( if (clickCount < 1) { return { code: "invalid_params", message: "click_count must be greater than zero" }; } - const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); - let automationBypassEnabled = false; - if (overlayBlocking && deps.bypassOverlay) { - try { - await deps.bypassOverlay(target.tabId, true); - automationBypassEnabled = true; - } catch (err) { - console.debug("[bsk interaction] overlay bypass enable failed", err); - } - } - - try { - const error = await dispatchClickAtPoint( - target.tabId, - centre, - params, - deps, - undefined, - markSent, - ); - if (error) return error; - } finally { - if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { - try { - await deps.bypassOverlay(target.tabId, false); - } catch (err) { - console.debug("[bsk interaction] overlay bypass disable failed", err); - } - } - } + const error = await withClickOverlay(target.tabId, centre, deps, (verifyOverlay) => + dispatchClickAtPoint(target.tabId, centre, params, deps, verifyOverlay, undefined, markSent), + ); + if (error) return error; return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, @@ -563,6 +533,7 @@ async function dispatchClickAtPoint( point: { x: number; y: number }, params: Pick, deps: InteractionDeps, + verifyOverlay: () => Promise, beforePress?: () => Promise, markSent?: () => void, ): Promise { @@ -581,7 +552,7 @@ async function dispatchClickAtPoint( modifiers, }); const failure = (error: RpcError): RpcError => - beforePress + beforePress || error.data?.reason === "input_not_ready" ? { ...error, data: { @@ -612,6 +583,9 @@ async function dispatchClickAtPoint( if (error) return failure(error); } if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); + const blocked = await verifyOverlay(); + if (blocked) return failure(blocked); + if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" }); deps.onInputSent?.(tabId); markSent?.(); attempted = true; @@ -683,40 +657,42 @@ async function clickVisualPoint( return changed(); return null; }; - let bypass = false; try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - if (deps.bypassOverlay) { - await deps.bypassOverlay(target.tabId, true); - bypass = true; - } - const invalid = await validate(); - if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } }; - const error = await dispatchClickAtPoint( + return await withClickOverlay( target.tabId, point, - params, deps, - async () => { - await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability. - return validate(); + async (verifyOverlay) => { + const invalid = await validate(); + if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } }; + const error = await dispatchClickAtPoint( + target.tabId, + point, + params, + deps, + verifyOverlay, + async () => { + await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability. + return validate(); + }, + markSent, + ); + if (error) return error; + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: capture.ref, + ...point, + }); }, - markSent, + true, // Visual target validation also needs the page blocker bypassed. ); - if (error) return error; - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: capture.ref, - ...point, - }); } catch (error) { return { code: deps.signal?.aborted || isAbortError(error) ? "cancelled" : "cdp_failed", message: error instanceof Error ? error.message : String(error), data: { effect_state: "none" }, }; - } finally { - if (bypass) await deps.bypassOverlay!(target.tabId, false).catch(() => {}); } }