mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-10-02 07:34:35 +08:00
fix(extension): prevent control overlay from swallowing clicks
This commit is contained in:
@@ -1,8 +1,13 @@
|
||||
// @vitest-environment node
|
||||
// Opt in with BSK_CLICK_CHROME; each test owns its browser, profile and HTTP server.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { createElement } from "react";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { CAPTURE_SUPPRESS, type CaptureSuppressSendToTab } from "@/lib/capture-suppress-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { prepareBackgroundExecution } from "../background-execution";
|
||||
import { handleClick, handlePress } from "../interaction";
|
||||
@@ -354,4 +359,165 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
it("clicks through the real control pill and fails if the closed shadow overlay still intercepts input", async () => {
|
||||
const { withChrome } = await import(
|
||||
new URL(
|
||||
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
|
||||
import.meta.url,
|
||||
).href
|
||||
);
|
||||
const css = readFileSync(new URL("../../content/overlay.css", import.meta.url), "utf8");
|
||||
await withChrome(
|
||||
{ executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 },
|
||||
async (send: Send) => {
|
||||
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
|
||||
url: "about:blank",
|
||||
});
|
||||
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId,
|
||||
flatten: true,
|
||||
});
|
||||
const local: Send = (method, params) => send(method, params, sessionId);
|
||||
const evaluate = async <T>(expression: string) => {
|
||||
const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression,
|
||||
returnByValue: true,
|
||||
awaitPromise: true,
|
||||
},
|
||||
);
|
||||
expect(reply.exceptionDetails).toBeUndefined();
|
||||
return reply.result.value;
|
||||
};
|
||||
await local("Page.bringToFront");
|
||||
await local("Emulation.setDeviceMetricsOverride", {
|
||||
width: 1280,
|
||||
height: 757,
|
||||
deviceScaleFactor: 1,
|
||||
mobile: false,
|
||||
});
|
||||
await evaluate(`document.body.innerHTML = '<button id="target" style="position:fixed;width:80px;height:30px">Page action</button>';
|
||||
window.clicks=[]; window.stops=0;
|
||||
document.querySelector('#target').onclick=e=>clicks.push(e.isTrusted);
|
||||
window.overlay=document.createElement('browser-skill-overlay');
|
||||
overlay.setAttribute('data-bsk-overlay',''); overlay.setAttribute('data-bsk-overlay-surface','');
|
||||
document.documentElement.append(overlay);
|
||||
window.overlayRoot=overlay.attachShadow({mode:'closed'});`);
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => ({ windowId: 100, initialTabIds: [] }),
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("overlay-click");
|
||||
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
const mouse: string[] = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent")
|
||||
mouse.push((params as { type: string }).type);
|
||||
return (await local(method, params)) as never;
|
||||
},
|
||||
};
|
||||
const sendToTab: CaptureSuppressSendToTab = async (_tabId, message) => {
|
||||
// Transport adapter: use the production CSS suppression marker, as the content script does.
|
||||
await evaluate(
|
||||
`overlay.toggleAttribute('data-bsk-capture-hidden', ${message.phase === "begin"})`,
|
||||
);
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
};
|
||||
// A retained hover bypass leaves the pill interactive even though the host is transparent.
|
||||
for (const automationBypass of [false, true]) {
|
||||
const markup = renderToStaticMarkup(
|
||||
createElement(ControlOverlay, {
|
||||
visible: true,
|
||||
interrupting: false,
|
||||
automationBypass,
|
||||
onInterrupt: () => {},
|
||||
}),
|
||||
);
|
||||
await evaluate(`overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass});
|
||||
overlayRoot.innerHTML = ${JSON.stringify(`<style>${css}</style>`)} + ${JSON.stringify(markup)};
|
||||
overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]').addEventListener('click',()=>stops++);`);
|
||||
for (const slot of ["control-overlay-pill", "control-overlay-stop-all"]) {
|
||||
const point = await evaluate<{ x: number; y: number }>(`(() => {
|
||||
const r=overlayRoot.querySelector('[data-slot="${slot}"]').getBoundingClientRect();
|
||||
const x=${slot === "control-overlay-pill" ? "r.x+25" : "r.x+r.width/2"}, y=r.y+r.height/2;
|
||||
Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'});
|
||||
clicks=[]; stops=0; return {x,y};
|
||||
})()`);
|
||||
expect(
|
||||
await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`),
|
||||
).toBe(true);
|
||||
const nativeClick = async () => {
|
||||
for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) {
|
||||
await local("Input.dispatchMouseEvent", {
|
||||
type,
|
||||
...point,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
};
|
||||
await nativeClick(); // Reproduce the interception before using the fixed click path.
|
||||
expect(await evaluate("clicks")).toEqual([]);
|
||||
await evaluate("stops=0");
|
||||
const { root } = await local<{ root: { nodeId: number } }>("DOM.getDocument");
|
||||
const { nodeId } = await local<{ nodeId: number }>("DOM.querySelector", {
|
||||
nodeId: root.nodeId,
|
||||
selector: "#target",
|
||||
});
|
||||
const { node } = await local<{ node: { backendNodeId: number } }>("DOM.describeNode", {
|
||||
nodeId,
|
||||
});
|
||||
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
|
||||
mouse.length = 0;
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{
|
||||
session_id: ctx.sessionId,
|
||||
...(automationBypass ? { ref: "e1" } : { selector: "#target" }),
|
||||
},
|
||||
{ cdp, tabsApi, sendToTab },
|
||||
);
|
||||
expect(result, JSON.stringify(result)).not.toHaveProperty("code");
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
expect(await evaluate("stops")).toBe(0);
|
||||
expect(mouse).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]);
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false);
|
||||
expect(
|
||||
await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`),
|
||||
).toBe(true);
|
||||
if (slot === "control-overlay-stop-all") {
|
||||
await nativeClick();
|
||||
expect(await evaluate("stops")).toBe(1);
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
}
|
||||
mouse.length = 0;
|
||||
const blocked = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, selector: "#target" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi,
|
||||
sendToTab: async () => {
|
||||
throw new Error("Content script unavailable");
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(blocked).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "input_not_ready", effect_state: "none" },
|
||||
});
|
||||
expect(mouse).toEqual(["mouseMoved"]);
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -341,7 +341,7 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||
it("hides and restores the complete control overlay for an upload trigger click", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -383,15 +383,8 @@ describe("ToolDispatcher", () => {
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string }).expression ?? "";
|
||||
if (expression.includes("hitIndex")) {
|
||||
return { result: { value: { hitIndex: 0 } } } as T;
|
||||
}
|
||||
if (expression.includes("overlayHostPresent")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||
},
|
||||
} as T;
|
||||
if (expression.startsWith("!!document.elementFromPoint")) {
|
||||
return { result: { value: false } } as T;
|
||||
}
|
||||
if (expression.includes("count:")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
@@ -433,12 +426,12 @@ describe("ToolDispatcher", () => {
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: true,
|
||||
type: "bsk/capture-suppress",
|
||||
phase: "begin",
|
||||
});
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: false,
|
||||
type: "bsk/capture-suppress",
|
||||
phase: "end",
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -64,6 +64,13 @@ function uploadCdp(
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).startsWith(
|
||||
"!!document.elementFromPoint",
|
||||
)
|
||||
)
|
||||
return { result: { value: false } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
@@ -594,6 +601,13 @@ describe("file transfer tools", () => {
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).startsWith(
|
||||
"!!document.elementFromPoint",
|
||||
)
|
||||
)
|
||||
return { result: { value: false } };
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { withInputReady } from "../input-readiness";
|
||||
@@ -34,6 +35,7 @@ function makeFakeCdp(
|
||||
rendered: () => boolean | Promise<boolean> = () => true,
|
||||
) {
|
||||
const sent: Array<{ tabId: number; method: string; params?: object }> = [];
|
||||
const overlayHit = vi.fn(async () => ({ result: { value: false } }));
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
if (
|
||||
@@ -42,6 +44,13 @@ function makeFakeCdp(
|
||||
)
|
||||
return { result: { value: visibility() } };
|
||||
if (method === "Page.captureScreenshot") return { data: (await rendered()) ? "pixel" : "" };
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).startsWith(
|
||||
"!!document.elementFromPoint",
|
||||
)
|
||||
)
|
||||
return overlayHit();
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Accessibility.getPartialAXTree") return { nodes: [] };
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
@@ -65,7 +74,7 @@ function makeFakeCdp(
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
return { cdp, tabsApi, sent };
|
||||
return { cdp, tabsApi, sent, overlayHit };
|
||||
}
|
||||
|
||||
describe("modifiersBitfield", () => {
|
||||
@@ -241,10 +250,25 @@ describe("handleClick", () => {
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
it.each([
|
||||
"success",
|
||||
"dispatch-error",
|
||||
"cancel",
|
||||
"still-covered",
|
||||
"suppression-failed",
|
||||
"hit-test-error",
|
||||
"invalid-hit-test",
|
||||
"no-content-script",
|
||||
])("scopes complete overlay suppression and checks the click point: %s", async (mode) => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
order.push(enabled ? "bypass-on" : "bypass-off");
|
||||
const controller = new AbortController();
|
||||
const bypassOverlay = vi.fn();
|
||||
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||
order.push(message.phase);
|
||||
if (mode === "suppression-failed" || mode === "no-content-script") {
|
||||
throw new Error("No receiving content script");
|
||||
}
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -252,146 +276,52 @@ describe("handleClick", () => {
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => {
|
||||
order.push("mouse");
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
const { type } = params as { type: string };
|
||||
order.push(type);
|
||||
if (type === "mouseMoved" && mode === "cancel") controller.abort();
|
||||
if (type === "mousePressed" && mode === "dispatch-error") throw new Error("Input failed");
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
fake.overlayHit.mockImplementation(async () => {
|
||||
order.push("hit-test");
|
||||
if (mode === "hit-test-error") throw new Error("Renderer unavailable");
|
||||
if (mode === "invalid-hit-test") return { exceptionDetails: {} } as never;
|
||||
return { result: { value: mode === "still-covered" || mode === "suppression-failed" } };
|
||||
});
|
||||
const result = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
{ ...fake, sendToTab, bypassOverlay, signal: controller.signal },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
expect(order.indexOf("bypass-on")).toBeLessThan(order.indexOf("mouse"));
|
||||
expect(order.lastIndexOf("bypass-off")).toBeGreaterThan(order.lastIndexOf("mouse"));
|
||||
});
|
||||
|
||||
it("disables overlay bypass when mouse dispatch throws", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
let mouseCalls = 0;
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => {
|
||||
mouseCalls += 1;
|
||||
if (mouseCalls === 2) throw new Error("mousePressed failed");
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
);
|
||||
expect(res).toMatchObject({ code: "cdp_failed" });
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("enables overlay bypass when the overlay host itself captures the point", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const host = document.createElement("browser-skill-overlay");
|
||||
host.setAttribute("data-bsk-overlay", "");
|
||||
host.style.cssText =
|
||||
"position:fixed;inset:0;width:100vw;height:100vh;pointer-events:auto;display:block";
|
||||
host.attachShadow({ mode: "closed" });
|
||||
document.body.append(host);
|
||||
Object.defineProperty(host, "getBoundingClientRect", {
|
||||
value: () => ({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 1280,
|
||||
bottom: 720,
|
||||
}),
|
||||
});
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
const value = new Function(`return (${expr})`)();
|
||||
return { result: { value } };
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
try {
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
} finally {
|
||||
host.remove();
|
||||
expect(order[0]).toBe("begin");
|
||||
expect(bypassOverlay).not.toHaveBeenCalled(); // Do not change a retained hover bypass.
|
||||
if (mode === "success" || mode === "no-content-script") {
|
||||
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||
expect(order.slice(1, 5)).toEqual([
|
||||
"mouseMoved",
|
||||
"hit-test",
|
||||
"mousePressed",
|
||||
"mouseReleased",
|
||||
]);
|
||||
} else {
|
||||
expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed");
|
||||
if (mode === "dispatch-error") {
|
||||
expect(order).toContain("mouseReleased");
|
||||
} else {
|
||||
expect(order).not.toContain("mousePressed");
|
||||
}
|
||||
if (mode === "still-covered" || mode === "suppression-failed") {
|
||||
expect(result).toMatchObject({ data: { reason: "input_not_ready", effect_state: "none" } });
|
||||
}
|
||||
}
|
||||
if (mode === "suppression-failed" || mode === "no-content-script") {
|
||||
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||
} else {
|
||||
expect(order.at(-1)).toBe("end");
|
||||
expect(sendToTab.mock.calls.map((call) => call[1].phase)).toEqual(["begin", "end"]);
|
||||
}
|
||||
});
|
||||
|
||||
it("skips overlay bypass when overlay does not block the click point", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: false, overlayHostConnected: false } } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
);
|
||||
expect(bypassOverlay).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves disabled click behavior to the browser without an AX preflight", async () => {
|
||||
|
||||
@@ -524,7 +524,7 @@ async function pointFixture(child = false, oopif = false) {
|
||||
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
|
||||
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {});
|
||||
const sendToTab = vi.fn(async (_tab: number, _message: { phase: string }) => ({}));
|
||||
const original = f.send.getMockImplementation()!;
|
||||
const input: Record<string, unknown>[] = [];
|
||||
const hitPoints: number[][] = [];
|
||||
@@ -539,6 +539,11 @@ async function pointFixture(child = false, oopif = false) {
|
||||
f.send.mockImplementation(async (target, method, params = {}) => {
|
||||
if (method === "Runtime.evaluate" && params.expression === "document.visibilityState")
|
||||
return { result: { value: control.visibility } };
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String(params.expression).startsWith("!!document.elementFromPoint")
|
||||
)
|
||||
return { result: { value: false } };
|
||||
if (method === "Runtime.evaluate" && params.awaitPromise) return { result: { value: true } };
|
||||
if (method === "Emulation.setFocusEmulationEnabled") {
|
||||
control.focusCommands.push(params.enabled as boolean);
|
||||
@@ -586,7 +591,7 @@ async function pointFixture(child = false, oopif = false) {
|
||||
input,
|
||||
hitPoints,
|
||||
pointControl: control,
|
||||
deps: { cdp: f.cdp, tabsApi, bypassOverlay },
|
||||
deps: { cdp: f.cdp, tabsApi, sendToTab },
|
||||
shot,
|
||||
};
|
||||
}
|
||||
@@ -608,9 +613,9 @@ it.each([
|
||||
"visual_capture_stale",
|
||||
);
|
||||
expect(f.input).toHaveLength(count);
|
||||
expect(f.deps.bypassOverlay.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
expect(f.deps.sendToTab.mock.calls.map(([tab, message]) => [tab, message.phase])).toEqual([
|
||||
[4, "begin"],
|
||||
[4, "end"],
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -15,6 +15,11 @@ import { isAbortError } from "./vom/capture-abort";
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
BlurParams,
|
||||
@@ -60,6 +65,8 @@ export interface InteractionDeps {
|
||||
/** Abort hook (full chain wired in M10.2). */
|
||||
signal?: AbortSignal;
|
||||
defaultTimeoutMs?: number;
|
||||
/** Hide the complete extension overlay for the bounded click operation. */
|
||||
sendToTab?: CaptureSuppressSendToTab;
|
||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||
@@ -517,36 +524,12 @@ export async function clickResolvedTarget(
|
||||
if (clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||
}
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
automationBypassEnabled = true;
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const error = await dispatchClickAtPoint(
|
||||
target.tabId,
|
||||
centre,
|
||||
params,
|
||||
deps,
|
||||
undefined,
|
||||
markSent,
|
||||
);
|
||||
if (error) return error;
|
||||
} finally {
|
||||
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
const error = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() => dispatchClickAtPoint(target.tabId, centre, params, deps, undefined, markSent),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (error) return error;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
@@ -612,6 +595,28 @@ async function dispatchClickAtPoint(
|
||||
if (error) return failure(error);
|
||||
}
|
||||
if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" });
|
||||
// A closed shadow root retargets its inner controls to the host here.
|
||||
// Check the real hit after mouseMoved, including when suppression failed.
|
||||
const hit = await deps.cdp.send<{
|
||||
result?: { value?: boolean };
|
||||
exceptionDetails?: unknown;
|
||||
}>(tabId, "Runtime.evaluate", {
|
||||
expression: `!!document.elementFromPoint(${point.x},${point.y})?.closest(${JSON.stringify(OVERLAY_HOST_SELECTOR)})`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (hit.exceptionDetails || hit.result?.value !== false) {
|
||||
return failure(
|
||||
rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_ready",
|
||||
hit.result?.value === true
|
||||
? "Extension overlay intercepts the click point"
|
||||
: "Could not verify that the click point is clear of the extension overlay",
|
||||
{ effect_state: attempted ? "unknown" : "none", pointer_moved: moved },
|
||||
),
|
||||
);
|
||||
}
|
||||
if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" });
|
||||
deps.onInputSent?.(tabId);
|
||||
markSent?.();
|
||||
attempted = true;
|
||||
@@ -683,40 +688,39 @@ async function clickVisualPoint(
|
||||
return changed();
|
||||
return null;
|
||||
};
|
||||
let bypass = false;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
if (deps.bypassOverlay) {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
bypass = true;
|
||||
}
|
||||
const invalid = await validate();
|
||||
if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } };
|
||||
const error = await dispatchClickAtPoint(
|
||||
return await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
point,
|
||||
params,
|
||||
deps,
|
||||
async () => {
|
||||
await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability.
|
||||
return validate();
|
||||
const invalid = await validate();
|
||||
if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } };
|
||||
const error = await dispatchClickAtPoint(
|
||||
target.tabId,
|
||||
point,
|
||||
params,
|
||||
deps,
|
||||
async () => {
|
||||
await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability.
|
||||
return validate();
|
||||
},
|
||||
markSent,
|
||||
);
|
||||
if (error) return error;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: capture.ref,
|
||||
...point,
|
||||
});
|
||||
},
|
||||
markSent,
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (error) return error;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: capture.ref,
|
||||
...point,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
code: deps.signal?.aborted || isAbortError(error) ? "cancelled" : "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
data: { effect_state: "none" },
|
||||
};
|
||||
} finally {
|
||||
if (bypass) await deps.bypassOverlay!(target.tabId, false).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user