mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-10-02 07:34:35 +08:00
Merge pull request #323 from Tencent/fix/control-overlay-click
fix(extension): prevent control overlay from swallowing clicks
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
INPUT_PASSTHROUGH,
|
||||
INPUT_PASSTHROUGH_ATTR,
|
||||
INPUT_PASSTHROUGH_TTL_MS,
|
||||
type InputPassthroughMessage,
|
||||
isInputPassthroughMessage,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
import { createInputPassthroughController } from "../input-passthrough";
|
||||
|
||||
describe("click input passthrough", () => {
|
||||
let host: HTMLElement;
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
host = document.createElement("div");
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
const message = (phase: "begin" | "end", id = "click-1"): InputPassthroughMessage => ({
|
||||
type: INPUT_PASSTHROUGH,
|
||||
phase,
|
||||
id,
|
||||
});
|
||||
|
||||
it("applies and acknowledges synchronously without hiding the host or scheduling frames", () => {
|
||||
const controller = createInputPassthroughController(() => host);
|
||||
const raf = vi.spyOn(window, "requestAnimationFrame");
|
||||
const ack = vi.fn(() => expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true));
|
||||
expect(controller.handleMessage(message("begin"), ack)).toBe(false);
|
||||
expect(ack).toHaveBeenCalledWith({ type: INPUT_PASSTHROUGH, ok: true });
|
||||
expect(raf).not.toHaveBeenCalled();
|
||||
expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(false);
|
||||
raf.mockRestore();
|
||||
});
|
||||
|
||||
it("counts independent clicks and ignores duplicate or unmatched messages", () => {
|
||||
const controller = createInputPassthroughController(() => host);
|
||||
for (const id of ["one", "one", "two"]) controller.handleMessage(message("begin", id), vi.fn());
|
||||
expect(controller.pendingCount).toBe(2);
|
||||
for (const id of ["missing", "one", "one"])
|
||||
controller.handleMessage(message("end", id), vi.fn());
|
||||
expect(controller.pendingCount).toBe(1);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true);
|
||||
controller.handleMessage(message("end", "two"), vi.fn());
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("reapplies pending clicks after host replacement and leaves screenshot state alone", () => {
|
||||
let current: HTMLElement | null = null;
|
||||
const controller = createInputPassthroughController(() => current);
|
||||
controller.handleMessage(message("begin"), vi.fn());
|
||||
current = host;
|
||||
host.setAttribute("data-bsk-capture-hidden", "");
|
||||
controller.onHostMounted(host);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true);
|
||||
controller.handleMessage(message("end"), vi.fn());
|
||||
expect(host.hasAttribute("data-bsk-capture-hidden")).toBe(true);
|
||||
current = document.createElement("div");
|
||||
controller.onHostMounted(current);
|
||||
expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts only paired click messages with an operation id", () => {
|
||||
expect(isInputPassthroughMessage(message("begin"))).toBe(true);
|
||||
expect(isInputPassthroughMessage(message("end"))).toBe(true);
|
||||
for (const invalid of [
|
||||
null,
|
||||
{},
|
||||
{ ...message("begin"), id: "" },
|
||||
{ ...message("begin"), phase: "reset" },
|
||||
{ ...message("begin"), type: "bsk/capture-suppress" },
|
||||
])
|
||||
expect(isInputPassthroughMessage(invalid)).toBe(false);
|
||||
});
|
||||
|
||||
it("expires a lost end without extending the lease on duplicate begin", () => {
|
||||
const controller = createInputPassthroughController(() => host);
|
||||
controller.handleMessage(message("begin"), vi.fn());
|
||||
vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1);
|
||||
controller.handleMessage(message("begin"), vi.fn());
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true);
|
||||
vi.advanceTimersByTime(1);
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("expires only the old click and ignores its late end while another click is active", () => {
|
||||
const controller = createInputPassthroughController(() => host);
|
||||
controller.handleMessage(message("begin", "old"), vi.fn());
|
||||
vi.advanceTimersByTime(1000);
|
||||
controller.handleMessage(message("begin", "new"), vi.fn());
|
||||
vi.advanceTimersByTime(INPUT_PASSTHROUGH_TTL_MS - 1000);
|
||||
expect(controller.pendingCount).toBe(1);
|
||||
controller.handleMessage(message("end", "old"), vi.fn());
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true);
|
||||
vi.advanceTimersByTime(1000);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("cancels timers on end and reset, and does not restore old leases on remount", () => {
|
||||
let current = host;
|
||||
const controller = createInputPassthroughController(() => current);
|
||||
controller.handleMessage(message("begin", "one"), vi.fn());
|
||||
controller.handleMessage(message("begin", "two"), vi.fn());
|
||||
controller.handleMessage(message("end", "one"), vi.fn());
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
controller.reset();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(host.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
current = document.createElement("div");
|
||||
controller.onHostMounted(current);
|
||||
expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
controller.handleMessage(message("begin", "new-session"), vi.fn());
|
||||
controller.handleMessage(message("end", "two"), vi.fn());
|
||||
expect(current.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not revive an expired lease when a frozen page remounts before timers run", () => {
|
||||
const controller = createInputPassthroughController(() => host);
|
||||
controller.handleMessage(message("begin"), vi.fn());
|
||||
vi.setSystemTime(Date.now() + INPUT_PASSTHROUGH_TTL_MS);
|
||||
const replacement = document.createElement("div");
|
||||
controller.onHostMounted(replacement);
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(replacement.hasAttribute(INPUT_PASSTHROUGH_ATTR)).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import {
|
||||
INPUT_PASSTHROUGH,
|
||||
INPUT_PASSTHROUGH_ATTR,
|
||||
INPUT_PASSTHROUGH_TTL_MS,
|
||||
type InputPassthroughAck,
|
||||
type InputPassthroughMessage,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
|
||||
export function createInputPassthroughController(getHost: () => HTMLElement | null) {
|
||||
const pending = new Map<string, { timer: ReturnType<typeof setTimeout>; expiresAt: number }>();
|
||||
const release = (id: string) => {
|
||||
const lease = pending.get(id);
|
||||
if (lease) clearTimeout(lease.timer);
|
||||
pending.delete(id);
|
||||
};
|
||||
const apply = (host: HTMLElement | null) => {
|
||||
// A suspended page can resume/remount before its overdue timers run.
|
||||
for (const [id, lease] of pending) {
|
||||
if (lease.expiresAt <= Date.now()) release(id);
|
||||
}
|
||||
host?.toggleAttribute(INPUT_PASSTHROUGH_ATTR, pending.size > 0);
|
||||
};
|
||||
|
||||
return {
|
||||
get pendingCount() {
|
||||
return pending.size;
|
||||
},
|
||||
handleMessage(
|
||||
message: InputPassthroughMessage,
|
||||
sendResponse: (ack: InputPassthroughAck) => void,
|
||||
): false {
|
||||
if (message.phase === "begin") {
|
||||
if (!pending.has(message.id)) {
|
||||
pending.set(message.id, {
|
||||
expiresAt: Date.now() + INPUT_PASSTHROUGH_TTL_MS,
|
||||
timer: setTimeout(() => {
|
||||
release(message.id);
|
||||
apply(getHost());
|
||||
}, INPUT_PASSTHROUGH_TTL_MS),
|
||||
});
|
||||
}
|
||||
} else release(message.id);
|
||||
apply(getHost());
|
||||
// Hit testing uses current styles; unlike screenshots it needs no compositor frame.
|
||||
sendResponse({ type: INPUT_PASSTHROUGH, ok: true });
|
||||
return false;
|
||||
},
|
||||
reset() {
|
||||
for (const id of pending.keys()) release(id);
|
||||
apply(getHost());
|
||||
},
|
||||
onHostMounted: apply,
|
||||
};
|
||||
}
|
||||
@@ -15,6 +15,15 @@
|
||||
pointer-events: auto !important;
|
||||
}
|
||||
|
||||
/* Click passthrough changes only control hit targets, not visibility or other UI.
|
||||
Keep this after the equally specific host blocking rule above. */
|
||||
:host([data-bsk-overlay-surface][data-bsk-input-passthrough]),
|
||||
:host([data-bsk-input-passthrough]) [data-slot="control-overlay-blocker"],
|
||||
:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"],
|
||||
:host([data-bsk-input-passthrough]) [data-slot="control-overlay-pill"] * {
|
||||
pointer-events: none !important;
|
||||
}
|
||||
|
||||
/* Screenshot suppression: the host lives in the main DOM, so hiding it
|
||||
removes the whole shadow subtree from compositing and captured frames
|
||||
only contain page content. */
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_ATTR } from "@/lib/input-passthrough-bridge";
|
||||
import {
|
||||
OVERLAY_AGENT_OVERLAY_RESET,
|
||||
OVERLAY_AGENT_STATE,
|
||||
type OverlayMode,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { RECORD_START, RECORD_STOP } from "@/lib/record-bridge";
|
||||
|
||||
vi.hoisted(() => {
|
||||
Object.assign(globalThis, { defineContentScript: (definition: unknown) => definition });
|
||||
});
|
||||
// Exercise the real content entrypoint/controller wiring without mounting React components.
|
||||
vi.mock("react-dom/client", () => ({
|
||||
default: { createRoot: () => ({ render() {}, unmount() {} }) },
|
||||
}));
|
||||
vi.mock("@/lib/instance-id", () => ({
|
||||
getControlHintsHidden: async () => false,
|
||||
STORAGE_KEYS: { CONTROL_HINTS_HIDDEN: "control_hints_hidden" },
|
||||
}));
|
||||
|
||||
import content from "../content";
|
||||
|
||||
let dispose: (() => void) | undefined;
|
||||
afterEach(() => {
|
||||
dispose?.();
|
||||
dispose = undefined;
|
||||
document.querySelectorAll("browser-skill-overlay").forEach((host) => host.remove());
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
vi.useFakeTimers();
|
||||
let receive: (message: unknown, sender: object, ack: () => void) => void;
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(async () => {
|
||||
throw new Error("Background unavailable");
|
||||
}),
|
||||
onMessage: {
|
||||
addListener: (listener: typeof receive) => {
|
||||
receive = listener;
|
||||
},
|
||||
removeListener: vi.fn(),
|
||||
},
|
||||
},
|
||||
storage: { onChanged: { addListener: vi.fn(), removeListener: vi.fn() } },
|
||||
});
|
||||
let host: HTMLElement;
|
||||
let mount: () => void;
|
||||
vi.stubGlobal(
|
||||
"createShadowRootUi",
|
||||
async (
|
||||
_ctx: unknown,
|
||||
options: {
|
||||
onMount(container: HTMLElement, shadow: ShadowRoot, host: HTMLElement): unknown;
|
||||
onRemove(root: unknown): void;
|
||||
},
|
||||
) => {
|
||||
let root: unknown;
|
||||
mount = () => {
|
||||
if (host) {
|
||||
options.onRemove(root);
|
||||
host.remove();
|
||||
}
|
||||
host = document.createElement("browser-skill-overlay");
|
||||
const shadow = host.attachShadow({ mode: "closed" });
|
||||
const container = document.createElement("div");
|
||||
shadow.append(container);
|
||||
document.documentElement.append(host);
|
||||
root = options.onMount(container, shadow, host);
|
||||
};
|
||||
return { mount };
|
||||
},
|
||||
);
|
||||
await (content.main as (ctx: { onInvalidated(fn: () => void): void }) => Promise<void>)({
|
||||
onInvalidated: (fn) => {
|
||||
dispose = fn;
|
||||
},
|
||||
});
|
||||
const send = (message: unknown) => receive(message, {}, vi.fn());
|
||||
const state = (sessionId = "one", mode: OverlayMode = "control") =>
|
||||
send({ type: OVERLAY_AGENT_STATE, sessionId, mode, generation: 0 });
|
||||
const begin = (id = "click-one") => send({ type: INPUT_PASSTHROUGH, phase: "begin", id });
|
||||
const active = () => host.hasAttribute(INPUT_PASSTHROUGH_ATTR);
|
||||
state();
|
||||
return { send, state, begin, active, remount: () => mount() };
|
||||
}
|
||||
|
||||
it.each([
|
||||
"paused",
|
||||
"hidden",
|
||||
"interrupting",
|
||||
] as const)("clears leases when control becomes %s", async (mode) => {
|
||||
const f = await fixture();
|
||||
f.begin();
|
||||
expect(f.active()).toBe(true);
|
||||
f.state("one", mode);
|
||||
expect(f.active()).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
f.state();
|
||||
f.remount();
|
||||
expect(f.active()).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves same-session control updates and remounts until the click ends", async () => {
|
||||
const f = await fixture();
|
||||
f.begin();
|
||||
f.state();
|
||||
f.remount();
|
||||
expect(f.active()).toBe(true);
|
||||
f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" });
|
||||
expect(f.active()).toBe(false);
|
||||
});
|
||||
|
||||
it("clears old session leases without letting stale reset/end clear the new session", async () => {
|
||||
const f = await fixture();
|
||||
f.begin();
|
||||
f.state("two");
|
||||
expect(f.active()).toBe(false);
|
||||
f.begin("click-two");
|
||||
f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "one" });
|
||||
f.send({ type: INPUT_PASSTHROUGH, phase: "end", id: "click-one" });
|
||||
expect(f.active()).toBe(true);
|
||||
f.send({ type: OVERLAY_AGENT_OVERLAY_RESET, sessionId: "two" });
|
||||
expect(f.active()).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
f.remount();
|
||||
expect(f.active()).toBe(false);
|
||||
});
|
||||
|
||||
it("clears leases when recording starts or finishes", async () => {
|
||||
const f = await fixture();
|
||||
f.begin();
|
||||
f.send({ type: RECORD_START, requestId: "record" });
|
||||
expect(f.active()).toBe(false);
|
||||
f.begin("late-click");
|
||||
f.send({ type: RECORD_STOP, requestId: "record" });
|
||||
expect(f.active()).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("removes passthrough and timers when the content context is invalidated", async () => {
|
||||
const f = await fixture();
|
||||
f.begin();
|
||||
dispose?.();
|
||||
expect(f.active()).toBe(false);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import { createHelpRequestData } from "@/content/help-request";
|
||||
import { createInputPassthroughController } from "@/content/input-passthrough";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
@@ -28,6 +29,11 @@ import {
|
||||
isHelpCancelMessage,
|
||||
isHelpRequestMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
import {
|
||||
type InputPassthroughAck,
|
||||
type InputPassthroughMessage,
|
||||
isInputPassthroughMessage,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id";
|
||||
import {
|
||||
isOverlayAgentOverlayResetMessage,
|
||||
@@ -87,6 +93,7 @@ export default defineContentScript({
|
||||
}
|
||||
|
||||
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
||||
const inputPassthrough = createInputPassthroughController(() => overlayHost);
|
||||
|
||||
const ui = await createShadowRootUi(ctx, {
|
||||
name: "browser-skill-overlay",
|
||||
@@ -101,6 +108,7 @@ export default defineContentScript({
|
||||
hostLossReported = false;
|
||||
// A host rebuilt mid-capture must stay hidden until `end` arrives.
|
||||
captureSuppress.onHostMounted(shadowHost);
|
||||
inputPassthrough.onHostMounted(shadowHost);
|
||||
const app = document.createElement("div");
|
||||
app.className = "bsk-overlay-root";
|
||||
container.append(app);
|
||||
@@ -166,6 +174,8 @@ export default defineContentScript({
|
||||
function renderReactOverlays(): void {
|
||||
const overlayState = overlays.snapshot();
|
||||
const controlOverlayVisible = shouldShowAgentControlOverlay(overlayState);
|
||||
// Leaving control (including help/record UI) must not retain click leases.
|
||||
if (!controlOverlayVisible) inputPassthrough.reset();
|
||||
const interactiveOverlayVisible =
|
||||
overlayState.borrowRequests.length > 0 ||
|
||||
overlayState.activeHelp !== null ||
|
||||
@@ -218,12 +228,16 @@ export default defineContentScript({
|
||||
}
|
||||
|
||||
function applyOverlayState(state: OverlayAgentStateMessage): void {
|
||||
if (overlays.snapshot().activeSessionId !== state.sessionId) inputPassthrough.reset();
|
||||
activeAgentState = state;
|
||||
overlays.applyAgentControlMode(state.sessionId, state.mode);
|
||||
renderAll();
|
||||
}
|
||||
|
||||
function resetAgentOverlayState(sessionId: string) {
|
||||
const activeSessionId = overlays.snapshot().activeSessionId;
|
||||
if (activeSessionId && activeSessionId !== sessionId) return;
|
||||
inputPassthrough.reset();
|
||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||
if (previousHelp) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
@@ -257,6 +271,7 @@ export default defineContentScript({
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| CaptureSuppressMessage
|
||||
| InputPassthroughMessage
|
||||
| RecordStartMessage
|
||||
| RecordStopMessage
|
||||
| RecordCancelMessage
|
||||
@@ -264,11 +279,16 @@ export default defineContentScript({
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||
sendResponse: (
|
||||
response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck | InputPassthroughAck,
|
||||
) => void,
|
||||
) => {
|
||||
if (isCaptureSuppressMessage(message)) {
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
if (isInputPassthroughMessage(message)) {
|
||||
return inputPassthrough.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordStartMessage(message)) {
|
||||
activeRecordRequestId = message.requestId;
|
||||
@@ -514,6 +534,7 @@ export default defineContentScript({
|
||||
hostObserver.observe(document.documentElement, { childList: true, subtree: false });
|
||||
|
||||
ctx.onInvalidated(() => {
|
||||
inputPassthrough.reset();
|
||||
hostObserver.disconnect();
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/** Click-only hit-test suppression; independent of screenshot visibility and hover bypass. */
|
||||
export const INPUT_PASSTHROUGH = "bsk/input-passthrough";
|
||||
export const INPUT_PASSTHROUGH_ATTR = "data-bsk-input-passthrough";
|
||||
/** Bound orphaned click leases even if the background never sends end. */
|
||||
export const INPUT_PASSTHROUGH_TTL_MS = 5_000;
|
||||
|
||||
export interface InputPassthroughMessage {
|
||||
type: typeof INPUT_PASSTHROUGH;
|
||||
phase: "begin" | "end";
|
||||
/** Identifies one click so cleanup after a lost ack cannot release another click. */
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface InputPassthroughAck {
|
||||
type: typeof INPUT_PASSTHROUGH;
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export function isInputPassthroughMessage(message: unknown): message is InputPassthroughMessage {
|
||||
if (!message || typeof message !== "object") return false;
|
||||
const m = message as Record<string, unknown>;
|
||||
return (
|
||||
m.type === INPUT_PASSTHROUGH &&
|
||||
(m.phase === "begin" || m.phase === "end") &&
|
||||
typeof m.id === "string" &&
|
||||
m.id.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
export type InputPassthroughSendToTab = (
|
||||
tabId: number,
|
||||
message: InputPassthroughMessage,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export const sendInputPassthrough: InputPassthroughSendToTab = (tabId, message) =>
|
||||
chrome.tabs.sendMessage(tabId, message);
|
||||
@@ -1,8 +1,18 @@
|
||||
// @vitest-environment node
|
||||
// Opt in with BSK_CLICK_CHROME; each test owns its browser, profile and HTTP server.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { createElement } from "react";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import ts from "typescript";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import {
|
||||
INPUT_PASSTHROUGH,
|
||||
INPUT_PASSTHROUGH_TTL_MS,
|
||||
type InputPassthroughSendToTab,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { prepareBackgroundExecution } from "../background-execution";
|
||||
import { handleClick, handlePress } from "../interaction";
|
||||
@@ -354,4 +364,289 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
it("preserves hover and visible controls while scoped passthrough clears closed shadow click targets", async () => {
|
||||
const { withChrome } = await import(
|
||||
new URL(
|
||||
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
|
||||
import.meta.url,
|
||||
).href
|
||||
);
|
||||
const css = readFileSync(new URL("../../content/overlay.css", import.meta.url), "utf8");
|
||||
// Run the actual content-side controller in the browser, including its lease bookkeeping.
|
||||
const moduleUrl = (source: string) =>
|
||||
`data:text/javascript;base64,${Buffer.from(
|
||||
ts.transpileModule(source, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ES2022 },
|
||||
}).outputText,
|
||||
).toString("base64")}`;
|
||||
const bridgeUrl = moduleUrl(
|
||||
readFileSync(new URL("../../lib/input-passthrough-bridge.ts", import.meta.url), "utf8"),
|
||||
);
|
||||
const controllerUrl = moduleUrl(
|
||||
readFileSync(new URL("../../content/input-passthrough.ts", import.meta.url), "utf8").replace(
|
||||
'"@/lib/input-passthrough-bridge"',
|
||||
JSON.stringify(bridgeUrl),
|
||||
),
|
||||
);
|
||||
await withChrome(
|
||||
{ executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 },
|
||||
async (send: Send) => {
|
||||
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
|
||||
url: "about:blank",
|
||||
});
|
||||
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId,
|
||||
flatten: true,
|
||||
});
|
||||
const local: Send = (method, params) => send(method, params, sessionId);
|
||||
const evaluate = async <T>(expression: string) => {
|
||||
const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression,
|
||||
returnByValue: true,
|
||||
awaitPromise: true,
|
||||
},
|
||||
);
|
||||
expect(reply.exceptionDetails).toBeUndefined();
|
||||
return reply.result.value;
|
||||
};
|
||||
await local("Page.bringToFront");
|
||||
await local("Emulation.setDeviceMetricsOverride", {
|
||||
width: 1280,
|
||||
height: 757,
|
||||
deviceScaleFactor: 1,
|
||||
mobile: false,
|
||||
});
|
||||
await evaluate(`(async () => { document.body.innerHTML = '<style>#target:hover { background: rgb(1, 2, 3); }</style><button id="target" style="position:fixed;width:80px;height:30px">Page action</button>';
|
||||
window.clicks=[]; window.stops=0; window.pageEvents=[];
|
||||
document.querySelector('#target').onclick=e=>clicks.push(e.isTrusted);
|
||||
for (const type of ['mouseover','mousedown','mouseup','click']) document.querySelector('#target').addEventListener(type,e=>pageEvents.push({type,trusted:e.isTrusted,hover:getComputedStyle(e.currentTarget).backgroundColor==='rgb(1, 2, 3)'}));
|
||||
window.overlay=document.createElement('browser-skill-overlay');
|
||||
overlay.setAttribute('data-bsk-overlay',''); overlay.setAttribute('data-bsk-overlay-surface','');
|
||||
document.documentElement.append(overlay);
|
||||
window.overlayRoot=overlay.attachShadow({mode:'closed'});
|
||||
window.passthroughController=(await import(${JSON.stringify(controllerUrl)})).createInputPassthroughController(()=>overlay); })()`);
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => ({ windowId: 100, initialTabIds: [] }),
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("overlay-click");
|
||||
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
const mouse: string[] = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
mouse.push((params as { type: string }).type);
|
||||
expect(await evaluate("getComputedStyle(overlay).display")).toBe("block");
|
||||
expect(
|
||||
await evaluate(
|
||||
"getComputedStyle(overlayRoot.querySelector('[data-slot=control-overlay-pill]')).opacity",
|
||||
),
|
||||
).toBe("1");
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false);
|
||||
}
|
||||
return (await local(method, params)) as never;
|
||||
},
|
||||
};
|
||||
const phases: string[] = [];
|
||||
const sendInputPassthrough: InputPassthroughSendToTab = async (_tabId, message) => {
|
||||
phases.push(message.phase);
|
||||
return evaluate(
|
||||
`new Promise(resolve=>passthroughController.handleMessage(${JSON.stringify(message)},resolve))`,
|
||||
);
|
||||
};
|
||||
const bypassOverlay = async (_tabId: number, enabled: boolean) => {
|
||||
await evaluate(`window.bypassCount += ${enabled ? 1 : -1};
|
||||
overlay.toggleAttribute('data-bsk-overlay-blocking',bypassCount===0);
|
||||
overlayRoot.querySelector('[data-slot="control-overlay-blocker"]').style.pointerEvents=bypassCount>0?'none':'auto';`);
|
||||
};
|
||||
// A retained hover bypass leaves the pill interactive even though the host is transparent.
|
||||
for (const automationBypass of [false, true]) {
|
||||
const markup = renderToStaticMarkup(
|
||||
createElement(ControlOverlay, {
|
||||
visible: true,
|
||||
interrupting: false,
|
||||
automationBypass,
|
||||
onInterrupt: () => {},
|
||||
}),
|
||||
);
|
||||
await evaluate(`window.bypassCount=${automationBypass ? 1 : 0}; overlay.toggleAttribute('data-bsk-overlay-blocking', ${!automationBypass});
|
||||
overlayRoot.innerHTML = ${JSON.stringify(`<style>${css}</style>`)} + ${JSON.stringify(markup)};
|
||||
// Static markup starts before the component's reveal effect; model its settled state.
|
||||
for (const control of overlayRoot.querySelectorAll('[data-slot^="control-overlay"]')) control.style.opacity='1';
|
||||
overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]').addEventListener('click',()=>stops++);`);
|
||||
await evaluate(
|
||||
"Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})",
|
||||
);
|
||||
phases.length = 0;
|
||||
const clear = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, selector: "#target" },
|
||||
{ cdp, tabsApi, sendInputPassthrough, bypassOverlay },
|
||||
);
|
||||
expect(clear).not.toHaveProperty("code");
|
||||
expect(phases).toEqual([]);
|
||||
expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0);
|
||||
for (const slot of ["control-overlay-pill", "control-overlay-stop-all"]) {
|
||||
const point = await evaluate<{ x: number; y: number }>(`(() => {
|
||||
const r=overlayRoot.querySelector('[data-slot="${slot}"]').getBoundingClientRect();
|
||||
const x=${slot === "control-overlay-pill" ? "r.x+25" : "r.x+r.width/2"}, y=r.y+r.height/2;
|
||||
Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'});
|
||||
clicks=[]; stops=0; return {x,y};
|
||||
})()`);
|
||||
expect(
|
||||
await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`),
|
||||
).toBe(true);
|
||||
const nativeClick = async () => {
|
||||
for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) {
|
||||
await local("Input.dispatchMouseEvent", {
|
||||
type,
|
||||
...point,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
};
|
||||
await nativeClick(); // Reproduce the interception before using the fixed click path.
|
||||
expect(await evaluate("clicks")).toEqual([]);
|
||||
await local("Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
|
||||
await evaluate("stops=0; pageEvents=[]");
|
||||
phases.length = 0;
|
||||
const { root } = await local<{ root: { nodeId: number } }>("DOM.getDocument");
|
||||
const { nodeId } = await local<{ nodeId: number }>("DOM.querySelector", {
|
||||
nodeId: root.nodeId,
|
||||
selector: "#target",
|
||||
});
|
||||
const { node } = await local<{ node: { backendNodeId: number } }>("DOM.describeNode", {
|
||||
nodeId,
|
||||
});
|
||||
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
|
||||
mouse.length = 0;
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{
|
||||
session_id: ctx.sessionId,
|
||||
...(automationBypass ? { ref: "e1" } : { selector: "#target" }),
|
||||
},
|
||||
{ cdp, tabsApi, sendInputPassthrough, bypassOverlay },
|
||||
);
|
||||
expect(result, JSON.stringify(result)).not.toHaveProperty("code");
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
expect(phases).toEqual(["begin", "end"]);
|
||||
expect(await evaluate("pageEvents")).toEqual(
|
||||
["mouseover", "mousedown", "mouseup", "click"].map((type) => ({
|
||||
type,
|
||||
trusted: true,
|
||||
hover: true,
|
||||
})),
|
||||
);
|
||||
expect(await evaluate("bypassCount")).toBe(automationBypass ? 1 : 0);
|
||||
expect(await evaluate("getComputedStyle(overlay).display")).toBe("block");
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(
|
||||
false,
|
||||
);
|
||||
expect(await evaluate("stops")).toBe(0);
|
||||
expect(mouse).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]);
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-capture-hidden')")).toBe(false);
|
||||
expect(
|
||||
await evaluate(`document.elementFromPoint(${point.x},${point.y})===overlay`),
|
||||
).toBe(true);
|
||||
if (slot === "control-overlay-stop-all") {
|
||||
await nativeClick();
|
||||
expect(await evaluate("stops")).toBe(1);
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
}
|
||||
mouse.length = 0;
|
||||
const blocked = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, selector: "#target" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi,
|
||||
bypassOverlay,
|
||||
sendInputPassthrough: async () => {
|
||||
throw new Error("Content script unavailable");
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(blocked).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "input_not_ready", effect_state: "none" },
|
||||
});
|
||||
expect(mouse).toEqual([]);
|
||||
expect(await evaluate("clicks")).toEqual([true]);
|
||||
}
|
||||
// Other extension controls stay hit-testable even while a click lease is active.
|
||||
await evaluate(
|
||||
`(() => { const extra=document.createElement('div'); extra.innerHTML='<button id="other" style="position:fixed;left:20px;top:20px;width:80px;height:30px;pointer-events:auto">Help or recording control</button>'; overlayRoot.append(extra); })()`,
|
||||
);
|
||||
await evaluate(
|
||||
"Object.assign(document.querySelector('#target').style,{left:'20px',top:'20px'})",
|
||||
);
|
||||
mouse.length = 0;
|
||||
const protectedControl = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, selector: "#target" },
|
||||
{ cdp, tabsApi, sendInputPassthrough, bypassOverlay },
|
||||
);
|
||||
expect(protectedControl).toHaveProperty("data.reason", "input_not_ready");
|
||||
expect(mouse).toEqual([]);
|
||||
await sendInputPassthrough(4, {
|
||||
type: INPUT_PASSTHROUGH,
|
||||
phase: "begin",
|
||||
id: "css-check",
|
||||
});
|
||||
expect(await evaluate("overlayRoot.elementFromPoint(40,30).id")).toBe("other");
|
||||
expect(await evaluate("getComputedStyle(overlay).display")).toBe("block");
|
||||
await evaluate(`(() => { const replacement=document.createElement('browser-skill-overlay');
|
||||
for(const attr of overlay.attributes) replacement.setAttribute(attr.name,attr.value);
|
||||
const replacementRoot=replacement.attachShadow({mode:'closed'});
|
||||
replacementRoot.innerHTML=overlayRoot.innerHTML;
|
||||
overlay.replaceWith(replacement); overlay=replacement; overlayRoot=replacementRoot;
|
||||
passthroughController.onHostMounted(overlay); })()`);
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(true);
|
||||
await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "end", id: "css-check" });
|
||||
expect(await evaluate("overlay.hasAttribute('data-bsk-input-passthrough')")).toBe(false);
|
||||
}
|
||||
// Losing end must restore the user's Stop button without another background message.
|
||||
const stopPoint = await evaluate<{ x: number; y: number }>(`(() => {
|
||||
const stop=overlayRoot.querySelector('[data-slot="control-overlay-stop-all"]');
|
||||
stop.addEventListener('click',()=>stops++);
|
||||
const r=stop.getBoundingClientRect(), x=r.x+r.width/2, y=r.y+r.height/2;
|
||||
Object.assign(document.querySelector('#target').style,{left:(x-40)+'px',top:(y-15)+'px'});
|
||||
stops=0; clicks=[]; return {x,y};
|
||||
})()`);
|
||||
await sendInputPassthrough(4, { type: INPUT_PASSTHROUGH, phase: "begin", id: "lost-end" });
|
||||
expect(await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y}).id`)).toBe(
|
||||
"target",
|
||||
);
|
||||
await expect
|
||||
.poll(() => evaluate("overlay.hasAttribute('data-bsk-input-passthrough')"), {
|
||||
timeout: INPUT_PASSTHROUGH_TTL_MS + 3_000,
|
||||
})
|
||||
.toBe(false);
|
||||
expect(await evaluate("passthroughController.pendingCount")).toBe(0);
|
||||
expect(
|
||||
await evaluate(`document.elementFromPoint(${stopPoint.x},${stopPoint.y})===overlay`),
|
||||
).toBe(true);
|
||||
for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) {
|
||||
await local("Input.dispatchMouseEvent", {
|
||||
type,
|
||||
...stopPoint,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
expect(await evaluate("stops")).toBe(1);
|
||||
expect(await evaluate("clicks")).toEqual([]);
|
||||
expect(await evaluate("getComputedStyle(overlay).display")).toBe("block");
|
||||
},
|
||||
);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -341,7 +341,7 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||
it("leaves overlay state alone for an unobstructed upload trigger click", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -383,15 +383,8 @@ describe("ToolDispatcher", () => {
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string }).expression ?? "";
|
||||
if (expression.includes("hitIndex")) {
|
||||
return { result: { value: { hitIndex: 0 } } } as T;
|
||||
}
|
||||
if (expression.includes("overlayHostPresent")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||
},
|
||||
} as T;
|
||||
if (expression.includes('return "absent"')) {
|
||||
return { result: { value: "clear" } } as T;
|
||||
}
|
||||
if (expression.includes("count:")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
@@ -432,14 +425,7 @@ describe("ToolDispatcher", () => {
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: true,
|
||||
});
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: false,
|
||||
});
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detaches CDP state before stopping a session", async () => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
@@ -58,12 +59,18 @@ function uploadCdp(
|
||||
multiple?: boolean;
|
||||
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||
pendingResolve?: boolean;
|
||||
overlayHit?: () => string;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).includes('return "absent"')
|
||||
)
|
||||
return { result: { value: options.overlayHit?.() ?? "clear" } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
@@ -161,11 +168,20 @@ function dropCdp(
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("uploads staged paths through the requested click (covered=%s)", async (covered) => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const { cdp, calls } = uploadCdp();
|
||||
let passthrough = false;
|
||||
const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => {
|
||||
passthrough = message.phase === "begin";
|
||||
});
|
||||
const { cdp, calls } = uploadCdp({
|
||||
overlayHit: () => (covered && !passthrough ? "covered" : "clear"),
|
||||
});
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
@@ -177,8 +193,12 @@ describe("file transfer tools", () => {
|
||||
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||
],
|
||||
},
|
||||
{ cdp, tabsApi: tabsApi() },
|
||||
{ cdp, tabsApi: tabsApi(), sendInputPassthrough },
|
||||
);
|
||||
expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(
|
||||
covered ? ["begin", "end"] : [],
|
||||
);
|
||||
expect(passthrough).toBe(false);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||
expect(calls[0]).toEqual({
|
||||
@@ -553,7 +573,7 @@ describe("file transfer tools", () => {
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
it.each([false, true])("routes an exact-target download (covered=%s)", async (covered) => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
@@ -593,7 +613,16 @@ describe("file transfer tools", () => {
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
let passthrough = false;
|
||||
const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => {
|
||||
passthrough = message.phase === "begin";
|
||||
});
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).includes('return "absent"')
|
||||
)
|
||||
return { result: { value: covered && !passthrough ? "covered" : "clear" } };
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
@@ -626,8 +655,12 @@ describe("file transfer tools", () => {
|
||||
const result = await handleDownload(
|
||||
manager,
|
||||
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||
{ cdp, tabsApi: tabsApi(), downloads },
|
||||
{ cdp, tabsApi: tabsApi(), downloads, sendInputPassthrough },
|
||||
);
|
||||
expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(
|
||||
covered ? ["begin", "end"] : [],
|
||||
);
|
||||
expect(passthrough).toBe(false);
|
||||
|
||||
expect(suggested).toEqual({
|
||||
filename: "BrowserSkill/tr_1/result.zip",
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
INPUT_PASSTHROUGH,
|
||||
INPUT_PASSTHROUGH_TTL_MS,
|
||||
type InputPassthroughMessage,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { withInputReady } from "../input-readiness";
|
||||
@@ -34,6 +39,7 @@ function makeFakeCdp(
|
||||
rendered: () => boolean | Promise<boolean> = () => true,
|
||||
) {
|
||||
const sent: Array<{ tabId: number; method: string; params?: object }> = [];
|
||||
const overlayHit = vi.fn(async () => ({ result: { value: "clear" } }));
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
if (
|
||||
@@ -42,6 +48,11 @@ function makeFakeCdp(
|
||||
)
|
||||
return { result: { value: visibility() } };
|
||||
if (method === "Page.captureScreenshot") return { data: (await rendered()) ? "pixel" : "" };
|
||||
if (
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression).includes('return "absent"')
|
||||
)
|
||||
return overlayHit();
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Accessibility.getPartialAXTree") return { nodes: [] };
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
@@ -65,7 +76,7 @@ function makeFakeCdp(
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
return { cdp, tabsApi, sent };
|
||||
return { cdp, tabsApi, sent, overlayHit };
|
||||
}
|
||||
|
||||
describe("modifiersBitfield", () => {
|
||||
@@ -241,10 +252,41 @@ describe("handleClick", () => {
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
it.each([
|
||||
"clear",
|
||||
"absent",
|
||||
"bypass-only",
|
||||
"covered",
|
||||
"press-fails",
|
||||
"cancel-after-move",
|
||||
"cancel-during-begin",
|
||||
"still-covered",
|
||||
"begin-fails",
|
||||
"lost-ack",
|
||||
"restore-fails",
|
||||
"probe-throws",
|
||||
"probe-invalid",
|
||||
"verify-throws",
|
||||
"late-covered",
|
||||
"late-unknown",
|
||||
"known-late-unknown",
|
||||
])("scopes click passthrough without changing a retained hover bypass: %s", async (mode) => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
order.push(enabled ? "bypass-on" : "bypass-off");
|
||||
const controller = new AbortController();
|
||||
let bypassCount = 1; // A hover owns this reference throughout the click helper.
|
||||
const bypassOverlay = vi.fn(async (_tab: number, enabled: boolean) => {
|
||||
order.push(enabled ? "bypass-begin" : "bypass-end");
|
||||
bypassCount += enabled ? 1 : -1;
|
||||
});
|
||||
let passthrough = false;
|
||||
const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => {
|
||||
order.push(message.phase);
|
||||
if (message.phase === "begin" && mode === "begin-fails") throw new Error("No receiver");
|
||||
passthrough = message.phase === "begin";
|
||||
if (passthrough && mode === "cancel-during-begin") controller.abort();
|
||||
if (passthrough && mode === "lost-ack") throw new Error("Response lost after application");
|
||||
if (!passthrough && mode === "restore-fails") throw new Error("Tab closed");
|
||||
return { type: INPUT_PASSTHROUGH, ok: true };
|
||||
});
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -252,146 +294,205 @@ describe("handleClick", () => {
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => {
|
||||
order.push("mouse");
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
const { type } = params as { type: string };
|
||||
order.push(type);
|
||||
if (type === "mouseMoved" && mode === "cancel-after-move") controller.abort();
|
||||
if (type === "mousePressed" && mode === "press-fails") throw new Error("Input failed");
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
const initiallyClear = [
|
||||
"clear",
|
||||
"absent",
|
||||
"probe-throws",
|
||||
"probe-invalid",
|
||||
"late-covered",
|
||||
"late-unknown",
|
||||
].includes(mode);
|
||||
fake.overlayHit.mockImplementation(async () => {
|
||||
order.push("probe");
|
||||
if (mode === "probe-throws" || (mode === "verify-throws" && passthrough))
|
||||
throw new Error("Renderer unavailable");
|
||||
if (mode === "probe-invalid") return { exceptionDetails: {} } as never;
|
||||
if (order.includes("mouseMoved") && ["late-unknown", "known-late-unknown"].includes(mode))
|
||||
return {} as never;
|
||||
const hit =
|
||||
mode === "absent"
|
||||
? "absent"
|
||||
: mode === "late-covered" && order.includes("mouseMoved")
|
||||
? "covered"
|
||||
: mode === "bypass-only" && bypassCount === 2
|
||||
? "clear"
|
||||
: initiallyClear || (passthrough && mode !== "still-covered")
|
||||
? "clear"
|
||||
: "covered";
|
||||
return { result: { value: hit } };
|
||||
});
|
||||
const result = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
{
|
||||
...fake,
|
||||
sendInputPassthrough,
|
||||
bypassOverlay,
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
expect(order.indexOf("bypass-on")).toBeLessThan(order.indexOf("mouse"));
|
||||
expect(order.lastIndexOf("bypass-off")).toBeGreaterThan(order.lastIndexOf("mouse"));
|
||||
});
|
||||
|
||||
it("disables overlay bypass when mouse dispatch throws", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
let mouseCalls = 0;
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => {
|
||||
mouseCalls += 1;
|
||||
if (mouseCalls === 2) throw new Error("mousePressed failed");
|
||||
return {};
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
const beforeMoveFailure = [
|
||||
"still-covered",
|
||||
"begin-fails",
|
||||
"verify-throws",
|
||||
"cancel-during-begin",
|
||||
].includes(mode);
|
||||
const noPress =
|
||||
beforeMoveFailure ||
|
||||
["cancel-after-move", "late-covered", "known-late-unknown"].includes(mode);
|
||||
const mouse = order.filter((step) => step.startsWith("mouse"));
|
||||
expect(mouse).toEqual(
|
||||
beforeMoveFailure
|
||||
? []
|
||||
: noPress
|
||||
? ["mouseMoved"]
|
||||
: ["mouseMoved", "mousePressed", "mouseReleased"],
|
||||
);
|
||||
expect(res).toMatchObject({ code: "cdp_failed" });
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("enables overlay bypass when the overlay host itself captures the point", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const host = document.createElement("browser-skill-overlay");
|
||||
host.setAttribute("data-bsk-overlay", "");
|
||||
host.style.cssText =
|
||||
"position:fixed;inset:0;width:100vw;height:100vh;pointer-events:auto;display:block";
|
||||
host.attachShadow({ mode: "closed" });
|
||||
document.body.append(host);
|
||||
Object.defineProperty(host, "getBoundingClientRect", {
|
||||
value: () => ({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 1280,
|
||||
bottom: 720,
|
||||
}),
|
||||
});
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
const value = new Function(`return (${expr})`)();
|
||||
return { result: { value } };
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
try {
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
if (noPress || mode === "press-fails") {
|
||||
expect(result).toHaveProperty(
|
||||
"code",
|
||||
mode.startsWith("cancel-") ? "cancelled" : "cdp_failed",
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, false);
|
||||
} finally {
|
||||
host.remove();
|
||||
if (
|
||||
[
|
||||
"still-covered",
|
||||
"begin-fails",
|
||||
"verify-throws",
|
||||
"late-covered",
|
||||
"known-late-unknown",
|
||||
].includes(mode)
|
||||
)
|
||||
expect(result).toMatchObject({
|
||||
data: {
|
||||
reason: "input_not_ready",
|
||||
effect_state: "none",
|
||||
pointer_moved: ["late-covered", "known-late-unknown"].includes(mode),
|
||||
},
|
||||
});
|
||||
} else expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||
if (initiallyClear || mode === "bypass-only") {
|
||||
expect(sendInputPassthrough).not.toHaveBeenCalled();
|
||||
} else {
|
||||
const messages = sendInputPassthrough.mock.calls.map(([, message]) => message);
|
||||
expect(messages).toEqual([
|
||||
{ type: INPUT_PASSTHROUGH, phase: "begin", id: expect.any(String) },
|
||||
{ type: INPUT_PASSTHROUGH, phase: "end", id: messages[0].id },
|
||||
]);
|
||||
if (mouse.length) expect(order.indexOf("begin")).toBeLessThan(order.indexOf("mouseMoved"));
|
||||
if (mouse.includes("mouseReleased"))
|
||||
expect(order.indexOf("end")).toBeGreaterThan(order.indexOf("mouseReleased"));
|
||||
expect(order.slice(-2)).toEqual(["end", "bypass-end"]);
|
||||
}
|
||||
if (mode === "clear" || mode === "absent")
|
||||
expect(order).toEqual(["probe", "mouseMoved", "probe", "mousePressed", "mouseReleased"]);
|
||||
if (mode === "covered")
|
||||
expect(order).toEqual([
|
||||
"probe",
|
||||
"bypass-begin",
|
||||
"probe",
|
||||
"begin",
|
||||
"probe",
|
||||
"mouseMoved",
|
||||
"probe",
|
||||
"mousePressed",
|
||||
"mouseReleased",
|
||||
"end",
|
||||
"bypass-end",
|
||||
]);
|
||||
expect(bypassCount).toBe(1);
|
||||
if (initiallyClear) expect(bypassOverlay).not.toHaveBeenCalled();
|
||||
else
|
||||
expect(bypassOverlay.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips overlay bypass when overlay does not block the click point", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: false, overlayHostConnected: false } } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
|
||||
);
|
||||
expect(bypassOverlay).not.toHaveBeenCalled();
|
||||
it.each([
|
||||
["begin", INPUT_PASSTHROUGH_TTL_MS],
|
||||
["move", INPUT_PASSTHROUGH_TTL_MS],
|
||||
["press", INPUT_PASSTHROUGH_TTL_MS],
|
||||
["release", INPUT_PASSTHROUGH_TTL_MS],
|
||||
["clear-slow", INPUT_PASSTHROUGH_TTL_MS],
|
||||
["begin", 4_000],
|
||||
["move", 4_000],
|
||||
["probe", 4_000],
|
||||
["move", 3_999],
|
||||
["press", 4_500],
|
||||
["release", 4_500],
|
||||
] as const)("handles %s delayed by %i ms without retrying input", async (phase, elapsed) => {
|
||||
let now = Date.now();
|
||||
const clock = vi.spyOn(Date, "now").mockImplementation(() => now);
|
||||
try {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await manager.start("expiry");
|
||||
ctx.refStore.set("e1", 100, { tabId: 4 });
|
||||
const mouse: string[] = [];
|
||||
let passthrough = false;
|
||||
const sendInputPassthrough = vi.fn(async (_tab: number, message: InputPassthroughMessage) => {
|
||||
passthrough = message.phase === "begin";
|
||||
if (passthrough && phase === "begin") now += elapsed;
|
||||
});
|
||||
const bypassOverlay = vi.fn(async () => {});
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 40, 0, 40, 20, 0, 20]] }),
|
||||
"Input.dispatchMouseEvent": (params) => {
|
||||
const type = (params as { type: string }).type;
|
||||
mouse.push(type);
|
||||
if (
|
||||
(phase === "move" && type === "mouseMoved") ||
|
||||
(["press", "clear-slow"].includes(phase) && type === "mousePressed") ||
|
||||
(phase === "release" && type === "mouseReleased")
|
||||
)
|
||||
now += elapsed;
|
||||
return {};
|
||||
},
|
||||
});
|
||||
// Another concurrent lease may keep the point clear after ours has expired.
|
||||
fake.overlayHit.mockImplementation(async () => {
|
||||
if (phase === "probe" && mouse.length > 0) now += elapsed;
|
||||
return { result: { value: passthrough || phase === "clear-slow" ? "clear" : "covered" } };
|
||||
});
|
||||
const result = await handleClick(
|
||||
manager,
|
||||
{ session_id: "expiry", ref: "e1" },
|
||||
{ ...fake, bypassOverlay, sendInputPassthrough },
|
||||
);
|
||||
const blockedBeforePress = ["begin", "move", "probe"].includes(phase) && elapsed >= 4_000;
|
||||
if (phase === "clear-slow" || (!blockedBeforePress && elapsed < INPUT_PASSTHROUGH_TTL_MS)) {
|
||||
expect(result).not.toHaveProperty("code");
|
||||
} else {
|
||||
expect(result).toMatchObject({
|
||||
data: {
|
||||
reason: blockedBeforePress ? "input_not_ready" : "input_outcome_unknown",
|
||||
effect_state: blockedBeforePress ? "none" : "unknown",
|
||||
},
|
||||
});
|
||||
}
|
||||
if (phase === "clear-slow") expect(sendInputPassthrough).not.toHaveBeenCalled();
|
||||
else {
|
||||
expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(["begin", "end"]);
|
||||
expect(bypassOverlay.mock.calls).toHaveLength(2);
|
||||
}
|
||||
expect(mouse).toEqual(
|
||||
blockedBeforePress && phase === "begin"
|
||||
? []
|
||||
: blockedBeforePress
|
||||
? ["mouseMoved"]
|
||||
: ["mouseMoved", "mousePressed", "mouseReleased"],
|
||||
);
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves disabled click behavior to the browser without an AX preflight", async () => {
|
||||
|
||||
@@ -524,21 +524,29 @@ async function pointFixture(child = false, oopif = false) {
|
||||
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
|
||||
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
const sendInputPassthrough = vi.fn(async (_tab: number, _message: { phase: string }) => ({}));
|
||||
const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {});
|
||||
const original = f.send.getMockImplementation()!;
|
||||
const input: Record<string, unknown>[] = [];
|
||||
const hitPoints: number[][] = [];
|
||||
const control = {
|
||||
hit: true,
|
||||
overlayCovered: false,
|
||||
onMove: () => {},
|
||||
failPress: false,
|
||||
visibility: "visible",
|
||||
onFocus: () => {},
|
||||
focusCommands: [] as boolean[],
|
||||
};
|
||||
sendInputPassthrough.mockImplementation(async (_tab, message) => {
|
||||
control.overlayCovered = message.phase !== "begin";
|
||||
return {};
|
||||
});
|
||||
f.send.mockImplementation(async (target, method, params = {}) => {
|
||||
if (method === "Runtime.evaluate" && params.expression === "document.visibilityState")
|
||||
return { result: { value: control.visibility } };
|
||||
if (method === "Runtime.evaluate" && String(params.expression).includes('return "absent"'))
|
||||
return { result: { value: control.overlayCovered ? "covered" : "clear" } };
|
||||
if (method === "Runtime.evaluate" && params.awaitPromise) return { result: { value: true } };
|
||||
if (method === "Emulation.setFocusEmulationEnabled") {
|
||||
control.focusCommands.push(params.enabled as boolean);
|
||||
@@ -550,7 +558,7 @@ async function pointFixture(child = false, oopif = false) {
|
||||
String(params.functionDeclaration).includes("elementFromPoint")
|
||||
) {
|
||||
hitPoints.push((params.arguments as { value: number }[]).map((a) => a.value));
|
||||
return { result: { value: control.hit } };
|
||||
return { result: { value: control.hit && !control.overlayCovered } };
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
input.push(params);
|
||||
@@ -586,7 +594,7 @@ async function pointFixture(child = false, oopif = false) {
|
||||
input,
|
||||
hitPoints,
|
||||
pointControl: control,
|
||||
deps: { cdp: f.cdp, tabsApi, bypassOverlay },
|
||||
deps: { cdp: f.cdp, tabsApi, sendInputPassthrough, bypassOverlay },
|
||||
shot,
|
||||
};
|
||||
}
|
||||
@@ -608,12 +616,52 @@ it.each([
|
||||
"visual_capture_stale",
|
||||
);
|
||||
expect(f.input).toHaveLength(count);
|
||||
expect(f.deps.sendInputPassthrough).not.toHaveBeenCalled();
|
||||
expect(f.deps.bypassOverlay.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"success",
|
||||
"press-fails",
|
||||
"cancel",
|
||||
"still-covered",
|
||||
"appears-during-bypass",
|
||||
])("scopes visual click passthrough: %s", async (mode) => {
|
||||
const f = await pointFixture();
|
||||
const abort = new AbortController();
|
||||
f.pointControl.overlayCovered = mode !== "appears-during-bypass";
|
||||
if (mode === "appears-during-bypass")
|
||||
f.deps.bypassOverlay.mockImplementation(async (_tab, enabled) => {
|
||||
if (enabled) f.pointControl.overlayCovered = true;
|
||||
});
|
||||
f.pointControl.failPress = mode === "press-fails";
|
||||
if (mode === "cancel") f.pointControl.onMove = () => abort.abort();
|
||||
if (mode === "still-covered") f.deps.sendInputPassthrough.mockImplementation(async () => ({}));
|
||||
const result = await handleClick(f.manager, f.params, { ...f.deps, signal: abort.signal });
|
||||
if (mode === "success" || mode === "appears-during-bypass")
|
||||
expect(result).not.toHaveProperty("code");
|
||||
else expect(result).toHaveProperty("code", mode === "cancel" ? "cancelled" : "cdp_failed");
|
||||
expect(f.input.map((e) => e.type)).toEqual(
|
||||
mode === "still-covered"
|
||||
? []
|
||||
: mode === "cancel"
|
||||
? ["mouseMoved"]
|
||||
: ["mouseMoved", "mousePressed", "mouseReleased"],
|
||||
);
|
||||
expect(f.deps.sendInputPassthrough.mock.calls.map(([, message]) => message.phase)).toEqual([
|
||||
"begin",
|
||||
"end",
|
||||
]);
|
||||
expect(f.deps.bypassOverlay.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
if (mode === "still-covered") expect(result).toHaveProperty("data.effect_state", "none");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"success",
|
||||
"geometry",
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import {
|
||||
INPUT_PASSTHROUGH,
|
||||
INPUT_PASSTHROUGH_TTL_MS,
|
||||
type InputPassthroughSendToTab,
|
||||
sendInputPassthrough,
|
||||
} from "@/lib/input-passthrough-bridge";
|
||||
import { OVERLAY_HOST_SELECTOR } from "@/lib/overlay-bridge";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
export interface ClickOverlayDeps {
|
||||
cdp: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
/** Acquire/release only this operation's automation bypass reference. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
sendInputPassthrough?: InputPassthroughSendToTab;
|
||||
}
|
||||
|
||||
type OverlayHit = "absent" | "clear" | "covered" | "unknown";
|
||||
|
||||
/** Prepare before mouseMoved, then check again before every press. */
|
||||
export async function withClickOverlay<T>(
|
||||
tabId: number,
|
||||
point: { x: number; y: number },
|
||||
deps: ClickOverlayDeps,
|
||||
click: (beforePress: () => Promise<RpcError | null>) => Promise<T>,
|
||||
alwaysBypass = false,
|
||||
): Promise<T | RpcError> {
|
||||
const send = deps.sendInputPassthrough ?? sendInputPassthrough;
|
||||
let bypass = false;
|
||||
let passthroughId: string | undefined;
|
||||
let passthroughExpiresAt = Infinity;
|
||||
const expired = () => Date.now() >= passthroughExpiresAt;
|
||||
// Reserve one second for input delivery; completed clicks still use the full lease.
|
||||
const pressDeadlineReached = () => Date.now() >= passthroughExpiresAt - 1_000;
|
||||
const notReady = () =>
|
||||
rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_ready",
|
||||
"Could not clear the extension overlay from the click point",
|
||||
{
|
||||
effect_state: "none",
|
||||
pointer_moved: false,
|
||||
},
|
||||
);
|
||||
const cancelled = (): RpcError => ({
|
||||
code: "cancelled",
|
||||
message: "click aborted",
|
||||
data: { effect_state: "none", pointer_moved: false },
|
||||
});
|
||||
const probe = async (): Promise<OverlayHit> => {
|
||||
try {
|
||||
const selector = JSON.stringify(OVERLAY_HOST_SELECTOR);
|
||||
const hit = await deps.cdp.send<{
|
||||
result?: { value?: unknown };
|
||||
exceptionDetails?: unknown;
|
||||
}>(tabId, "Runtime.evaluate", {
|
||||
expression: `(() => {
|
||||
const host = document.querySelector(${selector});
|
||||
if (!host) return "absent";
|
||||
return document.elementFromPoint(${point.x},${point.y})?.closest(${selector}) ? "covered" : "clear";
|
||||
})()`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const value = hit.result?.value;
|
||||
if (!hit.exceptionDetails && (value === "absent" || value === "clear" || value === "covered"))
|
||||
return value;
|
||||
console.debug("[bsk click] overlay hit-test returned no result", hit.exceptionDetails);
|
||||
} catch (error) {
|
||||
console.debug("[bsk click] overlay hit-test failed", error);
|
||||
}
|
||||
return "unknown";
|
||||
};
|
||||
try {
|
||||
if (deps.signal?.aborted) return cancelled();
|
||||
let hit = await probe();
|
||||
let covered = hit === "covered";
|
||||
if (deps.signal?.aborted) return cancelled();
|
||||
if ((alwaysBypass || covered) && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(tabId, true);
|
||||
bypass = true;
|
||||
} catch (error) {
|
||||
console.debug("[bsk click] overlay bypass enable failed", error);
|
||||
}
|
||||
// Enabling bypass can also render newly arrived control UI.
|
||||
hit = await probe();
|
||||
covered ||= hit === "covered";
|
||||
}
|
||||
if (deps.signal?.aborted) return cancelled();
|
||||
if (hit === "covered") {
|
||||
passthroughId = crypto.randomUUID();
|
||||
// Start before sending, so our deadline cannot outlive the content-side lease.
|
||||
passthroughExpiresAt = Date.now() + INPUT_PASSTHROUGH_TTL_MS;
|
||||
try {
|
||||
await send(tabId, { type: INPUT_PASSTHROUGH, phase: "begin", id: passthroughId });
|
||||
} catch (error) {
|
||||
console.debug("[bsk click] overlay passthrough enable failed", error);
|
||||
}
|
||||
hit = await probe();
|
||||
}
|
||||
if (deps.signal?.aborted) return cancelled();
|
||||
if (pressDeadlineReached() || hit === "covered" || (covered && hit === "unknown"))
|
||||
return notReady();
|
||||
const result = await click(async () => {
|
||||
if (pressDeadlineReached()) return notReady();
|
||||
const current = await probe();
|
||||
// Unknown probes preserve ordinary-page behavior, but cannot clear a known obstruction.
|
||||
return pressDeadlineReached() || current === "covered" || (covered && current === "unknown")
|
||||
? notReady()
|
||||
: null;
|
||||
});
|
||||
// The lease can expire while a slow press/release is in flight. Always release
|
||||
// the mouse, but do not claim success (or retry) when delivery is uncertain.
|
||||
if (expired() && !isRpcError(result)) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_outcome_unknown",
|
||||
"Overlay passthrough expired during click",
|
||||
{
|
||||
effect_state: "unknown",
|
||||
pointer_moved: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
// Release our lease even if begin's acknowledgement was lost.
|
||||
if (passthroughId) {
|
||||
try {
|
||||
await send(tabId, { type: INPUT_PASSTHROUGH, phase: "end", id: passthroughId });
|
||||
} catch (error) {
|
||||
console.debug("[bsk click] overlay passthrough restore failed", error);
|
||||
}
|
||||
}
|
||||
if (bypass) {
|
||||
try {
|
||||
await deps.bypassOverlay!(tabId, false);
|
||||
} catch (error) {
|
||||
console.debug("[bsk click] overlay bypass restore failed", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import type {
|
||||
SelectParams,
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { type ClickOverlayDeps, withClickOverlay } from "./click-overlay";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
@@ -52,16 +53,11 @@ import {
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
export interface InteractionDeps {
|
||||
export interface InteractionDeps extends ClickOverlayDeps {
|
||||
/** Arm short popup observation immediately before native input dispatch. */
|
||||
onInputSent?: (tabId: number) => void;
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
/** Abort hook (full chain wired in M10.2). */
|
||||
signal?: AbortSignal;
|
||||
defaultTimeoutMs?: number;
|
||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
@@ -517,36 +513,10 @@ export async function clickResolvedTarget(
|
||||
if (clickCount < 1) {
|
||||
return { code: "invalid_params", message: "click_count must be greater than zero" };
|
||||
}
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
automationBypassEnabled = true;
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const error = await dispatchClickAtPoint(
|
||||
target.tabId,
|
||||
centre,
|
||||
params,
|
||||
deps,
|
||||
undefined,
|
||||
markSent,
|
||||
);
|
||||
if (error) return error;
|
||||
} finally {
|
||||
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
const error = await withClickOverlay(target.tabId, centre, deps, (verifyOverlay) =>
|
||||
dispatchClickAtPoint(target.tabId, centre, params, deps, verifyOverlay, undefined, markSent),
|
||||
);
|
||||
if (error) return error;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
@@ -563,6 +533,7 @@ async function dispatchClickAtPoint(
|
||||
point: { x: number; y: number },
|
||||
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||
deps: InteractionDeps,
|
||||
verifyOverlay: () => Promise<RpcError | null>,
|
||||
beforePress?: () => Promise<RpcError | null>,
|
||||
markSent?: () => void,
|
||||
): Promise<RpcError | null> {
|
||||
@@ -581,7 +552,7 @@ async function dispatchClickAtPoint(
|
||||
modifiers,
|
||||
});
|
||||
const failure = (error: RpcError): RpcError =>
|
||||
beforePress
|
||||
beforePress || error.data?.reason === "input_not_ready"
|
||||
? {
|
||||
...error,
|
||||
data: {
|
||||
@@ -612,6 +583,9 @@ async function dispatchClickAtPoint(
|
||||
if (error) return failure(error);
|
||||
}
|
||||
if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" });
|
||||
const blocked = await verifyOverlay();
|
||||
if (blocked) return failure(blocked);
|
||||
if (deps.signal?.aborted) return failure({ code: "cancelled", message: "click aborted" });
|
||||
deps.onInputSent?.(tabId);
|
||||
markSent?.();
|
||||
attempted = true;
|
||||
@@ -683,40 +657,42 @@ async function clickVisualPoint(
|
||||
return changed();
|
||||
return null;
|
||||
};
|
||||
let bypass = false;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
if (deps.bypassOverlay) {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
bypass = true;
|
||||
}
|
||||
const invalid = await validate();
|
||||
if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } };
|
||||
const error = await dispatchClickAtPoint(
|
||||
return await withClickOverlay<ClickResult | RpcError>(
|
||||
target.tabId,
|
||||
point,
|
||||
params,
|
||||
deps,
|
||||
async () => {
|
||||
await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability.
|
||||
return validate();
|
||||
async (verifyOverlay) => {
|
||||
const invalid = await validate();
|
||||
if (invalid) return { ...invalid, data: { ...invalid.data, effect_state: "none" } };
|
||||
const error = await dispatchClickAtPoint(
|
||||
target.tabId,
|
||||
point,
|
||||
params,
|
||||
deps,
|
||||
verifyOverlay,
|
||||
async () => {
|
||||
await wait(32, deps.signal); // Scheduling opportunity, not a claim of page stability.
|
||||
return validate();
|
||||
},
|
||||
markSent,
|
||||
);
|
||||
if (error) return error;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: capture.ref,
|
||||
...point,
|
||||
});
|
||||
},
|
||||
markSent,
|
||||
true, // Visual target validation also needs the page blocker bypassed.
|
||||
);
|
||||
if (error) return error;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: capture.ref,
|
||||
...point,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
code: deps.signal?.aborted || isAbortError(error) ? "cancelled" : "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
data: { effect_state: "none" },
|
||||
};
|
||||
} finally {
|
||||
if (bypass) await deps.bypassOverlay!(target.tabId, false).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user