mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-10-02 07:34:35 +08:00
fix(vom): tolerate viewport rounding and honor AX modality
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// @vitest-environment node
|
||||
// Reuses the CI browser opt-in; every run owns its Chrome profile.
|
||||
import { createServer } from "node:http";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
@@ -24,6 +25,8 @@ const cases = [
|
||||
{ name: "ARIA modal", width: 64, role: "dialog", modal: true },
|
||||
{ name: "small native modal", width: 20, native: "showModal", small: true, modal: true },
|
||||
{ name: "small ARIA modal", width: 20, role: "dialog", small: true, modal: true },
|
||||
{ name: "ignored ARIA modal", width: 64, role: "dialog", ariaModal: true, ignored: true },
|
||||
{ name: "ARIA modal on a non-dialog", width: 64, ariaModal: true },
|
||||
{ name: "CSS backdrop", width: 64, backdrop: true },
|
||||
{ name: "pointer-transparent sidebar", width: 95, transparent: true },
|
||||
{ name: "iframe beside sidebar", width: 64, frame: true },
|
||||
@@ -39,6 +42,7 @@ function fixture(c: (typeof cases)[number]): string {
|
||||
width:${c.width}vw; height:${c.small ? "120px" : "100vh"}; max-width:none; max-height:none;
|
||||
background:white; border:0; padding:8px; z-index:10; ${c.transparent ? "pointer-events:none;" : ""} }
|
||||
#covered { position:absolute; top:200px; left:60vw; width:80px; }
|
||||
#offscreen { position:absolute; top:150vh; left:8px; width:120px; }
|
||||
#backdrop { position:fixed; inset:0; z-index:9; background:#0008; }
|
||||
iframe { width:100%; height:120px; border:0; }
|
||||
</style><main>
|
||||
@@ -46,139 +50,165 @@ function fixture(c: (typeof cases)[number]): string {
|
||||
<canvas aria-label="Main canvas" width="40" height="30"></canvas>
|
||||
${c.frame ? `<iframe srcdoc="<button id='child' onclick='parent.frameClicks++'>Frame action</button>"></iframe>` : ""}
|
||||
</main><button id="covered">Covered action</button>
|
||||
<button id="offscreen">Offscreen action</button>
|
||||
${c.backdrop ? '<div id="backdrop"></div>' : ""}
|
||||
<${tag} id="panel" ${c.role ? `role="${c.role}"` : ""} ${c.modal && !c.native ? 'aria-modal="true"' : c.role ? 'aria-modal="false"' : ""}>
|
||||
<${tag} id="panel" ${c.role ? `role="${c.role}"` : ""} ${c.ariaModal || (c.modal && !c.native) ? 'aria-modal="true"' : c.role ? 'aria-modal="false"' : ""} ${c.ignored ? 'aria-hidden="true"' : ""}>
|
||||
<button id="inside">Panel action</button>
|
||||
</${tag}>`;
|
||||
}
|
||||
|
||||
describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser observation layers", () => {
|
||||
it("retains usable page refs beside sidebars while preserving modal and backdrop behavior", async () => {
|
||||
it.each([
|
||||
{ deviceScale: 1, zoom: 1 },
|
||||
{ deviceScale: 1.25, zoom: 1 },
|
||||
{ deviceScale: 1.5, zoom: 1 },
|
||||
{ deviceScale: 1, zoom: 1.1 },
|
||||
])("preserves sidebar, modal and backdrop behavior at DPR $deviceScale / zoom $zoom", async ({
|
||||
deviceScale,
|
||||
zoom,
|
||||
}) => {
|
||||
const { withChrome } = await import(
|
||||
new URL(
|
||||
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
|
||||
import.meta.url,
|
||||
).href
|
||||
);
|
||||
await withChrome(
|
||||
{ executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 },
|
||||
async (send: Send) => {
|
||||
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
|
||||
url: "about:blank",
|
||||
});
|
||||
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId,
|
||||
flatten: true,
|
||||
});
|
||||
const local: Send = (method, params) => send(method, params, sessionId);
|
||||
const evaluate = async <T>(expression: string) => {
|
||||
const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>(
|
||||
"Runtime.evaluate",
|
||||
{ expression, returnByValue: true, awaitPromise: true },
|
||||
);
|
||||
expect(reply.exceptionDetails).toBeUndefined();
|
||||
return reply.result.value;
|
||||
};
|
||||
await local("Page.enable");
|
||||
await local("Page.bringToFront");
|
||||
await local("Emulation.setDeviceMetricsOverride", {
|
||||
width: 1280,
|
||||
height: 757,
|
||||
deviceScaleFactor: 1,
|
||||
mobile: false,
|
||||
});
|
||||
const graph = async (): Promise<CdpFrameGraph> => {
|
||||
const { frameTree } = await local<{ frameTree: FrameTree }>("Page.getFrameTree");
|
||||
const frames: CdpFrame[] = [];
|
||||
const visit = async (tree: FrameTree, parentFrameId?: string) => {
|
||||
const owner = parentFrameId
|
||||
? await local<{ backendNodeId: number }>("DOM.getFrameOwner", {
|
||||
frameId: tree.frame.id,
|
||||
})
|
||||
: undefined;
|
||||
frames.push({
|
||||
frameId: tree.frame.id,
|
||||
parentFrameId,
|
||||
target: { tabId: 4 },
|
||||
ownerBackendNodeId: owner?.backendNodeId,
|
||||
});
|
||||
for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id);
|
||||
// A normal origin is needed for Chrome's saved page zoom to apply.
|
||||
const server = createServer((_request, response) => {
|
||||
response.setHeader("Content-Type", "text/html");
|
||||
response.end("<!doctype html><title>Observation layers</title>");
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
|
||||
try {
|
||||
await withChrome(
|
||||
{ executable: process.env.BSK_CLICK_CHROME, deviceScale, zoom, startupTimeout: 30_000 },
|
||||
async (send: Send) => {
|
||||
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
|
||||
url: "about:blank",
|
||||
});
|
||||
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId,
|
||||
flatten: true,
|
||||
});
|
||||
const local: Send = (method, params) => send(method, params, sessionId);
|
||||
const evaluate = async <T>(expression: string) => {
|
||||
const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>(
|
||||
"Runtime.evaluate",
|
||||
{ expression, returnByValue: true, awaitPromise: true },
|
||||
);
|
||||
expect(reply.exceptionDetails).toBeUndefined();
|
||||
return reply.result.value;
|
||||
};
|
||||
await visit(frameTree);
|
||||
return { rootFrameId: frameTree.frame.id, frames };
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: (_tabId, method, params) => local(method, params),
|
||||
sendToTarget: (_target, method, params) => local(method, params),
|
||||
getAttachmentId: () => sessionId,
|
||||
getFrameGraph: graph,
|
||||
};
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => ({ windowId: 100, initialTabIds: [] }),
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("layers");
|
||||
const tab = { id: 4, windowId: 100, active: true, url: "about:blank" } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
for (const c of cases) {
|
||||
await evaluate(`document.open(); document.write(${JSON.stringify(fixture(c))}); document.close();
|
||||
await local("Page.enable");
|
||||
await local("Page.bringToFront");
|
||||
await local("Page.navigate", { url });
|
||||
await expect.poll(() => evaluate("document.title")).toBe("Observation layers");
|
||||
const metrics = await local<{ cssVisualViewport: { zoom: number } }>(
|
||||
"Page.getLayoutMetrics",
|
||||
);
|
||||
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(zoom, 4);
|
||||
expect(await evaluate<number>("devicePixelRatio")).toBeCloseTo(deviceScale * zoom, 4);
|
||||
const graph = async (): Promise<CdpFrameGraph> => {
|
||||
const { frameTree } = await local<{ frameTree: FrameTree }>("Page.getFrameTree");
|
||||
const frames: CdpFrame[] = [];
|
||||
const visit = async (tree: FrameTree, parentFrameId?: string) => {
|
||||
const owner = parentFrameId
|
||||
? await local<{ backendNodeId: number }>("DOM.getFrameOwner", {
|
||||
frameId: tree.frame.id,
|
||||
})
|
||||
: undefined;
|
||||
frames.push({
|
||||
frameId: tree.frame.id,
|
||||
parentFrameId,
|
||||
target: { tabId: 4 },
|
||||
ownerBackendNodeId: owner?.backendNodeId,
|
||||
});
|
||||
for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id);
|
||||
};
|
||||
await visit(frameTree);
|
||||
return { rootFrameId: frameTree.frame.id, frames };
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: (_tabId, method, params) => local(method, params),
|
||||
sendToTarget: (_target, method, params) => local(method, params),
|
||||
getAttachmentId: () => sessionId,
|
||||
getFrameGraph: graph,
|
||||
};
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => ({ windowId: 100, initialTabIds: [] }),
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("layers");
|
||||
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
for (const c of cases) {
|
||||
await evaluate(`document.open(); document.write(${JSON.stringify(fixture(c))}); document.close();
|
||||
window.mainClicks=[]; window.frameClicks=0;
|
||||
document.querySelector('#main').onclick=e=>mainClicks.push(e.isTrusted);
|
||||
document.querySelector('canvas').getContext('2d').fillRect(0,0,40,30);
|
||||
scrollTo(0,0);
|
||||
${c.native ? `document.querySelector('#panel').${c.native}();` : ""}`);
|
||||
if (c.frame)
|
||||
await expect
|
||||
.poll(() =>
|
||||
evaluate(
|
||||
"!!document.querySelector('iframe').contentDocument?.querySelector('#child')",
|
||||
),
|
||||
)
|
||||
.toBe(true);
|
||||
const result = await handleObserve(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId },
|
||||
{ cdp, tabsApi, conditionalSurfaceProbe: false },
|
||||
);
|
||||
expect(result, c.name + ": " + JSON.stringify(result)).not.toHaveProperty("code");
|
||||
if (!("text" in result)) throw new Error("Observation failed");
|
||||
const blocked = c.modal || c.backdrop || false;
|
||||
const mainRef = result.text.match(/(@e\d+) button "Main action"/)?.[1];
|
||||
expect(!!mainRef, c.name + "\n" + result.text).toBe(!blocked);
|
||||
expect(result.text.includes("@layers 2"), c.name + "\n" + result.text).toBe(blocked);
|
||||
expect(result.text.includes("[visual:screenshot]"), c.name + "\n" + result.text).toBe(
|
||||
!blocked,
|
||||
);
|
||||
if (!blocked) {
|
||||
const hit = await evaluate<boolean>(
|
||||
`(() => { const b=document.querySelector('#main'),r=b.getBoundingClientRect();return document.elementFromPoint(r.x+r.width/2,r.y+r.height/2)===b; })()`,
|
||||
);
|
||||
expect(hit, c.name).toBe(true);
|
||||
const clicked = await handleClick(
|
||||
if (c.frame)
|
||||
await expect
|
||||
.poll(() =>
|
||||
evaluate(
|
||||
"!!document.querySelector('iframe').contentDocument?.querySelector('#child')",
|
||||
),
|
||||
)
|
||||
.toBe(true);
|
||||
const result = await handleObserve(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, ref: mainRef },
|
||||
{ cdp, tabsApi },
|
||||
{ session_id: ctx.sessionId },
|
||||
{ cdp, tabsApi, conditionalSurfaceProbe: false },
|
||||
);
|
||||
expect(clicked, c.name + ": " + JSON.stringify(clicked)).not.toHaveProperty("code");
|
||||
expect(await evaluate("mainClicks"), c.name).toEqual([true]);
|
||||
}
|
||||
if (!c.transparent && !c.small && !blocked)
|
||||
expect(result.text, c.name).not.toContain('button "Covered action"');
|
||||
if (c.frame) {
|
||||
const frameRef = result.text.match(/(@e\d+) button "Frame action"/)?.[1];
|
||||
expect(frameRef, result.text).toBeDefined();
|
||||
const clicked = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, ref: frameRef },
|
||||
{ cdp, tabsApi },
|
||||
expect(result, c.name + ": " + JSON.stringify(result)).not.toHaveProperty("code");
|
||||
if (!("text" in result)) throw new Error("Observation failed");
|
||||
const blocked = c.modal || c.backdrop || false;
|
||||
const mainRef = result.text.match(/(@e\d+) button "Main action"/)?.[1];
|
||||
expect(!!mainRef, c.name + "\n" + result.text).toBe(!blocked);
|
||||
expect(result.text.includes("@layers 2"), c.name + "\n" + result.text).toBe(blocked);
|
||||
if (blocked) expect(result.text, c.name).not.toContain('button "Offscreen action"');
|
||||
if (c.backdrop) expect(result.text, c.name).toContain("L1 mask cover=100%");
|
||||
expect(result.text.includes("[visual:screenshot]"), c.name + "\n" + result.text).toBe(
|
||||
!blocked,
|
||||
);
|
||||
expect(clicked).not.toHaveProperty("code");
|
||||
expect(await evaluate("frameClicks")).toBe(1);
|
||||
if (!blocked) {
|
||||
const hit = await evaluate<boolean>(
|
||||
`(() => { const b=document.querySelector('#main'),r=b.getBoundingClientRect();return document.elementFromPoint(r.x+r.width/2,r.y+r.height/2)===b; })()`,
|
||||
);
|
||||
expect(hit, c.name).toBe(true);
|
||||
const clicked = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, ref: mainRef },
|
||||
{ cdp, tabsApi },
|
||||
);
|
||||
expect(clicked, c.name + ": " + JSON.stringify(clicked)).not.toHaveProperty("code");
|
||||
expect(await evaluate("mainClicks"), c.name).toEqual([true]);
|
||||
}
|
||||
if (!c.transparent && !c.small && !c.ignored && !blocked)
|
||||
expect(result.text, c.name).not.toContain('button "Covered action"');
|
||||
if (c.frame) {
|
||||
const frameRef = result.text.match(/(@e\d+) button "Frame action"/)?.[1];
|
||||
expect(frameRef, result.text).toBeDefined();
|
||||
const clicked = await handleClick(
|
||||
manager,
|
||||
{ session_id: ctx.sessionId, ref: frameRef },
|
||||
{ cdp, tabsApi },
|
||||
);
|
||||
expect(clicked).not.toHaveProperty("code");
|
||||
expect(await evaluate("frameClicks")).toBe(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
server.close((error) => (error ? reject(error) : resolve())),
|
||||
);
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
@@ -47,16 +47,25 @@ describe("semantic VOM graph", () => {
|
||||
{ tag: "dialog", role: "dialog", expected: false },
|
||||
{ tag: "div", role: "dialog", expected: false },
|
||||
{ tag: "div", role: "alertdialog", expected: false },
|
||||
{ tag: "div", role: "dialog", aria: "true", expected: true },
|
||||
{ tag: "div", role: "dialog", aria: "true", expected: false },
|
||||
{ tag: "div", role: "dialog", aria: "false", expected: false },
|
||||
{ tag: "div", role: "alertdialog", aria: "TRUE", expected: true },
|
||||
{ tag: "div", role: "alertdialog", aria: "TRUE", axModal: true, expected: true },
|
||||
{ tag: "div", role: "dialog", aria: "true", axModal: false, expected: false },
|
||||
{ tag: "dialog", role: "dialog", aria: "false", axModal: true, expected: true },
|
||||
{ tag: "dialog", role: "dialog", axModal: true, ignored: true, expected: false },
|
||||
])("resolves modality from state rather than dialog role/tag: %j", ({
|
||||
{ tag: "div", role: "dialog", aria: "true", ignored: true, expected: false },
|
||||
{ tag: "div", role: "dialog", aria: "true", axModal: true, ignored: true, expected: false },
|
||||
{ tag: "div", role: "dialog", aria: "true", missingAx: true, expected: true },
|
||||
{ tag: "div", role: "alertdialog", aria: "TRUE", missingAx: true, expected: true },
|
||||
{ tag: "dialog", role: "dialog", aria: "false", missingAx: true, expected: false },
|
||||
{ tag: "dialog", role: "dialog", missingAx: true, expected: false },
|
||||
])("uses AX modality when present, falling back to ARIA only without AX: %j", ({
|
||||
tag,
|
||||
role,
|
||||
axModal,
|
||||
aria,
|
||||
ignored,
|
||||
missingAx,
|
||||
expected,
|
||||
}) => {
|
||||
const scene = buildSemanticVomScene({
|
||||
@@ -65,18 +74,20 @@ describe("semantic VOM graph", () => {
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "dialog",
|
||||
backendDOMNodeId: 2,
|
||||
ignored,
|
||||
role: { type: "role", value: role },
|
||||
name: { type: "computedString", value: "Panel" },
|
||||
...(axModal !== undefined
|
||||
? { properties: [{ name: "modal", value: { value: axModal } }] }
|
||||
: {}),
|
||||
},
|
||||
],
|
||||
missingAx
|
||||
? []
|
||||
: [
|
||||
{
|
||||
nodeId: "dialog",
|
||||
backendDOMNodeId: 2,
|
||||
ignored,
|
||||
role: { type: "role", value: role },
|
||||
name: { type: "computedString", value: "Panel" },
|
||||
...(axModal !== undefined
|
||||
? { properties: [{ name: "modal", value: { value: axModal } }] }
|
||||
: {}),
|
||||
},
|
||||
],
|
||||
[dom(2, null, tag, { role, ...(aria ? { "aria-modal": aria } : {}) })],
|
||||
),
|
||||
],
|
||||
|
||||
@@ -505,9 +505,10 @@ export function resolveSemanticGraph(
|
||||
...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
|
||||
inputState: inputState(node, role, sourceValue, isSensitive),
|
||||
sensitive: isSensitive,
|
||||
modal:
|
||||
(node.ax?.ignored !== true && axProperty(node.ax, "modal") === "true") ||
|
||||
(attrs["aria-modal"] ?? "").toLowerCase() === "true",
|
||||
// An available AX result is authoritative, including ignored/nonmodal nodes.
|
||||
modal: node.ax
|
||||
? node.ax.ignored !== true && axProperty(node.ax, "modal") === "true"
|
||||
: (attrs["aria-modal"] ?? "").toLowerCase() === "true",
|
||||
disabled:
|
||||
(node.ax?.ignored !== true && axProperty(node.ax, "disabled") === "true") ||
|
||||
Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
|
||||
|
||||
@@ -93,6 +93,39 @@ describe("detectBlockingLayer", () => {
|
||||
expect(layer?.kind).toBe("mask");
|
||||
});
|
||||
|
||||
it("allows one CSS pixel of rounding at every viewport edge", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({
|
||||
id: 1,
|
||||
position: "fixed",
|
||||
rect: { x: 0.4, y: 0, w: 999.2, h: 799.7 },
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
expect(layer?.kind).toBe("mask");
|
||||
expect(layer?.coverage).toBeLessThan(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ x: 1.1, y: 0, w: 998.9, h: 800 },
|
||||
{ x: 0, y: 1.1, w: 1000, h: 798.9 },
|
||||
{ x: 0, y: 0, w: 998.9, h: 800 },
|
||||
{ x: 0, y: 0, w: 1000, h: 798.9 },
|
||||
])("does not fold a page with an edge beyond the rounding tolerance: %j", (rect) => {
|
||||
expect(detectBlockingLayer([node({ id: 1, position: "fixed", rect })], VP)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not use an area ratio to forgive a sidebar's larger pixel gap", () => {
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
[node({ id: 1, position: "fixed", rect: { x: 2, y: 0, w: 1998, h: 800 } })],
|
||||
{ width: 2000, height: 800 },
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("treats explicit modal nodes as modal without requiring a viewport cover", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
@@ -142,7 +175,7 @@ describe("detectBlockingLayer", () => {
|
||||
"absolute",
|
||||
"sticky",
|
||||
])("does not treat a wide %s sidebar as a global blocker", (position) => {
|
||||
for (const fraction of [0.36, 0.64, 0.95, 0.999]) {
|
||||
for (const fraction of [0.36, 0.64, 0.95, 0.99]) {
|
||||
for (const role of [undefined, "dialog", "alertdialog"]) {
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
|
||||
@@ -35,7 +35,7 @@ const action = { tag: "button", role: "button" };
|
||||
|
||||
describe("page and region occlusion", () => {
|
||||
it.each([
|
||||
0.36, 0.64, 0.95, 0.999,
|
||||
0.36, 0.64, 0.95, 0.99,
|
||||
])("retains uncovered refs beside a %s-width sidebar in both renderers", (fraction) => {
|
||||
const edge = viewport.width * (1 - fraction);
|
||||
for (const position of ["fixed", "absolute", "sticky"]) {
|
||||
@@ -152,14 +152,76 @@ describe("page and region occlusion", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("continues to fold background refs under a full-viewport CSS mask", () => {
|
||||
it("applies viewport rounding to child-frame region occlusion without folding the parent", () => {
|
||||
const scene: VomScene = {
|
||||
viewport,
|
||||
rootFrameId: "root",
|
||||
nodes: [
|
||||
node(1, null, { role: "RootWebArea" }),
|
||||
node(2, 1, { tag: "iframe", role: "Iframe", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node(3, 2, {
|
||||
...action,
|
||||
frameId: "child",
|
||||
name: "Covered child",
|
||||
rect: { x: 100, y: 20, w: 100, h: 30 },
|
||||
}),
|
||||
node(4, 2, {
|
||||
frameId: "child",
|
||||
position: "fixed",
|
||||
rect: { x: 0.4, y: 0, w: 999.2, h: 799.7 },
|
||||
}),
|
||||
node(5, 1, {
|
||||
...action,
|
||||
name: "Parent foreground",
|
||||
paintOrder: 2,
|
||||
rect: { x: 100, y: 20, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
};
|
||||
for (const result of rendered(scene)) {
|
||||
expect(result.text).toContain("L1 page");
|
||||
expect(result.ids).toEqual([5]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("requires explicit modality for an offscreen region (modal=%s)", (modal) => {
|
||||
const scene: VomScene = {
|
||||
viewport,
|
||||
rootFrameId: "root",
|
||||
nodes: [
|
||||
node(1, null, { role: "RootWebArea" }),
|
||||
node(2, 1, { ...action, name: "Offscreen", rect: { x: 10, y: 1200, w: 100, h: 30 } }),
|
||||
node(3, 1, {
|
||||
tag: "dialog",
|
||||
role: "dialog",
|
||||
modal,
|
||||
position: "absolute",
|
||||
paintOrder: 10,
|
||||
rect: { x: 0, y: 1100, w: 500, h: 200 },
|
||||
}),
|
||||
],
|
||||
};
|
||||
for (const result of rendered(scene)) {
|
||||
expect(result.text).toContain("L1 page");
|
||||
expect(result.ids).toEqual(modal ? [] : [2]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ x: 0, y: 0, w: 1000, h: 800 },
|
||||
{ x: 0.4, y: 0, w: 999.2, h: 799.7 },
|
||||
])("folds viewport and offscreen background refs under a CSS mask: %j", (rect) => {
|
||||
const scene: VomScene = {
|
||||
viewport,
|
||||
rootFrameId: "root",
|
||||
nodes: [
|
||||
node(1, null, { role: "RootWebArea" }),
|
||||
node(2, 1, { ...action, name: "Background", rect: { x: 10, y: 10, w: 100, h: 30 } }),
|
||||
node(3, 1, { position: "fixed", paintOrder: 10, rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node(3, 1, { position: "fixed", paintOrder: 10, rect }),
|
||||
node(4, 1, { ...action, name: "Offscreen", rect: { x: 10, y: 1200, w: 100, h: 30 } }),
|
||||
],
|
||||
};
|
||||
for (const result of rendered(scene)) {
|
||||
|
||||
+22
-14
@@ -1,7 +1,7 @@
|
||||
import type { BlockingLayer, Rect, Viewport, VomNode } from "./types";
|
||||
|
||||
/** At/above this coverage with no inputs we call it an opaque mask. */
|
||||
export const MASK_COVERAGE_THRESHOLD = 0.9;
|
||||
/** CSS pixels of layout rounding allowed at each viewport edge. */
|
||||
const VIEWPORT_EDGE_TOLERANCE = 1;
|
||||
|
||||
const POSITIONED = new Set(["fixed", "absolute", "sticky"]);
|
||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||
@@ -23,29 +23,37 @@ export function coverage(rect: Rect | null, vp: Viewport): number {
|
||||
return Math.min(1, overlap / (vp.width * vp.height));
|
||||
}
|
||||
|
||||
export function spansViewport(rect: Rect | null, vp: Viewport): boolean {
|
||||
if (!rect || vp.width <= 0 || vp.height <= 0) return false;
|
||||
const t = VIEWPORT_EDGE_TOLERANCE;
|
||||
return (
|
||||
rect.x <= t &&
|
||||
rect.y <= t &&
|
||||
rect.x + rect.w >= vp.width - t &&
|
||||
rect.y + rect.h >= vp.height - t
|
||||
);
|
||||
}
|
||||
|
||||
function isBlockingCandidate(node: VomNode): boolean {
|
||||
if (node.pointerEvents === "none") return false;
|
||||
return POSITIONED.has(node.position);
|
||||
}
|
||||
|
||||
function hasModalFeature(node: VomNode): boolean {
|
||||
/** Naming only: dialog roles/tags do not determine whether a layer blocks. */
|
||||
function looksLikeDialog(node: VomNode): boolean {
|
||||
const role = normalizedRole(node);
|
||||
const tag = normalizedTag(node);
|
||||
return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog";
|
||||
}
|
||||
|
||||
function classifyLayer(
|
||||
nodes: VomNode[],
|
||||
members: Set<number>,
|
||||
blockerCoverage: number,
|
||||
): BlockingLayer["kind"] {
|
||||
function classifyLayer(nodes: VomNode[], members: Set<number>): BlockingLayer["kind"] {
|
||||
for (const node of nodes) {
|
||||
if (!members.has(node.id)) continue;
|
||||
if (hasModalFeature(node)) return "modal";
|
||||
if (FORM_TAGS.has(normalizedTag(node))) return "modal";
|
||||
if (normalizedTag(node) === "iframe") return "modal";
|
||||
if (looksLikeDialog(node)) return "modal";
|
||||
const tag = normalizedTag(node);
|
||||
if (FORM_TAGS.has(tag) || tag === "iframe") return "modal";
|
||||
}
|
||||
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
|
||||
return "mask";
|
||||
}
|
||||
|
||||
export function detectBlockingLayer(
|
||||
@@ -64,7 +72,7 @@ export function detectBlockingLayer(
|
||||
const cov = coverage(node.rect, vp);
|
||||
// A large sidebar can leave usable page content beside it. Only actual
|
||||
// modality or a cover spanning the whole viewport can fold the base page.
|
||||
const qualifies = cov === 1 || (node.modal === true && cov > 0);
|
||||
const qualifies = spansViewport(node.rect, vp) || (node.modal === true && cov > 0);
|
||||
if (!qualifies) continue;
|
||||
if (
|
||||
blocker === null ||
|
||||
@@ -87,7 +95,7 @@ export function detectBlockingLayer(
|
||||
|
||||
return {
|
||||
rootId: blocker.node.id,
|
||||
kind: classifyLayer(nodes, members, blocker.coverage),
|
||||
kind: classifyLayer(nodes, members),
|
||||
coverage: blocker.coverage,
|
||||
members,
|
||||
};
|
||||
|
||||
+10
-26
@@ -1,4 +1,4 @@
|
||||
import { coverage, detectBlockingLayer } from "./layers";
|
||||
import { coverage, detectBlockingLayer, spansViewport } from "./layers";
|
||||
import type {
|
||||
ActiveScopeBlock,
|
||||
BlockingLayer,
|
||||
@@ -1024,19 +1024,6 @@ function interactionPoints(rect: Rect): Array<[number, number]> {
|
||||
];
|
||||
}
|
||||
|
||||
function isModalLike(node: VomNode): boolean {
|
||||
const role = normalizedRole(node);
|
||||
const tag = normalizedTag(node);
|
||||
return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog";
|
||||
}
|
||||
|
||||
function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number): number {
|
||||
if (isModalLike(node)) return 4;
|
||||
if (viewportCoverage >= 0.9) return 3;
|
||||
if (viewportCoverage >= 0.15) return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function paintLineageByFrame(
|
||||
node: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
@@ -1090,7 +1077,7 @@ function isBlockedByRegion(
|
||||
blocker: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
viewportCoverage: number,
|
||||
fullCover: boolean,
|
||||
lineageCache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): boolean {
|
||||
if (target.id === blocker.id) return false;
|
||||
@@ -1103,7 +1090,7 @@ function isBlockedByRegion(
|
||||
const points = interactionPoints(target.rect);
|
||||
|
||||
if (
|
||||
viewportCoverage < 1 &&
|
||||
!fullCover &&
|
||||
paintNodes.target.paintOrder >= paintNodes.blocker.paintOrder &&
|
||||
points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y))
|
||||
) {
|
||||
@@ -1152,15 +1139,12 @@ function applyActiveRegionPolicy(
|
||||
const lineageCache = new Map<number, Map<string | undefined, VomNode>>();
|
||||
const candidates = nodes
|
||||
.filter(isPositionedRegionCandidate)
|
||||
.map((node) => {
|
||||
const viewportCoverage = coverage(node.rect, scene.viewport);
|
||||
return {
|
||||
node,
|
||||
viewportCoverage,
|
||||
priority: activeRegionCandidatePriority(node, viewportCoverage),
|
||||
};
|
||||
})
|
||||
.filter((candidate) => candidate.priority > 1 || candidate.viewportCoverage > 0);
|
||||
.map((node) => ({
|
||||
node,
|
||||
viewportCoverage: coverage(node.rect, scene.viewport),
|
||||
fullCover: spansViewport(node.rect, scene.viewport),
|
||||
}))
|
||||
.filter((candidate) => candidate.viewportCoverage > 0 || candidate.node.modal === true);
|
||||
|
||||
const blockedRoots = new Set<number>();
|
||||
for (const target of nodes) {
|
||||
@@ -1171,7 +1155,7 @@ function applyActiveRegionPolicy(
|
||||
candidate.node,
|
||||
parentMap,
|
||||
nodesById,
|
||||
candidate.viewportCoverage,
|
||||
candidate.fullCover,
|
||||
lineageCache,
|
||||
),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user