diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76a83cb..af90985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,7 +207,7 @@ jobs: working-directory: apps/extension env: BSK_CLICK_CHROME: google-chrome - run: pnpm exec vitest run src/tools/__tests__/click.browser.test.ts src/debug/__tests__/*.browser.test.ts + run: pnpm exec vitest run src/tools/__tests__/click.browser.test.ts src/tools/__tests__/observation-layers.browser.test.ts src/debug/__tests__/*.browser.test.ts node-scripts: name: Node script tests diff --git a/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts b/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts new file mode 100644 index 0000000..7befb99 --- /dev/null +++ b/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts @@ -0,0 +1,214 @@ +// @vitest-environment node +// Reuses the CI browser opt-in; every run owns its Chrome profile. +import { createServer } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleClick } from "../interaction"; +import { handleObserve } from "../observation"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type FrameTree = { frame: { id: string }; childFrames?: FrameTree[] }; + +const cases = [ + { name: "sidebar 64%", width: 64 }, + { name: "sidebar 95%", width: 95 }, + { name: "nonmodal dialog role", width: 64, role: "dialog" }, + { name: "nonmodal alertdialog role", width: 64, role: "alertdialog" }, + { name: "native show", width: 64, native: "show" }, + { name: "native showModal", width: 64, native: "showModal", modal: true }, + { name: "ARIA modal", width: 64, role: "dialog", modal: true }, + { name: "small native modal", width: 20, native: "showModal", small: true, modal: true }, + { name: "small ARIA modal", width: 20, role: "dialog", small: true, modal: true }, + { name: "ignored ARIA modal", width: 64, role: "dialog", ariaModal: true, ignored: true }, + { name: "ARIA modal on a non-dialog", width: 64, ariaModal: true }, + { name: "CSS backdrop", width: 64, backdrop: true }, + { name: "pointer-transparent sidebar", width: 95, transparent: true }, + { name: "iframe beside sidebar", width: 64, frame: true }, +]; + +function fixture(c: (typeof cases)[number]): string { + const tag = c.native ? "dialog" : "aside"; + return `
+ + + ${c.frame ? `` : ""} +
+ + ${c.backdrop ? '
' : ""} + <${tag} id="panel" ${c.role ? `role="${c.role}"` : ""} ${c.ariaModal || (c.modal && !c.native) ? 'aria-modal="true"' : c.role ? 'aria-modal="false"' : ""} ${c.ignored ? 'aria-hidden="true"' : ""}> + + `; +} + +describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser observation layers", () => { + it.each([ + { deviceScale: 1, zoom: 1 }, + { deviceScale: 1.25, zoom: 1 }, + { deviceScale: 1.5, zoom: 1 }, + { deviceScale: 1, zoom: 1.1 }, + ])("preserves sidebar, modal and backdrop behavior at DPR $deviceScale / zoom $zoom", async ({ + deviceScale, + zoom, + }) => { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + // A normal origin is needed for Chrome's saved page zoom to apply. + const server = createServer((_request, response) => { + response.setHeader("Content-Type", "text/html"); + response.end("Observation layers"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + try { + await withChrome( + { executable: process.env.BSK_CLICK_CHROME, deviceScale, zoom, startupTimeout: 30_000 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + const local: Send = (method, params) => send(method, params, sessionId); + const evaluate = async (expression: string) => { + const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await local("Page.enable"); + await local("Page.bringToFront"); + await local("Page.navigate", { url }); + await expect.poll(() => evaluate("document.title")).toBe("Observation layers"); + const metrics = await local<{ cssVisualViewport: { zoom: number } }>( + "Page.getLayoutMetrics", + ); + expect(metrics.cssVisualViewport.zoom).toBeCloseTo(zoom, 4); + expect(await evaluate("devicePixelRatio")).toBeCloseTo(deviceScale * zoom, 4); + const graph = async (): Promise => { + const { frameTree } = await local<{ frameTree: FrameTree }>("Page.getFrameTree"); + const frames: CdpFrame[] = []; + const visit = async (tree: FrameTree, parentFrameId?: string) => { + const owner = parentFrameId + ? await local<{ backendNodeId: number }>("DOM.getFrameOwner", { + frameId: tree.frame.id, + }) + : undefined; + frames.push({ + frameId: tree.frame.id, + parentFrameId, + target: { tabId: 4 }, + ownerBackendNodeId: owner?.backendNodeId, + }); + for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); + }; + await visit(frameTree); + return { rootFrameId: frameTree.frame.id, frames }; + }; + const cdp: CdpRunner = { + send: (_tabId, method, params) => local(method, params), + sendToTarget: (_target, method, params) => local(method, params), + getAttachmentId: () => sessionId, + getFrameGraph: graph, + }; + const manager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("layers"); + const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab; + const tabsApi = { get: async () => tab, query: async () => [tab] }; + for (const c of cases) { + await evaluate(`document.open(); document.write(${JSON.stringify(fixture(c))}); document.close(); + window.mainClicks=[]; window.frameClicks=0; + document.querySelector('#main').onclick=e=>mainClicks.push(e.isTrusted); + document.querySelector('canvas').getContext('2d').fillRect(0,0,40,30); + scrollTo(0,0); + ${c.native ? `document.querySelector('#panel').${c.native}();` : ""}`); + if (c.frame) + await expect + .poll(() => + evaluate( + "!!document.querySelector('iframe').contentDocument?.querySelector('#child')", + ), + ) + .toBe(true); + const result = await handleObserve( + manager, + { session_id: ctx.sessionId }, + { cdp, tabsApi, conditionalSurfaceProbe: false }, + ); + expect(result, c.name + ": " + JSON.stringify(result)).not.toHaveProperty("code"); + if (!("text" in result)) throw new Error("Observation failed"); + const blocked = c.modal || c.backdrop || false; + const mainRef = result.text.match(/(@e\d+) button "Main action"/)?.[1]; + expect(!!mainRef, c.name + "\n" + result.text).toBe(!blocked); + expect(result.text.includes("@layers 2"), c.name + "\n" + result.text).toBe(blocked); + if (blocked) expect(result.text, c.name).not.toContain('button "Offscreen action"'); + if (c.backdrop) expect(result.text, c.name).toContain("L1 mask cover=100%"); + expect(result.text.includes("[visual:screenshot]"), c.name + "\n" + result.text).toBe( + !blocked, + ); + if (!blocked) { + const hit = await evaluate( + `(() => { const b=document.querySelector('#main'),r=b.getBoundingClientRect();return document.elementFromPoint(r.x+r.width/2,r.y+r.height/2)===b; })()`, + ); + expect(hit, c.name).toBe(true); + const clicked = await handleClick( + manager, + { session_id: ctx.sessionId, ref: mainRef }, + { cdp, tabsApi }, + ); + expect(clicked, c.name + ": " + JSON.stringify(clicked)).not.toHaveProperty("code"); + expect(await evaluate("mainClicks"), c.name).toEqual([true]); + } + if (!c.transparent && !c.small && !c.ignored && !blocked) + expect(result.text, c.name).not.toContain('button "Covered action"'); + if (c.frame) { + const frameRef = result.text.match(/(@e\d+) button "Frame action"/)?.[1]; + expect(frameRef, result.text).toBeDefined(); + const clicked = await handleClick( + manager, + { session_id: ctx.sessionId, ref: frameRef }, + { cdp, tabsApi }, + ); + expect(clicked).not.toHaveProperty("code"); + expect(await evaluate("frameClicks")).toBe(1); + } + } + }, + ); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }, 90_000); +}); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index dfcae6b..24336a8 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -2476,7 +2476,7 @@ describe("buildFrameVomScene", () => { expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text"); }); - it("marks captured dialog elements as modal without AX role or aria-modal", () => { + it("does not infer modality from a captured dialog tag without modal state", () => { const scene = buildTestScene([], { viewport: { width: 640, height: 480 }, excludedBackendNodeIds: new Set(), @@ -2497,7 +2497,7 @@ describe("buildFrameVomScene", () => { expect(scene.nodes[0]).toEqual( expect.objectContaining({ backendNodeId: 50, - modal: true, + modal: false, }), ); }); diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index 4505091..9449395 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -41,6 +41,60 @@ function document( } describe("semantic VOM graph", () => { + it.each([ + { tag: "dialog", role: "dialog", axModal: true, expected: true }, + { tag: "dialog", role: "dialog", axModal: false, expected: false }, + { tag: "dialog", role: "dialog", expected: false }, + { tag: "div", role: "dialog", expected: false }, + { tag: "div", role: "alertdialog", expected: false }, + { tag: "div", role: "dialog", aria: "true", expected: false }, + { tag: "div", role: "dialog", aria: "false", expected: false }, + { tag: "div", role: "alertdialog", aria: "TRUE", axModal: true, expected: true }, + { tag: "div", role: "dialog", aria: "true", axModal: false, expected: false }, + { tag: "dialog", role: "dialog", aria: "false", axModal: true, expected: true }, + { tag: "dialog", role: "dialog", axModal: true, ignored: true, expected: false }, + { tag: "div", role: "dialog", aria: "true", ignored: true, expected: false }, + { tag: "div", role: "dialog", aria: "true", axModal: true, ignored: true, expected: false }, + { tag: "div", role: "dialog", aria: "true", missingAx: true, expected: true }, + { tag: "div", role: "alertdialog", aria: "TRUE", missingAx: true, expected: true }, + { tag: "dialog", role: "dialog", aria: "false", missingAx: true, expected: false }, + { tag: "dialog", role: "dialog", missingAx: true, expected: false }, + ])("uses AX modality when present, falling back to ARIA only without AX: %j", ({ + tag, + role, + axModal, + aria, + ignored, + missingAx, + expected, + }) => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + missingAx + ? [] + : [ + { + nodeId: "dialog", + backendDOMNodeId: 2, + ignored, + role: { type: "role", value: role }, + name: { type: "computedString", value: "Panel" }, + ...(axModal !== undefined + ? { properties: [{ name: "modal", value: { value: axModal } }] } + : {}), + }, + ], + [dom(2, null, tag, { role, ...(aria ? { "aria-modal": aria } : {}) })], + ), + ], + }); + expect(scene.nodes.find((node) => node.backendNodeId === 2)?.modal).toBe(expected); + }); + it.each<{ ax: boolean; ignored: boolean; attrs: Record; disabled: boolean }>([ { ax: true, ignored: false, attrs: {}, disabled: true }, { ax: false, ignored: false, attrs: {}, disabled: false }, diff --git a/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts b/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts index 7ee1db2..cf6a71f 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts @@ -608,6 +608,7 @@ it.each([ f.scene.nodes.push(node(2, 1, kind === "frame" ? "iframe" : "canvas")); f.scene.nodes.push({ ...node(10, 1, "dialog", "Login"), + modal: true, position: "fixed", rect: { x: 0, y: 0, w: 800, h: 600 }, }); @@ -641,6 +642,7 @@ it("preserves modal Canvas with omitted semantics and admits proven foreground f node(1, null, "rootwebarea"), { ...node(10, 1, "dialog", "Login"), + modal: true, position: "fixed", rect: { x: 0, y: 0, w: 800, h: 600 }, }, @@ -798,6 +800,7 @@ it("keeps excluded Canvas out of every continuation page", async () => { f.scene.rootFrameId = "top"; f.scene.nodes.push({ ...node(10, 1, "dialog", "Login"), + modal: true, position: "fixed", rect: { x: 0, y: 0, w: 800, h: 600 }, }); diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 6d7aff6..8858a8e 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -505,10 +505,10 @@ export function resolveSemanticGraph( ...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}), inputState: inputState(node, role, sourceValue, isSensitive), sensitive: isSensitive, - modal: - ["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") || - tag === "dialog" || - (attrs["aria-modal"] ?? "").toLowerCase() === "true", + // An available AX result is authoritative, including ignored/nonmodal nodes. + modal: node.ax + ? node.ax.ignored !== true && axProperty(node.ax, "modal") === "true" + : (attrs["aria-modal"] ?? "").toLowerCase() === "true", disabled: (node.ax?.ignored !== true && axProperty(node.ax, "disabled") === "true") || Object.prototype.hasOwnProperty.call(attrs, "disabled") || diff --git a/packages/vom/src/__tests__/layers.test.ts b/packages/vom/src/__tests__/layers.test.ts index acc7dc8..4ecdaef 100644 --- a/packages/vom/src/__tests__/layers.test.ts +++ b/packages/vom/src/__tests__/layers.test.ts @@ -75,7 +75,7 @@ describe("detectBlockingLayer", () => { expect([...(layer?.members ?? [])].sort((a, b) => a - b)).toEqual([2, 3, 4]); }); - it("classifies a near-full blocker without modal signals or controls as a mask", () => { + it("classifies a full-viewport blocker without modal signals or controls as a mask", () => { const layer = detectBlockingLayer( [ node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), @@ -93,7 +93,40 @@ describe("detectBlockingLayer", () => { expect(layer?.kind).toBe("mask"); }); - it("treats explicit modal nodes as modal even below the generic coverage threshold", () => { + it("allows one CSS pixel of rounding at every viewport edge", () => { + const layer = detectBlockingLayer( + [ + node({ + id: 1, + position: "fixed", + rect: { x: 0.4, y: 0, w: 999.2, h: 799.7 }, + }), + ], + VP, + ); + expect(layer?.kind).toBe("mask"); + expect(layer?.coverage).toBeLessThan(1); + }); + + it.each([ + { x: 1.1, y: 0, w: 998.9, h: 800 }, + { x: 0, y: 1.1, w: 1000, h: 798.9 }, + { x: 0, y: 0, w: 998.9, h: 800 }, + { x: 0, y: 0, w: 1000, h: 798.9 }, + ])("does not fold a page with an edge beyond the rounding tolerance: %j", (rect) => { + expect(detectBlockingLayer([node({ id: 1, position: "fixed", rect })], VP)).toBeNull(); + }); + + it("does not use an area ratio to forgive a sidebar's larger pixel gap", () => { + expect( + detectBlockingLayer( + [node({ id: 1, position: "fixed", rect: { x: 2, y: 0, w: 1998, h: 800 } })], + { width: 2000, height: 800 }, + ), + ).toBeNull(); + }); + + it("treats explicit modal nodes as modal without requiring a viewport cover", () => { const layer = detectBlockingLayer( [ node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), @@ -115,7 +148,7 @@ describe("detectBlockingLayer", () => { expect(layer?.kind).toBe("modal"); }); - it("detects modal features regardless of role or tag casing", () => { + it("classifies an explicit modal regardless of role or tag casing", () => { const layer = detectBlockingLayer( [ node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), @@ -124,6 +157,7 @@ describe("detectBlockingLayer", () => { parentId: 1, tag: "DIV", role: "Dialog", + modal: true, rect: { x: 300, y: 200, w: 400, h: 300 }, paintOrder: 90, position: "fixed", @@ -136,6 +170,67 @@ describe("detectBlockingLayer", () => { expect(layer?.kind).toBe("modal"); }); + it.each([ + "fixed", + "absolute", + "sticky", + ])("does not treat a wide %s sidebar as a global blocker", (position) => { + for (const fraction of [0.36, 0.64, 0.95, 0.99]) { + for (const role of [undefined, "dialog", "alertdialog"]) { + expect( + detectBlockingLayer( + [ + node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), + node({ + id: 2, + parentId: 1, + tag: role ? "dialog" : "aside", + role, + modal: false, + position, + paintOrder: 10, + rect: { x: 1000 * (1 - fraction), y: 0, w: 1000 * fraction, h: 800 }, + }), + ], + VP, + ), + ).toBeNull(); + } + } + }); + + it.each(["fixed", "absolute"])("recognizes a small explicit %s modal", (position) => { + const layer = detectBlockingLayer( + [ + node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }), + node({ + id: 2, + parentId: 1, + modal: true, + position, + paintOrder: 10, + rect: { x: 400, y: 300, w: 100, h: 60 }, + }), + ], + VP, + ); + expect(layer).toMatchObject({ rootId: 2, kind: "modal" }); + }); + + it("keeps child-frame modality local and ignores offscreen modal nodes", () => { + const modal = node({ + id: 2, + frameId: "child", + modal: true, + position: "fixed", + rect: { x: 0, y: 0, w: 1000, h: 800 }, + }); + expect(detectBlockingLayer([modal], VP, "root")).toBeNull(); + expect( + detectBlockingLayer([{ ...modal, rect: { x: 1000, y: 0, w: 200, h: 100 } }], VP), + ).toBeNull(); + }); + it("ignores small toasts and pointer-events:none covers", () => { expect( detectBlockingLayer( diff --git a/packages/vom/src/__tests__/occlusion.test.ts b/packages/vom/src/__tests__/occlusion.test.ts new file mode 100644 index 0000000..0da5c65 --- /dev/null +++ b/packages/vom/src/__tests__/occlusion.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it } from "vitest"; +import { prepareObservationRender, renderVom } from "../render"; +import type { VomNode, VomScene } from "../types"; + +function node(id: number, parentId: number | null, props: Partial = {}): VomNode { + return { + id, + backendNodeId: id, + parentId, + frameId: "root", + tag: "div", + rect: null, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + ...props, + }; +} + +function rendered(scene: VomScene) { + const legacy = renderVom(scene, { activeRegionPolicy: true }); + const prepared = prepareObservationRender(scene, { activeRegionPolicy: true }); + const rows = Array.from({ [Symbol.iterator]: () => prepared.rows }); + return [ + { text: legacy.text, ids: legacy.refs.map((ref) => ref.backendNodeId) }, + { + text: [...prepared.headers, ...rows.map((row) => row.text)].join("\n"), + ids: rows.flatMap((row) => (row.ref ? [row.ref.backendNodeId] : [])), + }, + ]; +} + +const viewport = { width: 1000, height: 800 }; +const action = { tag: "button", role: "button" }; + +describe("page and region occlusion", () => { + it.each([ + 0.36, 0.64, 0.95, 0.99, + ])("retains uncovered refs beside a %s-width sidebar in both renderers", (fraction) => { + const edge = viewport.width * (1 - fraction); + for (const position of ["fixed", "absolute", "sticky"]) { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { + ...action, + name: "Main action", + rect: { x: edge / 4, y: 20, w: edge / 2, h: 30 }, + }), + node(3, 1, { + ...action, + name: "Covered action", + rect: { x: edge + 10, y: 100, w: 100, h: 30 }, + }), + node(4, 1, { + tag: "aside", + role: "complementary", + name: "Sidebar", + position, + paintOrder: 10, + rect: { x: edge, y: 0, w: viewport.width * fraction, h: viewport.height }, + }), + node(5, 4, { + ...action, + name: "Sidebar action", + paintOrder: 10, + rect: { x: edge + 10, y: 150, w: 100, h: 30 }, + }), + node(6, 1, { + ...action, + name: "Foreground action", + paintOrder: 11, + rect: { x: edge + 10, y: 200, w: 100, h: 30 }, + }), + ], + }; + for (const result of rendered(scene)) { + expect(result.text).toContain("L1 page"); + expect(result.text).not.toContain("occluded by L1"); + expect(result.ids).toEqual([2, 5, 6]); + } + } + }); + + it("keeps actions outside a near-full child-frame region", () => { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { tag: "iframe", role: "Iframe", rect: { x: 0, y: 0, w: 1000, h: 800 } }), + node(3, 2, { + ...action, + frameId: "child", + name: "Uncovered child", + paintOrder: 1, + rect: { x: 5, y: 20, w: 30, h: 30 }, + }), + node(4, 2, { + frameId: "child", + position: "fixed", + paintOrder: 10, + rect: { x: 50, y: 0, w: 950, h: 800 }, + }), + node(5, 2, { + ...action, + frameId: "child", + name: "Covered child", + paintOrder: 1, + rect: { x: 100, y: 20, w: 100, h: 30 }, + }), + node(6, 1, { + ...action, + name: "Parent foreground", + paintOrder: 2, + rect: { x: 100, y: 20, w: 100, h: 30 }, + }), + ], + }; + for (const result of rendered(scene)) expect(result.ids).toEqual([3, 6]); + }); + + it.each([false, true])("only folds the whole page for an explicit modal (modal=%s)", (modal) => { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { ...action, name: "Background", rect: { x: 10, y: 10, w: 100, h: 30 } }), + node(3, 1, { + tag: "dialog", + role: "dialog", + name: "Dialog", + modal, + position: "fixed", + paintOrder: 10, + rect: { x: 400, y: 300, w: 200, h: 100 }, + }), + node(4, 3, { + ...action, + name: "Dialog action", + paintOrder: 10, + rect: { x: 450, y: 320, w: 100, h: 30 }, + }), + ], + }; + for (const result of rendered(scene)) { + expect(result.ids).toEqual(modal ? [4] : [2, 4]); + expect(result.text.includes("L1 modal")).toBe(modal); + } + }); + + it("applies viewport rounding to child-frame region occlusion without folding the parent", () => { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { tag: "iframe", role: "Iframe", rect: { x: 0, y: 0, w: 1000, h: 800 } }), + node(3, 2, { + ...action, + frameId: "child", + name: "Covered child", + rect: { x: 100, y: 20, w: 100, h: 30 }, + }), + node(4, 2, { + frameId: "child", + position: "fixed", + rect: { x: 0.4, y: 0, w: 999.2, h: 799.7 }, + }), + node(5, 1, { + ...action, + name: "Parent foreground", + paintOrder: 2, + rect: { x: 100, y: 20, w: 100, h: 30 }, + }), + ], + }; + for (const result of rendered(scene)) { + expect(result.text).toContain("L1 page"); + expect(result.ids).toEqual([5]); + } + }); + + it.each([ + false, + true, + ])("requires explicit modality for an offscreen region (modal=%s)", (modal) => { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { ...action, name: "Offscreen", rect: { x: 10, y: 1200, w: 100, h: 30 } }), + node(3, 1, { + tag: "dialog", + role: "dialog", + modal, + position: "absolute", + paintOrder: 10, + rect: { x: 0, y: 1100, w: 500, h: 200 }, + }), + ], + }; + for (const result of rendered(scene)) { + expect(result.text).toContain("L1 page"); + expect(result.ids).toEqual(modal ? [] : [2]); + } + }); + + it.each([ + { x: 0, y: 0, w: 1000, h: 800 }, + { x: 0.4, y: 0, w: 999.2, h: 799.7 }, + ])("folds viewport and offscreen background refs under a CSS mask: %j", (rect) => { + const scene: VomScene = { + viewport, + rootFrameId: "root", + nodes: [ + node(1, null, { role: "RootWebArea" }), + node(2, 1, { ...action, name: "Background", rect: { x: 10, y: 10, w: 100, h: 30 } }), + node(3, 1, { position: "fixed", paintOrder: 10, rect }), + node(4, 1, { ...action, name: "Offscreen", rect: { x: 10, y: 1200, w: 100, h: 30 } }), + ], + }; + for (const result of rendered(scene)) { + expect(result.text).toContain("L1 mask cover=100%"); + expect(result.ids).toEqual([]); + } + }); +}); diff --git a/packages/vom/src/layers.ts b/packages/vom/src/layers.ts index a5a7d6b..b2ec630 100644 --- a/packages/vom/src/layers.ts +++ b/packages/vom/src/layers.ts @@ -1,9 +1,7 @@ import type { BlockingLayer, Rect, Viewport, VomNode } from "./types"; -/** Default fraction of the viewport an overlay must cover to block. */ -export const BLOCK_COVERAGE_THRESHOLD = 0.6; -/** At/above this coverage with no inputs we call it an opaque mask. */ -export const MASK_COVERAGE_THRESHOLD = 0.9; +/** CSS pixels of layout rounding allowed at each viewport edge. */ +const VIEWPORT_EDGE_TOLERANCE = 1; const POSITIONED = new Set(["fixed", "absolute", "sticky"]); const FORM_TAGS = new Set(["input", "textarea", "select"]); @@ -25,29 +23,37 @@ export function coverage(rect: Rect | null, vp: Viewport): number { return Math.min(1, overlap / (vp.width * vp.height)); } +export function spansViewport(rect: Rect | null, vp: Viewport): boolean { + if (!rect || vp.width <= 0 || vp.height <= 0) return false; + const t = VIEWPORT_EDGE_TOLERANCE; + return ( + rect.x <= t && + rect.y <= t && + rect.x + rect.w >= vp.width - t && + rect.y + rect.h >= vp.height - t + ); +} + function isBlockingCandidate(node: VomNode): boolean { if (node.pointerEvents === "none") return false; return POSITIONED.has(node.position); } -function hasModalFeature(node: VomNode): boolean { +/** Naming only: dialog roles/tags do not determine whether a layer blocks. */ +function looksLikeDialog(node: VomNode): boolean { const role = normalizedRole(node); const tag = normalizedTag(node); return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog"; } -function classifyLayer( - nodes: VomNode[], - members: Set, - blockerCoverage: number, -): BlockingLayer["kind"] { +function classifyLayer(nodes: VomNode[], members: Set): BlockingLayer["kind"] { for (const node of nodes) { if (!members.has(node.id)) continue; - if (hasModalFeature(node)) return "modal"; - if (FORM_TAGS.has(normalizedTag(node))) return "modal"; - if (normalizedTag(node) === "iframe") return "modal"; + if (looksLikeDialog(node)) return "modal"; + const tag = normalizedTag(node); + if (FORM_TAGS.has(tag) || tag === "iframe") return "modal"; } - return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal"; + return "mask"; } export function detectBlockingLayer( @@ -64,7 +70,9 @@ export function detectBlockingLayer( if (rootFrameId !== undefined && node.frameId !== rootFrameId) continue; if (!isBlockingCandidate(node)) continue; const cov = coverage(node.rect, vp); - const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15); + // A large sidebar can leave usable page content beside it. Only actual + // modality or a cover spanning the whole viewport can fold the base page. + const qualifies = spansViewport(node.rect, vp) || (node.modal === true && cov > 0); if (!qualifies) continue; if ( blocker === null || @@ -87,7 +95,7 @@ export function detectBlockingLayer( return { rootId: blocker.node.id, - kind: classifyLayer(nodes, members, blocker.coverage), + kind: classifyLayer(nodes, members), coverage: blocker.coverage, members, }; diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index c66e65d..6aa16a5 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -1,4 +1,4 @@ -import { coverage, detectBlockingLayer } from "./layers"; +import { coverage, detectBlockingLayer, spansViewport } from "./layers"; import type { ActiveScopeBlock, BlockingLayer, @@ -1024,19 +1024,6 @@ function interactionPoints(rect: Rect): Array<[number, number]> { ]; } -function isModalLike(node: VomNode): boolean { - const role = normalizedRole(node); - const tag = normalizedTag(node); - return node.modal === true || tag === "dialog" || role === "dialog" || role === "alertdialog"; -} - -function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number): number { - if (isModalLike(node)) return 4; - if (viewportCoverage >= 0.9) return 3; - if (viewportCoverage >= 0.15) return 2; - return 1; -} - function paintLineageByFrame( node: VomNode, parentMap: Map, @@ -1090,7 +1077,7 @@ function isBlockedByRegion( blocker: VomNode, parentMap: Map, nodesById: Map, - viewportCoverage: number, + fullCover: boolean, lineageCache: Map>, ): boolean { if (target.id === blocker.id) return false; @@ -1103,7 +1090,7 @@ function isBlockedByRegion( const points = interactionPoints(target.rect); if ( - viewportCoverage < 0.9 && + !fullCover && paintNodes.target.paintOrder >= paintNodes.blocker.paintOrder && points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)) ) { @@ -1117,8 +1104,7 @@ function isBlockedByRegion( ) { return false; } - if (viewportCoverage >= 0.9) return true; - + // Even a near-full region must overlap the target to occlude it. return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)); } @@ -1153,15 +1139,12 @@ function applyActiveRegionPolicy( const lineageCache = new Map>(); const candidates = nodes .filter(isPositionedRegionCandidate) - .map((node) => { - const viewportCoverage = coverage(node.rect, scene.viewport); - return { - node, - viewportCoverage, - priority: activeRegionCandidatePriority(node, viewportCoverage), - }; - }) - .filter((candidate) => candidate.priority > 1 || candidate.viewportCoverage > 0); + .map((node) => ({ + node, + viewportCoverage: coverage(node.rect, scene.viewport), + fullCover: spansViewport(node.rect, scene.viewport), + })) + .filter((candidate) => candidate.viewportCoverage > 0 || candidate.node.modal === true); const blockedRoots = new Set(); for (const target of nodes) { @@ -1172,7 +1155,7 @@ function applyActiveRegionPolicy( candidate.node, parentMap, nodesById, - candidate.viewportCoverage, + candidate.fullCover, lineageCache, ), );