From 8dfa097e55dbb6b4b18901fbdb57b70a721e59f4 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 19:16:47 +0800 Subject: [PATCH 1/7] refactor(observation): organize capture around document facts --- .../src/tools/__tests__/observation.test.ts | 398 ++++++----- apps/extension/src/tools/observation.ts | 233 +------ .../vom/__tests__/capture-coordinator.test.ts | 619 ++++++++++++++++++ .../src/tools/vom/__tests__/capture.test.ts | 220 ++----- .../src/tools/vom/__tests__/facts.test.ts | 156 +++++ .../tools/vom/__tests__/form-state.test.ts | 59 +- .../tools/vom/__tests__/frame-capture.test.ts | 283 -------- .../vom/__tests__/frame-document.test.ts | 63 +- .../src/tools/vom/capture-coordinator.ts | 268 ++++++++ apps/extension/src/tools/vom/capture.ts | 439 +------------ apps/extension/src/tools/vom/facts.ts | 210 ++++++ apps/extension/src/tools/vom/form-capture.ts | 154 +++++ apps/extension/src/tools/vom/frame-capture.ts | 178 ----- .../extension/src/tools/vom/frame-document.ts | 115 ++-- apps/extension/src/tools/vom/normalize.ts | 217 ++++++ .../src/tools/vom/record-safe-observation.ts | 5 +- .../src/tools/vom/semantic-graph/build.ts | 10 +- apps/extension/src/tools/vom/snapshot.ts | 230 +++++++ 18 files changed, 2361 insertions(+), 1496 deletions(-) create mode 100644 apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts create mode 100644 apps/extension/src/tools/vom/__tests__/facts.test.ts delete mode 100644 apps/extension/src/tools/vom/__tests__/frame-capture.test.ts create mode 100644 apps/extension/src/tools/vom/capture-coordinator.ts create mode 100644 apps/extension/src/tools/vom/facts.ts create mode 100644 apps/extension/src/tools/vom/form-capture.ts delete mode 100644 apps/extension/src/tools/vom/frame-capture.ts create mode 100644 apps/extension/src/tools/vom/normalize.ts create mode 100644 apps/extension/src/tools/vom/snapshot.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 76d989c..da52eb5 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; +import type { BuildVomSceneOptions, VomFrameDocument } from "../observation"; import { buildFrameVomScene, - buildVomScene, type CdpAxNode, captureVomObservation, handleGetHtml, @@ -17,7 +17,35 @@ import { type ScreenshotDeps, stripDataUrlPrefix, } from "../observation"; -import type { CapturedNode, CapturedViewModel } from "../vom/capture"; +import type { CapturedNode, CapturedSceneInput } from "../vom/facts"; + +interface SceneFixture extends CapturedSceneInput { + documents?: VomFrameDocument[]; +} + +// Only supplies the common root fixture; child documents declare their own ownership. +function buildTestScene( + axNodes: CdpAxNode[], + captured: SceneFixture, + options: BuildVomSceneOptions = {}, +) { + const rootFrameId = captured.rootFrameId ?? "root"; + return buildFrameVomScene( + [ + { + frameId: rootFrameId, + contextScopeId: rootFrameId, + target: { tabId: 7 }, + url: options.pageUrl, + domNodes: captured.nodes, + axNodes, + }, + ...(captured.documents ?? []), + ], + captured, + options, + ); +} function fakeAgentWindow(ids: number[]) { let i = 0; @@ -526,10 +554,10 @@ describe("handleScreenshot overlay suppression", () => { }); // --------------------------------------------------------------------------- -// buildVomScene +// Semantic scene fixtures // --------------------------------------------------------------------------- -describe("buildVomScene", () => { +describe("buildFrameVomScene", () => { it("merges multiple sibling and nested frames without mixing their targets", () => { const node = ( backendNodeId: number, @@ -567,25 +595,10 @@ describe("buildVomScene", () => { node(300, null, "nested", "body"), node(301, 300, "nested", "input", { placeholder: "Nested field" }), ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1200, height: 800 }, nodes: mainNodes, - iframeNodes: new Map([ - [10, firstNodes], - [20, secondNodes], - [210, nestedNodes], - ]), - frameNodes: new Map([ - ["main", mainNodes], - ["first", firstNodes], - ["second", secondNodes], - ["nested", nestedNodes], - ]), - frameOwnerBackendNodeIds: new Map([ - ["first", 10], - ["second", 20], - ["nested", 210], - ]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -684,11 +697,10 @@ describe("buildVomScene", () => { position: "static", pointerEvents: "auto", }; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 800, height: 600 }, nodes: [], - iframeNodes: new Map(), - frameNodes: new Map([["child", [childNode]]]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -750,14 +762,10 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 800, height: 600 }, nodes: mainNodes, - iframeNodes: new Map(), - frameNodes: new Map([ - ["main", mainNodes], - ["child", []], - ]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -833,9 +841,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 200, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -861,11 +868,11 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); expect(scene.viewport).toEqual({ width: 1000, height: 800 }); expect(scene.nodes[0]).toEqual( expect.objectContaining({ - id: 100, + backendNodeId: 100, parentId: null, role: "RootWebArea", name: "Example", @@ -877,14 +884,14 @@ describe("buildVomScene", () => { ); expect(scene.nodes[1]).toEqual( expect.objectContaining({ - id: 200, - parentId: 100, + backendNodeId: 200, + parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, role: "button", name: "Submit", tag: "button", rect: { x: 20, y: 20, w: 120, h: 40 }, - domParentId: 100, - domAncestorIds: [100], + domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, + domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id], attrs: {}, }), ); @@ -913,9 +920,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 300, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -951,10 +957,10 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); + const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); - expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); - expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); + expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org"); + expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined(); }); it("uses the nearest backend AX ancestor as parentId", () => { @@ -979,9 +985,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 400, height: 300 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1007,10 +1012,12 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]); - expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10); + expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]); + expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe( + scene.nodes.find((node) => node.backendNodeId === 10)!.id, + ); }); it("maps iframe sub-document controls to VomNodes", () => { @@ -1028,7 +1035,7 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1052,10 +1059,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 101, parentBackendNodeId: null, @@ -1077,26 +1089,25 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }; - const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) => - [101, 102].includes(node.id), - ); + const scene = buildTestScene(axNodes, captured); + const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!)); expect(iframeControls).toEqual([ expect.objectContaining({ - id: 101, - parentId: 20, + backendNodeId: 101, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "textbox", name: "请输入手机号", sensitive: false, }), expect.objectContaining({ - id: 102, - parentId: 20, + backendNodeId: 102, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "textbox", name: "密码", sensitive: true, @@ -1117,11 +1128,9 @@ describe("buildVomScene", () => { parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, - childIds: ["3"], }, { nodeId: "3", - parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "输入密码" }, value: { value: "iframe-secret" }, @@ -1129,7 +1138,7 @@ describe("buildVomScene", () => { properties: [{ name: "inputType", value: { value: "password" } }], }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1153,10 +1162,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [axNodes[2]], + domNodes: [ { backendNodeId: 102, parentBackendNodeId: null, @@ -1168,25 +1182,27 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }; - const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( + const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter( (node) => node.role === "textbox" && node.name === "输入密码", ); expect(passwordNodes).toHaveLength(1); - expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); + expect(passwordNodes[0]).toEqual( + expect.objectContaining({ backendNodeId: 202, sensitive: true }), + ); expect(passwordNodes[0].value).toBeUndefined(); - const rendered = renderVom(buildVomScene(axNodes, captured)).text; + const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text; expect(rendered).toContain('textbox "输入密码" [filled] ="•••"'); expect(rendered).not.toContain("iframe-secret"); }); it("keeps unnamed iframe controls but skips unnamed iframe links", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1225,10 +1241,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1250,25 +1271,25 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.id === 201)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual( expect.objectContaining({ - id: 201, - parentId: 20, + backendNodeId: 201, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "button", }), ); - expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name"); - expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined(); + expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name"); + expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined(); }); it("preserves captured-only iframe anchors for iframe sub-document controls", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1301,10 +1322,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1326,16 +1352,16 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.id === 20)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( expect.objectContaining({ - id: 20, - parentId: 10, + backendNodeId: 20, + parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id, role: "Iframe", name: "支付验证", }), @@ -1356,9 +1382,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 320, height: 240 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1379,9 +1404,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - id: 10, + backendNodeId: 10, sensitive: true, }), ); @@ -1396,9 +1421,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1415,9 +1439,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - id: 10, + backendNodeId: 10, role: "textbox", name: "+86", placeholder: "输入手机号", @@ -1435,9 +1459,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1454,8 +1477,13 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( - expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }), + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( + expect.objectContaining({ + backendNodeId: 10, + role: "textbox", + name: "Email", + value: "a@b.com", + }), ); }); @@ -1474,9 +1502,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1514,9 +1541,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( - expect.objectContaining({ role: "textbox", name: "验证码" }), - ); + expect( + buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), + ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" })); }); it("uses preceding AX static text as a form control label", () => { @@ -1540,9 +1567,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1559,13 +1585,13 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( - expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }), - ); + expect( + buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), + ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" })); }); it("marks runtime default input values without treating them as ordinary filled input", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1575,7 +1601,6 @@ describe("buildVomScene", () => { ], { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1619,9 +1644,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1671,9 +1695,14 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toEqual( - expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }), + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( + expect.objectContaining({ + backendNodeId: 20, + role: "button", + name: "close", + cursor: "pointer", + }), ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); @@ -1707,9 +1736,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1748,9 +1776,9 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined(); - expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link"); }); it("promotes only the outermost clickable in a nested pointer chain", () => { @@ -1775,9 +1803,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1816,11 +1843,15 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toEqual( - expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }), + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( + expect.objectContaining({ + backendNodeId: 20, + role: "button", + attrs: { "aria-label": "收藏" }, + }), ); - expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined(); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ @@ -1844,9 +1875,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1894,10 +1924,10 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); expect(scene.activeScopeBlocks).toEqual([ { - triggerId: 20, + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, label: "Reviews (12)", lines: ["Jane - ear cups are small", "Bob - great sound"], }, @@ -1921,11 +1951,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1958,7 +1987,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + triggerAction: "hover", + subItems: ["Shoes", "Bags"], + }, ]); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); @@ -1979,11 +2012,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2044,11 +2076,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2093,7 +2124,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id, + triggerAction: "hover", + subItems: ["My profile"], + }, ]); }); @@ -2113,11 +2148,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2168,7 +2202,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, ]); expect(renderVom(scene).text).toContain( '@e1 button "image" [hover first: My profile | Sign out]', @@ -2191,11 +2229,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 42, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2265,9 +2302,8 @@ describe("buildVomScene", () => { properties: [{ name: "inputType", value: { value: "credit-card" } }], }, ]; - const scene = buildVomScene(axNodes, { + const scene = buildTestScene(axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2314,7 +2350,7 @@ describe("buildVomScene", () => { ], }); - expect(scene.nodes.find((node) => node.id === 20)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( expect.objectContaining({ name: "Products [expanded]", attrs: expect.objectContaining({ @@ -2324,13 +2360,17 @@ describe("buildVomScene", () => { }), ); expect(scene.activeScopeBlocks).toEqual([ - { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + label: "Products [expanded]", + lines: ["Shoes Bags"], + }, ]); - expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); + expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true); }); it("aggregates AX virtual text into unnamed structural nodes", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -2354,7 +2394,6 @@ describe("buildVomScene", () => { ], { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2381,13 +2420,12 @@ describe("buildVomScene", () => { }, ); - expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); + expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text"); }); it("marks captured dialog elements as modal without AX role or aria-modal", () => { - const scene = buildVomScene([], { + const scene = buildTestScene([], { viewport: { width: 640, height: 480 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2405,7 +2443,7 @@ describe("buildVomScene", () => { expect(scene.nodes[0]).toEqual( expect.objectContaining({ - id: 50, + backendNodeId: 50, modal: true, }), ); @@ -2434,9 +2472,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 202, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set([200, 201, 202]), nodes: [ { @@ -2452,9 +2489,9 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined(); - expect(scene.nodes.find((n) => n.id === 2)).toEqual( + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual( expect.objectContaining({ role: "button", name: "Real Button" }), ); }); @@ -2482,13 +2519,12 @@ describe("buildVomScene", () => { backendDOMNodeId: 203, }, ]; - const scene = buildVomScene(axNodes, { + const scene = buildTestScene(axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set([200]), nodes: [], }); - expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined(); }); }); @@ -2601,6 +2637,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2696,6 +2733,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2794,6 +2832,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -3062,6 +3101,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeName: [i("html"), i("body"), i("div"), i("input")], @@ -3119,6 +3159,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1, 1], nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")], @@ -3270,12 +3311,17 @@ describe("handleSnapshot", () => { if (method === "Runtime.releaseObject") return {}; } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "DOMSnapshot.captureSnapshot") + return { ...snapshot, documents: [snapshot.documents[0]] }; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; throw new Error(`unexpected root CDP method ${method}`); }); const sendToTarget = vi.fn(async (_target, method: string) => { - if (method === "Accessibility.enable") return {}; + if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable") + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") + return { ...snapshot, documents: [snapshot.documents[1]] }; if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; throw new Error(`unexpected child CDP method ${method}`); }); @@ -3376,6 +3422,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 3], nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")], @@ -3700,6 +3747,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -3726,6 +3774,7 @@ describe("handleSnapshot", () => { }, // documents[1]: cross-origin login form { + frameId: "child", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -3904,6 +3953,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4, 5], nodeName: [ diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 6f3cae2..1c09abe 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1,3 +1,4 @@ +import type { CapturedSceneInput } from "./vom/facts"; // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // and semantic `tool.observe` (design §7). Each handler resolves the target // tab (defaulting to the Agent Window's active tab when omitted) and @@ -35,7 +36,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; import { screenshotPageRect } from "./geometry/coordinate-types"; -import { cssViewport, GeometryContext } from "./geometry/frame-context"; import { type ChromeTabsApi, enforceToolTargetScope, @@ -49,15 +49,9 @@ import { type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; -import { - type CapturedNode, - type CapturedSurfaceProbe, - type CapturedViewModel, - captureViewModel, - collectOverlayExcludedBackendIds, - probeHoverSurfaces, -} from "./vom/capture"; -import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture"; +import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator"; +import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document"; import { withOverlayBypass } from "./vom/hover-perception"; import { probeTooltipNames } from "./vom/name-enrichment"; import { @@ -531,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A function buildConditionalSurfaces( nodes: VomNode[], - captured: CapturedViewModel, + captured: CapturedSceneInput, probes: CapturedSurfaceProbe[], ): CondSurface[] { if (probes.length === 0) return []; @@ -661,161 +655,9 @@ export interface BuildVomSceneOptions { export type VomFrameDocument = CapturedFrameDocument; -function legacyFrameDocuments( - axNodes: CdpAxNode[], - captured: CapturedViewModel, - pageUrl?: string, - rootTarget: VomFrameDocument["target"] = { tabId: 0 }, -): VomFrameDocument[] { - const rootFrameId = captured.rootFrameId ?? "root"; - const documents: VomFrameDocument[] = [ - { - frameId: rootFrameId, - contextScopeId: rootFrameId, - target: rootTarget, - ...(pageUrl ? { url: pageUrl } : {}), - axNodes: [], - domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })), - }, - ]; - const pending = [...captured.iframeNodes.entries()]; - let progress = true; - let nextSyntheticFrame = 1; - while (pending.length > 0 && progress) { - progress = false; - for (let index = pending.length - 1; index >= 0; index -= 1) { - const [ownerBackendNodeId, domNodes] = pending[index]; - const parent = documents.find((document) => - document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId), - ); - if (!parent) continue; - const frameId = - domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`; - documents.push({ - frameId, - parentFrameId: parent.frameId, - ownerBackendNodeId, - contextScopeId: frameId, - target: parent.target, - axNodes: [], - domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })), - }); - pending.splice(index, 1); - progress = true; - } - } - - const documentByFrameId = new Map(documents.map((document) => [document.frameId, document])); - const backendFrame = new Map(); - const childFrameByOwner = new Map(); - for (const document of documents) { - if (document.ownerBackendNodeId !== undefined) { - childFrameByOwner.set(document.ownerBackendNodeId, document.frameId); - } - for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId); - } - const axById = new Map(axNodes.map((node) => [node.nodeId, node])); - const ownership = new Map(); - const resolving = new Set(); - const frameForAx = (node: CdpAxNode): string => { - const cached = ownership.get(node.nodeId); - if (cached) return cached; - let frameId: string | undefined; - if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId; - if (!frameId && typeof node.backendDOMNodeId === "number") { - frameId = backendFrame.get(node.backendDOMNodeId); - } - if (!frameId && node.parentId && !resolving.has(node.nodeId)) { - const parent = axById.get(node.parentId); - if (parent) { - frameId = - typeof parent.backendDOMNodeId === "number" - ? childFrameByOwner.get(parent.backendDOMNodeId) - : undefined; - if (!frameId) { - resolving.add(node.nodeId); - frameId = frameForAx(parent); - resolving.delete(node.nodeId); - } - } - } - frameId ??= rootFrameId; - ownership.set(node.nodeId, frameId); - return frameId; - }; - for (const node of axNodes) frameForAx(node); - for (const node of axNodes) { - const frameId = ownership.get(node.nodeId) ?? rootFrameId; - const document = documentByFrameId.get(frameId) ?? documents[0]; - document.axNodes.push({ - ...node, - frameId, - ...(node.parentId && ownership.get(node.parentId) === frameId - ? { parentId: node.parentId } - : { parentId: undefined }), - ...(node.childIds - ? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) } - : {}), - }); - } - return documents; -} - -function withLegacyBackendIds(scene: VomScene): VomScene { - const used = new Set(); - const idMap = new Map(); - let nextVirtualId = -1; - for (const node of scene.nodes) { - const preferred = node.backendNodeId; - const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--; - used.add(id); - idMap.set(node.id, id); - } - return { - ...scene, - nodes: scene.nodes.map((node) => ({ - ...node, - id: idMap.get(node.id) as number, - parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null), - ...(node.domParentId !== undefined - ? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) } - : {}), - ...(node.domAncestorIds - ? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) } - : {}), - })), - ...(scene.surfaces - ? { - surfaces: scene.surfaces.map((surface) => ({ - ...surface, - triggerId: idMap.get(surface.triggerId) ?? surface.triggerId, - })), - } - : {}), - ...(scene.activeScopeBlocks - ? { - activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({ - ...block, - triggerId: idMap.get(block.triggerId) ?? block.triggerId, - })), - } - : {}), - }; -} - -export function buildVomScene( - axNodes: CdpAxNode[], - captured: CapturedViewModel, - options: BuildVomSceneOptions = {}, -): VomScene { - return withLegacyBackendIds( - buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options), - ); -} - export function buildFrameVomScene( documents: VomFrameDocument[], - captured: CapturedViewModel, + captured: CapturedSceneInput, options: BuildVomSceneOptions = {}, ): VomScene { const scene = buildSemanticVomScene({ @@ -831,7 +673,7 @@ export function buildFrameVomScene( function attachCapturedSceneAnnotations( scene: VomScene, documents: VomFrameDocument[], - captured: CapturedViewModel, + captured: CapturedSceneInput, surfaceProbes: CapturedSurfaceProbe[], ): VomScene { const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); @@ -981,45 +823,6 @@ export async function handleGetHtml( } } -function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel { - return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; -} - -async function fallbackCapturedViewModel( - cdp: CdpRunner, - tabId: number, - geometry: GeometryContext, - signal?: AbortSignal, -): Promise { - throwIfAborted(signal, "observation"); - let viewport = { width: 0, height: 0 }; - try { - const metrics = await geometry.layoutMetrics({ tabId }); - throwIfAborted(signal, "observation"); - const source = cssViewport(metrics); - viewport = { width: source.width, height: source.height }; - } catch (error) { - if (isAbortError(error)) throw error; - } - const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); - throwIfAborted(signal, "observation"); - return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds }; -} - -async function captureForVom( - cdp: CdpRunner, - tabId: number, - options: CaptureVomObservationOptions, - geometry: GeometryContext, -): Promise { - try { - return await captureViewModel(cdp, tabId, { signal: options.signal, geometry }); - } catch (error) { - if (isAbortError(error)) throw error; - return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal); - } -} - export interface HoverProbeOutcome { /** Whether any active hover was dispatched during this observation. */ performed: boolean; @@ -1052,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = { async function runHoverProbes( cdp: CdpRunner, tabId: number, - captured: CapturedViewModel, + captured: CapturedSceneInput, documents: VomFrameDocument[], staticSemantics: ReturnType, options: CaptureVomObservationOptions, @@ -1091,21 +894,9 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); - const captured = await captureForVom(cdp, tabId, options, geometry); + const facts = await captureObservationFacts(cdp, tabId, options.signal); + const { captured, documents: normalizedDocuments } = semanticCapture(facts); throwIfAborted(options.signal, "observation"); - const documents = await captureFrameData( - cdp, - tabId, - captured, - options.signal, - geometry, - ); - throwIfAborted(options.signal, "observation"); - const normalizedDocuments = - documents.length === 1 && captured.iframeNodes.size > 0 - ? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target) - : documents; const semanticGraph = buildSemanticGraph({ documents: normalizedDocuments, viewport: captured.viewport, @@ -1135,8 +926,8 @@ export async function captureVomObservation( ); // Reserve space using the renderer's character-based token estimate. Like VOM // headers, this integrity notice remains visible even under a tiny token budget. - const geometryNotice = captured.frameGeometryIssues?.length - ? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n" + const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry") + ? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n" : ""; const rendered = renderVom(decoratedScene, { maxDepth: options.maxDepth, diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts new file mode 100644 index 0000000..0bff254 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -0,0 +1,619 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { cdpTargetKey } from "@/browser-driver/frame-graph"; +import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; +import type { CdpRunner } from "../../shared"; +import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; +import { buildSemanticGraph } from "../semantic-graph/build"; +import { REQUESTED_STYLES } from "../snapshot"; + +function fixture( + options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {}, +) { + const frames = options.frames ?? [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "same", + parentFrameId: "main", + ownerBackendNodeId: 3, + target: { tabId: 4 }, + }, + { + frameId: "nested", + parentFrameId: "same", + ownerBackendNodeId: 13, + target: { tabId: 4 }, + }, + { + frameId: "remote", + parentFrameId: "main", + ownerBackendNodeId: 4, + target: { tabId: 4, sessionId: "remote" }, + }, + ]; + const elements = new Map( + frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]), + ); + const logs: Array<{ target: CdpTarget; method: string; params: Record }> = []; + const send = vi.fn( + async (target: CdpTarget, method: string, params: object = {}): Promise => { + logs.push({ target, method, params: params as Record }); + const args = params as Record; + if (options.fail === `${target.sessionId ?? "main"}:${method}`) + throw new Error("fixture failure"); + let result: unknown = {}; + if (method === "Page.getLayoutMetrics") + result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + if (method === "DOMSnapshot.captureSnapshot") { + expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES); + result = { + strings: [ + "#document", + "html", + "button", + OVERLAY_HOST_MARKER_ATTR, + "", + "visible", + "1", + "static", + "auto", + ], + documents: frames + .filter( + (frame) => + cdpTargetKey(frame.target) === cdpTargetKey(target) && + frame.frameId !== options.omitDocument, + ) + .map((frame) => { + const element = elements.get(frame.frameId)!; + return { + frameId: frame.frameId, + nodes: { + backendNodeId: [element - 1, element + 10000, element, element + 1], + nodeName: [0, 1, 1, 2], + nodeType: [9, 10, 1, 1], + parentIndex: [-1, 0, 0, 2], + attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []], + }, + layout: { + nodeIndex: [2, 3], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 100, 40], + ], + styles: [ + [7, 8, 8, 5, 6], + [7, 8, 8, 5, 6], + ], + }, + }; + }), + }; + } + if (method === "Accessibility.getFullAXTree") { + const frameId = String(args.frameId); + result = { + nodes: [ + { + nodeId: `${frameId}-button`, + frameId, + backendDOMNodeId: elements.get(frameId)! + 1, + role: { value: "button" }, + name: { value: frameId }, + }, + ], + }; + } + return result as T; + }, + ); + const cdp: CdpRunner = { + getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }), + send: (tabId, method, params) => + (send as NonNullable)({ tabId }, method, params), + sendToTarget: send as NonNullable, + }; + return { cdp, logs, elements }; +} + +describe("captureObservationFacts", () => { + it("collects each target once and scopes equal backend IDs", async () => { + const { cdp, logs } = fixture(); + const facts = await captureObservationFacts(cdp, 4); + expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); + expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2); + expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4); + const main = facts.documents.find((doc) => doc.frame.frameId === "main")!; + const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; + expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2)); + expect(main.index.children.get(1)).toEqual([2]); + expect(main.index.nodes.get(10001)?.nodeType).toBe(10); + expect(remote.axNodes[0].frameId).toBe("remote"); + expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2)); + expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt); + }); + + it("does not retry failed or missing documents and retains valid AX-only semantics", async () => { + const { cdp, logs } = fixture({ + fail: "remote:DOMSnapshot.captureSnapshot", + omitDocument: "same", + }); + const facts = await captureObservationFacts(cdp, 4); + const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; + expect(remote.domNodes).toHaveLength(0); + expect(remote.axNodes).toHaveLength(1); + expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }), + ); + expect(facts.issues).toContainEqual( + expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }), + ); + }); + + it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => { + const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4); + const { documents } = semanticCapture(facts); + const graph = buildSemanticGraph({ + documents, + viewport: facts.viewport, + rootFrameId: facts.rootFrameId, + }); + expect( + [...graph.nodes.values()].find( + (node) => node.frameId === "remote" && node.backendNodeId === 2, + )?.excluded, + ).toBe(true); + expect( + [...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2) + ?.excluded, + ).toBe(false); + }); + + it("bounds collection across many targets and stops scheduling after cancellation", async () => { + const frames = Array.from({ length: 12 }, (_, i) => ({ + frameId: `f${i}`, + target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, + })); + const { cdp } = fixture({ frames }); + let active = 0, + peak = 0; + const original = cdp.sendToTarget!; + const send: NonNullable = async (target, method, params) => { + active++; + peak = Math.max(peak, active); + try { + await new Promise((resolve) => setTimeout(resolve, 1)); + return await original(target, method, params); + } finally { + active--; + } + }; + cdp.sendToTarget = send; + cdp.send = (tabId, method, params) => send({ tabId }, method, params); + await captureObservationFacts(cdp, 4); + // This fixture has no frame edges, so no concurrent owner requests. + expect(peak).toBeLessThanOrEqual(4); + expect(peak).toBeGreaterThan(1); + const controller = new AbortController(); + controller.abort(); + await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(active).toBe(0); + }); +}); + +function childSnapshot(frameId: string, backendNodeId: number) { + const strings = [frameId, "body", "button", "static", "auto", "pointer"]; + return { + strings, + documents: [ + { + frameId, + nodes: { + parentIndex: [-1, 0], + nodeName: [1, 2], + backendNodeId: [backendNodeId - 1, backendNodeId], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [3, 4, 4], + [3, 4, 5], + ], + bounds: [ + [0, 0, 300, 200], + [10, 20, 100, 40], + ], + paintOrders: [0, 1], + }, + }, + ], + }; +} + +describe("OOPIF capture", () => { + it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { + const sendToTarget = vi.fn(async (target, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return target.sessionId === "left-session" + ? childSnapshot("left", 101) + : childSnapshot("right", 201); + } + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + throw new Error(`unexpected ${method}`); + }); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") { + const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId; + const x = backendNodeId === 10 ? 50 : 500; + return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + throw new Error(`unexpected root ${method}`); + }) as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "left", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "left-session" }, + }, + { + frameId: "right", + parentFrameId: "main", + ownerBackendNodeId: 20, + target: { tabId: 4, sessionId: "right-session" }, + }, + ], + })), + }; + + const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4)); + + expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]); + expect( + trees + .find((doc) => doc.frameId === "left") + ?.domNodes?.find((node) => node.backendNodeId === 101)?.rect, + ).toEqual({ x: 60, y: 120, w: 100, h: 40 }); + expect( + trees + .find((doc) => doc.frameId === "right") + ?.domNodes?.find((node) => node.backendNodeId === 201)?.rect, + ).toEqual({ x: 510, y: 120, w: 100, h: 40 }); + expect( + new Map( + trees + .filter((doc) => doc.ownerBackendNodeId !== undefined) + .map((doc) => [doc.frameId, doc.ownerBackendNodeId]), + ), + ).toEqual( + new Map([ + ["left", 10], + ["right", 20], + ]), + ); + expect( + new Map( + trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]), + ), + ).toEqual( + new Map([ + ["left", "main"], + ["right", "main"], + ]), + ); + }); + + it("retains nested owner failures when merging a captured OOPIF", async () => { + const child = childSnapshot("child", 101); + const nested = childSnapshot("nested", 201); + const document = child.documents[0]; + const snapshot = { + strings: child.strings, + documents: [ + { + ...document, + nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } }, + }, + nested.documents[0], + ], + }; + const reply = async (_target: unknown, method: string) => { + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } }; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "DOM.getBoxModel") + return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; + if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); + return {}; + }; + const cdp: CdpRunner = { + send: vi.fn(reply) as CdpRunner["send"], + sendToTarget: vi.fn(reply) as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "nested", + parentFrameId: "child", + ownerBackendNodeId: 101, + target: { tabId: 4, sessionId: "child-session" }, + }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + const captured = await captureObservationFacts(cdp, 4); + expect( + captured.issues + .filter((issue) => issue.projectionIssue) + .map((issue) => issue.projectionIssue), + ).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" }, + ownerBackendNodeId: 101, + }, + ]); + expect( + captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1], + ).toMatchObject({ + rect: null, + localRect: { x: 10, y: 20, w: 100, h: 40 }, + }); + }); + + it("keeps OOPIF semantics when viewport projection is unavailable", async () => { + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + throw new Error(`unexpected root ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable"); + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101); + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "button", + backendDOMNodeId: 101, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Continue" }, + }, + ], + }; + } + throw new Error(`unexpected ${method}`); + }) as unknown as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const { documents } = semanticCapture(await captureObservationFacts(cdp, 4)); + const childDocument = documents.find((document) => document.frameId === "child"); + + expect(childDocument?.axNodes).toEqual([ + expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }), + ]); + expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual( + expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }), + ); + }); +}); + +// Six sibling frames and one nested frame exercise scheduling through the real capture entry. +function siblingCaptureFixture( + beforeReply: (method: string, params: Record) => Promise = async () => {}, +) { + const document = (id: number, owners: number[], childIndexes: number[]) => ({ + frameId: `frame-${id}`, + nodes: { + parentIndex: [-1, ...owners.map(() => 0)], + nodeName: [0, ...owners.map(() => 1)], + backendNodeId: [1000 + id, ...owners], + attributes: [[], ...owners.map(() => [])], + contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes }, + }, + layout: { + nodeIndex: [0, ...owners.map((_, i) => i + 1)], + bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])], + }, + }); + const snapshot = { + strings: ["body", "iframe"], + documents: [ + document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]), + document(1, [200], [7]), + ...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])), + ], + }; + let active = 0; + let peak = 0; + const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => { + const args = params as Record; + active++; + peak = Math.max(peak, active); + try { + await beforeReply(method, args); + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + if (method === "DOM.getBoxModel") + return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } }; + if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 200, height: 100 } } }; + return {}; + } finally { + active--; + } + }); + return { + cdp: { send: send as CdpRunner["send"] }, + send, + peak: () => peak, + active: () => active, + measured: () => + send.mock.calls + .filter(([, method]) => method === "DOM.getBoxModel") + .map(([, , params]) => (params as { backendNodeId: number }).backendNodeId), + }; +} + +describe("sibling frame measurement scheduling", () => { + it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => { + const pending = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200) + await new Promise((resolve) => pending.set(Number(params.backendNodeId), resolve)); + }); + const capture = captureObservationFacts(fixture.cdp, 4); + await vi.waitFor(() => expect(pending.size).toBe(4)); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + pending.get(103)!(); + await vi.waitFor(() => expect(pending.has(104)).toBe(true)); + pending.get(104)!(); + await vi.waitFor(() => expect(pending.has(105)).toBe(true)); + expect(fixture.measured()).not.toContain(200); + for (const id of [105, 102, 101, 100]) pending.get(id)!(); + const captured = await capture; + expect( + captured.issues + .filter((issue) => issue.stage === "geometry") + .map((issue) => issue.projectionIssue), + ).toEqual([]); + expect(fixture.peak()).toBe(4); + expect(fixture.active()).toBe(0); + expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]); + expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( + Array.from({ length: 8 }, (_, i) => `frame-${i}`), + ); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, + ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + for (const method of [ + "DOM.getBoxModel", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Runtime.releaseObject", + ]) + expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7); + }); + + it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => { + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode" && params.backendNodeId === 100) + throw new Error("owner replaced"); + }); + const captured = await captureObservationFacts(fixture.cdp, 4); + expect( + captured.issues + .filter((issue) => issue.stage === "geometry") + .map((issue) => issue.projectionIssue), + ).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4 }, frameId: "frame-1" }, + ownerBackendNodeId: 100, + }, + { + status: "blocked", + source: { target: { tabId: 4 }, frameId: "frame-7" }, + cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue, + }, + ]); + expect(fixture.measured()).not.toContain(200); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0], + ).toMatchObject({ + tag: "body", + rect: null, + localRect: { x: 0, y: 0, w: 200, h: 100 }, + }); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, + ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( + Array.from({ length: 8 }, (_, i) => `frame-${i}`), + ); + }); + + it("stops scheduling on cancellation and waits for resolved objects to be released", async () => { + const controller = new AbortController(); + const resolutions = new Map void>(); + const releases = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode") + await new Promise((resolve) => + resolutions.set(String(params.backendNodeId), resolve), + ); + if (method === "Runtime.releaseObject") + await new Promise((resolve) => releases.set(String(params.objectId), resolve)); + }); + let settled = false; + const capture = captureObservationFacts(fixture.cdp, 4, controller.signal); + const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); + void capture.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => expect(resolutions.size).toBe(4)); + controller.abort(); + for (const resolve of resolutions.values()) resolve(); + await vi.waitFor(() => expect(releases.size).toBe(4)); + expect(settled).toBe(false); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + for (const release of releases.values()) release(); + await rejected; + expect(fixture.active()).toBe(0); + expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe( + false, + ); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index b70ca2d..413512a 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; -import type { CdpRunner } from "../../shared"; -import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; +import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; +import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; // Minimal but format-accurate captureSnapshot reply: a body with one // fixed full-screen overlay div carrying a password input. @@ -27,6 +27,7 @@ function fakeSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeType: [1, 1, 1, 1], @@ -74,6 +75,7 @@ function hoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeType: [1, 1, 1], @@ -118,6 +120,7 @@ function twoHoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1], nodeType: [1, 1, 1, 1], @@ -171,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2, 3], nodeType: [1, 1, 1, 1, 1], @@ -211,11 +215,7 @@ function makeCdp(snapshot: unknown) { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { - return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - }; + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } if (method === "Runtime.evaluate") { return { @@ -248,13 +248,11 @@ function makeCdp(snapshot: unknown) { }; } -describe("captureViewModel", () => { +describe("snapshot capture and hover", () => { it("parses nodes, attrs, rects, paint order and styles", async () => { - const { nodes, viewport, iframeNodes } = await captureViewModel( - makeCdp(fakeSnapshotReply()), - 4, - ); - expect(iframeNodes.size).toBe(0); + const { nodes, viewport } = semanticCapture( + await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4), + ).captured; expect(viewport).toEqual({ width: 1000, height: 800 }); const div = nodes.find((n) => n.backendNodeId === 12); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); @@ -266,35 +264,11 @@ describe("captureViewModel", () => { it("never hovers the page while capturing", async () => { const cdp = makeCdp(fakeSnapshotReply()); - await captureViewModel(cdp, 4); + semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); }); - it("batch-enriches form controls without per-node object resolution", async () => { - const cdp = makeCdp(fakeSnapshotReply()); - const result = await captureViewModel(cdp, 4); - const input = result.nodes.find((node) => node.backendNodeId === 13); - - expect(input).toMatchObject({ - formState: "filled", - formPlaceholder: "secret", - attrs: { type: "password" }, - }); - expect(input?.formValue).toBeUndefined(); - expect(input?.formDefaultValue).toBeUndefined(); - expect(cdp.send).toHaveBeenCalledWith( - 4, - "Runtime.evaluate", - expect.objectContaining({ - serializationOptions: expect.objectContaining({ serialization: "deep" }), - }), - ); - expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything()); - expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything()); - expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything()); - }); - it("runs hover surface probes when explicitly enabled", async () => { let hoverStateCalls = 0; const cdp = { @@ -303,8 +277,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -336,7 +308,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -368,8 +340,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -398,7 +368,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -418,8 +388,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -445,7 +413,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; await probeHoverSurfaces(cdp, 4, captured.nodes); expect(hoverMoves).toBe(1); @@ -477,6 +445,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -507,7 +476,7 @@ describe("captureViewModel", () => { ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; const ids = nodes.map((n) => n.backendNodeId); // Real page nodes survive. expect(ids).toContain(12); @@ -538,6 +507,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -568,7 +538,9 @@ describe("captureViewModel", () => { ], }; - const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); }); @@ -589,6 +561,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], @@ -611,7 +584,9 @@ describe("captureViewModel", () => { ], }; - const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes, excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -634,6 +609,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -656,7 +632,9 @@ describe("captureViewModel", () => { ], }; - const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes, excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -679,6 +657,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -701,7 +680,7 @@ describe("captureViewModel", () => { }, ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); }); @@ -713,6 +692,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], @@ -739,7 +719,7 @@ describe("captureViewModel", () => { ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); @@ -753,66 +733,26 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 }, }; } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const div = nodes.find((n) => n.backendNodeId === 12); expect(div?.localRect?.y).toBe(-200); expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); - it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => { - const snapshot = { - strings: ["html", "button"], - documents: [ - { - frameId: "root", - scrollOffsetX: 40, - scrollOffsetY: 200, - nodes: { - parentIndex: [-1, 0], - nodeName: [0, 1], - backendNodeId: [10, 11], - attributes: [[], []], - }, - layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] }, - }, - ], - }; - const cdp: CdpRunner = { - send: vi.fn(async (_tab: number, method: string) => { - if (method === "DOMSnapshot.captureSnapshot") return snapshot; - if (method === "Page.getLayoutMetrics") - return { - visualViewport: { clientWidth: 2000 }, - cssVisualViewport: { clientWidth: 1000 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 }, - }; - return {}; - }) as CdpRunner["send"], - }; - const captured = await captureViewModel(cdp, 4); - expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({ - localRect: { x: 80, y: 300, w: 120, h: 40 }, - rect: { x: 80, y: 300, w: 120, h: 40 }, - rendered: true, - }); - expect(captured.frameGeometryIssues).toEqual([]); - }); - - it("normalizes raw snapshot bounds before viewport clipping", async () => { + it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -840,8 +780,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 2000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, layoutViewport: { clientWidth: 2000, clientHeight: 1600 }, }; @@ -849,7 +787,7 @@ describe("captureViewModel", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ x: 0, y: 0, @@ -859,8 +797,8 @@ describe("captureViewModel", () => { expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({ x: 0, y: 0, - w: 1000, - h: 800, + w: 2000, + h: 1600, }); }); @@ -885,6 +823,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -917,15 +856,13 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const btn = nodes.find((n) => n.backendNodeId === 12); expect(btn?.textContent).toBe("登录"); const link = nodes.find((n) => n.backendNodeId === 14); @@ -935,7 +872,7 @@ describe("captureViewModel", () => { expect(body?.textContent).toBeUndefined(); }); - it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { + it("retains explicit iframe document ownership and projected nodes", async () => { const S = [ "html", "body", @@ -1009,8 +946,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -1023,30 +958,21 @@ describe("captureViewModel", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { - nodes, - iframeNodes, - frameNodes, - frameOwnerBackendNodeIds, - frameParentIds, - rootFrameId, - } = await captureViewModel(cdp, 4); - // Main frame has 4 nodes; iframe is included - expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); - // iframeNodes keyed by the