mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-10-02 07:34:35 +08:00
Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
This commit is contained in:
Hannah Liu
@@ -672,6 +672,60 @@ describe("buildVomScene", () => {
|
||||
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
|
||||
});
|
||||
|
||||
it("excludes an overlay only in its owning document when target backend ids collide", () => {
|
||||
const owner = (id: number): CapturedNode => ({
|
||||
backendNodeId: id,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x: id * 10, y: 0, w: 200, h: 100 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
const owners = [owner(10), owner(20)];
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: owners,
|
||||
axNodes: [],
|
||||
},
|
||||
...["left", "right"].map((frameId, i) => ({
|
||||
frameId,
|
||||
contextScopeId: frameId,
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: owners[i].backendNodeId,
|
||||
target: { tabId: 7, sessionId: frameId },
|
||||
excludedBackendNodeIds: new Set(i === 0 ? [7] : []),
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
frameId,
|
||||
backendDOMNodeId: 7,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: frameId + " action" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
],
|
||||
{
|
||||
nodes: owners,
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
},
|
||||
);
|
||||
const rendered = renderVom(result);
|
||||
expect(rendered.text).not.toContain("left action");
|
||||
expect(rendered.text).toContain("right action");
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right");
|
||||
});
|
||||
|
||||
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
|
||||
const childNode: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
|
||||
@@ -676,6 +676,7 @@ function legacyFrameDocuments(
|
||||
...(pageUrl ? { url: pageUrl } : {}),
|
||||
axNodes: [],
|
||||
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId),
|
||||
},
|
||||
];
|
||||
const pending = [...captured.iframeNodes.entries()];
|
||||
@@ -699,6 +700,7 @@ function legacyFrameDocuments(
|
||||
target: parent.target,
|
||||
axNodes: [],
|
||||
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId),
|
||||
});
|
||||
pending.splice(index, 1);
|
||||
progress = true;
|
||||
|
||||
@@ -538,6 +538,9 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -568,8 +571,12 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel(
|
||||
makeCdp(snapshot),
|
||||
4,
|
||||
);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds);
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by tag name only", async () => {
|
||||
@@ -1168,6 +1175,8 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1175,8 +1184,6 @@ describe("captureViewModel", () => {
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [2] },
|
||||
},
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
@@ -1191,14 +1198,14 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [30, 31],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import { createCaptureCheckpoint } from "../capture-abort";
|
||||
import { buildDocumentIndex, type DecodedNode } from "../facts";
|
||||
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
|
||||
|
||||
function node(id: number, parent: number | null): DecodedNode {
|
||||
return {
|
||||
backendNodeId: id,
|
||||
parentBackendNodeId: parent,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
describe("document facts", () => {
|
||||
it.each([
|
||||
1_000, 10_000, 100_000,
|
||||
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
|
||||
for (const wide of [false, true]) {
|
||||
let reads = 0;
|
||||
const input = Array.from({ length: count }, (_, i) => {
|
||||
const item = node(i, i ? (wide ? 0 : i - 1) : null);
|
||||
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
|
||||
Object.defineProperty(item, "parentBackendNodeId", {
|
||||
get: () => {
|
||||
reads++;
|
||||
return i ? (wide ? 0 : i - 1) : null;
|
||||
},
|
||||
});
|
||||
return item;
|
||||
}).reverse(); // parent need not precede child
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect(index.nodes.size).toBe(count);
|
||||
expect(index.excludedBackendNodeIds.size).toBe(count);
|
||||
expect(reads).toBeLessThanOrEqual(count * 8);
|
||||
expect(index.nodes.get(count - 1)).toBe(input[0]);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
|
||||
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const input = [
|
||||
node(3, 2),
|
||||
{ ...node(2, 1), tag: "#document-fragment" },
|
||||
host,
|
||||
node(4, 99),
|
||||
node(5, 6),
|
||||
node(6, 5),
|
||||
node(7, 6),
|
||||
];
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
|
||||
expect(index.nodes.size).toBe(input.length);
|
||||
});
|
||||
|
||||
it("does not propagate a descendant overlay backwards into a cycle", async () => {
|
||||
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
|
||||
expect([...index.excludedBackendNodeIds]).toEqual([3]);
|
||||
});
|
||||
|
||||
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
|
||||
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
|
||||
const styles = REQUESTED_STYLES.map((style) =>
|
||||
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
|
||||
);
|
||||
const { nodes } = await decodeDocument(
|
||||
{
|
||||
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
|
||||
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
|
||||
},
|
||||
strings,
|
||||
);
|
||||
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
|
||||
expect(nodes[0].layout).toEqual({
|
||||
boundsSpace: "snapshot-document-layout",
|
||||
bounds: [10, 20, 120, 40],
|
||||
styles: {
|
||||
position: "static",
|
||||
"pointer-events": "auto",
|
||||
cursor: "auto",
|
||||
visibility: "visible",
|
||||
opacity: "0",
|
||||
},
|
||||
});
|
||||
expect(nodes[0]).not.toHaveProperty("rect");
|
||||
expect(nodes[0]).not.toHaveProperty("excluded");
|
||||
});
|
||||
|
||||
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
|
||||
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const checkpoint = createCaptureCheckpoint(controller.signal);
|
||||
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
|
||||
clock.mockReturnValue(9);
|
||||
const pending = checkpoint();
|
||||
expect(pending).toBeInstanceOf(Promise);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("checks cancellation at bounded indexing blocks", async () => {
|
||||
const controller = new AbortController();
|
||||
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
|
||||
let reads = 0;
|
||||
for (const item of input)
|
||||
Object.defineProperty(item, "attrs", {
|
||||
get: () => {
|
||||
reads++;
|
||||
if (reads === 100) controller.abort();
|
||||
return {};
|
||||
},
|
||||
});
|
||||
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(reads).toBeLessThanOrEqual(356);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
|
||||
import { captureViewModel } from "../capture";
|
||||
import { enrichFormControlStates } from "../form-capture";
|
||||
import { decodeDocument } from "../snapshot";
|
||||
|
||||
interface Control {
|
||||
id: number;
|
||||
@@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) {
|
||||
};
|
||||
}
|
||||
|
||||
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) {
|
||||
async function fakeCdp(
|
||||
groups: Control[][],
|
||||
evaluate: (params: Record<string, unknown>) => unknown,
|
||||
) {
|
||||
const snap = snapshot(groups);
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snap;
|
||||
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
|
||||
return {};
|
||||
});
|
||||
return { send, cdp: { send: send as CdpRunner["send"] } };
|
||||
const nodes = await decodeControls(snap);
|
||||
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
|
||||
}
|
||||
|
||||
async function decodeControls(snap: ReturnType<typeof snapshot>) {
|
||||
return Promise.all(
|
||||
snap.documents.map(async (doc) =>
|
||||
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
describe("form state identity", () => {
|
||||
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
|
||||
// The snapshot includes a shadow input. The runtime query skips it,
|
||||
// encounters a new input, and sees the remaining inputs in a new order.
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 10, value: "" },
|
||||
@@ -106,21 +118,21 @@ describe("form state identity", () => {
|
||||
[11, state("alice@example.com")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
formValue: "",
|
||||
formState: "empty",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "alice@example.com",
|
||||
});
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[],
|
||||
[{ id: 11, value: "inaccessible frame" }],
|
||||
@@ -133,8 +145,8 @@ describe("form state identity", () => {
|
||||
[21, state("second frame")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
const controls = [...captured.iframeNodes.values()].flat();
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const controls = nodes.slice(1).flat();
|
||||
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
|
||||
@@ -142,7 +154,6 @@ describe("form state identity", () => {
|
||||
|
||||
it("keeps identical backend ids in different OOPIF targets separate", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
|
||||
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
|
||||
return {};
|
||||
});
|
||||
@@ -151,8 +162,9 @@ describe("form state identity", () => {
|
||||
});
|
||||
const cdp = { send, sendToTarget } as unknown as CdpRunner;
|
||||
for (const sessionId of ["left-frame", "right-frame"]) {
|
||||
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 7, sessionId },
|
||||
"Runtime.releaseObjectGroup",
|
||||
@@ -168,7 +180,7 @@ describe("form state identity", () => {
|
||||
"malformed",
|
||||
"invalid-state",
|
||||
])("preserves snapshot data when enrichment is %s", async (mode) => {
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
if (mode === "unsupported") throw new Error("serialization unsupported");
|
||||
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
|
||||
const result = reply([
|
||||
@@ -178,8 +190,8 @@ describe("form state identity", () => {
|
||||
result.result.deepSerializedValue.value[0].value[1].value = "not json";
|
||||
return result;
|
||||
});
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "snapshot value",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
@@ -188,18 +200,34 @@ describe("form state identity", () => {
|
||||
|
||||
it("releases the batch's remote objects when capture is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
controller.abort();
|
||||
return reply([[11, state("new")]]);
|
||||
});
|
||||
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("does not report success when cancelled during object cleanup", async () => {
|
||||
const controller = new AbortController();
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
|
||||
if (method === "Runtime.releaseObjectGroup") controller.abort();
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("skips entries without node identity while still applying valid entries", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 11, value: "snapshot value" },
|
||||
@@ -216,20 +244,20 @@ describe("form state identity", () => {
|
||||
return result;
|
||||
},
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"password",
|
||||
"text",
|
||||
])("redacts a sensitive %s control after identity matching", async (type) => {
|
||||
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
reply([[11, { ...state("secret"), sensitive: true }]]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
const control = captured.nodes.find((n) => n.backendNodeId === 11);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const control = nodes[0].find((n) => n.backendNodeId === 11);
|
||||
expect(control?.formState).toBe("filled");
|
||||
expect(control?.formValue).toBeUndefined();
|
||||
expect(control?.formDefaultValue).toBeUndefined();
|
||||
@@ -246,7 +274,7 @@ describe("form state identity", () => {
|
||||
defaultValue: "",
|
||||
placeholder: "",
|
||||
}));
|
||||
const { cdp, send } = fakeCdp([controls], (params) => {
|
||||
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
|
||||
const doc = {
|
||||
querySelectorAll: (selector: string) =>
|
||||
selector === "input,textarea,select" ? elements : [],
|
||||
@@ -257,9 +285,9 @@ describe("form state identity", () => {
|
||||
expect(entries).toHaveLength(250);
|
||||
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
|
||||
});
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
expect(
|
||||
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
@@ -32,7 +33,7 @@ function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
attributes: [[], []] as number[][],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
@@ -140,9 +141,15 @@ describe("captureFrameData", () => {
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
if (target.sessionId !== "left-session") return childSnapshot("right", 201);
|
||||
const snapshot = childSnapshot("left", 101);
|
||||
const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1;
|
||||
const nodes = snapshot.documents[0].nodes;
|
||||
nodes.backendNodeId.push(201); // Same id as a real button in the other target.
|
||||
nodes.parentIndex.push(0);
|
||||
nodes.nodeName.push(2);
|
||||
nodes.attributes.push([marker, -1]);
|
||||
return snapshot;
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
@@ -188,6 +195,10 @@ describe("captureFrameData", () => {
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual(
|
||||
new Set([201]),
|
||||
);
|
||||
expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0);
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
@@ -274,10 +285,5 @@ describe("captureFrameData", () => {
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
|
||||
);
|
||||
expect(captured.frameGeometryIssues).toContainEqual({
|
||||
status: "unavailable",
|
||||
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "child" },
|
||||
reason: "snapshot-coordinates-unavailable",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -29,7 +29,7 @@ function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -54,7 +54,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -72,7 +72,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -92,7 +92,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -107,7 +107,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -129,7 +129,7 @@ describe("buildFrameDocuments", () => {
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -149,7 +149,7 @@ describe("buildFrameDocuments", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
@@ -164,11 +164,25 @@ describe("buildFrameDocuments", () => {
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
const documents = await buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
|
||||
const frame = { frameId: "main", target: { tabId: 4 } };
|
||||
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
|
||||
nodeId: String(i),
|
||||
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
|
||||
})).reverse();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames: [frame] },
|
||||
[{ frame, nodes }],
|
||||
captured(new Map()),
|
||||
);
|
||||
expect(documents[0].axNodes).toHaveLength(10000);
|
||||
expect(documents[0].axNodes[0].frameId).toBe("main");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Check at bounded work blocks; yield only after a processing slice is spent.
|
||||
* Each caller owns its deadline, so unrelated captures do not share state. */
|
||||
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
|
||||
throwIfAborted(signal);
|
||||
let deadline = performance.now() + 8;
|
||||
return () => {
|
||||
throwIfAborted(signal);
|
||||
if (performance.now() < deadline) return;
|
||||
return new Promise<void>((resolve) => {
|
||||
const channel = new MessageChannel();
|
||||
channel.port1.onmessage = () => {
|
||||
channel.port1.close();
|
||||
channel.port2.close();
|
||||
resolve();
|
||||
};
|
||||
channel.port2.postMessage(null);
|
||||
}).then(() => {
|
||||
throwIfAborted(signal);
|
||||
deadline = performance.now() + 8;
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
@@ -1,20 +1,14 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); requested computed styles are decoded through
|
||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||
// Coordinate DOMSnapshot acquisition, frame geometry and hover probes.
|
||||
// Snapshot decoding and form enrichment live in their own modules.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import {
|
||||
type FrameProjectionIssue,
|
||||
type FrameProjectionState,
|
||||
projectSnapshotRect,
|
||||
type SnapshotCoordinates,
|
||||
snapshotCoordinates,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import {
|
||||
GeometryContext,
|
||||
@@ -22,48 +16,20 @@ import {
|
||||
snapshotLayoutScale,
|
||||
} from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import { enrichFormControlStates } from "./form-capture";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { type FrameContext, normalizeDocument } from "./normalize";
|
||||
import {
|
||||
REQUESTED_STYLES,
|
||||
type SnapshotDocument,
|
||||
type SnapshotReply,
|
||||
snapshotFrameId,
|
||||
sparseIndexMap,
|
||||
} from "./snapshot";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
export type { CapturedNode } from "./capture-types";
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
@@ -80,6 +46,7 @@ export interface CapturedViewModel {
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameExcludedBackendNodeIds?: Map<string, ReadonlySet<number>>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
@@ -96,244 +63,9 @@ export interface CaptureViewModelOptions {
|
||||
target?: CdpTarget;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface DeepSerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const controls = [];
|
||||
const collect = (doc) => {
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
// Deep serialization supplies the node's backend id. Keep the
|
||||
// state as JSON so decoding needs no general-purpose V8 deserializer.
|
||||
controls.push([el, JSON.stringify(controlState(el))]);
|
||||
remaining -= 1;
|
||||
}
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
if (remaining <= 0) break;
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
if (childDoc) collect(childDoc);
|
||||
}
|
||||
};
|
||||
collect(document);
|
||||
return controls;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
|
||||
const states = new Map<number, CapturedFormState>();
|
||||
const serialized = result.result?.deepSerializedValue;
|
||||
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
|
||||
for (const entry of serialized.value as DeepSerializedValue[]) {
|
||||
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [element, json] = entry.value as DeepSerializedValue[];
|
||||
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
|
||||
try {
|
||||
const state = JSON.parse(json.value) as CapturedFormState | null;
|
||||
if (
|
||||
!state ||
|
||||
!["empty", "filled", "default"].includes(state.state ?? "") ||
|
||||
typeof state.sensitive !== "boolean" ||
|
||||
typeof state.placeholder !== "string" ||
|
||||
(!state.sensitive &&
|
||||
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
states.set(backendNodeId, state);
|
||||
} catch {
|
||||
// A malformed entry must not overwrite the snapshot's own state.
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
const state = states.get(node.backendNodeId);
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
// Backend ids are scoped to this capture's CDP target. OOPIF targets
|
||||
// use their own captureViewModel call and never share this lookup.
|
||||
const states = formStatesByBackendId(result);
|
||||
for (const nodes of frameNodeGroups) {
|
||||
applyFormStates(nodes, states);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
} finally {
|
||||
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
deepSerializedValue?: DeepSerializedValue;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -352,36 +84,6 @@ interface CdpDomNode {
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: FrameProjectionState;
|
||||
target: CdpTarget;
|
||||
coordinates: SnapshotCoordinates | null;
|
||||
layoutUnitsPerCssPixel: number | null;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -791,198 +493,12 @@ export async function collectOverlayExcludedBackendIds(
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let rendered = false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
const input = snapshotViewportRect(
|
||||
b,
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
context.coordinates,
|
||||
);
|
||||
if (input) {
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
localRect = { x, y, w, h };
|
||||
}
|
||||
const bounds =
|
||||
input && context.projection.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
rendered =
|
||||
!!b &&
|
||||
b.length >= 4 &&
|
||||
b.slice(0, 4).every(Number.isFinite) &&
|
||||
b[2] > 0 &&
|
||||
b[3] > 0 &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
frameGeometryIssues: FrameProjectionIssue[];
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
frameExcludedBackendNodeIds: Map<string, ReadonlySet<number>>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -999,6 +515,7 @@ async function parseChildFrameDocuments(
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const frameExcludedBackendNodeIds = new Map<string, ReadonlySet<number>>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
@@ -1007,6 +524,7 @@ async function parseChildFrameDocuments(
|
||||
iframeNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
frameExcludedBackendNodeIds,
|
||||
excludedBackendNodeIds,
|
||||
frameGeometryIssues,
|
||||
};
|
||||
@@ -1096,13 +614,14 @@ async function parseChildFrameDocuments(
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, childContext);
|
||||
const parsed = await normalizeDocument(childDoc, strings, childContext, signal);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = await parseChildFrameDocuments(
|
||||
documents,
|
||||
@@ -1123,6 +642,8 @@ async function parseChildFrameDocuments(
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds)
|
||||
frameExcludedBackendNodeIds.set(frameId, excluded);
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
@@ -1130,6 +651,7 @@ async function parseChildFrameDocuments(
|
||||
iframeNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
frameExcludedBackendNodeIds,
|
||||
excludedBackendNodeIds,
|
||||
frameGeometryIssues,
|
||||
};
|
||||
@@ -1201,9 +723,9 @@ export async function captureViewModel(
|
||||
coordinates,
|
||||
layoutUnitsPerCssPixel,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
|
||||
const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds);
|
||||
|
||||
const ownerIds = new Set<number>();
|
||||
for (const document of documents) {
|
||||
@@ -1222,6 +744,11 @@ export async function captureViewModel(
|
||||
options.signal,
|
||||
);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
if (topContext.frameId)
|
||||
frameParsed.frameExcludedBackendNodeIds.set(
|
||||
topContext.frameId,
|
||||
mainParsed.index.excludedBackendNodeIds,
|
||||
);
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
@@ -1251,6 +778,7 @@ export async function captureViewModel(
|
||||
...frameParsed.frameGeometryIssues,
|
||||
],
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
excludedBackendNodeIds,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { isOverlayHostNode } from "@/lib/overlay-bridge";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
/** Bounds retain their raw snapshot document layout units until normalization. */
|
||||
export interface SnapshotLayout {
|
||||
readonly boundsSpace: "snapshot-document-layout";
|
||||
bounds?: number[];
|
||||
styles: Readonly<Record<string, string>>;
|
||||
}
|
||||
|
||||
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface NodeFacts extends CapturedNode {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
|
||||
readonly nodes: ReadonlyMap<number, T>;
|
||||
readonly excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export interface DecodedDocument {
|
||||
nodes: DecodedNode[];
|
||||
}
|
||||
|
||||
/** All snapshot nodes participate, including document and shadow roots. This
|
||||
* preserves overlay propagation when the semantic adapter omits those nodes. */
|
||||
export async function buildDocumentIndex<T extends DecodedNode>(
|
||||
input: readonly T[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DocumentIndex<T>> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes = new Map<number, T>();
|
||||
const overlayByNode = new Map<number, boolean>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = input[i];
|
||||
nodes.set(node.backendNodeId, node);
|
||||
}
|
||||
let work = 0;
|
||||
for (const node of input) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (overlayByNode.has(node.backendNodeId)) continue;
|
||||
const path: DecodedNode[] = [];
|
||||
const visiting = new Set<number>();
|
||||
let current: DecodedNode | undefined = node;
|
||||
let overlay = false;
|
||||
while (current && !overlayByNode.has(current.backendNodeId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (visiting.has(current.backendNodeId)) {
|
||||
// Only an overlay inside the cycle may mark the cycle and its descendants.
|
||||
overlay = path
|
||||
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
|
||||
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
|
||||
break;
|
||||
}
|
||||
visiting.add(current.backendNodeId);
|
||||
path.push(current);
|
||||
if (current.parentBackendNodeId === null) {
|
||||
current = undefined;
|
||||
break;
|
||||
}
|
||||
current = nodes.get(current.parentBackendNodeId);
|
||||
}
|
||||
if (current && overlayByNode.has(current.backendNodeId))
|
||||
overlay = overlayByNode.get(current.backendNodeId)!;
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const item = path[i];
|
||||
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
|
||||
overlayByNode.set(item.backendNodeId, overlay);
|
||||
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
|
||||
}
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: { deepSerializedValue?: DeepSerializedValue };
|
||||
}
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface DeepSerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const controls = [];
|
||||
const collect = (doc) => {
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
// Deep serialization supplies the node's backend id. Keep the
|
||||
// state as JSON so decoding needs no general-purpose V8 deserializer.
|
||||
controls.push([el, JSON.stringify(controlState(el))]);
|
||||
remaining -= 1;
|
||||
}
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
if (remaining <= 0) break;
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
if (childDoc) collect(childDoc);
|
||||
}
|
||||
};
|
||||
collect(document);
|
||||
return controls;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
|
||||
const states = new Map<number, CapturedFormState>();
|
||||
const serialized = result.result?.deepSerializedValue;
|
||||
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
|
||||
for (const entry of serialized.value as DeepSerializedValue[]) {
|
||||
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [element, json] = entry.value as DeepSerializedValue[];
|
||||
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
|
||||
try {
|
||||
const state = JSON.parse(json.value) as CapturedFormState | null;
|
||||
if (
|
||||
!state ||
|
||||
!["empty", "filled", "default"].includes(state.state ?? "") ||
|
||||
typeof state.sensitive !== "boolean" ||
|
||||
typeof state.placeholder !== "string" ||
|
||||
(!state.sensitive &&
|
||||
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
states.set(backendNodeId, state);
|
||||
} catch {
|
||||
// A malformed entry must not overwrite the snapshot's own state.
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
|
||||
for (const node of nodes) {
|
||||
if (!["input", "textarea", "select"].includes(node.tag)) continue;
|
||||
const state = states.get(node.backendNodeId);
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (
|
||||
state.sensitive ||
|
||||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
|
||||
) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
|
||||
);
|
||||
if (!hasControls) return true;
|
||||
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
// Backend ids are scoped to this capture's CDP target. OOPIF targets
|
||||
// have separate enrichment calls and never share this lookup.
|
||||
const states = formStatesByBackendId(result);
|
||||
for (const nodes of frameNodeGroups) {
|
||||
applyFormStates(nodes, states);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
return false;
|
||||
} finally {
|
||||
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,11 @@ async function captureMissingFrameDocuments(
|
||||
geometry,
|
||||
target: frame.target,
|
||||
});
|
||||
if (child.frameExcludedBackendNodeIds?.size) {
|
||||
captured.frameExcludedBackendNodeIds ??= new Map();
|
||||
for (const [id, excluded] of child.frameExcludedBackendNodeIds)
|
||||
captured.frameExcludedBackendNodeIds.set(id, excluded);
|
||||
}
|
||||
if (child.frameGeometryIssues?.length) {
|
||||
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
|
||||
}
|
||||
@@ -174,5 +179,5 @@ export async function captureFrameData<T extends FrameAxNode>(
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
return buildFrameDocuments(graph, batches, captured, signal);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@ import {
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
@@ -23,6 +24,17 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
/** Explicit snapshot document data used to join DOM and AX ownership. */
|
||||
export interface FrameDomInput {
|
||||
nodes: CapturedNode[];
|
||||
rootFrameId?: string;
|
||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
|
||||
frameParentIds?: ReadonlyMap<string, string>;
|
||||
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
@@ -47,7 +59,7 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
captured: FrameDomInput,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
@@ -70,11 +82,14 @@ function frameList<T extends FrameOwnedAxNode>(
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
captured: FrameDomInput,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameDocument<T>[]> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
let work = 0;
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
@@ -83,54 +98,68 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
const nodeById = new Map<string, T>();
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
nodeById.set(node.nodeId, node);
|
||||
}
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
for (const node of batch.nodes) {
|
||||
const path: T[] = [];
|
||||
const visiting = new Set<string>();
|
||||
let current: T | undefined = node;
|
||||
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
while (current) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const cached = ownershipByNodeId.get(current.nodeId);
|
||||
if (cached) {
|
||||
ownership = cached;
|
||||
break;
|
||||
}
|
||||
if (visiting.has(current.nodeId)) break;
|
||||
visiting.add(current.nodeId);
|
||||
if (current.frameId && frameById.has(current.frameId)) {
|
||||
ownership = { frameId: current.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
const frameId =
|
||||
typeof current.backendDOMNodeId === "number"
|
||||
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
|
||||
: undefined;
|
||||
if (frameId) {
|
||||
ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
path.push(current);
|
||||
current = current.parentId ? nodeById.get(current.parentId) : undefined;
|
||||
}
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
|
||||
ownershipByNodeId.set(path[i].nodeId, ownership);
|
||||
}
|
||||
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
@@ -144,6 +173,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
@@ -169,5 +202,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type FrameProjectionState,
|
||||
projectSnapshotRect,
|
||||
type SnapshotCoordinates,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts";
|
||||
import { decodeDocument, type SnapshotDocument } from "./snapshot";
|
||||
|
||||
export interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: FrameProjectionState;
|
||||
target: CdpTarget;
|
||||
coordinates: SnapshotCoordinates | null;
|
||||
layoutUnitsPerCssPixel: number | null;
|
||||
}
|
||||
|
||||
export interface NormalizedDocument {
|
||||
nodes: NodeFacts[];
|
||||
index: DocumentIndex;
|
||||
}
|
||||
|
||||
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
|
||||
export async function normalizeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NormalizedDocument> {
|
||||
const decoded = await decodeDocument(doc, strings, signal);
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes: NodeFacts[] = [];
|
||||
for (let i = 0; i < decoded.nodes.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = decoded.nodes[i];
|
||||
const bounds = node.layout?.bounds ?? [];
|
||||
const input = snapshotViewportRect(
|
||||
bounds,
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
context.coordinates,
|
||||
);
|
||||
const local = input?.rect;
|
||||
const rect =
|
||||
input && context.projection.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
const visibility = node.layout?.styles.visibility || "visible";
|
||||
const opacity = node.layout?.styles.opacity || "1";
|
||||
nodes.push({
|
||||
...node,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
|
||||
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
|
||||
rendered:
|
||||
bounds.length >= 4 &&
|
||||
bounds.slice(0, 4).every(Number.isFinite) &&
|
||||
bounds[2] > 0 &&
|
||||
bounds[3] > 0 &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0,
|
||||
});
|
||||
}
|
||||
const index = await buildDocumentIndex(nodes, signal);
|
||||
return {
|
||||
nodes: nodes.filter(
|
||||
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
index,
|
||||
};
|
||||
}
|
||||
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
import type {
|
||||
FrameDocument as CapturedFrameDocument,
|
||||
FrameOwnedAxNode as FrameAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
|
||||
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
dom,
|
||||
sourceOrder: index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
||||
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
|
||||
};
|
||||
nodes.set(id, node);
|
||||
frameNodeIds.push(id);
|
||||
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
ax,
|
||||
sourceOrder: document.domNodes.length + index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
typeof ax.backendDOMNodeId === "number" &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
||||
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
|
||||
});
|
||||
frameNodeIds.push(id);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { DecodedDocument, DecodedNode } from "./facts";
|
||||
export const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
export interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
nodeType?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function decodeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DecodedDocument> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [] };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
layoutByNode.set(dl!.nodeIndex![i], i);
|
||||
}
|
||||
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string[]>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
if (existing) existing.push(text);
|
||||
else nodeTextContent.set(parentIdx, [text]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decode fields without assigning a coordinate projection or semantic policy.
|
||||
const nodes: DecodedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
const li = layoutByNode.get(n);
|
||||
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
|
||||
const styles: Record<string, string> = {};
|
||||
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
|
||||
const layout =
|
||||
li === undefined
|
||||
? undefined
|
||||
: {
|
||||
boundsSpace: "snapshot-document-layout" as const,
|
||||
bounds: dl?.bounds?.[li],
|
||||
styles,
|
||||
};
|
||||
|
||||
const textContent = nodeTextContent.get(n)?.join(" ");
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
nodeType: dn.nodeType?.[n],
|
||||
parentBackendNodeId,
|
||||
|
||||
tag,
|
||||
attrs,
|
||||
layout,
|
||||
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
|
||||
position: styles.position || "static",
|
||||
pointerEvents: styles["pointer-events"] || "auto",
|
||||
cursor: styles.cursor || "auto",
|
||||
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes };
|
||||
}
|
||||
Reference in New Issue
Block a user