Merge pull request #250 from Tencent/fix/background-tab-screenshots

fix(bsk): capture controlled tabs without activating them | 支持无需激活的受控后台标签页的视口截图

apps/extension/src/tools/__tests__/background-screenshot.browser.test.ts

@@ -0,0 +1,267 @@
+// @vitest-environment node
+// Opt in: BSK_BACKGROUND_CHROME=/path/to/chrome. Owns an isolated headed profile.
+import { createServer } from "node:http";
+import { describe, expect, it } from "vitest";
+import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import { SessionManager } from "@/session-manager/manager";
+import { handleClick } from "../interaction";
+import { handleObserve, handleScreenshot, handleSnapshot } from "../observation";
+
+type Send = <T = Record<string, unknown>>(
+  method: string,
+  params?: object,
+  sessionId?: string,
+) => Promise<T>;
+const html = `<!doctype html><style>body{margin:0;height:4000px;background:rgb(0,170,40)}button{display:block;width:200px;height:50px}canvas{display:block}</style>
+<button id="paint">Paint blue</button><canvas id="canvas" width="200" height="100"></canvas>
+<script>
+const canvas=document.querySelector('canvas'), ctx=canvas.getContext('2d');
+ctx.fillStyle='rgb(230,180,10)';ctx.fillRect(0,0,200,100);
+document.querySelector('button').onclick=()=>requestAnimationFrame(()=>{document.body.style.background='rgb(20,40,210)'});
+canvas.onclick=()=>requestAnimationFrame(()=>{ctx.fillStyle='rgb(180,30,90)';ctx.fillRect(0,0,200,100)});
+</script>`;
+
+function closeColor(actual: number[], expected: number[]) {
+  expected.forEach((value, index) =>
+    expect(Math.abs(actual[index] - value)).toBeLessThanOrEqual(3),
+  );
+}
+
+describe.skipIf(!process.env.BSK_BACKGROUND_CHROME)(
+  "background screenshot browser regression",
+  () => {
+    it.each([
+      { deviceScale: 1, zoom: 1 },
+      { deviceScale: 2, zoom: 1.25 },
+    ])("captures current target pixels and preserves ref interactions at $deviceScale/$zoom", async ({
+      deviceScale,
+      zoom,
+    }) => {
+      const server = createServer((_req, res) => {
+        res.setHeader("content-type", "text/html");
+        res.end(html);
+      });
+      await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+      const address = server.address();
+      if (!address || typeof address === "string") throw new Error("Missing fixture address");
+      const url = `http://127.0.0.1:${address.port}`;
+      try {
+        const { withChrome } = await import(
+          new URL(
+            "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
+            import.meta.url,
+          ).href
+        );
+        await withChrome(
+          { executable: process.env.BSK_BACKGROUND_CHROME, deviceScale, zoom, headless: false },
+          async (send: Send) => {
+            const { targetInfos } = await send<{
+              targetInfos: { targetId: string; type: string }[];
+            }>("Target.getTargets");
+            const control = targetInfos.find((target) => target.type === "page")!;
+            const controlSession = (
+              await send<{ sessionId: string }>("Target.attachToTarget", {
+                targetId: control.targetId,
+                flatten: true,
+              })
+            ).sessionId;
+            const targets = new Map<number, string>();
+            const sessions = new Map<number, string>();
+            const calls: string[] = [];
+            for (const id of [7, 8])
+              targets.set(
+                id,
+                (
+                  await send<{ targetId: string }>("Target.createTarget", {
+                    url: "about:blank",
+                    background: true,
+                  })
+                ).targetId,
+              );
+            const api: CdpDebuggerApi = {
+              attach: async ({ tabId }) => {
+                sessions.set(
+                  tabId!,
+                  (
+                    await send<{ sessionId: string }>("Target.attachToTarget", {
+                      targetId: targets.get(tabId!),
+                      flatten: true,
+                    })
+                  ).sessionId,
+                );
+              },
+              detach: async ({ tabId }) => {
+                await send("Target.detachFromTarget", { sessionId: sessions.get(tabId!) });
+                sessions.delete(tabId!);
+              },
+              sendCommand: async (target, method, params) => {
+                calls.push(method);
+                return send(method, params, target.sessionId ?? sessions.get(target.tabId!));
+              },
+              onEvent: {
+                addListener: () => {},
+                removeListener: () => {},
+              } as unknown as CdpDebuggerApi["onEvent"],
+              onDetach: {
+                addListener: () => {},
+                removeListener: () => {},
+              } as unknown as CdpDebuggerApi["onDetach"],
+            };
+            const cdp = new ChromiumCdp(api);
+            const evaluate = async <T>(expression: string, session = sessions.get(7)) => {
+              const result = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
+                "Runtime.evaluate",
+                { expression, awaitPromise: true, returnByValue: true },
+                session,
+              );
+              expect(result.exceptionDetails).toBeUndefined();
+              return result.result.value;
+            };
+            const paint = () =>
+              evaluate(
+                "new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(()=>r(true))))",
+              );
+            const pixels = (png: string) =>
+              evaluate<{ width: number; height: number; corner: number[]; center: number[] }>(
+                `(async()=>{const image=await createImageBitmap(await (await fetch('data:image/png;base64,${png}')).blob());const c=new OffscreenCanvas(image.width,image.height),x=c.getContext('2d');x.drawImage(image,0,0);return {width:image.width,height:image.height,corner:Array.from(x.getImageData(image.width-20,image.height-20,1,1).data),center:Array.from(x.getImageData(Math.floor(image.width/2),Math.floor(image.height/2),1,1).data)}})()`,
+                controlSession,
+              );
+            const manager = new SessionManager({
+              agentWindow: {
+                create: async () => 100,
+                remove: async () => {},
+                ensureActiveTab: async () => 1,
+              },
+            });
+            const ctx = await manager.start("agent");
+            const tabsApi = {
+              get: async (id: number) =>
+                ({ id, windowId: 100, active: false, url }) as chrome.tabs.Tab,
+              query: async () => [] as chrome.tabs.Tab[],
+            };
+            let windowCaptures = 0;
+            const deps = {
+              cdp,
+              tabsApi,
+              captureApi: {
+                ...tabsApi,
+                captureVisibleTab: async () => {
+                  windowCaptures++;
+                  throw new Error("Wrong backend");
+                },
+              },
+              sendToTab: async () => {},
+            };
+            const params = { session_id: "agent", tab_id: 7 };
+            try {
+              await evaluate("document.body.style.background='rgb(200,0,0)'", controlSession);
+              const controlBefore = await evaluate(
+                "({hidden:document.hidden,focus:document.hasFocus()})",
+                controlSession,
+              );
+              for (const id of [7, 8]) {
+                ctx.agentCreatedTabs.add(id);
+                await cdp.acquireBackgroundExecution("agent", id);
+                await cdp.send(id, "Page.navigate", { url });
+                await expect
+                  .poll(() =>
+                    evaluate(
+                      "document.readyState === 'complete' && !!document.querySelector('canvas')",
+                      sessions.get(id),
+                    ),
+                  )
+                  .toBe(true);
+              }
+              await paint();
+              const shot = await handleScreenshot(manager, params, deps);
+              if ("code" in shot) throw new Error(JSON.stringify(shot));
+              const first = await pixels(shot.image_base64);
+              closeColor(first.corner, [0, 170, 40, 255]);
+              const viewport = await evaluate<{ width: number; height: number; dpr: number }>(
+                "({width:innerWidth,height:innerHeight,dpr:devicePixelRatio})",
+              );
+              expect(Math.abs(shot.width - viewport.width * viewport.dpr)).toBeLessThanOrEqual(2);
+              expect(Math.abs(shot.height - viewport.height * viewport.dpr)).toBeLessThanOrEqual(2);
+              expect(shot).not.toHaveProperty("capture_id");
+              const snapshot = await handleSnapshot(manager, params, deps);
+              expect(snapshot).not.toHaveProperty("code");
+              const button = [...ctx.refStore.entries()].find(
+                ([, entry]) => entry.kind === "dom" && entry.name === "Paint blue",
+              );
+              expect(button).toBeDefined();
+              const element = await handleScreenshot(manager, { ...params, ref: button![0] }, deps);
+              if ("code" in element) throw new Error(JSON.stringify(element));
+              expect(element.width).toBeLessThan(shot.width);
+              expect(
+                await handleClick(manager, { ...params, ref: button![0] }, deps),
+              ).not.toHaveProperty("code");
+              await paint();
+              const blue = await handleScreenshot(manager, params, deps);
+              if ("code" in blue) throw new Error(JSON.stringify(blue));
+              closeColor((await pixels(blue.image_base64)).corner, [20, 40, 210, 255]);
+              const other = await handleScreenshot(manager, { ...params, tab_id: 8 }, deps);
+              if ("code" in other) throw new Error(JSON.stringify(other));
+              closeColor((await pixels(other.image_base64)).corner, [0, 170, 40, 255]);
+              expect(other.tab_id).toBe(8);
+              await handleObserve(manager, params, deps);
+              const canvas = [...ctx.refStore.entries()].find(
+                ([, entry]) => entry.kind === "visual-region",
+              );
+              expect(canvas).toBeDefined();
+              const canvasShot = await handleScreenshot(
+                manager,
+                { ...params, ref: canvas![0] },
+                deps,
+              );
+              if ("code" in canvasShot) throw new Error(JSON.stringify(canvasShot));
+              closeColor((await pixels(canvasShot.image_base64)).center, [230, 180, 10, 255]);
+              expect(canvasShot.capture_id).toBeTruthy();
+              expect(
+                await handleClick(
+                  manager,
+                  {
+                    ...params,
+                    ref: canvas![0],
+                    capture_id: canvasShot.capture_id,
+                    image_x: Math.floor(canvasShot.width / 2),
+                    image_y: Math.floor(canvasShot.height / 2),
+                  },
+                  deps,
+                ),
+              ).not.toHaveProperty("code");
+              await paint();
+              const paintedCanvas = await handleScreenshot(
+                manager,
+                { ...params, ref: canvas![0] },
+                deps,
+              );
+              if ("code" in paintedCanvas) throw new Error(JSON.stringify(paintedCanvas));
+              closeColor((await pixels(paintedCanvas.image_base64)).center, [180, 30, 90, 255]);
+              await evaluate("window.scrollTo(0,1000)");
+              await paint();
+              const scrolled = await handleScreenshot(manager, params, deps);
+              if ("code" in scrolled) throw new Error(JSON.stringify(scrolled));
+              expect(scrolled.height).toBe(shot.height);
+              expect(await evaluate("scrollY")).toBe(1000);
+              expect(
+                await evaluate(
+                  "({hidden:document.hidden,focus:document.hasFocus()})",
+                  controlSession,
+                ),
+              ).toEqual(controlBefore);
+              expect(windowCaptures).toBe(0);
+              expect(calls).not.toContain("Page.bringToFront");
+            } finally {
+              await cdp.detachAll();
+              cdp.dispose();
+            }
+          },
+        );
+      } finally {
+        await new Promise<void>((resolve, reject) =>
+          server.close((error) => (error ? reject(error) : resolve())),
+        );
+      }
+    }, 60_000);
+  },
+);

apps/extension/src/tools/__tests__/click.browser.test.ts

@@ -4,6 +4,7 @@ import { createServer } from "node:http";
 import { describe, expect, it, vi } from "vitest";
 import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
 import { SessionManager } from "@/session-manager/manager";
+import { prepareBackgroundExecution } from "../background-execution";
 import { handleClick, handlePress } from "../interaction";
 import { handleObserve } from "../observation";
 import type { CdpRunner } from "../shared";
@@ -230,14 +231,28 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
             if (event.method === "DOM.documentUpdated") documentUpdates++;
             for (const listener of listeners) listener({ tabId: 4 }, event.method, event.params);
           };
+          const readinessCommands: { method: string; elapsedMs?: number; state: string }[] = [];
           const api: CdpDebuggerApi = {
             // page() already attached the root debugger session.
             attach: async () => {},
             detach: async () => {
               await send("Target.detachFromTarget", { sessionId: target.sessionId });
             },
-            sendCommand: (debuggee, method, params) =>
-              send(method, params, debuggee.sessionId ?? target.sessionId),
+            sendCommand: async (debuggee, method, params) => {
+              const call = { method, state: "pending", elapsedMs: undefined as number | undefined };
+              readinessCommands.push(call);
+              const started = performance.now();
+              try {
+                const result = await send(method, params, debuggee.sessionId ?? target.sessionId);
+                call.state = "complete";
+                return result;
+              } catch (error) {
+                call.state = "failed";
+                throw error;
+              } finally {
+                call.elapsedMs = Math.round(performance.now() - started);
+              }
+            },
             onEvent: {
               addListener: (listener: CdpEventListener) => listeners.add(listener),
               removeListener: (listener: CdpEventListener) => listeners.delete(listener),
@@ -261,6 +276,22 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
             await cdp.send(4, "DOM.enable");
             const tab = { id: 4, windowId: 100, active: false, url } as chrome.tabs.Tab;
             const tabsApi = { get: async () => tab, query: async () => [tab] };
+            // Model the controlled request boundary used by the real dispatcher.
+            // The raw-runner cases above separately cover temporary hidden input.
+            ctx.agentCreatedTabs.add(4);
+            const prepare = (method: string) =>
+              prepareBackgroundExecution(
+                manager,
+                {
+                  id: "controlled-input",
+                  method,
+                  params: { session_id: ctx.sessionId, tab_id: 4 },
+                },
+                cdp,
+                tabsApi,
+                new AbortController().signal,
+              );
+            expect(await prepare("tool.observe")).toBeUndefined();
             const observed = await handleObserve(
               manager,
               { session_id: ctx.sessionId, tab_id: 4 },
@@ -272,12 +303,17 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
             )?.[0];
             expect(ref).toBeDefined();
             expect(documentChanges).toBe(0);
+            expect(await prepare("tool.click")).toBeUndefined();
+            readinessCommands.length = 0;
             const clicked = await handleClick(
               manager,
               { session_id: ctx.sessionId, tab_id: 4, ref: ref! },
               { cdp, tabsApi },
             );
-            expect(clicked, JSON.stringify(clicked)).not.toHaveProperty("code");
+            expect(
+              clicked,
+              JSON.stringify({ clicked, commands: readinessCommands }),
+            ).not.toHaveProperty("code");
             expect(await target.evaluate("window.clicks")).toEqual([true]);
             expect(documentChanges).toBe(0);
             expect(ctx.refStore.resolve(ref!, { tabId: 4 })).not.toBeNull();
@@ -290,10 +326,8 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
               { timeout: 5000 },
             );
 
-            // Simulate a leftover override and observe from another CDP session:
-            // ending the owning session must restore the hidden tab's real focus.
-            expect(await target.evaluate("document.hasFocus()")).toBe(false);
-            await cdp.send(4, "Emulation.setFocusEmulationEnabled", { enabled: true });
+            // The controlled policy persists across calls and navigation. Ending
+            // its owner must restore real focus, observed from another CDP session.
             expect(await target.evaluate("document.hasFocus()")).toBe(true);
             const observer = await send<{ sessionId: string }>("Target.attachToTarget", {
               targetId: target.targetId,
@@ -308,6 +342,7 @@ describe.skipIf(!process.env.BSK_CLICK_CHROME)("real browser click readiness", (
             expect(focus.result.value).toBe(false);
             expect(await foreground.evaluate("document.visibilityState")).toBe("visible");
           } finally {
+            ctx.agentCreatedTabs.delete(4);
             cdp.dispose();
             onEvent = undefined;
             await send("Target.closeTarget", { targetId: target.targetId });

apps/extension/src/tools/__tests__/observation.test.ts

@@ -4407,3 +4407,146 @@ describe("handleGetHtml", () => {
     expect(res.html.length).toBe(100);
   });
 });
+
+describe("controlled viewport screenshot routing", () => {
+  it.each([
+    false,
+    true,
+  ])("uses the exact controlled target instead of the window surface (active=%s)", async (active) => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await manager.start("aa11");
+    ctx.agentCreatedTabs.add(7);
+    const capture = vi.fn(async () => {
+      throw new Error("Window capture must not run");
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getFrameTree": () => ({ frameTree: { frame: { id: "main" } } }),
+      "Page.createIsolatedWorld": () => ({ executionContextId: 1 }),
+      "Runtime.evaluate": () => ({
+        result: { deepSerializedValue: { type: "node", value: { backendNodeId: 42 } } },
+      }),
+      "Runtime.releaseObjectGroup": () => ({}),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+    cdp.getAttachmentId = () => "attachment";
+    let nowActive = active;
+    const deps = makeScreenshotDeps({
+      cdp,
+      captureVisibleTab: capture,
+      get: vi.fn(async () => ({ id: 7, windowId: 100, active: nowActive }) as chrome.tabs.Tab),
+    });
+    deps.sendToTab = async () => {
+      nowActive = false;
+    };
+    const result = await handleScreenshot(manager, { session_id: "aa11", tab_id: 7 }, deps);
+    expect(result).toMatchObject({ tab_id: 7, image_base64: TINY_PNG, width: 1, height: 1 });
+    expect(capture).not.toHaveBeenCalled();
+    expect(sent.find((call) => call.method === "Page.captureScreenshot")?.params).toMatchObject({
+      captureBeyondViewport: false,
+    });
+  });
+
+  it("does not fall back to the window when controlled capture has no CDP", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await manager.start("aa11");
+    ctx.agentCreatedTabs.add(7);
+    const capture = vi.fn();
+    expect(
+      await handleScreenshot(
+        manager,
+        { session_id: "aa11", tab_id: 7 },
+        makeScreenshotDeps({ captureVisibleTab: capture }),
+      ),
+    ).toMatchObject({ code: "cdp_failed", data: { reason: "screenshot_capture_failed" } });
+    expect(capture).not.toHaveBeenCalled();
+  });
+});
+
+it.each([
+  "borrowed-default",
+  "background-explicit",
+] as const)("keeps screenshot targeting compatible with PR249 (%s)", async (mode) => {
+  const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+  const ctx = await manager.start("aa11");
+  ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 0 });
+  ctx.agentCreatedTabs.add(8);
+  const capture = vi.fn();
+  const { cdp } = makeFakeCdp({
+    "Page.getFrameTree": () => ({ frameTree: { frame: { id: "main" } } }),
+    "Page.createIsolatedWorld": () => ({ executionContextId: 1 }),
+    "Runtime.evaluate": () => ({
+      result: { deepSerializedValue: { type: "node", value: { backendNodeId: 42 } } },
+    }),
+    "Runtime.releaseObjectGroup": () => ({}),
+    "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+  });
+  cdp.getAttachmentId = () => "attachment";
+  let activeTab = mode === "borrowed-default" ? 7 : 8;
+  const tab = (id: number) => ({ id, windowId: 100, active: id === activeTab }) as chrome.tabs.Tab;
+  const query = vi.fn(async () => [tab(activeTab)]);
+  const deps = makeScreenshotDeps({
+    cdp,
+    get: async (id) => tab(id),
+    query,
+    captureVisibleTab: capture,
+  });
+  deps.sendToTab = async () => {
+    activeTab = 8;
+  };
+  const result = await handleScreenshot(
+    manager,
+    {
+      session_id: "aa11",
+      ...(mode === "background-explicit" ? { tab_id: 7 } : {}),
+    },
+    deps,
+  );
+  expect(result).toMatchObject({ tab_id: 7, image_base64: TINY_PNG });
+  expect(cdp.send).toHaveBeenCalledWith(7, "Page.captureScreenshot", expect.any(Object));
+  expect(capture).not.toHaveBeenCalled();
+  expect(activeTab).toBe(8);
+  if (mode === "borrowed-default")
+    expect(query).toHaveBeenCalledWith({ active: true, windowId: 100 });
+  else expect(query).not.toHaveBeenCalled();
+});
+
+it.each([
+  "returned",
+  "moved",
+  "session-ended",
+] as const)("discards a viewport result when target ownership changes during capture (%s)", async (kind) => {
+  const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+  const ctx = await manager.start("aa11");
+  ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 0 });
+  let windowId = 100;
+  const capture = vi.fn();
+  const { cdp } = makeFakeCdp({
+    "Page.getFrameTree": () => ({ frameTree: { frame: { id: "main" } } }),
+    "Page.createIsolatedWorld": () => ({ executionContextId: 1 }),
+    "Runtime.evaluate": () => ({
+      result: { deepSerializedValue: { type: "node", value: { backendNodeId: 42 } } },
+    }),
+    "Runtime.releaseObjectGroup": () => ({}),
+    "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+  });
+  cdp.getAttachmentId = () => "attachment";
+  const deps = makeScreenshotDeps({
+    cdp,
+    get: async () => ({ id: 7, windowId, active: false }) as chrome.tabs.Tab,
+    captureVisibleTab: capture,
+  });
+  const phases: string[] = [];
+  deps.sendToTab = async (_id, message) => {
+    phases.push(message.phase);
+    if (message.phase !== "end") return;
+    if (kind === "returned") ctx.borrowedTabs.delete(7);
+    if (kind === "moved") windowId = 200;
+    if (kind === "session-ended") await manager.stop("aa11");
+  };
+  expect(await handleScreenshot(manager, { session_id: "aa11", tab_id: 7 }, deps)).toMatchObject({
+    code: "not_found",
+    data: { reason: "visual_target_changed" },
+  });
+  expect(phases).toEqual(["begin", "end"]);
+  expect(capture).not.toHaveBeenCalled();
+});

apps/extension/src/tools/__tests__/viewport-screenshot.test.ts

@@ -0,0 +1,141 @@
+import { describe, expect, it, vi } from "vitest";
+import type { CaptureSuppressMessage } from "@/lib/capture-suppress-bridge";
+import type { CdpRunner } from "../shared";
+import { captureControlledViewport } from "../viewport-screenshot";
+
+const PNG =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg==";
+function fixture() {
+  const state = { attachment: "a", root: 42, controlled: true, png: PNG };
+  const hook = { command: async (_method: string) => {}, bridge: async (_phase: string) => {} };
+  const send = vi.fn(async (_tabId: number, method: string) => {
+    await hook.command(method);
+    if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
+    if (method === "Page.createIsolatedWorld") return { executionContextId: 1 };
+    if (method === "Runtime.evaluate")
+      return {
+        result: { deepSerializedValue: { type: "node", value: { backendNodeId: state.root } } },
+      };
+    if (method === "Page.captureScreenshot") return { data: state.png };
+    return {};
+  });
+  const cdp: CdpRunner = {
+    send: send as CdpRunner["send"],
+    getAttachmentId: () => state.attachment,
+  };
+  const bridge = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
+    await hook.bridge(message.phase);
+  });
+  const check = vi.fn(async () => state.controlled);
+  const run = (signal?: AbortSignal) => captureControlledViewport(cdp, 7, check, signal, bridge);
+  return { state, hook, send, cdp, bridge, check, run };
+}
+
+describe("controlled viewport capture", () => {
+  it("captures a valid viewport and restores the overlay without owning execution policy", async () => {
+    const f = fixture();
+    expect(await f.run()).toEqual({ image_base64: PNG, width: 1, height: 1 });
+    expect(f.send).toHaveBeenCalledWith(7, "Page.captureScreenshot", {
+      format: "png",
+      fromSurface: true,
+      captureBeyondViewport: false,
+    });
+    expect(f.bridge.mock.calls.map((call) => call[1].phase)).toEqual(["begin", "end"]);
+    const methods = f.send.mock.calls.map((call) => call[1]);
+    expect(methods).not.toContain("Emulation.setFocusEmulationEnabled");
+    expect(methods).not.toContain("Page.bringToFront");
+  });
+
+  it.each([
+    "document",
+    "attachment",
+    "control",
+  ])("discards a screenshot after %s changes during capture", async (kind) => {
+    const f = fixture();
+    f.hook.command = async (method) => {
+      if (method !== "Page.captureScreenshot") return;
+      if (kind === "document") f.state.root++; // Same tab and URL can contain a new document.
+      if (kind === "attachment") f.state.attachment = "b";
+      if (kind === "control") f.state.controlled = false;
+    };
+    expect(await f.run()).toMatchObject({
+      code: "not_found",
+      data: { reason: "visual_target_changed" },
+    });
+    expect(f.bridge.mock.calls.at(-1)?.[1].phase).toBe("end");
+  });
+
+  it("does not capture a document replaced while waiting for overlay suppression", async () => {
+    const f = fixture();
+    f.hook.bridge = async (phase) => {
+      if (phase === "begin") f.state.root++;
+    };
+    expect(await f.run()).toMatchObject({ code: "not_found" });
+    expect(f.send.mock.calls.some((call) => call[1] === "Page.captureScreenshot")).toBe(false);
+    expect(f.bridge.mock.calls.at(-1)?.[1].phase).toBe("end");
+  });
+
+  it("checks control again after restoring the overlay", async () => {
+    const f = fixture();
+    f.hook.bridge = async (phase) => {
+      if (phase === "end") f.state.controlled = false;
+    };
+    expect(await f.run()).toMatchObject({ code: "not_found" });
+  });
+
+  it("restores suppression and discards a cancelled capture", async () => {
+    const f = fixture();
+    const abort = new AbortController();
+    f.hook.command = async (method) => {
+      if (method === "Page.captureScreenshot") abort.abort();
+    };
+    expect(await f.run(abort.signal)).toMatchObject({ code: "cancelled" });
+    expect(f.bridge.mock.calls.at(-1)?.[1].phase).toBe("end");
+  });
+
+  it("reports capture failure without retrying another target or backend", async () => {
+    const f = fixture();
+    f.hook.command = async (method) => {
+      if (method === "Page.captureScreenshot") throw new Error("readback failed");
+    };
+    expect(await f.run()).toMatchObject({
+      code: "cdp_failed",
+      data: { reason: "screenshot_capture_failed" },
+      message: "readback failed",
+    });
+    expect(f.send.mock.calls.filter((call) => call[1] === "Page.captureScreenshot")).toHaveLength(
+      1,
+    );
+    expect(f.bridge.mock.calls.at(-1)?.[1].phase).toBe("end");
+  });
+
+  it.each([
+    "",
+    "not-png",
+    PNG.slice(0, 20),
+  ])("rejects missing or malformed image metadata", async (png) => {
+    const f = fixture();
+    f.state.png = png;
+    expect(await f.run()).toMatchObject({
+      code: "cdp_failed",
+      data: { reason: "screenshot_capture_failed" },
+    });
+  });
+
+  it("fails closed when the document cannot be identified", async () => {
+    const f = fixture();
+    f.state.attachment = "";
+    expect(await f.run()).toMatchObject({ code: "not_found" });
+    expect(f.bridge).not.toHaveBeenCalled();
+  });
+});
+
+it("does not send identity cleanup into a replacement attachment", async () => {
+  const f = fixture();
+  f.hook.command = async (method) => {
+    if (method === "Runtime.evaluate") f.state.attachment = "replacement";
+  };
+  expect(await f.run()).toMatchObject({ code: "not_found" });
+  expect(f.send.mock.calls.some((call) => call[1] === "Runtime.releaseObjectGroup")).toBe(false);
+  expect(f.send.mock.calls.some((call) => call[1] === "Page.captureScreenshot")).toBe(false);
+});

apps/extension/src/tools/observation.ts

@@ -34,7 +34,11 @@ import {
   type CaptureSuppressSendToTab,
   withExtensionOverlayHidden,
 } from "@/lib/capture-suppress-bridge";
-import type { SessionContext, SessionManager } from "@/session-manager/manager";
+import {
+  isAgentControlledTab,
+  type SessionContext,
+  type SessionManager,
+} from "@/session-manager/manager";
 import type {
   GetHtmlParams,
   GetHtmlResult,
@@ -63,6 +67,7 @@ import {
   type ToolEffect,
 } from "./shared";
 import { lookupRefTarget, resolveSnapshotRef } from "./snapshot-ref";
+import { captureControlledViewport } from "./viewport-screenshot";
 import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
 import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
 import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
@@ -352,6 +357,39 @@ export async function handleScreenshot(
     });
   }
 
+  // Pin controlled captures to their page, including when initially active:
+  // the user can select another tab while overlay suppression is pending.
+  if (isAgentControlledTab(ctx, target.tabId) && target.windowId === ctx.agentWindowId) {
+    const cdp = deps.cdp;
+    if (!cdp) {
+      return rpcError(
+        "cdp_failed",
+        "screenshot_capture_failed",
+        "Controlled viewport capture requires CDP",
+      );
+    }
+    cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
+    const captured = await captureControlledViewport(
+      cdp,
+      target.tabId,
+      async () => {
+        if (manager.get(ctx.sessionId) !== ctx || !isAgentControlledTab(ctx, target.tabId))
+          return false;
+        const tab = await deps.tabsApi.get(target.tabId);
+        return (
+          manager.get(ctx.sessionId) === ctx &&
+          isAgentControlledTab(ctx, target.tabId) &&
+          tab.id === target.tabId &&
+          tab.windowId === ctx.agentWindowId
+        );
+      },
+      signal,
+      deps.sendToTab,
+    );
+    if (isRpcError(captured)) return captured;
+    return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId });
+  }
+
   if (!target.active) {
     return rpcError(
       "invalid_params",

apps/extension/src/tools/viewport-screenshot.ts

@@ -0,0 +1,141 @@
+import {
+  type CaptureSuppressSendToTab,
+  withExtensionOverlayHidden,
+} from "@/lib/capture-suppress-bridge";
+import type { RpcError } from "@/transport/types";
+import { rpcError } from "./errors";
+import { parsePngDimensions } from "./png";
+import type { CdpRunner } from "./shared";
+import { isAbortError, throwIfAborted } from "./vom/capture-abort";
+import { verifyDocumentIdentity } from "./vom/document-identity";
+import type { DocumentIdentity } from "./vom/facts";
+
+function changed(): RpcError {
+  return rpcError(
+    "not_found",
+    "visual_target_changed",
+    "Screenshot target changed or its document could not be verified; capture the current target again",
+  );
+}
+
+/** Cleanup and identity probes must not auto-attach to a replacement connection. */
+function boundToAttachment(cdp: CdpRunner, tabId: number, attachmentId: string): CdpRunner {
+  return {
+    getAttachmentId: (id) => cdp.getAttachmentId?.(id),
+    send: <T>(id: number, method: string, params?: object) => {
+      if (id !== tabId || cdp.getAttachmentId?.(id) !== attachmentId) {
+        return Promise.reject(new Error("Screenshot attachment changed"));
+      }
+      return cdp.send<T>(id, method, params);
+    },
+  };
+}
+
+/** Read only the main document identity, not a VOM/AX snapshot or frame graph. */
+async function readDocumentIdentity(cdp: CdpRunner, tabId: number, signal?: AbortSignal) {
+  throwIfAborted(signal);
+  const tree = await cdp.send<{ frameTree?: { frame: { id: string } } }>(
+    tabId,
+    "Page.getFrameTree",
+  );
+  const attachmentId = cdp.getAttachmentId?.(tabId);
+  const frameId = tree.frameTree?.frame.id;
+  if (!attachmentId || !frameId) return null;
+  const bound = boundToAttachment(cdp, tabId, attachmentId);
+  const objectGroup = `bsk-viewport-identity-${crypto.randomUUID()}`;
+  try {
+    throwIfAborted(signal);
+    const world = await bound.send<{ executionContextId: number }>(
+      tabId,
+      "Page.createIsolatedWorld",
+      {
+        frameId,
+        worldName: "bsk-document-identity",
+      },
+    );
+    throwIfAborted(signal);
+    const root = await bound.send<{
+      result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
+    }>(tabId, "Runtime.evaluate", {
+      expression: "document.documentElement",
+      contextId: world.executionContextId,
+      objectGroup,
+      serializationOptions: {
+        serialization: "deep",
+        additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
+      },
+    });
+    const node = root.result?.deepSerializedValue;
+    const backendNodeId = node?.type === "node" ? node.value?.backendNodeId : undefined;
+    if (backendNodeId === undefined || cdp.getAttachmentId?.(tabId) !== attachmentId) return null;
+    return {
+      attachmentId,
+      frameId,
+      target: { tabId },
+      documentElementBackendNodeId: backendNodeId,
+    } satisfies DocumentIdentity;
+  } finally {
+    await bound.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => {});
+  }
+}
+
+/** Capture a controlled target's viewport without using the window's selected tab.
+ * The caller owns execution policy; this operation neither acquires nor releases it. */
+export async function captureControlledViewport(
+  cdp: CdpRunner,
+  tabId: number,
+  stillControlled: () => Promise<boolean>,
+  signal?: AbortSignal,
+  sendToTab?: CaptureSuppressSendToTab,
+): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
+  try {
+    throwIfAborted(signal);
+    if (!(await stillControlled())) return changed();
+    const identity = await readDocumentIdentity(cdp, tabId, signal);
+    if (!identity) return changed();
+    const bound = boundToAttachment(cdp, tabId, identity.attachmentId);
+    const current = async () => {
+      throwIfAborted(signal);
+      return (
+        (await stillControlled()) &&
+        (await verifyDocumentIdentity(bound, identity, signal)) === "current" &&
+        cdp.getAttachmentId?.(tabId) === identity.attachmentId
+      );
+    };
+    const shot = await withExtensionOverlayHidden(
+      tabId,
+      async () => {
+        if (!(await current())) return null;
+        return bound.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
+          format: "png",
+          fromSurface: true,
+          captureBeyondViewport: false,
+        });
+      },
+      sendToTab,
+    );
+    throwIfAborted(signal);
+    // Validate after overlay restoration as well: navigation/return can race it.
+    if (!shot || !(await current()) || !(await stillControlled())) return changed();
+    throwIfAborted(signal);
+    if (cdp.getAttachmentId?.(tabId) !== identity.attachmentId) return changed();
+    const dims = shot.data ? parsePngDimensions(shot.data) : null;
+    if (!dims) {
+      return rpcError(
+        "cdp_failed",
+        "screenshot_capture_failed",
+        "Viewport capture returned no valid PNG dimensions",
+      );
+    }
+    return { image_base64: shot.data!, ...dims };
+  } catch (error) {
+    if (signal?.aborted || isAbortError(error)) {
+      return { code: "cancelled", message: "screenshot aborted" };
+    }
+    return rpcError(
+      "cdp_failed",
+      "screenshot_capture_failed",
+      error instanceof Error ? error.message : String(error),
+    );
+  }
+}

crates/bsk-cli/skill/SKILL.md

@@ -141,9 +141,12 @@ window. For a background-created tab (`tab create --no-active`), retain the retu
 `tab_id` and pass `--tab-id <tab-id>` to observation, navigation and input commands.
 Created and borrowed web pages continue running while controlled even after they
 move into the background. A default created tab starts at `about:blank`.
-Ordinary viewport and full-page screenshots still require an active tab; do not
-activate a background task just to work around that limitation. Prefer semantic
-observation, and report the limitation when an image is required.
+Ordinary viewport screenshots of controlled tabs also work in the background,
+using the same explicit `--tab-id`. Prefer semantic observation first and take a
+screenshot when the task needs image content. Full-page screenshots still require
+an active tab; do not activate a background task just to work around that
+limitation. A viewport screenshot does not issue a Canvas `capture_id`; use the
+existing `--ref` flow for screenshot-bound Canvas clicks.
 
 Never invent tab IDs or keep a user tab across unrelated work. Do not repeat
 pending, denied or timed-out borrows. For `borrow_outcome_unknown`, inspect tab/

skill/SKILL.md

@@ -141,9 +141,12 @@ window. For a background-created tab (`tab create --no-active`), retain the retu
 `tab_id` and pass `--tab-id <tab-id>` to observation, navigation and input commands.
 Created and borrowed web pages continue running while controlled even after they
 move into the background. A default created tab starts at `about:blank`.
-Ordinary viewport and full-page screenshots still require an active tab; do not
-activate a background task just to work around that limitation. Prefer semantic
-observation, and report the limitation when an image is required.
+Ordinary viewport screenshots of controlled tabs also work in the background,
+using the same explicit `--tab-id`. Prefer semantic observation first and take a
+screenshot when the task needs image content. Full-page screenshots still require
+an active tab; do not activate a background task just to work around that
+limitation. A viewport screenshot does not issue a Canvas `capture_id`; use the
+existing `--ref` flow for screenshot-bound Canvas clicks.
 
 Never invent tab IDs or keep a user tab across unrelated work. Do not repeat
 pending, denied or timed-out borrows. For `borrow_outcome_unknown`, inspect tab/