Merge pull request #259 from Tencent/fix/download-popup-capture

fix(extension): fix agent download capture for attachments opened in new tabs | 修复新标签页附件下载无法被 Agent 捕获
This commit is contained in:
Zhang GH
2026-09-24 12:28:05 +08:00
committed by GitHub
4 changed files with 340 additions and 21 deletions
@@ -39,6 +39,76 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
};
}
function popupDownloadFakes() {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const onCreatedNavigationTarget =
fakeEvent<(details: chrome.webNavigation.WebNavigationSourceCallbackDetails) => void>();
const completed = new Map<number, chrome.downloads.DownloadItem>();
const downloads: DownloadsApi = {
onCreated,
onChanged: fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(),
onDeterminingFilename,
search: vi.fn(async ({ id }: chrome.downloads.DownloadQuery) => {
const item = id === undefined ? undefined : completed.get(id);
return item ? [item] : [];
}),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
return {
downloads,
navigationTargets: { onCreatedNavigationTarget },
item: (id: number, url: string, finalUrl = url) =>
({
id,
url,
finalUrl,
filename: `report-${id}.csv`,
state: "in_progress",
fileSize: -1,
totalBytes: 4,
}) as chrome.downloads.DownloadItem,
/** Resolves with the suggestion Chrome would receive for this candidate. */
offer: (item: chrome.downloads.DownloadItem) =>
new Promise<chrome.downloads.DownloadFilenameSuggestion | undefined>((resolve) => {
onDeterminingFilename.emit(item, resolve);
}),
finish: (item: chrome.downloads.DownloadItem) => {
const done = {
...item,
filename: `/profile/Downloads/${item.filename}`,
state: "complete",
fileSize: 4,
} as chrome.downloads.DownloadItem;
completed.set(item.id, done);
onCreated.emit(done);
},
popup: (sourceTabId: number, url: string) =>
onCreatedNavigationTarget.emit({
sourceTabId,
sourceFrameId: 0,
sourceProcessId: 1,
tabId: 99,
url,
timeStamp: 0,
} as chrome.webNavigation.WebNavigationSourceCallbackDetails),
};
}
function silentCdp(): CdpRunner {
return {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: () => ({ dispose: vi.fn() }),
};
}
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
return {
tab: { tabId: 4, windowId: 100, active: true },
@@ -655,7 +725,13 @@ describe("file transfer tools", () => {
const result = await handleDownload(
manager,
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
{ cdp, tabsApi: tabsApi(), downloads, sendInputPassthrough },
{
cdp,
tabsApi: tabsApi(),
downloads,
navigationTargets: popupDownloadFakes().navigationTargets,
sendInputPassthrough,
},
);
expect(sendInputPassthrough.mock.calls.map(([, m]) => m.phase)).toEqual(
covered ? ["begin", "end"] : [],
@@ -1020,4 +1096,201 @@ describe("file transfer tools", () => {
data: { effect_state: "unknown", phase: "attribution" },
});
});
it("routes an attachment that the clicked link opens in a new tab", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const fakes = popupDownloadFakes();
const download = fakes.item(50, "https://example.test/export?id=50");
let suggested: Promise<chrome.downloads.DownloadFilenameSuggestion | undefined> | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed"
) {
fakes.popup(4, download.url);
suggested = fakes.offer(download);
fakes.finish(download);
}
return {};
});
const result = await handleDownload(
manager,
{
session_id: "s1",
ref: "@e3",
browser_relative_dir: "BrowserSkill/tr_50",
timeout_ms: 1_000,
},
{
cdp: { ...silentCdp(), send: send as unknown as CdpRunner["send"] },
tabsApi: tabsApi(),
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
},
);
await expect(suggested).resolves.toEqual({
filename: "BrowserSkill/tr_50/report-50.csv",
conflictAction: "overwrite",
});
expect(result).toMatchObject({
tab_id: 4,
suggested_filename: "report-50.csv",
browser_path: "/profile/Downloads/report-50.csv",
});
});
it.each([
"navigation-first",
"candidate-first",
])("correlates a new-tab download in either arrival order (%s)", async (order) => {
const fakes = popupDownloadFakes();
const download = fakes.item(
51,
"https://example.test/export?id=51",
"https://cdn.example.test/report.csv",
);
let suggested: Promise<chrome.downloads.DownloadFilenameSuggestion | undefined> | undefined;
const result = await captureBrowserDownload({
cdp: silentCdp(),
target: { tabId: 4 },
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
browserRelativeDir: "BrowserSkill/tr_51",
timeoutMs: 1_000,
trigger: async (markDispatched) => {
markDispatched();
if (order === "navigation-first") fakes.popup(4, download.url);
suggested = fakes.offer(download);
if (order === "candidate-first") fakes.popup(4, download.url);
fakes.finish(download);
return { tab_id: 4, x: 10, y: 10 };
},
});
await expect(suggested).resolves.toEqual({
filename: "BrowserSkill/tr_51/report-51.csv",
conflictAction: "overwrite",
});
expect(result).toMatchObject({ item: { id: 51, state: "complete" } });
});
it("ignores new tabs opened before mouse press or by another tab", async () => {
const fakes = popupDownloadFakes();
const url = "https://example.test/export?id=52";
let suggested: Promise<chrome.downloads.DownloadFilenameSuggestion | undefined> | undefined;
const result = await captureBrowserDownload({
cdp: silentCdp(),
target: { tabId: 4 },
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
browserRelativeDir: "BrowserSkill/tr_52",
timeoutMs: 100,
trigger: async (markDispatched) => {
fakes.popup(4, url);
markDispatched();
fakes.popup(5, url);
suggested = fakes.offer(fakes.item(52, url));
return { tab_id: 4, x: 10, y: 10 };
},
});
await expect(suggested).resolves.toBeUndefined();
expect(fakes.downloads.cancel).not.toHaveBeenCalled();
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "download_capture_failed", phase: "attribution" },
});
});
it("does not attribute a same-URL download observed before mouse press", async () => {
const fakes = popupDownloadFakes();
const url = "https://example.test/export?id=53";
const own = fakes.item(54, url);
let earlier: Promise<chrome.downloads.DownloadFilenameSuggestion | undefined> | undefined;
let suggested: Promise<chrome.downloads.DownloadFilenameSuggestion | undefined> | undefined;
const result = await captureBrowserDownload({
cdp: silentCdp(),
target: { tabId: 4 },
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
browserRelativeDir: "BrowserSkill/tr_54",
timeoutMs: 1_000,
trigger: async (markDispatched) => {
earlier = fakes.offer(fakes.item(53, url));
markDispatched();
fakes.popup(4, url);
suggested = fakes.offer(own);
fakes.finish(own);
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({ item: { id: 54, state: "complete" } });
await expect(suggested).resolves.toMatchObject({
filename: "BrowserSkill/tr_54/report-54.csv",
});
await expect(earlier).resolves.toBeUndefined();
expect(fakes.downloads.cancel).not.toHaveBeenCalled();
});
it("rejects ambiguous new-tab attribution without cancelling either download", async () => {
const fakes = popupDownloadFakes();
const url = "https://example.test/export?id=55";
const suggestions: Array<Promise<chrome.downloads.DownloadFilenameSuggestion | undefined>> = [];
const result = await captureBrowserDownload({
cdp: silentCdp(),
target: { tabId: 4 },
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
browserRelativeDir: "BrowserSkill/tr_55",
timeoutMs: 1_000,
trigger: async (markDispatched) => {
markDispatched();
fakes.popup(4, url);
suggestions.push(fakes.offer(fakes.item(55, url)), fakes.offer(fakes.item(56, url)));
return { tab_id: 4, x: 10, y: 10 };
},
});
await expect(Promise.all(suggestions)).resolves.toEqual([undefined, undefined]);
expect(fakes.downloads.cancel).not.toHaveBeenCalled();
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "unknown", phase: "attribution" },
});
});
it("reports an unknown effect when the click fails after opening a new tab", async () => {
const fakes = popupDownloadFakes();
const result = await captureBrowserDownload({
cdp: silentCdp(),
target: { tabId: 4 },
downloads: fakes.downloads,
navigationTargets: fakes.navigationTargets,
browserRelativeDir: "BrowserSkill/tr_57",
timeoutMs: 1_000,
trigger: async (markDispatched) => {
markDispatched();
fakes.popup(4, "https://example.test/export?id=57");
return { code: "cdp_failed", message: "mouseReleased failed" };
},
});
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "unknown", phase: "trigger" },
});
});
});
+54 -16
View File
@@ -1,6 +1,9 @@
// Order-independent coordinator for one browser download. CDP supplies the
// exact target/frame intent while chrome.downloads supplies the download id
// and filename routing hook; neither event is assumed to arrive first.
// and filename routing hook; neither event is assumed to arrive first. A click
// that opens a new tab (for example `target="_blank"` to an attachment) emits
// no CDP intent on the clicked target, so a navigation target that the clicked
// tab opens after mouse press supplies the intent URL instead.
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
@@ -45,16 +48,30 @@ export const chromeDownloadsApi: DownloadsApi = {
removeFile: (id) => chrome.downloads.removeFile(id),
};
export interface NavigationTargetsApi {
onCreatedNavigationTarget: ListenerEvent<
(details: chrome.webNavigation.WebNavigationSourceCallbackDetails) => void
>;
}
export const chromeNavigationTargetsApi: NavigationTargetsApi = {
get onCreatedNavigationTarget() {
return chrome.webNavigation.onCreatedNavigationTarget;
},
};
export interface DownloadCaptureOptions {
cdp: CdpRunner;
target: CdpTarget;
expectedFrameId?: string;
downloads: DownloadsApi;
navigationTargets?: NavigationTargetsApi;
browserRelativeDir: string;
maxByteSize?: number;
timeoutMs: number;
signal?: AbortSignal;
trigger(): Promise<ClickResult | RpcError>;
/** `markDispatched` must be called immediately before the mouse press is sent. */
trigger(markDispatched: () => void): Promise<ClickResult | RpcError>;
}
export interface DownloadCaptureResult {
@@ -72,6 +89,7 @@ interface DownloadCandidate {
item: chrome.downloads.DownloadItem;
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
suggested: boolean;
afterDispatch: boolean;
graceTimer: ReturnType<typeof setTimeout>;
}
@@ -89,6 +107,10 @@ function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadInte
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
}
function matchesPopupUrl(item: chrome.downloads.DownloadItem, popupUrls: Set<string>): boolean {
return popupUrls.has(item.url) || popupUrls.has(item.finalUrl);
}
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
if (item.fileSize >= 0) return item.fileSize;
if (item.totalBytes >= 0) return item.totalBytes;
@@ -138,6 +160,8 @@ export async function captureBrowserDownload(
): Promise<DownloadCaptureResult | RpcError> {
let click: ClickResult | undefined;
let intent: DownloadIntent | undefined;
let dispatched = false;
const popupUrls = new Set<string>();
let capturedId: number | undefined;
let settled = false;
let succeeded = false;
@@ -174,18 +198,18 @@ export async function captureBrowserDownload(
candidate.suggested = true;
candidate.suggest();
};
const matchingCandidates = (): DownloadCandidate[] => {
const currentIntent = intent;
return currentIntent
? [...candidates.values()].filter(
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
)
: [];
};
const matchesCdpIntent = (candidate: DownloadCandidate): boolean =>
intent !== undefined && matchesIntent(candidate.item, intent);
// Pre-dispatch candidates cannot come from the popup opened by this click.
const attributable = (candidate: DownloadCandidate): boolean =>
matchesCdpIntent(candidate) ||
(candidate.afterDispatch && matchesPopupUrl(candidate.item, popupUrls));
const matchingCandidates = (): DownloadCandidate[] =>
[...candidates.values()].filter((candidate) => !candidate.suggested && attributable(candidate));
const claimUnique = () => {
uniquenessTimer = undefined;
if (settled || capturedId !== undefined || !intent) return;
if (settled || capturedId !== undefined) return;
const matches = matchingCandidates();
if (matches.length !== 1) {
if (matches.length > 1) {
@@ -198,8 +222,10 @@ export async function captureBrowserDownload(
candidate.suggested = true;
clearTimeout(candidate.graceTimer);
capturedId = candidate.item.id;
const filename =
intent && matchesCdpIntent(candidate) ? intent.suggestedFilename : candidate.item.filename;
candidate.suggest({
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
filename: `${options.browserRelativeDir}/${safeBasename(filename)}`,
conflictAction: "overwrite",
});
const size = knownSize(candidate.item);
@@ -215,7 +241,7 @@ export async function captureBrowserDownload(
}
};
const reconcile = () => {
if (settled || capturedId !== undefined || !intent) return;
if (settled || capturedId !== undefined) return;
const matches = matchingCandidates();
if (matches.length > 1) {
for (const candidate of matches) suggestDefault(candidate);
@@ -232,10 +258,11 @@ export async function captureBrowserDownload(
item,
suggest,
suggested: false,
afterDispatch: dispatched,
graceTimer: setTimeout(() => {
suggestDefault(candidate);
candidates.delete(item.id);
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
if (attributable(candidate) && capturedId === undefined) {
fail(new Error("download correlation grace elapsed before unique attribution"));
}
}, CORRELATION_GRACE_MS),
@@ -270,6 +297,13 @@ export async function captureBrowserDownload(
}
};
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
const navigationTargetListener = (
details: chrome.webNavigation.WebNavigationSourceCallbackDetails,
) => {
if (!dispatched || details.sourceTabId !== options.target.tabId) return;
popupUrls.add(details.url);
reconcile();
};
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
@@ -296,6 +330,7 @@ export async function captureBrowserDownload(
options.downloads.onDeterminingFilename.addListener(determiningListener);
options.downloads.onCreated.addListener(createdListener);
options.downloads.onChanged.addListener(changedListener);
options.navigationTargets?.onCreatedNavigationTarget.addListener(navigationTargetListener);
options.signal?.addEventListener("abort", onAbort, { once: true });
operationTimer = setTimeout(
() => fail(new Error("download did not complete before timeout")),
@@ -315,11 +350,13 @@ export async function captureBrowserDownload(
}, SIZE_POLL_MS);
try {
const triggered = await options.trigger();
const triggered = await options.trigger(() => {
dispatched = true;
});
if (isRpcError(triggered)) {
void completion.catch(() => undefined);
const effect: TransferEffectState =
capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
capturedId !== undefined ? "committed" : intent || popupUrls.size > 0 ? "unknown" : "none";
failureResult = {
...triggered,
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
@@ -348,6 +385,7 @@ export async function captureBrowserDownload(
options.downloads.onDeterminingFilename.removeListener(determiningListener);
options.downloads.onCreated.removeListener(createdListener);
options.downloads.onChanged.removeListener(changedListener);
options.navigationTargets?.onCreatedNavigationTarget.removeListener(navigationTargetListener);
cdpSubscription.dispose();
for (const candidate of candidates.values()) {
clearTimeout(candidate.graceTimer);
+11 -3
View File
@@ -3,16 +3,23 @@
import type { SessionManager } from "@/session-manager/manager";
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
import {
captureBrowserDownload,
chromeDownloadsApi,
chromeNavigationTargetsApi,
type DownloadsApi,
type NavigationTargetsApi,
} from "./download-capture";
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
let downloadActive = false;
export type { DownloadsApi } from "./download-capture";
export type { DownloadsApi, NavigationTargetsApi } from "./download-capture";
export interface DownloadDeps extends InteractionDeps {
downloads?: DownloadsApi;
navigationTargets?: NavigationTargetsApi;
}
export async function handleDownload(
@@ -39,12 +46,13 @@ export async function handleDownload(
cdp: deps.cdp,
target: address.cdpTarget,
downloads: deps.downloads ?? chromeDownloadsApi,
navigationTargets: deps.navigationTargets ?? chromeNavigationTargetsApi,
browserRelativeDir: params.browser_relative_dir,
maxByteSize: params.max_byte_size,
timeoutMs: params.timeout_ms ?? 120_000,
signal: deps.signal,
expectedFrameId: address.frameId,
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
trigger: (markDispatched) => clickResolvedTarget(ctx, address, {}, deps, markDispatched),
});
if (isRpcError(capture)) return capture;
const { click, item } = capture;
+1 -1
View File
@@ -173,7 +173,7 @@ return `unsupported`; screenshots and other RPC content results remain available
- The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path.
- The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path.
- The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes.
- The extension owns only the browser transaction. Every transfer resolves one `ResolvedActionTarget`. The default upload mechanism arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's document; one verified input is committed with `DOM.setFileInputFiles`, while a non-input picker is rejected immediately. Explicit drop mode performs no click and never falls back to the chooser mechanism: after geometry resolution it temporarily excludes BrowserSkill's own overlay, verifies that the resolved drop zone still owns its local action point, and sends one native `dragEnter` / `dragOver` / `drop` transaction to that node's CDP target before restoring the overlay. OOPIF drops use target-local coordinates rather than top-level click coordinates. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate.
- The extension owns only the browser transaction. Every transfer resolves one `ResolvedActionTarget`. The default upload mechanism arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's document; one verified input is committed with `DOM.setFileInputFiles`, while a non-input picker is rejected immediately. Explicit drop mode performs no click and never falls back to the chooser mechanism: after geometry resolution it temporarily excludes BrowserSkill's own overlay, verifies that the resolved drop zone still owns its local action point, and sends one native `dragEnter` / `dragOver` / `drop` transaction to that node's CDP target before restoring the overlay. OOPIF drops use target-local coordinates rather than top-level click coordinates. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate. When the click opens a new tab instead (for example a `target="_blank"` attachment link), the clicked target receives no CDP intent; the URL of a `webNavigation` navigation target whose source is the clicked tab and that appears after the mouse press is then the intent, and only candidates observed after the press can match it.
- Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect.
- Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate.