fix: return coordinate mapping with every screenshot

mobile_take_screenshot downscales by default since #418 but returned only
the image. A model reads a position off the smaller image and taps it in
full-size screen coordinates, so every screenshot-derived tap lands short.

The tool result now carries a text block ahead of the image that states the
screenshot size, the screen size, and the multiplier to apply before tapping.
When both agree it says so. When either size is unknown the text is omitted
rather than risk a wrong instruction.

Screenshot dimensions are read from the image bytes: png through the
existing PNG class, jpeg through a small SOF-marker parser (baseline and
progressive). Screen dimensions come from getScreenSize, which is already the
space taps use, so iOS points vs pixels needs no special casing.

Closes #29, closes #163.

src/coordinate-mapping.ts

@@ -0,0 +1,30 @@
+import { Dimensions } from "./robot";
+
+const RATIO_DECIMALS = 3;
+
+const formatRatio = (ratio: number): string => Number(ratio.toFixed(RATIO_DECIMALS)).toString();
+
+const isPositive = (size: Dimensions): boolean => size.width > 0 && size.height > 0;
+
+/**
+ * Plain-words instruction for a model that reads pixel positions off a
+ * screenshot and taps in screen coordinates. The screenshot is usually
+ * downscaled, and on iOS the screen is measured in points while a full-size
+ * screenshot is in pixels, so the two spaces rarely agree.
+ *
+ * Returns null when either size is unknown, in which case no instruction is
+ * better than a wrong one.
+ */
+export const describeCoordinateMapping = (screenshot: Dimensions, screen: Dimensions): string | null => {
+	if (!isPositive(screenshot) || !isPositive(screen)) {
+		return null;
+	}
+
+	const x = formatRatio(screen.width / screenshot.width);
+	const y = formatRatio(screen.height / screenshot.height);
+	if (x === "1" && y === "1") {
+		return `Screenshot is ${screenshot.width}x${screenshot.height} and its coordinates match the screen.`;
+	}
+
+	return `Screenshot is ${screenshot.width}x${screenshot.height}. Screen coordinates are ${screen.width}x${screen.height}. To tap something you see in this screenshot, multiply its x by ${x} and y by ${y}.`;
+};

src/jpeg.ts

@@ -0,0 +1,41 @@
+import { ActionableError, Dimensions } from "./robot";
+
+const JPEG_SOI = 0xffd8;
+const MARKER_PREFIX = 0xff;
+
+// SOFn markers carry the frame header with the image dimensions. 0xc4 (DHT),
+// 0xc8 (JPG) and 0xcc (DAC) sit in the same range but are not frame headers.
+const isStartOfFrameMarker = (marker: number): boolean =>
+	marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
+
+export const getJpegDimensions = (buffer: Buffer): Dimensions => {
+	if (buffer.length < 4 || buffer.readUInt16BE(0) !== JPEG_SOI) {
+		throw new ActionableError("Invalid JPEG");
+	}
+
+	let offset = 2;
+	while (offset + 4 <= buffer.length) {
+		if (buffer[offset] !== MARKER_PREFIX) {
+			throw new ActionableError("Invalid JPEG");
+		}
+
+		const marker = buffer[offset + 1];
+		const segmentLength = buffer.readUInt16BE(offset + 2);
+
+		if (isStartOfFrameMarker(marker)) {
+			// segment: length(2) precision(1) height(2) width(2) ...
+			if (offset + 9 > buffer.length) {
+				throw new ActionableError("Invalid JPEG");
+			}
+
+			return {
+				height: buffer.readUInt16BE(offset + 5),
+				width: buffer.readUInt16BE(offset + 7),
+			};
+		}
+
+		offset += 2 + segmentLength;
+	}
+
+	throw new ActionableError("Invalid JPEG");
+};

src/server.ts

@@ -8,14 +8,18 @@ import { ChildProcess } from "node:child_process";
 
 import { error, trace } from "./logger";
 import { AndroidRobot, AndroidDeviceManager } from "./android";
-import { ActionableError, Robot, ScreenshotOptions } from "./robot";
+import { ActionableError, Dimensions, Robot, ScreenshotOptions } from "./robot";
 import { IosManager, IosRobot } from "./ios";
 import { PNG } from "./png";
+import { getJpegDimensions } from "./jpeg";
+import { describeCoordinateMapping } from "./coordinate-mapping";
 import { Mobilecli } from "./mobilecli";
 import { MobileDevice } from "./mobile-device";
 import { validateOutputPath, validateFileExtension } from "./utils";
 import { formatElements } from "./format-elements";
 
+type ScreenshotContent = { type: "text", text: string } | { type: "image", data: string, mimeType: string };
+
 const ALLOWED_LOG_EXTENSIONS = [".log", ".txt", ".jsonl"];
 const DEFAULT_DEVICE_LOG_ENTRIES = 100;
 const MAX_DEVICE_LOG_ENTRIES = 10000;
@@ -806,7 +810,7 @@ export const createMcpServer = (): McpServer => {
 		"mobile_take_screenshot",
 		{
 			title: "Take Screenshot",
-			description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.",
+			description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",
 			inputSchema: {
 				device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
 				maxSize: z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
@@ -836,29 +840,41 @@ export const createMcpServer = (): McpServer => {
 
 				const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
 				let mimeType = "image/jpeg";
-				if (!isJpeg) {
+				let screenshotSize: Dimensions;
+				if (isJpeg) {
+					screenshotSize = getJpegDimensions(screenshot);
+				} else {
 					mimeType = "image/png";
 
 					// validate we received a png, will throw exception otherwise
 					const image = new PNG(screenshot);
-					const pngSize = image.getDimensions();
-					if (pngSize.width <= 0 || pngSize.height <= 0) {
-						throw new ActionableError("Screenshot is invalid. Please try again.");
-					}
-
+					screenshotSize = image.getDimensions();
 				}
 
+				if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
+					throw new ActionableError("Screenshot is invalid. Please try again.");
+				}
+
+				// the screenshot is downscaled (and on ios the screen is in points while pixels are not),
+				// so tell the model how to map what it sees onto tap coordinates. see #29.
+				const screenSize = await robot.getScreenSize();
+				const mapping = describeCoordinateMapping(screenshotSize, screenSize);
+
 				const screenshot64 = screenshot.toString("base64");
-				trace(`Screenshot taken: ${screenshot.length} bytes`);
+				trace(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
 				posthog("tool_invoked", {
 					"ToolName": "mobile_take_screenshot",
 					"ScreenshotFilesize": screenshot64.length,
 					"ScreenshotMimeType": mimeType,
 				}).then();
 
-				return {
-					content: [{ type: "image", data: screenshot64, mimeType }]
-				};
+				const content: ScreenshotContent[] = [];
+				if (mapping !== null) {
+					content.push({ type: "text", text: mapping });
+				}
+
+				content.push({ type: "image", data: screenshot64, mimeType });
+				return { content };
 			} catch (err: any) {
 				error(`Error taking screenshot: ${err.message} ${err.stack}`);
 				return {

test/coordinate-mapping.ts

@@ -0,0 +1,27 @@
+import { test, expect } from "@playwright/test";
+import { describeCoordinateMapping } from "../src/coordinate-mapping";
+
+test.describe("coordinate mapping", () => {
+	test("tells the model to multiply when the screenshot is smaller than the screen", () => {
+		const text = describeCoordinateMapping({ width: 590, height: 1278 }, { width: 1179, height: 2556 });
+		expect(text).toBe("Screenshot is 590x1278. Screen coordinates are 1179x2556. To tap something you see in this screenshot, multiply its x by 1.998 and y by 2.");
+	});
+
+	test("says coordinates match when the screenshot is the same size as the screen", () => {
+		const text = describeCoordinateMapping({ width: 1080, height: 2400 }, { width: 1080, height: 2400 });
+		expect(text).toBe("Screenshot is 1080x2400 and its coordinates match the screen.");
+	});
+
+	test("handles a full-size pixel screenshot of a screen measured in points", () => {
+		const text = describeCoordinateMapping({ width: 1179, height: 2556 }, { width: 393, height: 852 });
+		expect(text).toBe("Screenshot is 1179x2556. Screen coordinates are 393x852. To tap something you see in this screenshot, multiply its x by 0.333 and y by 0.333.");
+	});
+
+	test("returns nothing when the screen size is unknown", () => {
+		expect(describeCoordinateMapping({ width: 590, height: 1278 }, { width: 0, height: 0 })).toBeNull();
+	});
+
+	test("returns nothing when the screenshot size is unknown", () => {
+		expect(describeCoordinateMapping({ width: 0, height: 0 }, { width: 1080, height: 2400 })).toBeNull();
+	});
+});

test/jpeg.ts

@@ -0,0 +1,29 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { test, expect } from "@playwright/test";
+import { getJpegDimensions } from "../src/jpeg";
+
+const loadFixture = (name: string): Buffer => readFileSync(join(__dirname, "fixtures", name));
+
+test.describe("jpeg", () => {
+	test("reads width and height of a baseline jpeg", () => {
+		const dimensions = getJpegDimensions(loadFixture("baseline.jpg"));
+		expect(dimensions).toEqual({ width: 96, height: 64 });
+	});
+
+	test("reads width and height of a progressive jpeg", () => {
+		const dimensions = getJpegDimensions(loadFixture("progressive.jpg"));
+		expect(dimensions).toEqual({ width: 30, height: 50 });
+	});
+
+	test("rejects a buffer that is not a jpeg", () => {
+		const notAJpeg = Buffer.from("IAMADUCKIAMADUCKIAMADUCKIAMADUCKIAMADUCK");
+		expect(() => getJpegDimensions(notAJpeg)).toThrow("Invalid JPEG");
+	});
+
+	test("rejects a jpeg that is cut off before its frame header", () => {
+		const truncated = loadFixture("baseline.jpg").subarray(0, 20);
+		expect(() => getJpegDimensions(truncated)).toThrow("Invalid JPEG");
+	});
+});