mirror of
https://github.com/mobile-next/mobile-mcp.git
synced 2026-10-02 01:05:27 +08:00
feat: new ios simulator support
This commit is contained in:
Gil Megidish
@@ -1,2 +1,3 @@
|
||||
**/.DS_Store
|
||||
node_modules
|
||||
lib
|
||||
|
||||
Executable
+1
@@ -0,0 +1 @@
|
||||
npm run lint
|
||||
Generated
+1165
File diff suppressed because it is too large
Load Diff
+10
-8
@@ -13,16 +13,14 @@
|
||||
"scripts": {
|
||||
"build": "tsc && chmod +x lib/index.js",
|
||||
"lint": "eslint .",
|
||||
"test": "mocha --require ts-node/register test/*.ts",
|
||||
"watch": "tsc --watch",
|
||||
"clean": "rm -rf lib"
|
||||
},
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"types": "./index.d.ts",
|
||||
"default": "./index.js"
|
||||
}
|
||||
"clean": "rm -rf lib",
|
||||
"prepare": "husky"
|
||||
},
|
||||
"files": [
|
||||
"lib"
|
||||
],
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.6.1",
|
||||
"fast-xml-parser": "^5.0.9",
|
||||
@@ -33,6 +31,7 @@
|
||||
"@eslint/eslintrc": "^3.2.0",
|
||||
"@eslint/js": "^9.19.0",
|
||||
"@stylistic/eslint-plugin": "^3.0.1",
|
||||
"@types/mocha": "^10.0.10",
|
||||
"@types/node": "^22.13.10",
|
||||
"@typescript-eslint/eslint-plugin": "^8.28.0",
|
||||
"@typescript-eslint/parser": "^8.26.1",
|
||||
@@ -41,6 +40,9 @@
|
||||
"eslint-plugin": "^1.0.1",
|
||||
"eslint-plugin-import": "^2.31.0",
|
||||
"eslint-plugin-notice": "^1.0.0",
|
||||
"husky": "^9.1.7",
|
||||
"mocha": "^11.1.0",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.8.2"
|
||||
},
|
||||
"main": "index.js",
|
||||
|
||||
+183
-156
@@ -1,17 +1,188 @@
|
||||
import { execSync } from "child_process";
|
||||
import { execFileSync, execSync } from "child_process";
|
||||
import * as xml from "fast-xml-parser";
|
||||
import { readFileSync, unlinkSync } from "fs";
|
||||
import { Bounds, Button, Dimensions, ElementCoordinates, Robot, SwipeDirection } from "./robot";
|
||||
|
||||
interface Bounds {
|
||||
left: number;
|
||||
top: number;
|
||||
right: number;
|
||||
bottom: number;
|
||||
interface UiAutomatorXmlNode {
|
||||
node: UiAutomatorXmlNode[];
|
||||
text?: string;
|
||||
bounds?: string;
|
||||
"content-desc"?: string;
|
||||
}
|
||||
|
||||
interface ElementCoordinates {
|
||||
x: number,
|
||||
y: number;
|
||||
interface UiAutomatorXml {
|
||||
hierarchy: {
|
||||
node: UiAutomatorXmlNode;
|
||||
};
|
||||
}
|
||||
|
||||
export class AndroidRobot implements Robot {
|
||||
|
||||
public async getScreenSize(): Promise<Dimensions> {
|
||||
const screenSize = execSync("adb shell wm size")
|
||||
.toString()
|
||||
.split(" ")
|
||||
.pop();
|
||||
|
||||
if (!screenSize) {
|
||||
throw new Error("Failed to get screen size");
|
||||
}
|
||||
|
||||
const [width, height] = screenSize.split("x").map(Number);
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
public adb(args: string[]): Buffer {
|
||||
return execFileSync("adb", args, {
|
||||
maxBuffer: 1024 * 1024 * 4,
|
||||
timeout: 30000,
|
||||
});
|
||||
}
|
||||
|
||||
public async listApps(): Promise<string[]> {
|
||||
const result = this.adb(["shell", "cmd", "package", "query-activities", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER"])
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.startsWith("packageName="))
|
||||
.map(line => line.substring("packageName=".length))
|
||||
.filter((value, index, self) => self.indexOf(value) === index);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async launchApp(packageName: string): Promise<void> {
|
||||
this.adb(["shell", "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1"]);
|
||||
}
|
||||
|
||||
public async swipe(direction: SwipeDirection): Promise<void> {
|
||||
const screenSize = await this.getScreenSize();
|
||||
const centerX = screenSize.width >> 1;
|
||||
// const centerY = screenSize[1] >> 1;
|
||||
|
||||
let x0: number, y0: number, x1: number, y1: number;
|
||||
|
||||
switch (direction) {
|
||||
case "up":
|
||||
x0 = x1 = centerX;
|
||||
y0 = Math.floor(screenSize.height * 0.80);
|
||||
y1 = Math.floor(screenSize.height * 0.20);
|
||||
break;
|
||||
case "down":
|
||||
x0 = x1 = centerX;
|
||||
y0 = Math.floor(screenSize.height * 0.20);
|
||||
y1 = Math.floor(screenSize.height * 0.80);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Swipe direction "${direction}" is not supported`);
|
||||
}
|
||||
|
||||
this.adb(["shell", "input", "swipe", `${x0}`, `${y0}`, `${x1}`, `${y1}`, "1000"]);
|
||||
}
|
||||
|
||||
public async getScreenshot(): Promise<Buffer> {
|
||||
return this.adb(["shell", "screencap", "-p"]);
|
||||
}
|
||||
|
||||
private collectElements(node: UiAutomatorXmlNode, screenSize: Dimensions): any[] {
|
||||
const elements: any[] = [];
|
||||
|
||||
const getCoordinates = (element: UiAutomatorXmlNode): Bounds => {
|
||||
const bounds = String(element.bounds);
|
||||
|
||||
const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
|
||||
return { left, top, right, bottom };
|
||||
};
|
||||
|
||||
const getCenter = (coordinates: Bounds): ElementCoordinates => {
|
||||
return {
|
||||
x: Math.floor((coordinates.left + coordinates.right) / 2),
|
||||
y: Math.floor((coordinates.top + coordinates.bottom) / 2),
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeCoordinates = (coordinates: ElementCoordinates, screenSize: Dimensions): ElementCoordinates => {
|
||||
return {
|
||||
x: Number((coordinates.x / screenSize.width).toFixed(3)),
|
||||
y: Number((coordinates.y / screenSize.height).toFixed(3)),
|
||||
};
|
||||
};
|
||||
|
||||
if (node.node) {
|
||||
if (Array.isArray(node.node)) {
|
||||
for (const childNode of node.node) {
|
||||
elements.push(...this.collectElements(childNode, screenSize));
|
||||
}
|
||||
} else {
|
||||
elements.push(...this.collectElements(node.node, screenSize));
|
||||
}
|
||||
}
|
||||
|
||||
if (node.text) {
|
||||
elements.push({
|
||||
"text": node.text,
|
||||
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
||||
});
|
||||
}
|
||||
|
||||
if (node["content-desc"]) {
|
||||
elements.push({
|
||||
"text": node["content-desc"],
|
||||
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
||||
});
|
||||
}
|
||||
|
||||
return elements;
|
||||
}
|
||||
|
||||
public async getElementsOnScreen(): Promise<any[]> {
|
||||
const dump = this.adb(["exec-out", "uiautomator", "dump", "/dev/tty"]);
|
||||
|
||||
const parser = new xml.XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: ""
|
||||
});
|
||||
|
||||
const parsedXml = parser.parse(dump) as UiAutomatorXml;
|
||||
const hierarchy = parsedXml.hierarchy;
|
||||
|
||||
const screenSize = await this.getScreenSize();
|
||||
const elements = this.collectElements(hierarchy.node, screenSize);
|
||||
return elements;
|
||||
}
|
||||
|
||||
public async terminateApp(packageName: string): Promise<void> {
|
||||
this.adb(["shell", "am", "force-stop", packageName]);
|
||||
}
|
||||
|
||||
public async openUrl(url: string): Promise<void> {
|
||||
this.adb(["shell", "am", "start", "-a", "android.intent.action.VIEW", "-d", url]);
|
||||
}
|
||||
|
||||
public async sendKeys(text: string): Promise<void> {
|
||||
// adb shell requires some escaping
|
||||
const _text = text.replace(/ /g, "\\ ");
|
||||
this.adb(["shell", "input", "text", _text]);
|
||||
}
|
||||
|
||||
public async pressButton(button: Button) {
|
||||
const _map = {
|
||||
"BACK": "KEYCODE_BACK",
|
||||
"HOME": "KEYCODE_HOME",
|
||||
"VOLUME_UP": "KEYCODE_VOLUME_UP",
|
||||
"VOLUME_DOWN": "KEYCODE_VOLUME_DOWN",
|
||||
"ENTER": "KEYCODE_ENTER",
|
||||
};
|
||||
|
||||
if (!_map[button]) {
|
||||
throw new Error(`Button "${button}" is not supported`);
|
||||
}
|
||||
|
||||
this.adb(["shell", "input", "keyevent", _map[button]]);
|
||||
}
|
||||
|
||||
public async tap(x: number, y: number): Promise<void> {
|
||||
this.adb(["shell", "input", "tap", `${x}`, `${y}`]);
|
||||
}
|
||||
}
|
||||
|
||||
export const getConnectedDevices = (): string[] => {
|
||||
@@ -19,150 +190,6 @@ export const getConnectedDevices = (): string[] => {
|
||||
.toString()
|
||||
.split("\n")
|
||||
.filter(line => !line.startsWith("List of devices attached"))
|
||||
.filter(line => line.trim() !== "");
|
||||
};
|
||||
|
||||
export const resolveLaunchableActivities = (packageName: string): string[] => {
|
||||
return execSync(`adb shell cmd package resolve-activity ${packageName}`)
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.startsWith("name="))
|
||||
.map(line => line.substring("name=".length));
|
||||
};
|
||||
|
||||
export const getScreenSize = (): [number, number] => {
|
||||
const screenSize = execSync("adb shell wm size")
|
||||
.toString()
|
||||
.split(" ")
|
||||
.pop();
|
||||
|
||||
if (!screenSize) {
|
||||
throw new Error("Failed to get screen size");
|
||||
}
|
||||
|
||||
const [width, height] = screenSize.split("x").map(Number);
|
||||
return [width, height];
|
||||
};
|
||||
|
||||
const collectElements = (node: any, screenSize: [number, number]): any[] => {
|
||||
const elements: any[] = [];
|
||||
|
||||
const getCoordinates = (element: any): Bounds => {
|
||||
const bounds = String(element.bounds);
|
||||
|
||||
const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
|
||||
return { left, top, right, bottom };
|
||||
};
|
||||
|
||||
const getCenter = (coordinates: Bounds): ElementCoordinates => {
|
||||
return {
|
||||
x: Math.floor((coordinates.left + coordinates.right) / 2),
|
||||
y: Math.floor((coordinates.top + coordinates.bottom) / 2),
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeCoordinates = (coordinates: ElementCoordinates, screenSize: [number, number]): ElementCoordinates => {
|
||||
return {
|
||||
x: Number((coordinates.x / screenSize[0]).toFixed(3)),
|
||||
y: Number((coordinates.y / screenSize[1]).toFixed(3)),
|
||||
};
|
||||
};
|
||||
|
||||
if (node.node) {
|
||||
if (Array.isArray(node.node)) {
|
||||
for (const childNode of node.node) {
|
||||
elements.push(...collectElements(childNode, screenSize));
|
||||
}
|
||||
} else {
|
||||
elements.push(...collectElements(node.node, screenSize));
|
||||
}
|
||||
}
|
||||
|
||||
if (node.text) {
|
||||
elements.push({
|
||||
"text": node.text,
|
||||
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
||||
});
|
||||
}
|
||||
|
||||
if (node["content-desc"]) {
|
||||
elements.push({
|
||||
"text": node["content-desc"],
|
||||
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
||||
});
|
||||
}
|
||||
|
||||
return elements;
|
||||
};
|
||||
|
||||
export const getElementsOnScreen = (): any[] => {
|
||||
const dump = execSync(`adb exec-out uiautomator dump /dev/tty`);
|
||||
|
||||
const parser = new xml.XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: ""
|
||||
});
|
||||
|
||||
const parsedXml = parser.parse(dump);
|
||||
const hierarchy = parsedXml.hierarchy;
|
||||
|
||||
const screenSize = getScreenSize();
|
||||
const elements = collectElements(hierarchy, screenSize);
|
||||
return elements;
|
||||
};
|
||||
|
||||
export const swipe = (direction: "up" | "down" | "left" | "right") => {
|
||||
|
||||
const screenSize = getScreenSize();
|
||||
const centerX = screenSize[0] >> 1;
|
||||
// const centerY = screenSize[1] >> 1;
|
||||
|
||||
let x0, y0, x1, y1: number;
|
||||
|
||||
switch (direction) {
|
||||
case "up":
|
||||
x0 = x1 = centerX;
|
||||
y0 = Math.floor(screenSize[1] * 0.80);
|
||||
y1 = Math.floor(screenSize[1] * 0.20);
|
||||
break;
|
||||
case "down":
|
||||
x0 = x1 = centerX;
|
||||
y0 = Math.floor(screenSize[1] * 0.20);
|
||||
y1 = Math.floor(screenSize[1] * 0.80);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Swipe direction "${direction}" is not supported`);
|
||||
}
|
||||
|
||||
execSync(`adb shell input swipe ${x0} ${y0} ${x1} ${y1} 1000`);
|
||||
};
|
||||
|
||||
export const takeScreenshot = async (): Promise<Buffer> => {
|
||||
const randomFilename = `screenshot-${Date.now()}.png`;
|
||||
|
||||
// take screenshot and save on device
|
||||
const remoteFilename = `/sdcard/Download/${randomFilename}`;
|
||||
execSync(`adb shell screencap -p ${remoteFilename}`);
|
||||
|
||||
// pull the file locally
|
||||
const localFilename = `/tmp/${randomFilename}`;
|
||||
execSync(`adb pull ${remoteFilename} ${localFilename}`);
|
||||
execSync(`adb shell rm ${remoteFilename}`);
|
||||
|
||||
const screenshot = readFileSync(localFilename);
|
||||
unlinkSync(localFilename);
|
||||
return screenshot;
|
||||
};
|
||||
|
||||
export const listApps = (): string[] => {
|
||||
const result = execSync(`adb shell cmd package query-activities -a android.intent.action.MAIN -c android.intent.category.LAUNCHER`)
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.startsWith("packageName="))
|
||||
.map(line => line.substring("packageName=".length))
|
||||
.filter((value, index, self) => self.indexOf(value) === index);
|
||||
|
||||
return result;
|
||||
.filter(line => line.trim() !== "")
|
||||
.map(line => line.split("\t")[0]);
|
||||
};
|
||||
|
||||
+368
-31
@@ -1,43 +1,380 @@
|
||||
import { execSync } from "child_process";
|
||||
import { execFileSync, execSync } from "child_process";
|
||||
import { Button, Dimensions, Robot, SwipeDirection } from "./robot";
|
||||
|
||||
export interface Simulator {
|
||||
name: string;
|
||||
uuid: string;
|
||||
name: string;
|
||||
uuid: string;
|
||||
state: string;
|
||||
}
|
||||
|
||||
export const getConnectedDevices = (): Simulator[] => {
|
||||
return execSync(`xcrun simctl list devices`)
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map(line => {
|
||||
// extract device name and UUID from the line
|
||||
const match = line.match(/(.*?)\s+\(([\w-]+)\)\s+\(Booted\)/);
|
||||
if (!match) {
|
||||
return null;
|
||||
interface SourceTreeElement {
|
||||
type: string;
|
||||
label?: string;
|
||||
name?: string;
|
||||
rawIdentifier?: string;
|
||||
rect: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
children?: Array<SourceTreeElement>;
|
||||
}
|
||||
|
||||
interface SourceTree {
|
||||
value: SourceTreeElement;
|
||||
}
|
||||
|
||||
|
||||
interface AppInfo {
|
||||
ApplicationType: string;
|
||||
Bundle: string;
|
||||
CFBundleDisplayName: string;
|
||||
CFBundleExecutable: string;
|
||||
CFBundleIdentifier: string;
|
||||
CFBundleName: string;
|
||||
CFBundleVersion: string;
|
||||
DataContainer: string;
|
||||
GroupContainers: Record<string, string>;
|
||||
Path: string;
|
||||
SBAppTags: string[];
|
||||
}
|
||||
|
||||
|
||||
export class Simctl implements Robot {
|
||||
constructor(private readonly simulatorUuid: string) {
|
||||
}
|
||||
|
||||
private simctl(...args: string[]): Buffer {
|
||||
return execFileSync(
|
||||
"xcrun",
|
||||
["simctl", ...args],
|
||||
{ maxBuffer: 1024 * 1024 * 4 });
|
||||
}
|
||||
|
||||
public async getScreenshot(): Promise<Buffer> {
|
||||
return this.simctl("io", this.simulatorUuid, "screenshot", "-");
|
||||
}
|
||||
|
||||
public async openUrl(url: string) {
|
||||
this.simctl("openurl", this.simulatorUuid, url);
|
||||
}
|
||||
|
||||
public async launchApp(packageName: string) {
|
||||
this.simctl("launch", this.simulatorUuid, packageName);
|
||||
}
|
||||
|
||||
public async terminateApp(packageName: string) {
|
||||
this.simctl("terminate", this.simulatorUuid, packageName);
|
||||
}
|
||||
|
||||
private parseIOSAppData(inputText: string): Array<AppInfo> {
|
||||
const result: Array<AppInfo> = [];
|
||||
|
||||
// Remove leading and trailing characters if needed
|
||||
const cleanText = inputText.trim();
|
||||
|
||||
// Extract each app section
|
||||
const appRegex = /"([^"]+)"\s+=\s+\{([^}]+)\};/g;
|
||||
let appMatch;
|
||||
|
||||
while ((appMatch = appRegex.exec(cleanText)) !== null) {
|
||||
// const bundleId = appMatch[1];
|
||||
const appContent = appMatch[2];
|
||||
|
||||
const appInfo: Partial<AppInfo> = {
|
||||
GroupContainers: {},
|
||||
SBAppTags: []
|
||||
};
|
||||
|
||||
// parse simple key-value pairs
|
||||
const keyValueRegex = /\s+(\w+)\s+=\s+([^;]+);/g;
|
||||
let keyValueMatch;
|
||||
|
||||
while ((keyValueMatch = keyValueRegex.exec(appContent)) !== null) {
|
||||
const key = keyValueMatch[1];
|
||||
let value = keyValueMatch[2].trim();
|
||||
|
||||
// Handle quoted string values
|
||||
if (value.startsWith('"') && value.endsWith('"')) {
|
||||
value = value.substring(1, value.length - 1);
|
||||
}
|
||||
|
||||
if (key !== "GroupContainers" && key !== "SBAppTags") {
|
||||
(appInfo as any)[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const deviceName = match[1].trim();
|
||||
const deviceUuid = match[2];
|
||||
// parse GroupContainers
|
||||
const groupContainersMatch = appContent.match(/GroupContainers\s+=\s+\{([^}]+)\};/);
|
||||
if (groupContainersMatch) {
|
||||
const groupContainersContent = groupContainersMatch[1];
|
||||
const groupRegex = /"([^"]+)"\s+=\s+"([^"]+)"/g;
|
||||
let groupMatch;
|
||||
|
||||
while ((groupMatch = groupRegex.exec(groupContainersContent)) !== null) {
|
||||
const groupId = groupMatch[1];
|
||||
const groupPath = groupMatch[2];
|
||||
appInfo.GroupContainers![groupId] = groupPath;
|
||||
}
|
||||
}
|
||||
|
||||
// parse SBAppTags
|
||||
const sbAppTagsMatch = appContent.match(/SBAppTags\s+=\s+\(\s*(.*?)\s*\);/);
|
||||
if (sbAppTagsMatch) {
|
||||
const tagsContent = sbAppTagsMatch[1].trim();
|
||||
if (tagsContent) {
|
||||
const tagRegex = /"([^"]+)"/g;
|
||||
let tagMatch;
|
||||
|
||||
while ((tagMatch = tagRegex.exec(tagsContent)) !== null) {
|
||||
appInfo.SBAppTags!.push(tagMatch[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.push(appInfo as AppInfo);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async listApps(): Promise<string[]> {
|
||||
const text = this.simctl("listapps", this.simulatorUuid).toString();
|
||||
const apps = this.parseIOSAppData(text);
|
||||
return apps.map(app => app.CFBundleIdentifier);
|
||||
}
|
||||
|
||||
public async getScreenSize(): Promise<Dimensions> {
|
||||
return this.withinSession(async (port, sessionId) => {
|
||||
const url = `http://localhost:${port}/session/${sessionId}/window/size`;
|
||||
const response = await fetch(url);
|
||||
const json = await response.json();
|
||||
return {
|
||||
name: deviceName,
|
||||
uuid: deviceUuid,
|
||||
width: json.value.width,
|
||||
height: json.value.height
|
||||
};
|
||||
})
|
||||
.filter(line => line !== null);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export const getScreenshot = (simulatorUuid: string): Buffer => {
|
||||
return execSync(`xcrun simctl io "${simulatorUuid}" screenshot -`);
|
||||
};
|
||||
public async sendKeys(keys: string) {
|
||||
await this.withinSession(async (port, sessionId) => {
|
||||
const url = `http://localhost:${port}/session/${sessionId}/wda/keys`;
|
||||
await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ value: [keys] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const openUrl = (simulatorUuid: string, url: string) => {
|
||||
return execSync(`xcrun simctl openurl "${simulatorUuid}" "${url}"`);
|
||||
};
|
||||
public async swipe(direction: SwipeDirection) {
|
||||
await this.withinSession(async (port, sessionId) => {
|
||||
|
||||
export const launchApp = (simulatorUuid: string, packageName: string) => {
|
||||
return execSync(`xcrun simctl launch "${simulatorUuid}" "${packageName}"`);
|
||||
};
|
||||
const x0 = 200;
|
||||
let y0 = 600;
|
||||
const x1 = 200;
|
||||
let y1 = 200;
|
||||
|
||||
export const listApps = (simulatorUuid: string) => {
|
||||
return execSync(`xcrun simctl list apps "${simulatorUuid}"`);
|
||||
};
|
||||
if (direction === "up") {
|
||||
const tmp = y0;
|
||||
y0 = y1;
|
||||
y1 = tmp;
|
||||
}
|
||||
|
||||
const url = `http://localhost:${port}/session/${sessionId}/actions`;
|
||||
await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
actions: [
|
||||
{
|
||||
type: "pointer",
|
||||
id: "finger1",
|
||||
parameters: { pointerType: "touch" },
|
||||
actions: [
|
||||
{ type: "pointerMove", duration: 0, x: x0, y: y0 },
|
||||
{ type: "pointerDown", button: 0 },
|
||||
{ type: "pointerMove", duration: 0, x: x1, y: y1 },
|
||||
{ type: "pause", duration: 1000 },
|
||||
{ type: "pointerUp", button: 0 }
|
||||
]
|
||||
}
|
||||
]
|
||||
}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public async tap(x: number, y: number) {
|
||||
await this.withinSession(async (port, sessionId) => {
|
||||
const url = `http://localhost:${port}/session/${sessionId}/actions`;
|
||||
await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
actions: [
|
||||
{
|
||||
type: "pointer",
|
||||
id: "finger1",
|
||||
parameters: { pointerType: "touch" },
|
||||
actions: [
|
||||
{ type: "pointerMove", duration: 0, x, y },
|
||||
{ type: "pointerDown", button: 0 },
|
||||
{ type: "pause", duration: 100 },
|
||||
{ type: "pointerUp", button: 0 }
|
||||
]
|
||||
}
|
||||
]
|
||||
}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public async pressButton(button: Button) {
|
||||
const _map = {
|
||||
"HOME": "home",
|
||||
"VOLUME_UP": "volumeup",
|
||||
"VOLUME_DOWN": "volumedown",
|
||||
};
|
||||
|
||||
if (button === "ENTER") {
|
||||
await this.sendKeys("\n");
|
||||
return;
|
||||
}
|
||||
|
||||
// Type assertion to check if button is a key of _map
|
||||
if (!(button in _map)) {
|
||||
throw new Error(`Button "${button}" is not supported`);
|
||||
}
|
||||
|
||||
await this.withinSession(async (port, sessionId) => {
|
||||
const url = `http://localhost:${port}/session/${sessionId}/wda/pressButton`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: button,
|
||||
}),
|
||||
});
|
||||
|
||||
return response.json();
|
||||
});
|
||||
}
|
||||
|
||||
private async createSession(port: number) {
|
||||
const url = `http://localhost:${port}/session`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ capabilities: { alwaysMatch: { platformName: "iOS" } } }),
|
||||
});
|
||||
|
||||
const json = await response.json();
|
||||
return json.value.sessionId;
|
||||
}
|
||||
|
||||
private async deleteSession(port: number, sessionId: string) {
|
||||
const url = `http://localhost:${port}/session/${sessionId}`;
|
||||
const response = await fetch(url, { method: "DELETE" });
|
||||
return response.json();
|
||||
}
|
||||
|
||||
private async withinSession(fn: (port: number, sessionId: string) => Promise<any>) {
|
||||
const port = 8100;
|
||||
const sessionId = await this.createSession(port);
|
||||
const result = await fn(port, sessionId);
|
||||
await this.deleteSession(port, sessionId);
|
||||
return result;
|
||||
}
|
||||
|
||||
private filterSourceElements(source: SourceTreeElement): Array<any> {
|
||||
|
||||
const output: any[] = [];
|
||||
|
||||
console.error("gilm " + JSON.stringify(source));
|
||||
if (["TextField", "Button", "Switch"].includes(source.type)) {
|
||||
output.push({
|
||||
type: source.type,
|
||||
label: source.label,
|
||||
name: source.name,
|
||||
rect: {
|
||||
x0: source.rect.x,
|
||||
y0: source.rect.y,
|
||||
x1: source.rect.x + source.rect.width,
|
||||
y1: source.rect.y + source.rect.height,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (source.children) {
|
||||
for (const child of source.children) {
|
||||
output.push(...this.filterSourceElements(child));
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
public async getPageSource(): Promise<SourceTree> {
|
||||
const port = 8100;
|
||||
const url = `http://localhost:${port}/source/?format=json`;
|
||||
const response = await fetch(url);
|
||||
const json = await response.json();
|
||||
return json as SourceTree;
|
||||
}
|
||||
|
||||
public async getElementsOnScreen(): Promise<any[]> {
|
||||
const source = await this.getPageSource();
|
||||
return this.filterSourceElements(source.value);
|
||||
}
|
||||
}
|
||||
|
||||
export class SimctlManager {
|
||||
|
||||
private parseSimulator(line: string): Simulator | null {
|
||||
// extract device name and UUID from the line
|
||||
const match = line.match(/(.*?)\s+\(([\w-]+)\)\s+\((\w+)\)/);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const deviceName = match[1].trim();
|
||||
const deviceUuid = match[2];
|
||||
const deviceState = match[3];
|
||||
|
||||
return {
|
||||
name: deviceName,
|
||||
uuid: deviceUuid,
|
||||
state: deviceState,
|
||||
};
|
||||
}
|
||||
|
||||
public listSimulators(): Simulator[] {
|
||||
return execSync(`xcrun simctl list devices`)
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map(line => this.parseSimulator(line))
|
||||
.filter(simulator => simulator !== null);
|
||||
}
|
||||
|
||||
public listBootedSimulators(): Simulator[] {
|
||||
return this.listSimulators()
|
||||
.filter(simulator => simulator.state === "Booted");
|
||||
}
|
||||
|
||||
public getSimulator(uuid: string): Simctl {
|
||||
return new Simctl(uuid);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
export interface Bounds {
|
||||
left: number;
|
||||
top: number;
|
||||
right: number;
|
||||
bottom: number;
|
||||
}
|
||||
|
||||
export interface ElementCoordinates {
|
||||
x: number,
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface Dimensions {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type SwipeDirection = "up" | "down" | "left" | "right";
|
||||
|
||||
export type Button = "HOME" | "BACK" | "VOLUME_UP" | "VOLUME_DOWN" | "ENTER";
|
||||
|
||||
export interface Robot {
|
||||
/**
|
||||
* Get the screen size of the device in pixels.
|
||||
*/
|
||||
getScreenSize(): Promise<Dimensions>;
|
||||
|
||||
/**
|
||||
* Swipe in a direction.
|
||||
*/
|
||||
swipe(direction: SwipeDirection): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get a screenshot of the screen. Returns a Buffer that contains
|
||||
* a PNG image of the screen. Will be same dimensions as getScreenSize().
|
||||
*/
|
||||
getScreenshot(): Promise<Buffer>;
|
||||
|
||||
/**
|
||||
* List all installed apps on the device. Returns an array of package names (or
|
||||
* bundle identifiers in iOS) for all installed apps.
|
||||
*/
|
||||
listApps(): Promise<string[]>;
|
||||
|
||||
/**
|
||||
* Launch an app.
|
||||
*/
|
||||
launchApp(packageName: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Terminate an app. If app was already terminated (or non existent) then this
|
||||
* is a no-op.
|
||||
*/
|
||||
terminateApp(packageName: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Open a URL in the device's web browser. Can be an https:// url, or a
|
||||
* custom scheme (e.g. "myapp://").
|
||||
*/
|
||||
openUrl(url: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Send keys to the device, simulating keyboard input.
|
||||
*/
|
||||
sendKeys(text: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Press a button on the device, simulating a physical button press.
|
||||
*/
|
||||
pressButton(button: Button): Promise<void>;
|
||||
|
||||
/**
|
||||
* Tap on a specific coordinate on the screen.
|
||||
*/
|
||||
tap(x: number, y: number): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get all elements on the screen. Works only on native apps (not webviews). Will
|
||||
* return a filtered list of elements that make sense to interact with.
|
||||
*/
|
||||
getElementsOnScreen(): Promise<any[]>;
|
||||
}
|
||||
+85
-24
@@ -1,13 +1,12 @@
|
||||
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { CallToolResult } from "@modelcontextprotocol/sdk/types";
|
||||
|
||||
import { execSync } from "child_process";
|
||||
import { error, trace } from "./logger";
|
||||
import { z, ZodRawShape, ZodTypeAny } from "zod";
|
||||
import { getElementsOnScreen, getScreenSize, listApps, swipe, takeScreenshot } from "./android";
|
||||
|
||||
import sharp from "sharp";
|
||||
|
||||
import { error, trace } from "./logger";
|
||||
import { AndroidRobot } from "./android";
|
||||
import { Robot } from "./robot";
|
||||
|
||||
const getAgentVersion = (): string => {
|
||||
const json = require("../package.json");
|
||||
return json.version;
|
||||
@@ -45,24 +44,34 @@ export const createMcpServer = (): McpServer => {
|
||||
server.tool(name, description, paramsSchema, args => wrappedCb(args));
|
||||
};
|
||||
|
||||
const robot: Robot = new AndroidRobot();
|
||||
|
||||
tool(
|
||||
"mobile_list_apps",
|
||||
"List all the installed apps on the device",
|
||||
{},
|
||||
async ({}) => {
|
||||
const result = listApps();
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
const result = await robot.listApps();
|
||||
return `Found these packages on device: ${result.join(",")}`;
|
||||
}
|
||||
);
|
||||
|
||||
tool(
|
||||
"mobile_launch_app",
|
||||
"Launch an app on mobile device",
|
||||
"Launch an app on mobile device. Use this to open a specific app. You can find the package name of the app by calling list_apps_on_device.",
|
||||
{
|
||||
packageName: z.string().describe("The package name of the app to launch"),
|
||||
},
|
||||
async ({ packageName }) => {
|
||||
execSync(`adb shell monkey -p "${packageName}" -c android.intent.category.LAUNCHER 1`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
await robot.launchApp(packageName);
|
||||
return `Launched app ${packageName}`;
|
||||
}
|
||||
);
|
||||
@@ -74,7 +83,11 @@ export const createMcpServer = (): McpServer => {
|
||||
packageName: z.string().describe("The package name of the app to terminate"),
|
||||
},
|
||||
async ({ packageName }) => {
|
||||
execSync(`adb shell am force-stop "${packageName}"`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
await robot.terminateApp(packageName);
|
||||
return `Terminated app ${packageName}`;
|
||||
}
|
||||
);
|
||||
@@ -84,8 +97,12 @@ export const createMcpServer = (): McpServer => {
|
||||
"Get the screen size of the mobile device in pixels",
|
||||
{},
|
||||
async ({}) => {
|
||||
const screenSize = getScreenSize();
|
||||
return `Screen size is ${screenSize[0]}x${screenSize[1]} pixels`;
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
const screenSize = await robot.getScreenSize();
|
||||
return `Screen size is ${screenSize.width}x${screenSize.height} pixels`;
|
||||
}
|
||||
);
|
||||
|
||||
@@ -97,10 +114,14 @@ export const createMcpServer = (): McpServer => {
|
||||
y: z.number().describe("The y coordinate to click between 0 and 1"),
|
||||
},
|
||||
async ({ x, y }) => {
|
||||
const screenSize = getScreenSize();
|
||||
const x0 = Math.floor(screenSize[0] * x);
|
||||
const y0 = Math.floor(screenSize[1] * y);
|
||||
execSync(`adb shell input tap ${x0} ${y0}`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
const screenSize = await robot.getScreenSize();
|
||||
const x0 = Math.floor(screenSize.width * x);
|
||||
const y0 = Math.floor(screenSize.height * y);
|
||||
await robot.tap(x0, y0);
|
||||
return `Clicked on screen at coordinates: ${x}, ${y}`;
|
||||
}
|
||||
);
|
||||
@@ -111,8 +132,29 @@ export const createMcpServer = (): McpServer => {
|
||||
{
|
||||
},
|
||||
async ({}) => {
|
||||
const elements = getElementsOnScreen();
|
||||
return `Found these elements on screen: ${JSON.stringify(elements)}`;
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
const screenSize = await robot.getScreenSize();
|
||||
const elements = await robot.getElementsOnScreen();
|
||||
|
||||
const result = [];
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
elements[i].rect.x0 = elements[i].rect.x0 / screenSize.width;
|
||||
elements[i].rect.y0 = elements[i].rect.y0 / screenSize.height;
|
||||
elements[i].rect.x1 = elements[i].rect.x1 / screenSize.width;
|
||||
elements[i].rect.y1 = elements[i].rect.y1 / screenSize.height;
|
||||
result.push({
|
||||
text: elements[i].label,
|
||||
coordinates: {
|
||||
x: (elements[i].rect.x0 + elements[i].rect.x1) / 2,
|
||||
y: (elements[i].rect.y0 + elements[i].rect.y1) / 2,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return `Found these elements on screen: ${JSON.stringify(result)}`;
|
||||
}
|
||||
);
|
||||
|
||||
@@ -120,10 +162,14 @@ export const createMcpServer = (): McpServer => {
|
||||
"mobile_press_button",
|
||||
"Press a button on device",
|
||||
{
|
||||
button: z.string().describe("The button to press. Supported buttons: KEYCODE_BACK, KEYCODE_HOME, KEYCODE_MENU, KEYCODE_VOLUME_UP, KEYCODE_VOLUME_DOWN, KEYCODE_ENTER"),
|
||||
button: z.string().describe("The button to press. Supported buttons: BACK, HOME, VOLUME_UP, VOLUME_DOWN, ENTER"),
|
||||
},
|
||||
async ({ button }) => {
|
||||
execSync(`adb shell input keyevent ${button}`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
robot.pressButton(button);
|
||||
return `Pressed the button: ${button}`;
|
||||
}
|
||||
);
|
||||
@@ -135,7 +181,11 @@ export const createMcpServer = (): McpServer => {
|
||||
url: z.string().describe("The URL to open"),
|
||||
},
|
||||
async ({ url }) => {
|
||||
execSync(`adb shell am start -a android.intent.action.VIEW -d "${url}"`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
robot.openUrl(url);
|
||||
return `Opened URL: ${url}`;
|
||||
}
|
||||
);
|
||||
@@ -147,7 +197,11 @@ export const createMcpServer = (): McpServer => {
|
||||
direction: z.enum(["up", "down"]).describe("The direction to swipe"),
|
||||
},
|
||||
async ({ direction }) => {
|
||||
swipe(direction);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
robot.swipe(direction);
|
||||
return `Swiped ${direction} on screen`;
|
||||
}
|
||||
);
|
||||
@@ -159,8 +213,11 @@ export const createMcpServer = (): McpServer => {
|
||||
text: z.string().describe("The text to type"),
|
||||
},
|
||||
async ({ text }) => {
|
||||
const _text = text.replace(/ /g, "\\ ");
|
||||
execSync(`adb shell input text "${_text}"`);
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
robot.sendKeys(text);
|
||||
return `Typed text: ${text}`;
|
||||
}
|
||||
);
|
||||
@@ -170,8 +227,12 @@ export const createMcpServer = (): McpServer => {
|
||||
"Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.",
|
||||
{},
|
||||
async ({}) => {
|
||||
if (!robot) {
|
||||
throw new Error("No device selected");
|
||||
}
|
||||
|
||||
try {
|
||||
const screenshot = await takeScreenshot();
|
||||
const screenshot = await robot.getScreenshot();
|
||||
|
||||
// Scale down the screenshot by 50%
|
||||
const image = sharp(screenshot);
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
|
||||
import assert from "assert";
|
||||
import { AndroidRobot } from "../src/android";
|
||||
import sharp from "sharp";
|
||||
|
||||
describe("android", () => {
|
||||
const android = new AndroidRobot();
|
||||
|
||||
it("should be able to get the screen size", async () => {
|
||||
const screenSize = await android.getScreenSize();
|
||||
assert.ok(screenSize.width > 1024);
|
||||
assert.ok(screenSize.height > 1024);
|
||||
assert.equal(Object.keys(screenSize).length, 2, "screenSize should have exactly 2 properties");
|
||||
});
|
||||
|
||||
it("should be able to take screenshot", async () => {
|
||||
const screenSize = await android.getScreenSize();
|
||||
|
||||
const screenshot = await android.getScreenshot();
|
||||
assert.ok(screenshot.length > 64 * 1024);
|
||||
|
||||
// must be a valid png image that matches the screen size
|
||||
const image = sharp(screenshot);
|
||||
const metadata = await image.metadata();
|
||||
assert.equal(metadata.width, screenSize.width);
|
||||
assert.equal(metadata.height, screenSize.height);
|
||||
});
|
||||
|
||||
it("should be able to list apps", async () => {
|
||||
const apps = await android.listApps();
|
||||
assert.ok(apps.includes("com.android.settings"));
|
||||
});
|
||||
|
||||
it("should be able to open a url", async () => {
|
||||
await android.adb(["shell", "input", "keyevent", "KEYCODE_HOME"]);
|
||||
await android.openUrl("https://www.example.com");
|
||||
});
|
||||
|
||||
it("should be able to list elements on screen", async () => {
|
||||
await android.adb(["shell", "input", "keyevent", "KEYCODE_HOME"]);
|
||||
await android.openUrl("https://www.example.com");
|
||||
const elements = await android.getElementsOnScreen();
|
||||
|
||||
const foundTitle = elements.find(element => element.text.includes("This domain is for use in illustrative examples in documents"));
|
||||
assert.ok(foundTitle);
|
||||
|
||||
// make sure navbar is present
|
||||
const foundNavbar = elements.find(element => element.text === "example.com");
|
||||
assert.ok(foundNavbar);
|
||||
|
||||
// this is an icon, but has accessibility text
|
||||
const foundSecureIcon = elements.find(element => element.text === "Connection is secure");
|
||||
assert.ok(foundSecureIcon);
|
||||
}).timeout(5000);
|
||||
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import assert from "assert";
|
||||
import { SimctlManager } from "../src/iphone-simulator";
|
||||
|
||||
describe("ios", () => {
|
||||
|
||||
const manager = new SimctlManager();
|
||||
const bootedSimulators = manager.listBootedSimulators();
|
||||
assert.ok(bootedSimulators.length === 1, "should have exactly one booted simulator");
|
||||
const simctl = manager.getSimulator(bootedSimulators[0].uuid);
|
||||
|
||||
it("should be able to get the screen size", async () => {
|
||||
const screenSize = await simctl.getScreenSize();
|
||||
assert.ok(screenSize.width > 256);
|
||||
assert.ok(screenSize.height > 256);
|
||||
assert.equal(Object.keys(screenSize).length, 2, "screenSize should have exactly 2 properties");
|
||||
});
|
||||
|
||||
it("should be able to list apps", async () => {
|
||||
const apps = await simctl.listApps();
|
||||
assert.ok(apps.includes("com.apple.mobilesafari"));
|
||||
assert.ok(apps.includes("com.apple.reminders"));
|
||||
assert.ok(apps.includes("com.apple.Preferences"));
|
||||
});
|
||||
|
||||
it("should be able to get elements on screen", async () => {
|
||||
const elements = await simctl.getElementsOnScreen();
|
||||
assert.ok(elements.length > 0);
|
||||
}).timeout(10000);
|
||||
});
|
||||
Reference in New Issue
Block a user