From 3bb16e522722553eea5865fab4877efcd0fc93ff Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 00:10:00 +0100 Subject: [PATCH 1/7] container: stream the screen over MJPEG and expose WebViews over CDP The supervisor now keeps two more services alive next to the WDA forward: - 127.0.0.1:9100: WebDriverAgent's MJPEG screen stream - 127.0.0.1:9222: pymobiledevice3's Chrome DevTools Protocol bridge for Safari tabs and inspectable WebViews Forwards are tracked per UDID and port, so restarting one no longer kills the others. Both ports bind to 127.0.0.1 only and can be turned off with MJPEG_PORT=0 / CDP_PORT=0. doctor reports whether the stream is up and how many pages are inspectable. Co-Authored-By: Claude Opus 5.5 --- .env.example | 3 +++ container/doctor.sh | 20 ++++++++++++++++++++ container/lib.sh | 3 +++ container/supervisor.sh | 26 ++++++++++++++++++++------ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index ba4394f..1360e17 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,9 @@ PGID=1000 # WDA 端口,只监听 127.0.0.1 WDA_PORT=8100 +# Live screen stream (MJPEG, used by iphonectl viewer) and WebView debugging (Chrome DevTools Protocol), 127.0.0.1 only; 0 = off +MJPEG_PORT=9100 +CDP_PORT=9222 # 接了多台手机时指定要控制的那台;留空用第一台 IPHONE_UDID= # WDA 不可用、且从没连上过 WDA 时,HID 后备路线换算坐标用的屏幕尺寸(点);一般留空 diff --git a/container/doctor.sh b/container/doctor.sh index 905bce7..a063e2f 100755 --- a/container/doctor.sh +++ b/container/doctor.sh @@ -10,6 +10,7 @@ warn() { echo "⚠️ $1${2:+ $2}"; } bad() { echo "❌ $1${2:+ $2}"; [ -n "${3:-}" ] && echo " → $3"; fails=$((fails + 1)); } skip() { echo "⏭ $1 (跳过:$blocked)"; } check() { [ -z "$blocked" ]; } +off() { echo "⏭ $1 (off: $2=0)"; } echo "== 设备连接" if pgrep -x usbmuxd >/dev/null; then ok "usbmuxd 在运行"; else @@ -83,6 +84,25 @@ if check && [ -n "$bundle" ]; then fi else skip "WDA"; fi +echo "== Live view and WebView debugging" +if [ "$MJPEG_PORT" -eq 0 ]; then off "Live view" MJPEG_PORT +elif check && [ -n "$bundle" ]; then + # the MJPEG stream never ends: an HTTP 200 before the timeout means it works + code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$MJPEG_PORT") + if [ "$code" = 200 ]; then ok "Live view" "http://127.0.0.1:$MJPEG_PORT, open it on the host with iphonectl viewer" + else warn "Live view unavailable" "served by WDA, so only once WDA is ready; see data/logs/forward.log"; fi +else skip "Live view"; fi + +if [ "$CDP_PORT" -eq 0 ]; then off "WebView debugging" CDP_PORT +elif check; then + if pages=$(curl -sf -m 5 "http://127.0.0.1:$CDP_PORT/json"); then + n=$(printf '%s' "$pages" | grep -o '"webSocketDebuggerUrl"' | wc -l) + if [ "$n" -gt 0 ]; then ok "WebView debugging" "$n inspectable page(s), listed at http://127.0.0.1:$CDP_PORT/json" + else warn "WebView debugging: no inspectable pages" \ + "turn on Settings → Apps → Safari → Advanced → Web Inspector; an app's WKWebView needs isInspectable"; fi + else warn "WebView debugging unavailable" "see data/logs/cdp.log"; fi +else skip "WebView debugging"; fi + echo "== 附加功能" if [ -n "${TG_BOT_TOKEN:-}${BARK_KEY:-}" ]; then ok "通知已配置" "测试:iphonectl notify-test"; else warn "没有配置通知" "重签失败、手机离线、需要输密码时不会提醒你"; fi diff --git a/container/lib.sh b/container/lib.sh index 3dfdeeb..b184bfb 100644 --- a/container/lib.sh +++ b/container/lib.sh @@ -9,6 +9,9 @@ STATE_DIR=$DATA/state SECRETS_DIR=$DATA/secrets PLUME_DIR=$DATA/plume WDA_PORT=${WDA_PORT:-8100} +# Live screen (WDA's MJPEG stream) and WebView debugging (Chrome DevTools Protocol) ports, 127.0.0.1 only; 0 = off +MJPEG_PORT=${MJPEG_PORT:-9100} +CDP_PORT=${CDP_PORT:-9222} WDA_UNSIGNED=/opt/ictl/wda/WDA-unsigned.ipa PROFILE=$CERT_DIR/free-dev.mobileprovision PUID=${PUID:-1000} diff --git a/container/supervisor.sh b/container/supervisor.sh index 88a2bd5..6669897 100755 --- a/container/supervisor.sh +++ b/container/supervisor.sh @@ -31,11 +31,23 @@ start_tunnel() { log "隧道已(重新)启动" } # 不用 ios forward:它监听所有网卡,而 WDA 没有认证;pymobiledevice3 的转发默认只绑 127.0.0.1 -start_forward() { - pkill -f 'usbmux forward'; sleep 1 - pymobiledevice3 usbmux forward --udid "$UDID" "$WDA_PORT" "$WDA_PORT" >> "$LOG_DIR/forward.log" 2>&1 & - fwd_udid=$UDID - log "端口转发 127.0.0.1:$WDA_PORT 已启动" +# One forward process per port, identified by UDID and port: restarted when the phone changes, independent of each other. +ensure_forward() { + [ "$1" -gt 0 ] || return 0 + running "usbmux forward --udid $UDID $1 $1\$" && return 0 + pkill -f "usbmux forward --udid .* $1 $1\$"; sleep 1 + pymobiledevice3 usbmux forward --udid "$UDID" "$1" "$1" >> "$LOG_DIR/forward.log" 2>&1 & + log "Port forward 127.0.0.1:$1 started" +} + +# WebView debugging: serves the phone's Safari tabs and inspectable WebViews over the Chrome DevTools Protocol. +# Needs Settings → Apps → Safari → Advanced → Web Inspector on the phone, and isInspectable on an app's WKWebView. +ensure_cdp() { + [ "$CDP_PORT" -gt 0 ] || return 0 + running "webinspector cdp --udid $UDID " && return 0 + pkill -f 'webinspector cdp'; sleep 1 + pymobiledevice3 webinspector cdp --udid "$UDID" --host 127.0.0.1 --port "$CDP_PORT" >> "$LOG_DIR/cdp.log" 2>&1 & + log "CDP bridge 127.0.0.1:$CDP_PORT started" } # 手机重启后开发者镜像会掉;已挂载时 auto-mount 约 1 秒直接跳过 start_wda() { @@ -105,7 +117,9 @@ while :; do fi fi - if ! running 'usbmux forward' || [ "${fwd_udid:-}" != "$UDID" ]; then start_forward; fi + ensure_forward "$WDA_PORT" + ensure_forward "$MJPEG_PORT" + ensure_cdp bundle=$(wda_bundle) if [ -z "$bundle" ]; then From 00da95a64db1f46c0031a717751ccf86a1751013 Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 00:10:00 +0100 Subject: [PATCH 2/7] mcp: join the active WebDriverAgent session instead of replacing it WebDriverAgent runs a single session and opening a new one ends the current one. Reuse the session reported by /status, so the MCP server and other clients (such as iphonectl viewer) can work side by side. A stale session is still replaced once. Co-Authored-By: Claude Opus 5.5 --- src/iphone_mcp/wda.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/iphone_mcp/wda.py b/src/iphone_mcp/wda.py index d0f4d3d..0a31cef 100644 --- a/src/iphone_mcp/wda.py +++ b/src/iphone_mcp/wda.py @@ -46,13 +46,20 @@ def request(method, path, body=None, timeout=30): _sid = None +def _open_session(join_existing): + """WDA has a single session and opening one ends the current one (e.g. iphonectl viewer's), so join it first.""" + if join_existing and (sid := request("GET", "/status").get("sessionId")): + return sid + r = request("POST", "/session", {"capabilities": {"alwaysMatch": {}}}) + return r.get("sessionId") or r["value"]["sessionId"] + + def session(method, path, body=None, timeout=30): """带会话的 WDA 请求;WDA 重启后旧会话失效,自动重建一次。""" global _sid for attempt in range(2): if _sid is None: - r = request("POST", "/session", {"capabilities": {"alwaysMatch": {}}}) - _sid = r.get("sessionId") or r["value"]["sessionId"] + _sid = _open_session(join_existing=attempt == 0) try: return request(method, f"/session/{_sid}{path}", body, timeout) except WDAError as e: From 9336510f0bb8b19be4cb32e4ada01bb967beaeef Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 00:10:00 +0100 Subject: [PATCH 3/7] Add iphonectl viewer: live screen with mouse and keyboard control A Tk window on the host shows the MJPEG stream and turns input into touches: click/hold/drag/wheel for tap, long press, swipe and scroll, keyboard for typing, plus Home and Close app buttons for the system gestures iOS does not accept from outside. The window is freely resizable. Stream parsing, window-to-point mapping, gesture classification and session handling are unit tested. Co-Authored-By: Claude Opus 5.5 --- bin/iphonectl | 12 +- pyproject.toml | 2 + src/iphone_viewer/__init__.py | 4 + src/iphone_viewer/__main__.py | 34 ++++++ src/iphone_viewer/app.py | 210 ++++++++++++++++++++++++++++++++++ src/iphone_viewer/geometry.py | 66 +++++++++++ src/iphone_viewer/mjpeg.py | 27 +++++ src/iphone_viewer/wda.py | 102 +++++++++++++++++ tests/unit/test_viewer.py | 126 ++++++++++++++++++++ 9 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 src/iphone_viewer/__init__.py create mode 100644 src/iphone_viewer/__main__.py create mode 100644 src/iphone_viewer/app.py create mode 100644 src/iphone_viewer/geometry.py create mode 100644 src/iphone_viewer/mjpeg.py create mode 100644 src/iphone_viewer/wda.py create mode 100644 tests/unit/test_viewer.py diff --git a/bin/iphonectl b/bin/iphonectl index 990ffbb..0db0d45 100755 --- a/bin/iphonectl +++ b/bin/iphonectl @@ -57,6 +57,15 @@ cmd_shell() { need_running; inct bash; } cmd_login() { need_running; inct plumesign account login "$@"; } cmd_resign() { need_running; inc /opt/ictl/resign-wda.sh "$@"; } cmd_notify_test() { need_running; inc /opt/ictl/notify.sh --test; } + +# Live view window. Runs on the host (needs Python with Tk and Pillow) and drives the phone through WDA and the MJPEG port on 127.0.0.1 +cmd_viewer() { + need_running + local mjpeg; mjpeg=$(env_get MJPEG_PORT); mjpeg=${mjpeg:-9100} + [ "$mjpeg" -gt 0 ] || die "The live view is turned off (MJPEG_PORT=0 in .env)" + PYTHONPATH="$ROOT/src${PYTHONPATH:+:$PYTHONPATH}" exec python3 -m iphone_viewer \ + --wda "http://127.0.0.1:$PORT" --mjpeg "http://127.0.0.1:$mjpeg" "$@" +} cmd_unlock_reset() { need_running; inc rm -f /data/logs/UNLOCK-DISABLED; say "已恢复自动解锁"; } cmd_status() { @@ -198,6 +207,7 @@ usage() { passcode set|clear 设置/删除锁屏密码(自动解锁) unlock-reset 自动解锁因输错被停用后,核对密码再恢复 mcp [install] 显示/执行注册到 Claude Code 的命令 + viewer [options] Live view of the screen, controlled with mouse and keyboard (needs python3-tk and Pillow; --help) 容器 up | down | restart | pull | build | shell @@ -210,7 +220,7 @@ EOF cmd=${1:-help}; shift || true case $cmd in - setup|up|down|restart|pull|build|status|doctor|logs|resign|login|shell|passcode|mcp) + setup|up|down|restart|pull|build|status|doctor|logs|resign|login|shell|passcode|mcp|viewer) "cmd_$cmd" "$@" ;; notify-test) cmd_notify_test ;; unlock-reset) cmd_unlock_reset ;; diff --git a/pyproject.toml b/pyproject.toml index 0ac4d1a..83eb234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ iphone-mcp = "iphone_mcp.server:main" [tool.setuptools.packages.find] where = ["src"] +# iphone_viewer runs on the host (iphonectl viewer), not in the container +include = ["iphone_mcp*"] [tool.pytest.ini_options] testpaths = ["tests/unit"] diff --git a/src/iphone_viewer/__init__.py b/src/iphone_viewer/__init__.py new file mode 100644 index 0000000..4186635 --- /dev/null +++ b/src/iphone_viewer/__init__.py @@ -0,0 +1,4 @@ +"""Desktop viewer for the iPhone: live screen from WebDriverAgent's MJPEG stream, driven by mouse and keyboard. + +Runs on the host (not in the container) and only needs Python with Tk and Pillow. Start it with `iphonectl viewer`. +""" diff --git a/src/iphone_viewer/__main__.py b/src/iphone_viewer/__main__.py new file mode 100644 index 0000000..36d0be3 --- /dev/null +++ b/src/iphone_viewer/__main__.py @@ -0,0 +1,34 @@ +import argparse +import sys + +from .wda import WDAError + +MISSING_GUI = """The viewer needs Tk and Pillow's Tk support ({error}). Install them, for example: + Debian / Ubuntu: sudo apt install python3-tk python3-pil.imagetk + Arch / Manjaro: sudo pacman -S tk python-pillow + Fedora: sudo dnf install python3-tkinter python3-pillow-tk""" + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(prog="iphonectl viewer", + description="Live view of the iPhone screen, controlled with mouse and keyboard.") + parser.add_argument("--wda", default="http://127.0.0.1:8100", help="WebDriverAgent URL") + parser.add_argument("--mjpeg", default="http://127.0.0.1:9100", help="MJPEG stream URL") + parser.add_argument("--fps", type=int, default=30, help="stream frame rate (default: 30)") + parser.add_argument("--scale", type=int, default=50, + help="stream resolution in percent of the native screen (default: 50)") + parser.add_argument("--quality", type=int, default=50, help="JPEG quality 1-100 (default: 50)") + args = parser.parse_args(argv) + + try: + from .app import run + except ImportError as e: # tkinter or PIL.ImageTk missing + sys.exit(MISSING_GUI.format(error=e)) + try: + run(args.wda, args.mjpeg, args.fps, args.scale, args.quality) + except WDAError as e: + sys.exit(f"{e}\nIs the container running and WebDriverAgent ready? Check with: iphonectl doctor") + + +if __name__ == "__main__": + main() diff --git a/src/iphone_viewer/app.py b/src/iphone_viewer/app.py new file mode 100644 index 0000000..f67fd27 --- /dev/null +++ b/src/iphone_viewer/app.py @@ -0,0 +1,210 @@ +"""Tk window: shows the live stream and turns mouse and keyboard input into touches on the phone.""" +import io +import queue +import threading +import time +import tkinter as tk +import urllib.request +from collections.abc import Callable + +from PIL import Image, ImageTk + +from .geometry import LongPress, Swipe, Tap, Viewport, classify +from .mjpeg import iter_frames +from .wda import WDAClient, WDAError + +SCROLL_POINTS = 120 +RECONNECT_SECONDS = 2 +FRAME_POLL_MS = 15 + +HELP = ("Click: tap · hold: long press · drag: swipe · wheel: scroll · keys: type · Esc / right click: Home. " + "iOS ignores the swipe up from the bottom edge when it does not come from a finger, use the buttons.") + + +class FrameReader(threading.Thread): + """Reads the MJPEG stream and keeps only the newest frame (or the last connection error), reconnecting.""" + + def __init__(self, url: str): + super().__init__(daemon=True) + self.url = url + self.latest: queue.Queue[bytes | Exception] = queue.Queue(maxsize=1) + self.stopped = threading.Event() + + def _publish(self, item: bytes | Exception) -> None: + try: + self.latest.get_nowait() + except queue.Empty: + pass + self.latest.put_nowait(item) + + def run(self) -> None: + while not self.stopped.is_set(): + try: + with urllib.request.urlopen(self.url, timeout=10) as stream: + for frame in iter_frames(stream): + if self.stopped.is_set(): + return + self._publish(frame) + except OSError as e: + self._publish(e) + self.stopped.wait(RECONNECT_SECONDS) + + +class ActionWorker(threading.Thread): + """Runs phone actions one after another off the UI thread, so typed text keeps its order.""" + + def __init__(self, on_error: Callable[[Exception], None]): + super().__init__(daemon=True) + self.jobs: queue.Queue[Callable[[], None]] = queue.Queue() + self.on_error = on_error + + def submit(self, job: Callable[[], None]) -> None: + self.jobs.put(job) + + def run(self) -> None: + while True: + job = self.jobs.get() + try: + job() + except WDAError as e: + self.on_error(e) + + +class Viewer: + def __init__(self, client: WDAClient, mjpeg_url: str): + self.client = client + self.screen_w, self.screen_h = client.screen_size() + + self.root = tk.Tk() + self.root.title("iPhone") + self.root.configure(bg="#111") + self.root.minsize(180, 300) + + bar = tk.Frame(self.root, bg="#111") + bar.pack(side="bottom", fill="x") + button = {"bg": "#2a2a2a", "fg": "#eee", "activebackground": "#444", "activeforeground": "#fff", + "relief": "flat", "bd": 0, "highlightthickness": 0, "padx": 10, "pady": 4} + tk.Button(bar, text="⌂ Home", command=self.home, **button).pack(side="left", padx=(6, 3), pady=6) + tk.Button(bar, text="✕ Close app", command=self.close_app, **button).pack(side="left", padx=3, pady=6) + self.status = tk.Label(bar, text="connecting…", fg="#888", bg="#111", anchor="e") + self.status.pack(side="right", padx=6) + + start_scale = min((self.root.winfo_screenheight() - 180) / self.screen_h, 1.6) + self.canvas = tk.Canvas(self.root, width=round(self.screen_w * start_scale), + height=round(self.screen_h * start_scale), bg="#000", highlightthickness=0, + cursor="hand2") + self.canvas.pack(fill="both", expand=True) + self.viewport = Viewport.fit(self.screen_w, self.screen_h, int(self.canvas["width"]), + int(self.canvas["height"])) + self.image_item = self.canvas.create_image(0, 0, anchor="nw") + self.photo: ImageTk.PhotoImage | None = None + self.last_frame: bytes | None = None + self.press: tuple[tuple[float, float], float] | None = None + self.frames_shown, self.fps_since = 0, time.monotonic() + + self.canvas.bind("", self.on_resize) + self.canvas.bind("", self.on_press) + self.canvas.bind("", self.on_release) + self.canvas.bind("", lambda e: self.home()) + self.canvas.bind("", lambda e: self.scroll(e, 1)) # X11 wheel up + self.canvas.bind("", lambda e: self.scroll(e, -1)) # X11 wheel down + self.canvas.bind("", lambda e: self.scroll(e, 1 if e.delta > 0 else -1)) + self.root.bind("", self.on_key) + self.root.bind("", lambda e: self.home()) + self.root.protocol("WM_DELETE_WINDOW", self.close) + + self.worker = ActionWorker(lambda e: self.root.after(0, self.show_status, f"phone: {e}")) + self.worker.start() + self.reader = FrameReader(mjpeg_url) + self.reader.start() + self.root.after(FRAME_POLL_MS, self.poll_frames) + + # ---- drawing ---- + + def on_resize(self, event: tk.Event) -> None: + self.viewport = Viewport.fit(self.screen_w, self.screen_h, event.width, event.height) + self.canvas.coords(self.image_item, self.viewport.x, self.viewport.y) + if self.last_frame: + self.draw(self.last_frame) + + def draw(self, jpeg: bytes) -> None: + image = Image.open(io.BytesIO(jpeg)).convert("RGB") + image = image.resize((self.viewport.width, self.viewport.height), Image.Resampling.BILINEAR) + self.photo = ImageTk.PhotoImage(image) + self.canvas.itemconfig(self.image_item, image=self.photo) + + def poll_frames(self) -> None: + try: + item = self.reader.latest.get_nowait() + except queue.Empty: + item = None + if isinstance(item, Exception): + self.show_status(f"no stream: {item}") + elif item is not None: + self.last_frame = item + self.draw(item) + self.frames_shown += 1 + elapsed = time.monotonic() - self.fps_since + if elapsed >= 1: + self.show_status(f"{self.frames_shown / elapsed:.0f} fps") + self.frames_shown, self.fps_since = 0, time.monotonic() + self.root.after(FRAME_POLL_MS, self.poll_frames) + + def show_status(self, text: str) -> None: + self.status.config(text=text) + + # ---- input ---- + + def on_press(self, event: tk.Event) -> None: + self.press = (self.viewport.to_points(event.x, event.y), time.monotonic()) + + def on_release(self, event: tk.Event) -> None: + if self.press is None: + return + start, pressed_at = self.press + self.press = None + gesture = classify(start, self.viewport.to_points(event.x, event.y), time.monotonic() - pressed_at) + match gesture: + case Tap(x, y): + self.worker.submit(lambda: self.client.tap(x, y)) + case LongPress(x, y, seconds): + self.worker.submit(lambda: self.client.long_press(x, y, seconds)) + case Swipe(x0, y0, x1, y1, ms): + self.worker.submit(lambda: self.client.swipe(x0, y0, x1, y1, ms)) + + def scroll(self, event: tk.Event, direction: int) -> None: + x, y = self.viewport.to_points(event.x, event.y) + y1 = min(max(y + direction * SCROLL_POINTS, 1), self.screen_h - 1) + self.worker.submit(lambda: self.client.swipe(x, y, x, y1, 150)) + + def on_key(self, event: tk.Event) -> None: + if event.keysym == "BackSpace": + text = "\b" + elif event.keysym in ("Return", "KP_Enter"): + text = "\n" + elif event.char and event.char.isprintable(): + text = event.char + else: + return + self.worker.submit(lambda: self.client.type_text(text)) + + def home(self) -> None: + self.worker.submit(self.client.home) + + def close_app(self) -> None: + self.worker.submit(self.client.close_foreground_app) + + def close(self) -> None: + self.reader.stopped.set() + self.root.destroy() + + +def run(wda_url: str, mjpeg_url: str, fps: int, scale_percent: int, quality: int) -> None: + client = WDAClient(wda_url) + try: + client.configure_stream(fps, scale_percent, quality) + except WDAError as e: + print(f"warning: could not configure the stream ({e}), using WebDriverAgent's defaults") + viewer = Viewer(client, mjpeg_url) + print(HELP) + viewer.root.mainloop() diff --git a/src/iphone_viewer/geometry.py b/src/iphone_viewer/geometry.py new file mode 100644 index 0000000..ca12f51 --- /dev/null +++ b/src/iphone_viewer/geometry.py @@ -0,0 +1,66 @@ +"""Mapping between the viewer window and the phone screen, and turning mouse input into touch gestures. + +Phone coordinates are in points (what WebDriverAgent expects), window coordinates in pixels. +""" +from dataclasses import dataclass + +# a press that moves less than this (in points) is a tap, not a swipe +TAP_SLOP = 6 +LONG_PRESS_SECONDS = 0.5 +SWIPE_MS = (100, 800) + + +@dataclass(frozen=True) +class Viewport: + """Where the phone screen is drawn inside the canvas: scaled to fit, aspect ratio kept, centered.""" + + screen_w: float + screen_h: float + scale: float # canvas pixels per phone point + x: int + y: int + width: int + height: int + + @classmethod + def fit(cls, screen_w: float, screen_h: float, canvas_w: int, canvas_h: int) -> "Viewport": + scale = max(min(canvas_w / screen_w, canvas_h / screen_h), 1e-6) + width, height = max(1, round(screen_w * scale)), max(1, round(screen_h * scale)) + return cls(screen_w, screen_h, scale, (canvas_w - width) // 2, (canvas_h - height) // 2, width, height) + + def to_points(self, px: float, py: float) -> tuple[float, float]: + """Canvas pixel to phone point, clamped to the screen so presses on the margin hit the nearest edge.""" + x = (px - self.x) / self.scale + y = (py - self.y) / self.scale + return min(max(x, 0.0), self.screen_w), min(max(y, 0.0), self.screen_h) + + +@dataclass(frozen=True) +class Tap: + x: float + y: float + + +@dataclass(frozen=True) +class LongPress: + x: float + y: float + seconds: float + + +@dataclass(frozen=True) +class Swipe: + x0: float + y0: float + x1: float + y1: float + ms: int + + +def classify(start: tuple[float, float], end: tuple[float, float], seconds: float) -> Tap | LongPress | Swipe: + """Turn a mouse press (start and end in points, how long it was held) into a touch gesture.""" + (x0, y0), (x1, y1) = start, end + if abs(x1 - x0) < TAP_SLOP and abs(y1 - y0) < TAP_SLOP: + return LongPress(x0, y0, seconds) if seconds >= LONG_PRESS_SECONDS else Tap(x0, y0) + lo, hi = SWIPE_MS + return Swipe(x0, y0, x1, y1, min(max(round(seconds * 1000), lo), hi)) diff --git a/src/iphone_viewer/mjpeg.py b/src/iphone_viewer/mjpeg.py new file mode 100644 index 0000000..c897396 --- /dev/null +++ b/src/iphone_viewer/mjpeg.py @@ -0,0 +1,27 @@ +"""Reader for multipart MJPEG streams such as WebDriverAgent's MJPEG server.""" +from collections.abc import Iterator +from typing import BinaryIO + + +def iter_frames(stream: BinaryIO) -> Iterator[bytes]: + """Yield the JPEG frames of a multipart/x-mixed-replace stream until it ends. + + Each part has to carry a Content-Length header (WebDriverAgent always sends one); boundary lines and other + headers are skipped. A frame cut off by the end of the stream is dropped. + """ + length = None + while line := stream.readline(): + line = line.strip() + if line: + name, sep, value = line.partition(b":") + if sep and name.strip().lower() == b"content-length": + length = int(value) + continue + # a blank line ends the headers of a part; without a length it is just the gap before the next boundary + if length is None: + continue + frame = stream.read(length) + if len(frame) < length: + return + yield frame + length = None diff --git a/src/iphone_viewer/wda.py b/src/iphone_viewer/wda.py new file mode 100644 index 0000000..9480a54 --- /dev/null +++ b/src/iphone_viewer/wda.py @@ -0,0 +1,102 @@ +"""Minimal WebDriverAgent client for the viewer. + +WebDriverAgent runs a single session: opening a new one ends the current one, which would break the MCP server's +session. So the client joins the session that is already active and only opens one when there is none. +""" +import json +import threading +import urllib.error +import urllib.request + +SPRINGBOARD = "com.apple.springboard" + + +class WDAError(Exception): + pass + + +class WDAClient: + def __init__(self, base_url: str, timeout: float = 15): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self._sid: str | None = None + self._lock = threading.Lock() + + def request(self, method: str, path: str, body: dict | None = None) -> dict: + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(self.base_url + path, data=data, method=method, + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as r: + resp = json.loads(r.read()) + except urllib.error.HTTPError as e: + try: + resp = json.loads(e.read()) + except ValueError: + raise WDAError(f"HTTP {e.code}") from None + except (urllib.error.URLError, OSError, ValueError) as e: + raise WDAError(f"WebDriverAgent not reachable at {self.base_url}: {e}") from None + value = resp.get("value") + if isinstance(value, dict) and "error" in value: + raise WDAError(f"{value['error']}: {(value.get('message') or '').splitlines()[0]}") + return resp + + def _open_session(self, join_existing: bool) -> str: + if join_existing and (sid := self.request("GET", "/status").get("sessionId")): + return sid + resp = self.request("POST", "/session", {"capabilities": {"alwaysMatch": {}}}) + return resp.get("sessionId") or resp["value"]["sessionId"] + + def session_request(self, method: str, path: str, body: dict | None = None) -> dict: + """Request within the session; if the session is gone (WebDriverAgent restarted), open a new one once.""" + with self._lock: + for attempt in range(2): + if self._sid is None: + self._sid = self._open_session(join_existing=attempt == 0) + try: + return self.request(method, f"/session/{self._sid}{path}", body) + except WDAError as e: + if attempt or "session" not in str(e): + raise + self._sid = None + raise AssertionError("unreachable") + + def screen_size(self) -> tuple[float, float]: + size = self.request("GET", "/wda/screen")["value"]["screenSize"] + return size["width"], size["height"] + + def configure_stream(self, fps: int, scale_percent: int, quality: int) -> None: + self.session_request("POST", "/appium/settings", {"settings": { + "mjpegServerFramerate": fps, + "mjpegScalingFactor": scale_percent, + "mjpegServerScreenshotQuality": quality, + }}) + + def tap(self, x: float, y: float) -> None: + self.session_request("POST", "/wda/tap", {"x": x, "y": y}) + + def long_press(self, x: float, y: float, seconds: float) -> None: + self.session_request("POST", "/wda/touchAndHold", {"x": x, "y": y, "duration": seconds}) + + def swipe(self, x0: float, y0: float, x1: float, y1: float, ms: int) -> None: + self.session_request("POST", "/actions", {"actions": [{ + "type": "pointer", "id": "finger1", "parameters": {"pointerType": "touch"}, + "actions": [ + {"type": "pointerMove", "duration": 0, "x": x0, "y": y0}, + {"type": "pointerDown", "button": 0}, + {"type": "pointerMove", "duration": ms, "x": x1, "y": y1}, + {"type": "pointerUp", "button": 0}, + ], + }]}) + + def type_text(self, text: str) -> None: + self.session_request("POST", "/wda/keys", {"value": list(text)}) + + def home(self) -> None: + self.session_request("POST", "/wda/pressButton", {"name": "home"}) + + def close_foreground_app(self) -> None: + """Terminate the app in the foreground, like swiping it away in the app switcher.""" + bundle_id = self.request("GET", "/wda/activeAppInfo")["value"]["bundleId"] + if bundle_id != SPRINGBOARD: + self.session_request("POST", "/wda/apps/terminate", {"bundleId": bundle_id}) diff --git a/tests/unit/test_viewer.py b/tests/unit/test_viewer.py new file mode 100644 index 0000000..091c909 --- /dev/null +++ b/tests/unit/test_viewer.py @@ -0,0 +1,126 @@ +import io + +import pytest + +from iphone_viewer.geometry import LongPress, Swipe, Tap, Viewport, classify +from iphone_viewer.mjpeg import iter_frames +from iphone_viewer.wda import WDAClient, WDAError + + +def part(jpeg, header=b"Content-Length"): + return (b"--BoundaryString\r\nContent-type: image/jpeg\r\n" + header + b": %d\r\n\r\n" % len(jpeg) + + jpeg + b"\r\n") + + +# ---- MJPEG ---- + +def test_frames_are_split_by_content_length(): + first, second = b"\xff\xd8one\r\n\r\nwith blank lines\xff\xd9", b"\xff\xd8two\xff\xd9" + assert list(iter_frames(io.BytesIO(part(first) + part(second)))) == [first, second] + + +def test_header_name_is_case_insensitive(): + assert list(iter_frames(io.BytesIO(part(b"jpeg", header=b"content-length")))) == [b"jpeg"] + + +def test_truncated_last_frame_is_dropped(): + stream = part(b"complete") + part(b"truncated")[:-8] + assert list(iter_frames(io.BytesIO(stream))) == [b"complete"] + + +def test_empty_stream_yields_nothing(): + assert list(iter_frames(io.BytesIO(b""))) == [] + + +# ---- geometry ---- + +def test_tall_window_letterboxes_vertically(): + v = Viewport.fit(393, 852, 393 * 2, 852 * 2 + 100) + assert (v.scale, v.width, v.height, v.x, v.y) == (2, 786, 1704, 0, 50) + + +def test_wide_window_centers_horizontally(): + v = Viewport.fit(393, 852, 1000, 852) + assert (v.scale, v.width, v.x, v.y) == (1, 393, (1000 - 393) // 2, 0) + + +def test_pixels_map_back_to_points(): + v = Viewport.fit(393, 852, 393 * 2, 852 * 2 + 100) + assert v.to_points(v.x + 200, v.y + 300) == (100, 150) + + +def test_presses_outside_the_screen_are_clamped(): + v = Viewport.fit(393, 852, 1000, 852) + assert v.to_points(0, -5) == (0, 0) + assert v.to_points(999, 2000) == (393, 852) + + +def test_short_still_press_is_a_tap(): + assert classify((10, 10), (12, 13), 0.1) == Tap(10, 10) + + +def test_long_still_press_is_a_long_press(): + assert classify((10, 10), (10, 10), 1.2) == LongPress(10, 10, 1.2) + + +def test_moving_press_is_a_swipe_with_clamped_duration(): + assert classify((100, 700), (100, 200), 0.05) == Swipe(100, 700, 100, 200, 100) + assert classify((100, 700), (100, 200), 5) == Swipe(100, 700, 100, 200, 800) + assert classify((100, 700), (100, 200), 0.3).ms == 300 + + +# ---- WDA session handling ---- + +class FakeWDA(WDAClient): + """Records requests; `responses` maps "METHOD path" to a response dict or an exception.""" + + def __init__(self, responses): + super().__init__("http://wda") + self.responses, self.calls = responses, [] + + def request(self, method, path, body=None): + self.calls.append(f"{method} {path}") + resp = self.responses[f"{method} {path}"] + if isinstance(resp, list): + resp = resp.pop(0) + if isinstance(resp, Exception): + raise resp + return resp + + +def test_joins_the_active_session_instead_of_replacing_it(): + wda = FakeWDA({"GET /status": {"sessionId": "mcp"}, "POST /session/mcp/wda/tap": {"value": None}}) + wda.tap(1, 2) + assert wda.calls == ["GET /status", "POST /session/mcp/wda/tap"] + + +def test_opens_a_session_when_none_is_active(): + wda = FakeWDA({"GET /status": {"value": {}}, "POST /session": {"sessionId": "new"}, + "POST /session/new/wda/tap": {"value": None}}) + wda.tap(1, 2) + assert "POST /session" in wda.calls + + +def test_stale_session_is_replaced_once(): + wda = FakeWDA({ + "GET /status": {"sessionId": "stale"}, + "POST /session/stale/wda/tap": WDAError("invalid session id: gone"), + "POST /session": {"sessionId": "fresh"}, + "POST /session/fresh/wda/tap": {"value": None}, + }) + wda.tap(1, 2) + assert wda.calls[-2:] == ["POST /session", "POST /session/fresh/wda/tap"] + + +def test_other_errors_are_not_retried(): + wda = FakeWDA({"GET /status": {"sessionId": "s"}, + "POST /session/s/wda/tap": WDAError("no such element: x")}) + with pytest.raises(WDAError): + wda.tap(1, 2) + assert wda.calls.count("POST /session/s/wda/tap") == 1 + + +def test_close_app_leaves_the_home_screen_alone(): + wda = FakeWDA({"GET /wda/activeAppInfo": {"value": {"bundleId": "com.apple.springboard"}}}) + wda.close_foreground_app() + assert wda.calls == ["GET /wda/activeAppInfo"] From 11ce38d813976741b9454678a9a96f56bb1dee60 Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 00:10:00 +0100 Subject: [PATCH 4/7] docs: live viewer and WebView debugging English README only. Includes the Web Inspector setting on the phone and the Capacitor setting needed when building without Xcode (e.g. with xtool). Co-Authored-By: Claude Opus 5.5 --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ docs/architecture.md | 3 +++ 2 files changed, 43 insertions(+) diff --git a/README.md b/README.md index dd4e813..65c9fd9 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ Claude Code / any MCP client - **Runs unattended**: recovers from replugging, phone reboots, tunnel drops and WebDriverAgent crashes; re-signs WebDriverAgent with a free Apple ID before the 7-day profile expires - **Alerts**: Telegram or Bark notifications when re-signing fails, the phone goes offline, or a passcode is needed after a reboot - **Optional auto-unlock**: the phone can lock normally; every passcode digit is verified, and a wrong passcode disables auto-unlock after one attempt +- **Live viewer**: `iphonectl viewer` opens a desktop window with the phone's screen (up to ~25 fps over USB) that you control with mouse and keyboard, while the agent keeps working +- **WebView debugging**: Safari tabs and inspectable WebViews (Capacitor, Cordova, your own `WKWebView`) are exposed over the Chrome DevTools Protocol, so tools can read the DOM and console directly instead of going through screenshots - **Diagnostics**: `iphonectl doctor` checks every layer from USB to WebDriverAgent and tells you how to fix each failure ## Quick Start @@ -109,12 +111,49 @@ To build the image from source instead of pulling it, set `IMAGE=iphone-control- Screenshot pixels are tap coordinates, so the agent can tap what it sees. Games, maps and video players draw their own UI and have no element tree — use screenshots and coordinates there. +## Live Viewer + +```bash +bin/iphonectl viewer +``` + +A window on the host shows the phone's screen live, streamed from WebDriverAgent's MJPEG server, and passes your input on to the phone. It runs next to the MCP server and shares its WebDriverAgent session, so you can watch the agent work or take over. + +| Input | On the phone | +| --- | --- | +| Click / hold | Tap / long press | +| Drag, mouse wheel | Swipe, scroll | +| Keyboard | Types into the focused field | +| Esc, right click, **⌂ Home** | Home button | +| **✕ Close app** | Ends the foreground app, like swiping it away in the app switcher | + +iOS ignores system gestures that do not come from a finger, such as the swipe up from the bottom edge, so the window has buttons for them. Resize it freely. Options such as frame rate and resolution: `iphonectl viewer --help`. + +It needs Python 3.11+ with Tk and Pillow on the host: `sudo apt install python3-tk python3-pil.imagetk` (Debian/Ubuntu) or `sudo pacman -S tk python-pillow` (Arch). + +## Debugging WebViews + +The container serves the phone's Safari tabs and inspectable WebViews over the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) on `127.0.0.1:9222`: `curl 127.0.0.1:9222/json` lists them, and anything that speaks CDP can attach to read the DOM, run JavaScript, or follow console output and network requests. For hybrid apps that is faster and more precise than the element tree or screenshots. + +1. On the phone, turn on **Settings → Apps → Safari → Advanced → Web Inspector**. +2. Your app's `WKWebView` must be inspectable (`isInspectable = true`, iOS 16.4+). Safari is always inspectable. + +> [!TIP] +> **Capacitor apps built without Xcode** (for example with [xtool](https://github.com/xtool-org/xtool)): Capacitor only makes the WebView inspectable in debug builds, which it detects through a `CAPACITOR_DEBUG` Info.plist key that only Xcode's build settings fill in. Set it explicitly in `capacitor.config.ts`, for development builds only: +> +> ```ts +> ios: { webContentsDebuggingEnabled: !isReleaseBuild }, +> ``` + +`iphonectl doctor` reports how many pages are inspectable. Set `MJPEG_PORT=0` or `CDP_PORT=0` in `.env` to turn the stream or the bridge off. Both ports listen on `127.0.0.1` only and have no authentication, like WebDriverAgent itself. To view the phone from another machine, use SSH port forwarding rather than binding them to other interfaces. + ## Everyday Commands | Command | What it does | | --- | --- | | `iphonectl status` | Container status and recent supervisor log | | `iphonectl doctor` | Layer-by-layer health check with a fix for each failure | +| `iphonectl viewer` | Live view of the screen, controlled with mouse and keyboard | | `iphonectl logs [name] [-f]` | Logs: `supervisor`, `wda`, `tunnel`, `resign`, `notify`, … | | `iphonectl resign [--force]` | Re-sign WebDriverAgent if it expires soon (or now) | | `iphonectl login` | Sign in to the Apple ID again when the session expires | @@ -140,6 +179,7 @@ The detailed docs are in Chinese for now. - Free Apple ID profiles last 7 days. Re-signing is automatic, but an expired Apple ID session needs `iphonectl login` - After the phone reboots, you must enter the passcode once on the phone to allow UI automation — iOS requires this - Elements hidden behind a popup may still be listed, because visibility is judged by position +- System gestures (home indicator swipe, Control Center, Notification Center) cannot be injected from outside; use the Home button, closing apps by bundle ID, and URL schemes instead > [!WARNING] > This toolkit can fully control your phone. An agent may tap payment, delete or send buttons — keep tool-call confirmation on in your MCP client. Not affiliated with Apple; use it only with devices you own. diff --git a/docs/architecture.md b/docs/architecture.md index 03689bf..f621b7d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,6 +5,7 @@ ``` ┌─────────────── 宿主机(Linux)────────────────────────────────────────────┐ │ bin/iphonectl ── docker compose / docker exec │ +│ iphonectl viewer (Tk) ── stream 127.0.0.1:9100, input 127.0.0.1:8100 │ │ Claude Code ── docker exec -i <容器> /opt/ictl/mcp-server.sh (stdio) │ │ │ │ ┌──────────── 容器(privileged,host 网络,不对外开放端口)──────────────┐ │ @@ -12,6 +13,8 @@ │ │ ├─ usbmuxd USB 通讯 │ │ │ │ ├─ ios tunnel --userspace iOS 17+ 开发者服务隧道(不建网卡) │ │ │ │ ├─ pymobiledevice3 usbmux forward WDA 端口 → 127.0.0.1:8100 │ │ +│ │ ├─ pymobiledevice3 usbmux forward MJPEG stream → 127.0.0.1:9100 │ │ +│ │ ├─ pymobiledevice3 webinspector cdp WebViews → 127.0.0.1:9222 │ │ │ │ ├─ ios runwda 启动手机上的 WebDriverAgent │ │ │ │ └─ resign-wda.sh(定期) 免费 Apple ID 重签 │ │ │ │ iphone-mcp(Python,MCP 服务) │ │ From 193a8a6aae11b971eade3ae6bec073028312912d Mon Sep 17 00:00:00 2001 From: Derrors Date: Sat, 7 Nov 2026 09:16:06 +0800 Subject: [PATCH 5/7] Fix session fights, proxy use and silent failures in the viewer Sessions: once a cached session went stale, the retry always opened a new one, which ended the other client's session, so after a WebDriverAgent restart the MCP server and the viewer kept ending each other's sessions. Both now join the active session on retry too, and open one only if there is none or the active one is the one that just went stale. Viewer: - Talk to WDA and the stream directly: urllib sends even 127.0.0.1 through HTTP_PROXY on the host. - Keep the action worker, frame reader and frame polling alive after an unexpected error. Before, input or the picture stopped without any notice. - A long status message no longer widens the window, which moved the screen under the pointer and made a click land on the wrong spot. - Explain a missing display instead of showing a traceback. Co-Authored-By: Claude Opus 5.5 --- src/iphone_mcp/wda.py | 13 +++-- src/iphone_viewer/__main__.py | 7 +++ src/iphone_viewer/app.py | 26 ++++++--- src/iphone_viewer/wda.py | 22 ++++--- tests/unit/test_viewer.py | 103 +++++++++++++++++++++++++++++++++ tests/unit/test_wda_session.py | 63 ++++++++++++++++++++ 6 files changed, 213 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_wda_session.py diff --git a/src/iphone_mcp/wda.py b/src/iphone_mcp/wda.py index 0a31cef..c2f00b0 100644 --- a/src/iphone_mcp/wda.py +++ b/src/iphone_mcp/wda.py @@ -46,9 +46,11 @@ def request(method, path, body=None, timeout=30): _sid = None -def _open_session(join_existing): - """WDA has a single session and opening one ends the current one (e.g. iphonectl viewer's), so join it first.""" - if join_existing and (sid := request("GET", "/status").get("sessionId")): +def _open_session(stale=None): + """WDA 只有一个会话,新建会结束当前会话(比如 iphonectl viewer 的),所以先加入; + 只有没有活动会话,或活动的就是刚失效的那个时才新建,否则两个客户端会一直互相踢掉对方。""" + sid = request("GET", "/status").get("sessionId") + if sid and sid != stale: return sid r = request("POST", "/session", {"capabilities": {"alwaysMatch": {}}}) return r.get("sessionId") or r["value"]["sessionId"] @@ -57,14 +59,15 @@ def _open_session(join_existing): def session(method, path, body=None, timeout=30): """带会话的 WDA 请求;WDA 重启后旧会话失效,自动重建一次。""" global _sid + stale = None for attempt in range(2): if _sid is None: - _sid = _open_session(join_existing=attempt == 0) + _sid = _open_session(stale) try: return request(method, f"/session/{_sid}{path}", body, timeout) except WDAError as e: if attempt == 0 and "session" in e.error: - _sid = None + stale, _sid = _sid, None continue raise diff --git a/src/iphone_viewer/__main__.py b/src/iphone_viewer/__main__.py index 36d0be3..d065be8 100644 --- a/src/iphone_viewer/__main__.py +++ b/src/iphone_viewer/__main__.py @@ -8,6 +8,9 @@ Arch / Manjaro: sudo pacman -S tk python-pillow Fedora: sudo dnf install python3-tkinter python3-pillow-tk""" +NO_DISPLAY = """Cannot open the viewer window ({error}). It needs a desktop session. On a headless host, either use +X forwarding (ssh -X), or run it on your desktop and forward the ports, see "Live Viewer" in the README.""" + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(prog="iphonectl viewer", @@ -21,6 +24,8 @@ def main(argv: list[str] | None = None) -> None: args = parser.parse_args(argv) try: + import tkinter + from .app import run except ImportError as e: # tkinter or PIL.ImageTk missing sys.exit(MISSING_GUI.format(error=e)) @@ -28,6 +33,8 @@ def main(argv: list[str] | None = None) -> None: run(args.wda, args.mjpeg, args.fps, args.scale, args.quality) except WDAError as e: sys.exit(f"{e}\nIs the container running and WebDriverAgent ready? Check with: iphonectl doctor") + except tkinter.TclError as e: + sys.exit(NO_DISPLAY.format(error=e)) if __name__ == "__main__": diff --git a/src/iphone_viewer/app.py b/src/iphone_viewer/app.py index f67fd27..a07b74d 100644 --- a/src/iphone_viewer/app.py +++ b/src/iphone_viewer/app.py @@ -1,17 +1,17 @@ """Tk window: shows the live stream and turns mouse and keyboard input into touches on the phone.""" +import http.client import io import queue import threading import time import tkinter as tk -import urllib.request from collections.abc import Callable from PIL import Image, ImageTk from .geometry import LongPress, Swipe, Tap, Viewport, classify from .mjpeg import iter_frames -from .wda import WDAClient, WDAError +from .wda import WDAClient, WDAError, open_direct SCROLL_POINTS = 120 RECONNECT_SECONDS = 2 @@ -40,12 +40,12 @@ def _publish(self, item: bytes | Exception) -> None: def run(self) -> None: while not self.stopped.is_set(): try: - with urllib.request.urlopen(self.url, timeout=10) as stream: + with open_direct(self.url, timeout=10) as stream: for frame in iter_frames(stream): if self.stopped.is_set(): return self._publish(frame) - except OSError as e: + except (OSError, http.client.HTTPException, ValueError) as e: self._publish(e) self.stopped.wait(RECONNECT_SECONDS) @@ -66,7 +66,7 @@ def run(self) -> None: job = self.jobs.get() try: job() - except WDAError as e: + except Exception as e: # one failed action must not end the thread, or all further input is lost self.on_error(e) @@ -86,8 +86,9 @@ def __init__(self, client: WDAClient, mjpeg_url: str): "relief": "flat", "bd": 0, "highlightthickness": 0, "padx": 10, "pady": 4} tk.Button(bar, text="⌂ Home", command=self.home, **button).pack(side="left", padx=(6, 3), pady=6) tk.Button(bar, text="✕ Close app", command=self.close_app, **button).pack(side="left", padx=3, pady=6) - self.status = tk.Label(bar, text="connecting…", fg="#888", bg="#111", anchor="e") - self.status.pack(side="right", padx=6) + # width=1 plus fill: a long message is clipped instead of widening the window, which would move the screen + self.status = tk.Label(bar, text="connecting…", fg="#888", bg="#111", anchor="w", width=1) + self.status.pack(side="left", fill="x", expand=True, padx=6) start_scale = min((self.root.winfo_screenheight() - 180) / self.screen_h, 1.6) self.canvas = tk.Canvas(self.root, width=round(self.screen_w * start_scale), @@ -134,6 +135,14 @@ def draw(self, jpeg: bytes) -> None: self.canvas.itemconfig(self.image_item, image=self.photo) def poll_frames(self) -> None: + try: + self.show_next_frame() + except Exception as e: # e.g. a corrupt JPEG: skip it, the next frame replaces it + self.show_status(f"bad frame: {e}") + finally: + self.root.after(FRAME_POLL_MS, self.poll_frames) + + def show_next_frame(self) -> None: try: item = self.reader.latest.get_nowait() except queue.Empty: @@ -141,14 +150,13 @@ def poll_frames(self) -> None: if isinstance(item, Exception): self.show_status(f"no stream: {item}") elif item is not None: - self.last_frame = item self.draw(item) + self.last_frame = item self.frames_shown += 1 elapsed = time.monotonic() - self.fps_since if elapsed >= 1: self.show_status(f"{self.frames_shown / elapsed:.0f} fps") self.frames_shown, self.fps_since = 0, time.monotonic() - self.root.after(FRAME_POLL_MS, self.poll_frames) def show_status(self, text: str) -> None: self.status.config(text=text) diff --git a/src/iphone_viewer/wda.py b/src/iphone_viewer/wda.py index 9480a54..4714975 100644 --- a/src/iphone_viewer/wda.py +++ b/src/iphone_viewer/wda.py @@ -3,6 +3,7 @@ WebDriverAgent runs a single session: opening a new one ends the current one, which would break the MCP server's session. So the client joins the session that is already active and only opens one when there is none. """ +import http.client import json import threading import urllib.error @@ -10,6 +11,9 @@ SPRINGBOARD = "com.apple.springboard" +# The host may have HTTP_PROXY set and urllib would send even 127.0.0.1 through it +open_direct = urllib.request.build_opener(urllib.request.ProxyHandler({})).open + class WDAError(Exception): pass @@ -27,22 +31,25 @@ def request(self, method: str, path: str, body: dict | None = None) -> dict: req = urllib.request.Request(self.base_url + path, data=data, method=method, headers={"Content-Type": "application/json"}) try: - with urllib.request.urlopen(req, timeout=self.timeout) as r: + with open_direct(req, timeout=self.timeout) as r: resp = json.loads(r.read()) except urllib.error.HTTPError as e: try: resp = json.loads(e.read()) except ValueError: raise WDAError(f"HTTP {e.code}") from None - except (urllib.error.URLError, OSError, ValueError) as e: + except (OSError, http.client.HTTPException, ValueError) as e: raise WDAError(f"WebDriverAgent not reachable at {self.base_url}: {e}") from None value = resp.get("value") if isinstance(value, dict) and "error" in value: - raise WDAError(f"{value['error']}: {(value.get('message') or '').splitlines()[0]}") + message = (value.get("message") or "").partition("\n")[0] + raise WDAError(f"{value['error']}: {message}") return resp - def _open_session(self, join_existing: bool) -> str: - if join_existing and (sid := self.request("GET", "/status").get("sessionId")): + def _open_session(self, stale: str | None) -> str: + """Join the active session; open one only if there is none or the active one is the one that just went stale.""" + sid = self.request("GET", "/status").get("sessionId") + if sid and sid != stale: return sid resp = self.request("POST", "/session", {"capabilities": {"alwaysMatch": {}}}) return resp.get("sessionId") or resp["value"]["sessionId"] @@ -50,15 +57,16 @@ def _open_session(self, join_existing: bool) -> str: def session_request(self, method: str, path: str, body: dict | None = None) -> dict: """Request within the session; if the session is gone (WebDriverAgent restarted), open a new one once.""" with self._lock: + stale = None for attempt in range(2): if self._sid is None: - self._sid = self._open_session(join_existing=attempt == 0) + self._sid = self._open_session(stale) try: return self.request(method, f"/session/{self._sid}{path}", body) except WDAError as e: if attempt or "session" not in str(e): raise - self._sid = None + stale, self._sid = self._sid, None raise AssertionError("unreachable") def screen_size(self) -> tuple[float, float]: diff --git a/tests/unit/test_viewer.py b/tests/unit/test_viewer.py index 091c909..55530db 100644 --- a/tests/unit/test_viewer.py +++ b/tests/unit/test_viewer.py @@ -1,7 +1,12 @@ +import http.server import io +import json +import threading +import urllib.request import pytest +from iphone_viewer import wda as viewer_wda from iphone_viewer.geometry import LongPress, Swipe, Tap, Viewport, classify from iphone_viewer.mjpeg import iter_frames from iphone_viewer.wda import WDAClient, WDAError @@ -124,3 +129,101 @@ def test_close_app_leaves_the_home_screen_alone(): wda = FakeWDA({"GET /wda/activeAppInfo": {"value": {"bundleId": "com.apple.springboard"}}}) wda.close_foreground_app() assert wda.calls == ["GET /wda/activeAppInfo"] + + +class SimulatedWDA: + """WebDriverAgent's session rule: one active session, and opening a new one ends it.""" + + def __init__(self): + self.active, self.opened = None, 0 + + def open(self): + self.opened += 1 + self.active = f"s{self.opened}" + return {"sessionId": self.active} + + def restart(self): + self.active = None + + def request(self, method, path, body=None): + if path == "/status": + return {"sessionId": self.active} + if path == "/session": + return self.open() + if path.split("/")[2] != self.active: + raise WDAError("invalid session id: gone") + return {"value": None} + + +class ClientOf(WDAClient): + def __init__(self, wda): + super().__init__("http://wda") + self.wda = wda + + def request(self, method, path, body=None): + return self.wda.request(method, path, body) + + +def test_stale_cached_session_joins_the_new_active_one(): + wda = SimulatedWDA() + client = ClientOf(wda) + client.tap(1, 2) + wda.restart() + wda.open() # another client got here first + client.tap(1, 2) + assert client._sid == wda.active == "s2" + assert wda.opened == 2 + + +def test_clients_do_not_end_each_others_sessions_after_a_restart(): + wda = SimulatedWDA() + a, b = ClientOf(wda), ClientOf(wda) + a.tap(1, 2) + b.tap(1, 2) + wda.restart() + opened_before = wda.opened + for _ in range(5): + a.tap(1, 2) + b.tap(1, 2) + assert wda.opened == opened_before + 1 + + +# ---- errors and proxies ---- + +def test_error_without_message_is_a_wda_error(monkeypatch): + class Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + pass + + body = json.dumps({"value": {"error": "unknown error", "message": ""}}).encode() + monkeypatch.setattr(viewer_wda, "open_direct", lambda *a, **k: Response(body)) + with pytest.raises(WDAError, match="unknown error"): + WDAClient("http://wda").request("GET", "/x") + + +def test_requests_to_the_phone_ignore_the_hosts_proxy(monkeypatch): + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps({"value": {"screenSize": {"width": 393, "height": 852}}}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setenv("http_proxy", "http://127.0.0.1:1") + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.setattr(urllib.request, "_opener", None) # urlopen caches the proxy settings it first saw + try: + assert WDAClient(f"http://127.0.0.1:{server.server_port}").screen_size() == (393, 852) + finally: + server.shutdown() + server.server_close() diff --git a/tests/unit/test_wda_session.py b/tests/unit/test_wda_session.py new file mode 100644 index 0000000..d292810 --- /dev/null +++ b/tests/unit/test_wda_session.py @@ -0,0 +1,63 @@ +"""iphone_mcp.wda's session handling. The unit tests run without the mcp SDK, so ToolError is stubbed.""" +import importlib +import sys +import types + +import pytest + + +@pytest.fixture +def wda(monkeypatch): + exceptions = types.ModuleType("mcp.server.mcpserver.exceptions") + exceptions.ToolError = type("ToolError", (Exception,), {}) + for name in ("mcp", "mcp.server", "mcp.server.mcpserver"): + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + monkeypatch.setitem(sys.modules, "mcp.server.mcpserver.exceptions", exceptions) + monkeypatch.delitem(sys.modules, "iphone_mcp.wda", raising=False) + module = importlib.import_module("iphone_mcp.wda") + yield module + sys.modules.pop("iphone_mcp.wda", None) + + +def fake_requests(monkeypatch, wda, responses): + calls = [] + + def request(method, path, body=None, timeout=30): + calls.append(f"{method} {path}") + resp = responses[f"{method} {path}"] + if isinstance(resp, Exception): + raise resp + return resp + + monkeypatch.setattr(wda, "request", request) + return calls + + +def test_joins_the_active_session(monkeypatch, wda): + calls = fake_requests(monkeypatch, wda, {"GET /status": {"sessionId": "viewer"}, + "GET /session/viewer/x": {"value": 1}}) + assert wda.session("GET", "/x") == {"value": 1} + assert "POST /session" not in calls + + +def test_stale_cached_session_joins_the_active_one(monkeypatch, wda): + wda._sid = "old" + calls = fake_requests(monkeypatch, wda, { + "GET /session/old/x": wda.WDAError("invalid session id", "gone"), + "GET /status": {"sessionId": "viewer"}, + "GET /session/viewer/x": {"value": 1}, + }) + assert wda.session("GET", "/x") == {"value": 1} + assert "POST /session" not in calls + + +def test_opens_a_session_when_the_active_one_is_the_stale_one(monkeypatch, wda): + wda._sid = "old" + calls = fake_requests(monkeypatch, wda, { + "GET /session/old/x": wda.WDAError("invalid session id", "gone"), + "GET /status": {"sessionId": "old"}, + "POST /session": {"sessionId": "new"}, + "GET /session/new/x": {"value": 1}, + }) + assert wda.session("GET", "/x") == {"value": 1} + assert calls[-2:] == ["POST /session", "GET /session/new/x"] From c1323a4fc91546cb2bd9f9e15613fdf88869395c Mon Sep 17 00:00:00 2001 From: Derrors Date: Sat, 7 Nov 2026 09:16:06 +0800 Subject: [PATCH 6/7] Turn WebView debugging off by default, tell the stream from other services The CDP bridge gives any process on the host access to the phone's Safari tabs, including logged-in sites, and has no authentication or origin check, so it is now opt-in (CDP_PORT=0) and the README warns about it. doctor checks that the MJPEG port answers with a multipart stream: the container uses host networking and 9100 is a common port, so a plain HTTP 200 from another service used to count as a working live view. README: how to run the viewer on a desktop when the phone is on a headless host. Also fixes the width of a line in the architecture diagram. Co-Authored-By: Claude Opus 5.5 --- .env.example | 6 ++++-- README.md | 16 +++++++++++++--- container/doctor.sh | 10 ++++++---- container/lib.sh | 5 +++-- docs/architecture.md | 4 ++-- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 1360e17..3a66368 100644 --- a/.env.example +++ b/.env.example @@ -13,9 +13,11 @@ PGID=1000 # WDA 端口,只监听 127.0.0.1 WDA_PORT=8100 -# Live screen stream (MJPEG, used by iphonectl viewer) and WebView debugging (Chrome DevTools Protocol), 127.0.0.1 only; 0 = off +# Live screen stream (MJPEG, used by iphonectl viewer), 127.0.0.1 only; 0 = off MJPEG_PORT=9100 -CDP_PORT=9222 +# WebView debugging (Chrome DevTools Protocol), 127.0.0.1 only. Off by default: anything on this host could read and +# script the phone's logged-in Safari tabs. Set a port (usually 9222) to turn it on +CDP_PORT=0 # 接了多台手机时指定要控制的那台;留空用第一台 IPHONE_UDID= # WDA 不可用、且从没连上过 WDA 时,HID 后备路线换算坐标用的屏幕尺寸(点);一般留空 diff --git a/README.md b/README.md index 65c9fd9..45159b1 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Claude Code / any MCP client - **Alerts**: Telegram or Bark notifications when re-signing fails, the phone goes offline, or a passcode is needed after a reboot - **Optional auto-unlock**: the phone can lock normally; every passcode digit is verified, and a wrong passcode disables auto-unlock after one attempt - **Live viewer**: `iphonectl viewer` opens a desktop window with the phone's screen (up to ~25 fps over USB) that you control with mouse and keyboard, while the agent keeps working -- **WebView debugging**: Safari tabs and inspectable WebViews (Capacitor, Cordova, your own `WKWebView`) are exposed over the Chrome DevTools Protocol, so tools can read the DOM and console directly instead of going through screenshots +- **Optional WebView debugging**: Safari tabs and inspectable WebViews (Capacitor, Cordova, your own `WKWebView`) are exposed over the Chrome DevTools Protocol, so tools can read the DOM and console directly instead of going through screenshots - **Diagnostics**: `iphonectl doctor` checks every layer from USB to WebDriverAgent and tells you how to fix each failure ## Quick Start @@ -131,9 +131,19 @@ iOS ignores system gestures that do not come from a finger, such as the swipe up It needs Python 3.11+ with Tk and Pillow on the host: `sudo apt install python3-tk python3-pil.imagetk` (Debian/Ubuntu) or `sudo pacman -S tk python-pillow` (Arch). +If the phone is plugged into a headless host, run the viewer on your desktop from a checkout of this repository: + +```bash +ssh -N -L 8100:127.0.0.1:8100 -L 9100:127.0.0.1:9100 your-host & +PYTHONPATH=src python3 -m iphone_viewer +``` + ## Debugging WebViews -The container serves the phone's Safari tabs and inspectable WebViews over the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) on `127.0.0.1:9222`: `curl 127.0.0.1:9222/json` lists them, and anything that speaks CDP can attach to read the DOM, run JavaScript, or follow console output and network requests. For hybrid apps that is faster and more precise than the element tree or screenshots. +This is off by default. Set `CDP_PORT=9222` in `.env` and run `iphonectl up`, and the container serves the phone's Safari tabs and inspectable WebViews over the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) on `127.0.0.1:9222`: `curl 127.0.0.1:9222/json` lists them, and anything that speaks CDP can attach to read the DOM, run JavaScript, or follow console output and network requests. For hybrid apps that is faster and more precise than the element tree or screenshots. + +> [!WARNING] +> While it is on, any process on the host can read and script the phone's Safari tabs, including sites you are logged in to. The bridge has no authentication and does not check where a request comes from, so turn it on only while you need it and only on a host you trust. 1. On the phone, turn on **Settings → Apps → Safari → Advanced → Web Inspector**. 2. Your app's `WKWebView` must be inspectable (`isInspectable = true`, iOS 16.4+). Safari is always inspectable. @@ -145,7 +155,7 @@ The container serves the phone's Safari tabs and inspectable WebViews over the [ > ios: { webContentsDebuggingEnabled: !isReleaseBuild }, > ``` -`iphonectl doctor` reports how many pages are inspectable. Set `MJPEG_PORT=0` or `CDP_PORT=0` in `.env` to turn the stream or the bridge off. Both ports listen on `127.0.0.1` only and have no authentication, like WebDriverAgent itself. To view the phone from another machine, use SSH port forwarding rather than binding them to other interfaces. +`iphonectl doctor` reports how many pages are inspectable. Both the bridge and the live stream listen on `127.0.0.1` only and have no authentication, like WebDriverAgent itself; set `MJPEG_PORT=0` in `.env` to turn the stream off. To reach them from another machine, use SSH port forwarding rather than binding them to other interfaces. ## Everyday Commands diff --git a/container/doctor.sh b/container/doctor.sh index a063e2f..f9ae8bf 100755 --- a/container/doctor.sh +++ b/container/doctor.sh @@ -87,10 +87,12 @@ else skip "WDA"; fi echo "== Live view and WebView debugging" if [ "$MJPEG_PORT" -eq 0 ]; then off "Live view" MJPEG_PORT elif check && [ -n "$bundle" ]; then - # the MJPEG stream never ends: an HTTP 200 before the timeout means it works - code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$MJPEG_PORT") - if [ "$code" = 200 ]; then ok "Live view" "http://127.0.0.1:$MJPEG_PORT, open it on the host with iphonectl viewer" - else warn "Live view unavailable" "served by WDA, so only once WDA is ready; see data/logs/forward.log"; fi + # the MJPEG stream never ends, so curl always times out; the headers tell it apart from another service on the port + head=$(curl -s -m 2 -o /dev/null -w '%{http_code} %{content_type}' "http://127.0.0.1:$MJPEG_PORT") + case $head in + "200 multipart/"*) ok "Live view" "http://127.0.0.1:$MJPEG_PORT, open it on the host with iphonectl viewer" ;; + *) warn "Live view unavailable" "served by WDA, so only once WDA is ready; is another service on port $MJPEG_PORT? see data/logs/forward.log" ;; + esac else skip "Live view"; fi if [ "$CDP_PORT" -eq 0 ]; then off "WebView debugging" CDP_PORT diff --git a/container/lib.sh b/container/lib.sh index b184bfb..60afd5e 100644 --- a/container/lib.sh +++ b/container/lib.sh @@ -9,9 +9,10 @@ STATE_DIR=$DATA/state SECRETS_DIR=$DATA/secrets PLUME_DIR=$DATA/plume WDA_PORT=${WDA_PORT:-8100} -# Live screen (WDA's MJPEG stream) and WebView debugging (Chrome DevTools Protocol) ports, 127.0.0.1 only; 0 = off +# Live screen (WDA's MJPEG stream) and WebView debugging (Chrome DevTools Protocol) ports, 127.0.0.1 only; 0 = off. +# CDP stays off unless asked for: it gives access to the phone's logged-in Safari tabs MJPEG_PORT=${MJPEG_PORT:-9100} -CDP_PORT=${CDP_PORT:-9222} +CDP_PORT=${CDP_PORT:-0} WDA_UNSIGNED=/opt/ictl/wda/WDA-unsigned.ipa PROFILE=$CERT_DIR/free-dev.mobileprovision PUID=${PUID:-1000} diff --git a/docs/architecture.md b/docs/architecture.md index f621b7d..e0793d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,7 +5,7 @@ ``` ┌─────────────── 宿主机(Linux)────────────────────────────────────────────┐ │ bin/iphonectl ── docker compose / docker exec │ -│ iphonectl viewer (Tk) ── stream 127.0.0.1:9100, input 127.0.0.1:8100 │ +│ iphonectl viewer (Tk) ── stream 127.0.0.1:9100, input 127.0.0.1:8100 │ │ Claude Code ── docker exec -i <容器> /opt/ictl/mcp-server.sh (stdio) │ │ │ │ ┌──────────── 容器(privileged,host 网络,不对外开放端口)──────────────┐ │ @@ -14,7 +14,7 @@ │ │ ├─ ios tunnel --userspace iOS 17+ 开发者服务隧道(不建网卡) │ │ │ │ ├─ pymobiledevice3 usbmux forward WDA 端口 → 127.0.0.1:8100 │ │ │ │ ├─ pymobiledevice3 usbmux forward MJPEG stream → 127.0.0.1:9100 │ │ -│ │ ├─ pymobiledevice3 webinspector cdp WebViews → 127.0.0.1:9222 │ │ +│ │ ├─ pymobiledevice3 webinspector cdp WebViews → :9222(默认关闭) │ │ │ │ ├─ ios runwda 启动手机上的 WebDriverAgent │ │ │ │ └─ resign-wda.sh(定期) 免费 Apple ID 重签 │ │ │ │ iphone-mcp(Python,MCP 服务) │ │ From 8e0bec1e73011a0f4c1e0617c4f87abb9b2c8118 Mon Sep 17 00:00:00 2001 From: Derrors Date: Sat, 7 Nov 2026 09:25:10 +0800 Subject: [PATCH 7/7] docs: sync the Chinese docs with the live viewer and WebView debugging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README.zh-CN.md gets the features, the "实时画面" and "调试 WebView" sections, the viewer command and the system gesture limitation from the English README. The detailed docs now cover the MJPEG and CDP ports (security), the shared WebDriverAgent session and per-port forwards (architecture), common viewer and CDP problems (troubleshooting) and src/iphone_viewer/ (development). .env.example comments are in Chinese like the rest of the file. Co-Authored-By: Claude Opus 5.5 --- .env.example | 6 ++--- README.zh-CN.md | 50 +++++++++++++++++++++++++++++++++++++++++ docs/architecture.md | 17 +++++++++++++- docs/development.md | 7 ++++++ docs/security.md | 4 ++++ docs/troubleshooting.md | 11 +++++++++ 6 files changed, 91 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 3a66368..d4346e5 100644 --- a/.env.example +++ b/.env.example @@ -13,10 +13,10 @@ PGID=1000 # WDA 端口,只监听 127.0.0.1 WDA_PORT=8100 -# Live screen stream (MJPEG, used by iphonectl viewer), 127.0.0.1 only; 0 = off +# 实时画面的 MJPEG 流端口(iphonectl viewer 用),只监听 127.0.0.1;0 = 关闭 MJPEG_PORT=9100 -# WebView debugging (Chrome DevTools Protocol), 127.0.0.1 only. Off by default: anything on this host could read and -# script the phone's logged-in Safari tabs. Set a port (usually 9222) to turn it on +# WebView 调试(Chrome DevTools Protocol)端口,只监听 127.0.0.1。默认关闭:开启后本机任何进程都能读取、 +# 操作手机上已登录的 Safari 标签页。设成端口号(一般是 9222)开启 CDP_PORT=0 # 接了多台手机时指定要控制的那台;留空用第一台 IPHONE_UDID= diff --git a/README.zh-CN.md b/README.zh-CN.md index 52fa2fc..fac8fb6 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -65,6 +65,8 @@ Claude Code / 任意 MCP 客户端 - **无人值守**:USB 重插、手机重启、隧道断开、WebDriverAgent 崩溃后都能自动恢复;免费 Apple ID 的描述文件 7 天到期前自动重签 - **通知**:重签失败、手机离线、重启后需要输入密码时,推送 Telegram 或 Bark - **可选自动解锁**:手机可以正常锁屏;每一位密码都会核对,密码配置错误时输错一次就停用自动解锁 +- **实时画面**:`iphonectl viewer` 在桌面上打开一个窗口,实时显示手机屏幕(USB 下最高约 25 fps),可以用鼠标和键盘操作,AI 照常工作 +- **可选的 WebView 调试**:通过 Chrome DevTools Protocol 开放 Safari 标签页和可检查的 WebView(Capacitor、Cordova、你自己的 `WKWebView`),工具可以直接读 DOM 和控制台,不用靠截图 - **诊断**:`iphonectl doctor` 从 USB 到 WebDriverAgent 逐层检查,每项失败都给出修复办法 ## 快速开始 @@ -109,12 +111,59 @@ bin/iphonectl mcp install # 注册到 Claude Code 截图的像素坐标就是点击坐标,AI 看到哪里就能点哪里。游戏、地图、视频播放器自己绘制界面,没有元素树,只能看截图按坐标操作。 +## 实时画面 + +```bash +bin/iphonectl viewer +``` + +在宿主机上打开一个窗口,实时显示手机屏幕(来自 WebDriverAgent 的 MJPEG 流),并把你的操作转发给手机。它和 MCP 服务同时运行、共用同一个 WebDriverAgent 会话,你可以看着 AI 操作,也可以随时接手。 + +| 操作 | 手机上的效果 | +| --- | --- | +| 单击 / 按住 | 点击 / 长按 | +| 拖动、滚轮 | 滑动、滚动 | +| 键盘 | 在当前输入框里输入 | +| Esc、右键、**⌂ Home** | Home 键 | +| **✕ Close app** | 结束前台 App,相当于在多任务界面里把它划掉 | + +iOS 会忽略不是手指发出的系统手势,比如从屏幕底部上滑,所以窗口提供了对应的按钮。窗口大小可以随意调整。帧率、分辨率等选项见 `iphonectl viewer --help`。 + +宿主机需要 Python 3.11+,以及 Tk 和 Pillow:`sudo apt install python3-tk python3-pil.imagetk`(Debian/Ubuntu)或 `sudo pacman -S tk python-pillow`(Arch)。 + +手机接在没有图形界面的主机(比如 NAS)上时,在桌面电脑上克隆本仓库后运行: + +```bash +ssh -N -L 8100:127.0.0.1:8100 -L 9100:127.0.0.1:9100 你的主机 & +PYTHONPATH=src python3 -m iphone_viewer +``` + +## 调试 WebView + +默认关闭。在 `.env` 里设置 `CDP_PORT=9222` 后运行 `iphonectl up`,容器就会在 `127.0.0.1:9222` 上通过 [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) 开放手机的 Safari 标签页和可检查的 WebView:`curl 127.0.0.1:9222/json` 列出它们,任何支持 CDP 的工具都可以连上去读 DOM、执行 JavaScript、查看控制台输出和网络请求。对混合开发的 App 来说,这比元素树和截图更快也更准。 + +> [!WARNING] +> 开启期间,宿主机上的任何进程都能读取和操作手机的 Safari 标签页,包括你已登录的网站。桥接没有认证,也不检查请求来源,请只在需要时开启,并且只在你信任的主机上开启。 + +1. 在手机上打开 **设置 → App → Safari 浏览器 → 高级 → 网页检查器**。 +2. App 里的 `WKWebView` 必须可检查(`isInspectable = true`,iOS 16.4+)。Safari 始终可检查。 + +> [!TIP] +> **不用 Xcode 构建的 Capacitor App**(比如用 [xtool](https://github.com/xtool-org/xtool)):Capacitor 只在调试构建里让 WebView 可检查,而它是通过 Info.plist 里的 `CAPACITOR_DEBUG` 判断调试构建的,这个键只有 Xcode 的构建设置会填。请在 `capacitor.config.ts` 里显式设置,并且只用于开发构建: +> +> ```ts +> ios: { webContentsDebuggingEnabled: !isReleaseBuild }, +> ``` + +`iphonectl doctor` 会报告有几个可检查的页面。CDP 桥接和实时画面都只监听 `127.0.0.1`,和 WebDriverAgent 一样没有认证;不需要实时画面时,在 `.env` 里设置 `MJPEG_PORT=0` 关闭。要从其他机器访问,请用 SSH 端口转发,不要把它们绑到其他网卡上。 + ## 常用命令 | 命令 | 作用 | | --- | --- | | `iphonectl status` | 容器状态和最近的守护日志 | | `iphonectl doctor` | 逐层诊断,每项失败都给出修复办法 | +| `iphonectl viewer` | 实时查看屏幕,用鼠标和键盘操作 | | `iphonectl logs [名字] [-f]` | 看日志:`supervisor`、`wda`、`tunnel`、`resign`、`notify` 等 | | `iphonectl resign [--force]` | 快过期时重签 WebDriverAgent;`--force` 立即重签 | | `iphonectl login` | Apple ID 登录过期时重新登录 | @@ -138,6 +187,7 @@ bin/iphonectl mcp install # 注册到 Claude Code - 免费 Apple ID 的描述文件 7 天过期。重签是自动的,但 Apple ID 登录过期后要运行 `iphonectl login` - 手机重启后,要在手机上输一次锁屏密码才能启用 UI 自动化,这是 iOS 的要求 - 可见性按坐标判断,被弹窗挡住的元素也可能被列出来 +- 系统手势(底部上滑、控制中心、通知中心)无法从外部注入;请改用 Home 键、按 Bundle ID 结束 App 和 URL Scheme > [!WARNING] > 这个工具能完全控制你的手机。AI 可能点到付款、删除、发送等按钮,请在 MCP 客户端里保持工具调用确认。本项目与 Apple 无关,请只用于你自己的设备。 diff --git a/docs/architecture.md b/docs/architecture.md index e0793d7..a29f85e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -38,6 +38,20 @@ MCP 工具优先走 WDA;WDA 连不上时,截图、点击、滑动、按键、英文输入自动改走 HID。 坐标统一用「点」(pt):截图已缩放到点尺寸,图上的像素坐标可以直接拿去点。 +## WDA 会话 + +WDA 同一时间只有一个会话,新建会话会结束当前的会话。MCP 服务和 `iphonectl viewer` 都要用它,所以两边都先从 +`/status` 读出当前活动的会话并加入;只有没有活动会话,或者活动的正是自己刚失效的那个时,才新建一个。 +否则 WDA 重启后两边会不停地结束对方的会话。 + +## 实时画面和 WebView 调试 + +- **实时画面**:WDA 自带 MJPEG 服务(手机上的 9100 端口),supervisor 把它转发到 `127.0.0.1:9100`。 + `iphonectl viewer` 在宿主机上运行(`src/iphone_viewer/`,不打进镜像),从这里取画面,操作走 WDA 端口。 + 帧率、缩放和画质通过 WDA 的 `/appium/settings` 设置。 +- **WebView 调试**:默认关闭。开启后 supervisor 运行 `pymobiledevice3 webinspector cdp`,把手机的 Web Inspector + 协议翻译成 Chrome DevTools Protocol,监听 `127.0.0.1:$CDP_PORT`。 + ## 守护(supervisor.sh) | 检测到 | 处理 | @@ -45,7 +59,8 @@ MCP 工具优先走 WDA;WDA 连不上时,截图、点击、滑动、按键、英 | USB 上有 Apple 设备,但 usbmuxd 看不到 | 30 秒后重启 usbmuxd(容器里没有 udev,收不到热插拔) | | 手机离线超过 `OFFLINE_ALERT_MIN` 分钟 | 推送提醒;重新连上再推一条 | | 隧道 90 秒没建立 | 重启隧道(go-ios 失败后有时不再重试) | -| 端口转发不在,或换了手机 | 重新拉起 | +| 某个端口转发(WDA、MJPEG 各一个)不在,或换了手机 | 只重新拉起那一个,不影响其他转发 | +| 开启了 WebView 调试,CDP 桥接进程不在 | 重新拉起 | | WDA 连续 30 秒不响应 | 先补挂开发者镜像,再重启 WDA;失败后间隔 1→2→4→5 分钟 | | WDA 卡在「Enable UI Automation」密码框 | 推送「请输锁屏密码」,保持每分钟重试;恢复后推送「已恢复」 | | 本机 runwda 进程不在 | 即使手机上残留的 WDA 还在响应,也视为不可用(它几十秒后会自己退出) | diff --git a/docs/development.md b/docs/development.md index ff4fdda..f42aeb5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,6 +13,11 @@ src/iphone_mcp/ MCP 服务 hid.py HID 后备路线 unlock.py 自动解锁 server.py MCP 工具定义 +src/iphone_viewer/ 实时画面窗口(iphonectl viewer),在宿主机上运行,不打进镜像 + mjpeg.py MJPEG 流解析(纯函数) + geometry.py 窗口像素和点的换算、鼠标操作转成手势(纯函数) + wda.py 精简的 WDA 客户端,加入已有会话 + app.py Tk 界面 docker/ Dockerfile 和 Python 依赖锁定 scripts/ 发布用:收集 GPL/LGPL 组件的对应源码 tests/unit/ 不碰设备的单元测试 @@ -37,6 +42,8 @@ bin/iphonectl test-device '[["iphone_status", {}], ["iphone_screenshot", {}]]' 截图保存在 `data/test-shots/`。 +`iphone_viewer/app.py` 的 Tk 界面没有单元测试(CI 里不一定有 Tk),改了它要在桌面上实际跑一次 `iphonectl viewer`。 + 改了 `src/` 或 `container/` 后需要重建镜像:`bin/iphonectl up`(会自动构建)。 ## 升级上游版本 diff --git a/docs/security.md b/docs/security.md index b313104..6a0e253 100644 --- a/docs/security.md +++ b/docs/security.md @@ -7,6 +7,8 @@ | 风险 | 说明 | 默认措施 | |---|---|---| | WebDriverAgent 没有认证 | 能访问 WDA 端口的人就能操作手机 | 端口转发只监听 `127.0.0.1`(不用 go-ios 的 `ios forward`,它监听所有网卡);MCP 走 `docker exec` 的 stdio,不开网络端口 | +| 实时画面没有认证 | 能访问 MJPEG 端口的人就能看到手机屏幕,包括正在输入的内容 | 端口转发只监听 `127.0.0.1`;不需要时在 `.env` 里设 `MJPEG_PORT=0` 关闭 | +| WebView 调试(CDP)没有认证 | 本机任何进程都能读取和操作 Safari 标签页和可检查的 WebView,包括已登录的网站;桥接不检查请求来源 | **默认关闭**(`CDP_PORT=0`),要显式开启;只监听 `127.0.0.1` | | 容器以 privileged 运行 | 为了直接访问 USB 设备 | 隧道用用户态网络栈,不建内核网卡、不改宿主机网络;容器只挂 USB 设备和数据目录 | | 数据目录里有敏感材料 | 配对记录、Apple ID 登录状态、开发私钥、锁屏密码 | 敏感子目录权限 700;`.gitignore` 排除 `data/` 和 `.env`;CI 跑 gitleaks | | 自动解锁 | 能登录宿主机、或能调用这个 MCP 的人,都能解锁手机 | **默认关闭**,要显式 `iphonectl passcode set`;密码不进日志,也不会出现在返回给 AI 的内容里 | @@ -19,6 +21,8 @@ - 只在你信任、只有你能登录的机器上部署。 - 备份 `data/`,但备份也要按敏感数据对待。 +- 要从其他机器看实时画面或调试 WebView,用 SSH 端口转发,不要把端口绑到其他网卡。 +- WebView 调试用完就把 `CDP_PORT` 改回 `0` 并 `iphonectl up`。 - 不用时可以 `iphonectl down`,或在手机上关闭开发者模式。 - 停用自动解锁:`iphonectl passcode clear`,再把手机的「自动锁定」改回「永不」。 - 换手机或不再使用时:在 Apple ID 的开发者设备列表里删掉这台手机,删掉 `data/`。 diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 32117df..67c35f2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -32,6 +32,17 @@ | 游戏里 `iphone_list_elements` 返回空 | 游戏引擎把画面画在一块画布上,没有控件 | 用截图加坐标;小图标用区域放大 | | 被弹窗挡住的元素也列出来了 | 可见性按坐标判断 | 截图核对 | +## 实时画面和 WebView 调试 + +| 现象 | 原因 | 处理 | +|---|---|---| +| `iphonectl viewer` 提示缺少 Tk 和 Pillow | 宿主机的 Python 没装 Tk 支持 | 按提示安装,比如 `sudo apt install python3-tk python3-pil.imagetk` | +| 提示 `Cannot open the viewer window` | 宿主机没有图形界面(比如 NAS) | 在桌面电脑上运行,用 SSH 转发 8100 和 9100 端口,见 README「实时画面」 | +| 窗口里显示 `no stream` | MJPEG 流由 WDA 提供,WDA 就绪前没有画面 | 等 WDA 就绪;跑 `iphonectl doctor`,看 `data/logs/forward.log` | +| WDA 正常,doctor 却报 `Live view unavailable` | 容器用 host 网络,9100 被宿主机上的其他服务占用(比如 node_exporter) | 停掉占用端口的服务,或设 `MJPEG_PORT=0` 关闭实时画面 | +| 窗口里从底部上滑、下拉控制中心没反应 | iOS 忽略不是手指发出的系统手势 | 用窗口的 Home、Close app 按钮,或 URL Scheme | +| doctor 报 `no inspectable pages` | 手机没开网页检查器,或 App 的 `WKWebView` 不可检查 | 设置 → App → Safari 浏览器 → 高级 → 网页检查器;App 要设 `isInspectable`,Capacitor 见 README | + ## 自动解锁 | 现象 | 原因 | 处理 |