|
| 1 | +"""devbox ios ui <command>: the app on the device driven through WebDriverAgent. |
| 2 | +
|
| 3 | +WebDriverAgent answers on http://127.0.0.1:8100 and streams the screen on |
| 4 | +:9100 once `devbox ios ui start` runs. Points, not pixels: a tap at 195 785 |
| 5 | +is the middle of the bottom of a 390x844 screen (labels shows every rect). |
| 6 | +
|
| 7 | + session <bundle id> opens a session on the app, launched if needed; |
| 8 | + every command below uses it |
| 9 | + activate <bundle id> brings an installed app back to the foreground |
| 10 | + (a link opened in Safari or Chrome, for example) |
| 11 | + labels what is on screen: type, rect x,y wxh, label |
| 12 | + find <class> the elements of an XCUIElementType (TextField, |
| 13 | + Button...) with their rects, for what labels |
| 14 | + leaves out (a field inside a sheet) |
| 15 | + tap <text> [n] taps the n-th (0) element whose label or name |
| 16 | + contains <text> |
| 17 | + tapxy <x> <y> taps a point |
| 18 | + swipe <x1> <y1> <x2> <y2> drags from one point to the other |
| 19 | + type <text> types into the focused field (tap it first) |
| 20 | + alert [accept|dismiss] the text of the system alert shown, or answers it |
| 21 | + shot <file.png> screenshot, full resolution |
| 22 | + record <seconds> <dir> the screen at about 30 images a second (MJPEG, |
| 23 | + half size), one JPEG a frame named |
| 24 | + <n>-<time>.jpg: for an animation or a jump |
| 25 | + that screenshots, a second apart, miss |
| 26 | +
|
| 27 | +The session is kept in $IOS_UI_DIR/session (devbox ios ui sets it). |
| 28 | +""" |
| 29 | +import base64 |
| 30 | +import json |
| 31 | +import os |
| 32 | +import sys |
| 33 | +import time |
| 34 | +import urllib.error |
| 35 | +import urllib.request |
| 36 | + |
| 37 | +PORT = int(os.environ.get("IOS_UI_PORT", "8100")) |
| 38 | +MJPEG_PORT = int(os.environ.get("IOS_UI_MJPEG_PORT", "9100")) |
| 39 | +BASE = f"http://127.0.0.1:{PORT}" |
| 40 | +STATE = os.environ.get("IOS_UI_DIR", os.path.expanduser("~/.cache/dev-box/ios-ui")) |
| 41 | +SESSION_FILE = os.path.join(STATE, "session") |
| 42 | + |
| 43 | + |
| 44 | +def call(method, path, body=None, timeout=60, fail=True): |
| 45 | + data = json.dumps(body).encode() if body is not None else None |
| 46 | + request = urllib.request.Request(BASE + path, data=data, method=method, headers={"Content-Type": "application/json"}) |
| 47 | + try: |
| 48 | + with urllib.request.urlopen(request, timeout=timeout) as response: |
| 49 | + return json.loads(response.read() or b"{}") |
| 50 | + except urllib.error.HTTPError as error: |
| 51 | + reply = json.loads(error.read() or b"{}") |
| 52 | + if not fail: |
| 53 | + return {} |
| 54 | + sys.exit(f"WebDriverAgent: {reply.get('value', {}).get('message', reply)}") |
| 55 | + |
| 56 | + |
| 57 | +def session(): |
| 58 | + try: |
| 59 | + return open(SESSION_FILE).read().strip() |
| 60 | + except OSError: |
| 61 | + sys.exit("no session: devbox ios ui session <bundle id> first") |
| 62 | + |
| 63 | + |
| 64 | +def in_session(method, path, body=None, timeout=60): |
| 65 | + return call(method, f"/session/{session()}{path}", body, timeout).get("value") |
| 66 | + |
| 67 | + |
| 68 | +def element_id(element): |
| 69 | + return element.get("ELEMENT") or element.get("element-6066-11e4-a52e-4f735466cecf") |
| 70 | + |
| 71 | + |
| 72 | +def walk(node, out): |
| 73 | + if node.get("label") or node.get("name"): |
| 74 | + out.append(node) |
| 75 | + for child in node.get("children") or []: |
| 76 | + walk(child, out) |
| 77 | + |
| 78 | + |
| 79 | +def short_type(kind): |
| 80 | + kind = kind or "" |
| 81 | + return kind[len("XCUIElementType"):] if kind.startswith("XCUIElementType") else kind |
| 82 | + |
| 83 | + |
| 84 | +def rect_text(r): |
| 85 | + return f'{r.get("x", 0):>5},{r.get("y", 0):>5} {r.get("width", 0):>4}x{r.get("height", 0):<4}' |
| 86 | + |
| 87 | + |
| 88 | +def cmd_session(bundle_id): |
| 89 | + reply = call("POST", "/session", {"capabilities": {"alwaysMatch": {"bundleId": bundle_id, "shouldWaitForQuiescence": False}}}, timeout=120) |
| 90 | + sid = reply.get("sessionId") or reply.get("value", {}).get("sessionId") |
| 91 | + if not sid: |
| 92 | + sys.exit(f"no session: {reply}") |
| 93 | + os.makedirs(STATE, exist_ok=True) |
| 94 | + open(SESSION_FILE, "w").write(sid) |
| 95 | + # The MJPEG stream of record: close to 30 images a second, half size |
| 96 | + in_session("POST", "/appium/settings", {"settings": {"mjpegServerFramerate": 30, "mjpegScalingFactor": 50, "mjpegServerScreenshotQuality": 40}}) |
| 97 | + print("session", sid, "on", bundle_id) |
| 98 | + |
| 99 | + |
| 100 | +def cmd_activate(bundle_id): |
| 101 | + in_session("POST", "/wda/apps/activate", {"bundleId": bundle_id}) |
| 102 | + print("foreground:", bundle_id) |
| 103 | + |
| 104 | + |
| 105 | +def cmd_labels(): |
| 106 | + nodes = [] |
| 107 | + walk(call("GET", "/source?format=json")["value"], nodes) |
| 108 | + for n in nodes: |
| 109 | + print(f'{short_type(n.get("type")):<14} {rect_text(n.get("rect") or {})} {(n.get("label") or n.get("name"))[:90]}') |
| 110 | + |
| 111 | + |
| 112 | +def cmd_find(kind): |
| 113 | + kind = kind if kind.startswith("XCUIElementType") else "XCUIElementType" + kind |
| 114 | + for element in in_session("POST", "/elements", {"using": "class name", "value": kind}): |
| 115 | + rect = in_session("GET", f"/element/{element_id(element)}/rect") |
| 116 | + print(short_type(kind), rect_text(rect)) |
| 117 | + |
| 118 | + |
| 119 | +def cmd_tap(text, index="0"): |
| 120 | + quoted = text.replace("\\", "\\\\").replace("'", "\\'") |
| 121 | + found = in_session("POST", "/elements", {"using": "predicate string", "value": f"label CONTAINS '{quoted}' OR name CONTAINS '{quoted}'"}) |
| 122 | + if len(found) <= int(index): |
| 123 | + sys.exit(f"not on screen: {text} (devbox ios ui labels)") |
| 124 | + in_session("POST", f"/element/{element_id(found[int(index)])}/click", {}) |
| 125 | + print("tapped:", text) |
| 126 | + |
| 127 | + |
| 128 | +def cmd_tapxy(x, y): |
| 129 | + in_session("POST", "/wda/tap", {"x": float(x), "y": float(y)}) |
| 130 | + print("tapped", x, y) |
| 131 | + |
| 132 | + |
| 133 | +def cmd_swipe(x1, y1, x2, y2): |
| 134 | + in_session("POST", "/wda/dragfromtoforduration", {"fromX": float(x1), "fromY": float(y1), "toX": float(x2), "toY": float(y2), "duration": 0.2}) |
| 135 | + print("swiped") |
| 136 | + |
| 137 | + |
| 138 | +def cmd_type(text): |
| 139 | + in_session("POST", "/wda/keys", {"value": list(text)}) |
| 140 | + print("typed") |
| 141 | + |
| 142 | + |
| 143 | +def cmd_alert(answer=None): |
| 144 | + if answer is None: |
| 145 | + reply = call("GET", f"/session/{session()}/alert/text", fail=False) |
| 146 | + print(reply.get("value") or "no alert") |
| 147 | + elif answer in ("accept", "dismiss"): |
| 148 | + in_session("POST", f"/alert/{answer}", {}) |
| 149 | + print("alert:", answer) |
| 150 | + else: |
| 151 | + sys.exit("alert [accept|dismiss]") |
| 152 | + |
| 153 | + |
| 154 | +def cmd_shot(path): |
| 155 | + open(path, "wb").write(base64.b64decode(call("GET", "/screenshot")["value"])) |
| 156 | + print(path) |
| 157 | + |
| 158 | + |
| 159 | +def cmd_record(seconds, out): |
| 160 | + os.makedirs(out, exist_ok=True) |
| 161 | + stream = urllib.request.urlopen(f"http://127.0.0.1:{MJPEG_PORT}", timeout=10) |
| 162 | + buf, count, end = b"", 0, time.time() + float(seconds) |
| 163 | + while time.time() < end: |
| 164 | + buf += stream.read(4096) |
| 165 | + start = buf.find(b"\xff\xd8") |
| 166 | + stop = buf.find(b"\xff\xd9", start + 2) if start != -1 else -1 |
| 167 | + if start != -1 and stop != -1: |
| 168 | + with open(os.path.join(out, f"{count:04d}-{time.time():.3f}.jpg"), "wb") as frame: |
| 169 | + frame.write(buf[start:stop + 2]) |
| 170 | + count += 1 |
| 171 | + buf = buf[stop + 2:] |
| 172 | + print(count, "frames in", out) |
| 173 | + |
| 174 | + |
| 175 | +COMMANDS = { |
| 176 | + "session": cmd_session, "activate": cmd_activate, "labels": cmd_labels, "find": cmd_find, |
| 177 | + "tap": cmd_tap, "tapxy": cmd_tapxy, "swipe": cmd_swipe, "type": cmd_type, "alert": cmd_alert, |
| 178 | + "shot": cmd_shot, "record": cmd_record, |
| 179 | +} |
| 180 | + |
| 181 | +if __name__ == "__main__": |
| 182 | + if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: |
| 183 | + sys.exit(__doc__) |
| 184 | + try: |
| 185 | + COMMANDS[sys.argv[1]](*sys.argv[2:]) |
| 186 | + except TypeError: |
| 187 | + sys.exit(__doc__) |
0 commit comments