From 2a6e57d983afb3cfbdfe18970971e77114f53b37 Mon Sep 17 00:00:00 2001 From: FunctionFreak Date: Mon, 29 Jun 2026 22:52:22 +0100 Subject: [PATCH] added ocr in the mac os --- .gitignore | 1 + Auto_Use/macOS_use/tree/element.py | 243 ++++++++++++++++++++++++++++- Auto_Use/macOS_use/tree/ocr.py | 102 ++++++++++++ mac_requirements.txt | 3 + 4 files changed, 344 insertions(+), 5 deletions(-) create mode 100644 Auto_Use/macOS_use/tree/ocr.py diff --git a/.gitignore b/.gitignore index c4b9a01..d0e82a9 100644 --- a/.gitignore +++ b/.gitignore @@ -122,3 +122,4 @@ screen_record.py # the code (service.py / __init__.py) is tracked, the saved chats are not. /Auto_Use/agent_conversation/index.json /Auto_Use/agent_conversation/chat_*/ +automation_grant.json diff --git a/Auto_Use/macOS_use/tree/element.py b/Auto_Use/macOS_use/tree/element.py index b17fee1..e36d16c 100644 --- a/Auto_Use/macOS_use/tree/element.py +++ b/Auto_Use/macOS_use/tree/element.py @@ -40,6 +40,7 @@ import io import time import base64 +import threading import numpy as np from collections import namedtuple from PIL import Image, ImageDraw, ImageFont @@ -62,12 +63,19 @@ AXIsProcessTrusted, kAXErrorSuccess, ) +try: + from .ocr import OCRScanner +except ImportError: # allow `python element.py` standalone + from ocr import OCRScanner + # ========== CONFIGURATION ========== # Toggle switches — same semantics as Windows element.py SCREENSHOT = True # Set to False to only generate element tree without screenshot DEBUG = False # Set to True to save files to debug folders, False for direct LLM only FRONTEND = True # Set to True when running from app.py to send images to frontend +OCR = True # Set to False to disable Apple Vision OCR merge (canvas/label-less text) +OCR_RECOGNITION_LEVEL = "accurate" # "accurate" (default) or "fast" (lower latency) # Define Rect namedtuple matching Windows format (left, top, right, bottom) Rect = namedtuple('Rect', ['left', 'top', 'right', 'bottom']) @@ -495,6 +503,15 @@ def build_label(element, cfg): CLIP_ROLES = frozenset({"AXScrollArea", "AXList", "AXOutline", "AXTable"}) +# Cleaned ("AX"-stripped) roles that are structural wrappers — they never claim +# screen space for OCR overlap purposes; gaps between their children are exactly +# where OCR fills in. Only "Group" actually reaches the tree today (the rest are +# CLIP_ROLES that aren't tracked as nodes); listing them is future-proofing. +OCR_STRUCTURAL_CONTAINER_TYPES = frozenset({ + "Group", "ScrollArea", "List", "Outline", "Table", + "WebArea", "SplitGroup", "TabGroup", "Toolbar", +}) + def _overlaps(a, b): """Return True if rect a overlaps rect b.""" @@ -1335,6 +1352,15 @@ def __init__(self, config, frontend_callback=None): self.found_elements = {} # Dictionary to store elements by type self._debug_iteration = 0 # Debug iteration counter + # OCR state + self._screenshot = None # Single captured PIL screenshot (pixels) + self._scale = 2.0 # Backing scale factor of the capture + self._screen_size = (0, 0) # (width, height) in CG points + self._ocr_backdrop_max_area = float("inf") # leaves bigger than this don't suppress OCR + self.ocr_words = [] # Raw OCR lines (CG points, filtered later) + self.ocr_scanner = None # OCRScanner instance + self.ocr_thread = None # Thread handle for parallel OCR scan + def scan_elements(self): """Scan the active window and menu bar for configured element types.""" @@ -1350,6 +1376,10 @@ def scan_elements(self): self.top_layer_info = None self.second_layer_info = None self.found_elements = {} + self._screenshot = None + self.ocr_words = [] + self.ocr_scanner = None + self.ocr_thread = None # Check accessibility permission if not AXIsProcessTrusted(): @@ -1359,12 +1389,30 @@ def scan_elements(self): # Get screen info screen = get_screen() + self._screen_size = (screen["width"], screen["height"]) # Run the macOS AX scan _seen_roles.clear() app_info, menu_items, elements = extract_all(screen) elements.sort(key=lambda e: (e["y"], e["x"])) + # Capture the screen ONCE, now that extract_all has settled the frame + # (it activates the app + toggles AXEnhancedUserInterface with a sleep). + # The same image is reused for OCR overlap-filtering and final annotation, + # so OCR boxes, AX boxes, and the displayed screenshot share one frame. + if SCREENSHOT or OCR: + self._screenshot, self._scale = take_screenshot(screen) + + # Start OCR in a parallel thread — it overlaps the (main-thread) tree build. + if OCR and self._screenshot is not None: + self.ocr_scanner = OCRScanner( + self._screenshot.copy(), self._scale, + (screen["x"], screen["y"]), + recognition_level=OCR_RECOGNITION_LEVEL, + ) + self.ocr_thread = threading.Thread(target=self.ocr_scanner.scan) + self.ocr_thread.start() + # Store application name if app_info: self.application_name = app_info["name"] @@ -1432,6 +1480,16 @@ def scan_elements(self): # ----- Build element tree from flat elements using spatial containment ----- self.element_tree = self._build_hierarchical_tree(elements) + # ----- Merge OCR survivors into the tree ----- + # Join the parallel OCR scan, then nest any text the AX tree missed + # (canvas / label-less controls) as OCR_TEXT nodes that don't overlap + # an already-detected element. + if self.ocr_thread is not None: + self.ocr_thread.join() + self.ocr_words = self.ocr_scanner.get_lines() + self.ocr_thread = None + self._filter_and_merge_ocr() + # ----- Capture and annotate screenshot ----- self._debug_iteration += 1 if SCREENSHOT and self.elements_to_draw: @@ -1577,9 +1635,157 @@ def _cleanup(node_list): return roots + # ========== OCR MERGE (ported from windows_use/tree/element.py) ========== + + def _collect_leaf_rects(self, tree_list, rects): + """Recursively collect rects that actually claim screen space, for OCR + overlap checking. Structural wrappers (Group, ScrollArea, ...) never claim + space — the gaps between their children are exactly where OCR fills in — + so we recurse into them without adding their own rect. Every other element + type claims its rect, EXCEPT backdrop-sized leaves (a full-window AXImage + for a map / canvas, a big background) which shouldn't suppress the text + drawn on top of them.""" + for item in tree_list: + has_children = bool(item.get("children")) + if item["type"] in OCR_STRUCTURAL_CONTAINER_TYPES: + if has_children: + self._collect_leaf_rects(item["children"], rects) + else: + rect = item.get("rect") or item.get("visible_rect") + if rect: + area = (rect.right - rect.left) * (rect.bottom - rect.top) + if area <= self._ocr_backdrop_max_area: + rects.append(rect) + if has_children: + self._collect_leaf_rects(item["children"], rects) + + def _find_deepest_container(self, tree_list, cx, cy): + """Find the deepest element whose rect contains point (cx, cy) and return + its children list (so the OCR node nests there). Uses center-point + containment — forgiving of OCR boxes that poke a pixel outside their + visual parent. Returns the top-level tree_list if nothing claims the point.""" + for item in tree_list: + rect = item.get("rect") or item.get("visible_rect") + if not rect: + continue + if rect.left <= cx <= rect.right and rect.top <= cy <= rect.bottom: + # Inside this element — try to go deeper among its children. + if item.get("children"): + deeper = self._find_deepest_container(item["children"], cx, cy) + if deeper is not item["children"]: + return deeper + # Children exist but none claimed the point — nest as a sibling. + return item["children"] + # Leaf element that contains the point — nest as its child. + return item.setdefault("children", []) + # No element in this list contains the point — caller decides. + return tree_list + + def _filter_and_merge_ocr(self): + """Filter OCR lines against detected elements, nest survivors into the tree. + + Only leaf (space-claiming) element rects suppress OCR; structural wrappers + don't. An OCR line whose center sits inside a leaf rect is already covered + by the AX scan and is discarded. Surviving lines (canvas / label-less text + the AX tree missed) are nested into the deepest matching container as + OCR_TEXT nodes and registered for clicking (coordinate-click fallback, + ax_element=None) + drawing.""" + if not self.ocr_words: + return + + # Backdrop guard: a leaf element covering a large fraction of the screen + # is a canvas/background (e.g. Apple Maps' full-window "Map" AXImage), not + # the label for any specific word. Such leaves must NOT suppress OCR text + # drawn over them, or we lose every label on a map / large image / canvas. + sw, sh = self._screen_size + self._ocr_backdrop_max_area = (0.25 * sw * sh) if (sw and sh) else float("inf") + + # Collect rects that claim screen space (top layer + menu bar). + leaf_rects = [] + self._collect_leaf_rects(self.element_tree, leaf_rects) + self._collect_leaf_rects(self.menu_bar_tree, leaf_rects) + + # Drop OCR lines overlapping a detected leaf; also dedupe OCR-vs-OCR. + kept_lines = [] + seen = set() + for line in self.ocr_words: + cx = (line["left"] + line["right"]) // 2 + cy = (line["top"] + line["bottom"]) // 2 + + overlaps_leaf = any( + r.left <= cx <= r.right and r.top <= cy <= r.bottom + for r in leaf_rects + ) + if overlaps_leaf: + continue + + key = (line["text"], round(cx / 5), round(cy / 5)) + if key in seen: + continue + seen.add(key) + kept_lines.append((line, cx, cy)) + + # Nest each surviving OCR line into the deepest matching container. + for line, cx, cy in kept_lines: + self.element_index += 1 + line_rect = Rect(line["left"], line["top"], line["right"], line["bottom"]) + + node = { + "element": None, + "name": line["text"], + "aria_role": "", + "type": "OCR_TEXT", + "active": True, + "index": self.element_index, + "value": None, + "actions": None, + "visibility": "full", + "clipped_by": None, + "rect": line_rect, + "visible_rect": line_rect, + "children": [], + "browser_top_layer": None, + "browser_second_layer": None, + "source": "ocr", + } + target_list = self._find_deepest_container(self.element_tree, cx, cy) + target_list.append(node) + + # Controller mapping — ax_element=None routes to the coordinate-click fallback. + self.elements_mapping[str(self.element_index)] = { + 'element': None, + 'rect': line_rect, + 'visible_rect': line_rect, + 'name': line["text"], + 'aria_role': '', + 'type': 'OCR_TEXT', + 'value': None, + 'visibility': 'full', + 'clipped_by': None, + 'ax_element': None, + } + + # Track found elements by type (mirrors _assign_indices bookkeeping). + self.found_elements.setdefault("OCR_TEXT", []).append(node) + + if SCREENSHOT: + self.elements_to_draw.append({ + "rect": line_rect, + "index": self.element_index, + "depth": 0, + "visibility": "full", + "source": "ocr", + }) + def _capture_and_annotate(self, screen): - """Capture screenshot, draw bounding boxes, store as base64.""" - screenshot, scale = take_screenshot(screen) + """Annotate the screenshot captured earlier in scan_elements, store as base64.""" + # Reuse the single frame captured (after extract_all settled) so OCR boxes, + # AX boxes, and the displayed screenshot are all consistent. Fall back to a + # fresh capture only if it wasn't taken (e.g. called outside scan_elements). + if self._screenshot is not None: + screenshot, scale = self._screenshot.copy(), self._scale + else: + screenshot, scale = take_screenshot(screen) if screenshot is None: self._annotated_image_base64 = None self._plain_screenshot = None @@ -1608,6 +1814,7 @@ def _capture_and_annotate(self, screen): for item in self.elements_to_draw: rect = item["rect"] index = str(item["index"]) + is_ocr = item.get("source") == "ocr" # Convert from CG coordinates to pixel coordinates box = ( @@ -1617,6 +1824,11 @@ def _capture_and_annotate(self, screen): int((rect.bottom - oy) * scale), ) + # OCR boxes get slight padding for breathing room around tight text + if is_ocr: + pad = int(3 * scale) + box = (box[0] - pad, box[1] - pad, box[2] + pad, box[3] + pad) + draw.rectangle(box, outline=BOX_COLOR, width=2) label = f"[{index}]" @@ -1628,9 +1840,14 @@ def _capture_and_annotate(self, screen): text_width = len(label) * 8 text_height = 15 - # Position label at top-left inside box - text_x = box[0] + 4 - text_y = box[1] + 3 + if is_ocr: + # OCR: position label above the box (text usually fills the box) + text_x = box[0] + text_y = box[1] - text_height - 2 + else: + # Position label at top-left inside box + text_x = box[0] + 4 + text_y = box[1] + 3 # White outline for readability outline_color = (255, 255, 255) @@ -1735,6 +1952,14 @@ def _get_tree_text_recursive(self, tree_list, depth): indent = " " * depth for item in tree_list: + # OCR_TEXT elements use a distinct format (text the AX tree missed) + if item.get("source") == "ocr": + text = _xml_escape(item['name']) + result += f'{indent}[{item["index"]}]\n' + if item.get("children"): + result += self._get_tree_text_recursive(item["children"], depth + 1) + continue + name = _xml_escape(item['name']) visibility = item.get('visibility', 'full') @@ -1806,6 +2031,14 @@ def _write_tree_recursive(self, file, tree_list, depth): indent = " " * depth for item in tree_list: + # OCR_TEXT elements use a distinct format (text the AX tree missed) + if item.get("source") == "ocr": + text = _xml_escape(item['name']) + file.write(f'{indent}[{item["index"]}]\n') + if item.get("children"): + self._write_tree_recursive(file, item["children"], depth + 1) + continue + name = _xml_escape(item['name']) visibility = item.get('visibility', 'full') diff --git a/Auto_Use/macOS_use/tree/ocr.py b/Auto_Use/macOS_use/tree/ocr.py new file mode 100644 index 0000000..646d7fc --- /dev/null +++ b/Auto_Use/macOS_use/tree/ocr.py @@ -0,0 +1,102 @@ +# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# If you build on this project, please keep this header and credit +# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works. +# A small attribution goes a long way toward a healthy open-source +# community — thank you for contributing. + +""" +OCR Detection — macOS Apple Vision OCR scanner (ocrmac). + +Mirrors the Windows OCRScanner (windows_use/tree/ocr_detection.py): captures +nothing itself — it OCRs a PRE-CAPTURED screenshot so it shares the exact frame +the AX scan + annotation use — and returns a raw line list. Filtering/merging +against the AX element tree happens in the caller (element.py). + +Designed to run as a parallel thread alongside the AX walk. The returned line +boxes are already converted to CG/AX POINTS (top-left origin) so they live in +the same coordinate space as AX element rects. +""" + +from ocrmac import ocrmac + +# Drop low-confidence detections (passed straight to ocrmac) and stray +# single-character / blank lines that add noise without being actionable. +MIN_CONFIDENCE = 0.4 +MIN_TEXT_LEN = 2 + + +class OCRScanner: + """Run Apple Vision OCR over a pre-captured PIL screenshot. + + Stores a raw line list in CG/AX POINTS: + [{text, left, top, right, bottom, confidence}, ...] + Thread-safe for parallel execution — each ocrmac call creates its own + Vision request and autorelease pool, with no shared global state. + """ + + def __init__(self, pil_image, scale, screen_origin, recognition_level="accurate"): + self._image = pil_image # PIL image in PIXELS (already captured) + self._scale = scale # backing scale factor (retina) + self._ox, self._oy = screen_origin # screen["x"], screen["y"] in CG points + self._level = recognition_level + self.lines = [] + + def scan(self): + """OCR the image and store results in self.lines. Zero-arg so it can be + used directly as a threading.Thread target.""" + if self._image is None: + self.lines = [] + return + try: + # px=True returns pixel coords already flipped to top-left origin, + # so we only need the pixel -> CG-point conversion below. + annotations = ocrmac.OCR( + self._image, + recognition_level=self._level, + confidence_threshold=MIN_CONFIDENCE, + ).recognize(px=True) + except Exception as e: + print(f"OCR error: {e}") + self.lines = [] + return + + s, ox, oy = self._scale, self._ox, self._oy + out = [] + for text, conf, (x1, y1, x2, y2) in annotations: + t = (text or "").strip() + if len(t) < MIN_TEXT_LEN: + continue + # pixel -> CG points: inverse of element.py's (rect.left - ox) * scale + left = int(round(ox + x1 / s)) + top = int(round(oy + y1 / s)) + right = int(round(ox + x2 / s)) + bottom = int(round(oy + y2 / s)) + if right <= left or bottom <= top: + continue + out.append({ + "text": t, + "left": left, + "top": top, + "right": right, + "bottom": bottom, + "confidence": float(conf), + }) + self.lines = out + + def get_lines(self): + """Return raw line list: [{text, left, top, right, bottom, confidence}, ...] + in CG/AX points.""" + return self.lines diff --git a/mac_requirements.txt b/mac_requirements.txt index 903e73f..9c5fb65 100644 --- a/mac_requirements.txt +++ b/mac_requirements.txt @@ -10,6 +10,9 @@ Pillow opencv-python numpy scipy +ocrmac +pyobjc-framework-Vision +pyobjc-framework-Quartz # HTTP Requests & API clients requests