diff --git a/.gitignore b/.gitignore
index c4b9a01..d0e82a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -122,3 +122,4 @@ screen_record.py
# the code (service.py / __init__.py) is tracked, the saved chats are not.
/Auto_Use/agent_conversation/index.json
/Auto_Use/agent_conversation/chat_*/
+automation_grant.json
diff --git a/Auto_Use/macOS_use/tree/element.py b/Auto_Use/macOS_use/tree/element.py
index b17fee1..e36d16c 100644
--- a/Auto_Use/macOS_use/tree/element.py
+++ b/Auto_Use/macOS_use/tree/element.py
@@ -40,6 +40,7 @@
import io
import time
import base64
+import threading
import numpy as np
from collections import namedtuple
from PIL import Image, ImageDraw, ImageFont
@@ -62,12 +63,19 @@
AXIsProcessTrusted, kAXErrorSuccess,
)
+try:
+ from .ocr import OCRScanner
+except ImportError: # allow `python element.py` standalone
+ from ocr import OCRScanner
+
# ========== CONFIGURATION ==========
# Toggle switches — same semantics as Windows element.py
SCREENSHOT = True # Set to False to only generate element tree without screenshot
DEBUG = False # Set to True to save files to debug folders, False for direct LLM only
FRONTEND = True # Set to True when running from app.py to send images to frontend
+OCR = True # Set to False to disable Apple Vision OCR merge (canvas/label-less text)
+OCR_RECOGNITION_LEVEL = "accurate" # "accurate" (default) or "fast" (lower latency)
# Define Rect namedtuple matching Windows format (left, top, right, bottom)
Rect = namedtuple('Rect', ['left', 'top', 'right', 'bottom'])
@@ -495,6 +503,15 @@ def build_label(element, cfg):
CLIP_ROLES = frozenset({"AXScrollArea", "AXList", "AXOutline", "AXTable"})
+# Cleaned ("AX"-stripped) roles that are structural wrappers — they never claim
+# screen space for OCR overlap purposes; gaps between their children are exactly
+# where OCR fills in. Only "Group" actually reaches the tree today (the rest are
+# CLIP_ROLES that aren't tracked as nodes); listing them is future-proofing.
+OCR_STRUCTURAL_CONTAINER_TYPES = frozenset({
+ "Group", "ScrollArea", "List", "Outline", "Table",
+ "WebArea", "SplitGroup", "TabGroup", "Toolbar",
+})
+
def _overlaps(a, b):
"""Return True if rect a overlaps rect b."""
@@ -1335,6 +1352,15 @@ def __init__(self, config, frontend_callback=None):
self.found_elements = {} # Dictionary to store elements by type
self._debug_iteration = 0 # Debug iteration counter
+ # OCR state
+ self._screenshot = None # Single captured PIL screenshot (pixels)
+ self._scale = 2.0 # Backing scale factor of the capture
+ self._screen_size = (0, 0) # (width, height) in CG points
+ self._ocr_backdrop_max_area = float("inf") # leaves bigger than this don't suppress OCR
+ self.ocr_words = [] # Raw OCR lines (CG points, filtered later)
+ self.ocr_scanner = None # OCRScanner instance
+ self.ocr_thread = None # Thread handle for parallel OCR scan
+
def scan_elements(self):
"""Scan the active window and menu bar for configured element types."""
@@ -1350,6 +1376,10 @@ def scan_elements(self):
self.top_layer_info = None
self.second_layer_info = None
self.found_elements = {}
+ self._screenshot = None
+ self.ocr_words = []
+ self.ocr_scanner = None
+ self.ocr_thread = None
# Check accessibility permission
if not AXIsProcessTrusted():
@@ -1359,12 +1389,30 @@ def scan_elements(self):
# Get screen info
screen = get_screen()
+ self._screen_size = (screen["width"], screen["height"])
# Run the macOS AX scan
_seen_roles.clear()
app_info, menu_items, elements = extract_all(screen)
elements.sort(key=lambda e: (e["y"], e["x"]))
+ # Capture the screen ONCE, now that extract_all has settled the frame
+ # (it activates the app + toggles AXEnhancedUserInterface with a sleep).
+ # The same image is reused for OCR overlap-filtering and final annotation,
+ # so OCR boxes, AX boxes, and the displayed screenshot share one frame.
+ if SCREENSHOT or OCR:
+ self._screenshot, self._scale = take_screenshot(screen)
+
+ # Start OCR in a parallel thread — it overlaps the (main-thread) tree build.
+ if OCR and self._screenshot is not None:
+ self.ocr_scanner = OCRScanner(
+ self._screenshot.copy(), self._scale,
+ (screen["x"], screen["y"]),
+ recognition_level=OCR_RECOGNITION_LEVEL,
+ )
+ self.ocr_thread = threading.Thread(target=self.ocr_scanner.scan)
+ self.ocr_thread.start()
+
# Store application name
if app_info:
self.application_name = app_info["name"]
@@ -1432,6 +1480,16 @@ def scan_elements(self):
# ----- Build element tree from flat elements using spatial containment -----
self.element_tree = self._build_hierarchical_tree(elements)
+ # ----- Merge OCR survivors into the tree -----
+ # Join the parallel OCR scan, then nest any text the AX tree missed
+ # (canvas / label-less controls) as OCR_TEXT nodes that don't overlap
+ # an already-detected element.
+ if self.ocr_thread is not None:
+ self.ocr_thread.join()
+ self.ocr_words = self.ocr_scanner.get_lines()
+ self.ocr_thread = None
+ self._filter_and_merge_ocr()
+
# ----- Capture and annotate screenshot -----
self._debug_iteration += 1
if SCREENSHOT and self.elements_to_draw:
@@ -1577,9 +1635,157 @@ def _cleanup(node_list):
return roots
+ # ========== OCR MERGE (ported from windows_use/tree/element.py) ==========
+
+ def _collect_leaf_rects(self, tree_list, rects):
+ """Recursively collect rects that actually claim screen space, for OCR
+ overlap checking. Structural wrappers (Group, ScrollArea, ...) never claim
+ space — the gaps between their children are exactly where OCR fills in —
+ so we recurse into them without adding their own rect. Every other element
+ type claims its rect, EXCEPT backdrop-sized leaves (a full-window AXImage
+ for a map / canvas, a big background) which shouldn't suppress the text
+ drawn on top of them."""
+ for item in tree_list:
+ has_children = bool(item.get("children"))
+ if item["type"] in OCR_STRUCTURAL_CONTAINER_TYPES:
+ if has_children:
+ self._collect_leaf_rects(item["children"], rects)
+ else:
+ rect = item.get("rect") or item.get("visible_rect")
+ if rect:
+ area = (rect.right - rect.left) * (rect.bottom - rect.top)
+ if area <= self._ocr_backdrop_max_area:
+ rects.append(rect)
+ if has_children:
+ self._collect_leaf_rects(item["children"], rects)
+
+ def _find_deepest_container(self, tree_list, cx, cy):
+ """Find the deepest element whose rect contains point (cx, cy) and return
+ its children list (so the OCR node nests there). Uses center-point
+ containment — forgiving of OCR boxes that poke a pixel outside their
+ visual parent. Returns the top-level tree_list if nothing claims the point."""
+ for item in tree_list:
+ rect = item.get("rect") or item.get("visible_rect")
+ if not rect:
+ continue
+ if rect.left <= cx <= rect.right and rect.top <= cy <= rect.bottom:
+ # Inside this element — try to go deeper among its children.
+ if item.get("children"):
+ deeper = self._find_deepest_container(item["children"], cx, cy)
+ if deeper is not item["children"]:
+ return deeper
+ # Children exist but none claimed the point — nest as a sibling.
+ return item["children"]
+ # Leaf element that contains the point — nest as its child.
+ return item.setdefault("children", [])
+ # No element in this list contains the point — caller decides.
+ return tree_list
+
+ def _filter_and_merge_ocr(self):
+ """Filter OCR lines against detected elements, nest survivors into the tree.
+
+ Only leaf (space-claiming) element rects suppress OCR; structural wrappers
+ don't. An OCR line whose center sits inside a leaf rect is already covered
+ by the AX scan and is discarded. Surviving lines (canvas / label-less text
+ the AX tree missed) are nested into the deepest matching container as
+ OCR_TEXT nodes and registered for clicking (coordinate-click fallback,
+ ax_element=None) + drawing."""
+ if not self.ocr_words:
+ return
+
+ # Backdrop guard: a leaf element covering a large fraction of the screen
+ # is a canvas/background (e.g. Apple Maps' full-window "Map" AXImage), not
+ # the label for any specific word. Such leaves must NOT suppress OCR text
+ # drawn over them, or we lose every label on a map / large image / canvas.
+ sw, sh = self._screen_size
+ self._ocr_backdrop_max_area = (0.25 * sw * sh) if (sw and sh) else float("inf")
+
+ # Collect rects that claim screen space (top layer + menu bar).
+ leaf_rects = []
+ self._collect_leaf_rects(self.element_tree, leaf_rects)
+ self._collect_leaf_rects(self.menu_bar_tree, leaf_rects)
+
+ # Drop OCR lines overlapping a detected leaf; also dedupe OCR-vs-OCR.
+ kept_lines = []
+ seen = set()
+ for line in self.ocr_words:
+ cx = (line["left"] + line["right"]) // 2
+ cy = (line["top"] + line["bottom"]) // 2
+
+ overlaps_leaf = any(
+ r.left <= cx <= r.right and r.top <= cy <= r.bottom
+ for r in leaf_rects
+ )
+ if overlaps_leaf:
+ continue
+
+ key = (line["text"], round(cx / 5), round(cy / 5))
+ if key in seen:
+ continue
+ seen.add(key)
+ kept_lines.append((line, cx, cy))
+
+ # Nest each surviving OCR line into the deepest matching container.
+ for line, cx, cy in kept_lines:
+ self.element_index += 1
+ line_rect = Rect(line["left"], line["top"], line["right"], line["bottom"])
+
+ node = {
+ "element": None,
+ "name": line["text"],
+ "aria_role": "",
+ "type": "OCR_TEXT",
+ "active": True,
+ "index": self.element_index,
+ "value": None,
+ "actions": None,
+ "visibility": "full",
+ "clipped_by": None,
+ "rect": line_rect,
+ "visible_rect": line_rect,
+ "children": [],
+ "browser_top_layer": None,
+ "browser_second_layer": None,
+ "source": "ocr",
+ }
+ target_list = self._find_deepest_container(self.element_tree, cx, cy)
+ target_list.append(node)
+
+ # Controller mapping — ax_element=None routes to the coordinate-click fallback.
+ self.elements_mapping[str(self.element_index)] = {
+ 'element': None,
+ 'rect': line_rect,
+ 'visible_rect': line_rect,
+ 'name': line["text"],
+ 'aria_role': '',
+ 'type': 'OCR_TEXT',
+ 'value': None,
+ 'visibility': 'full',
+ 'clipped_by': None,
+ 'ax_element': None,
+ }
+
+ # Track found elements by type (mirrors _assign_indices bookkeeping).
+ self.found_elements.setdefault("OCR_TEXT", []).append(node)
+
+ if SCREENSHOT:
+ self.elements_to_draw.append({
+ "rect": line_rect,
+ "index": self.element_index,
+ "depth": 0,
+ "visibility": "full",
+ "source": "ocr",
+ })
+
def _capture_and_annotate(self, screen):
- """Capture screenshot, draw bounding boxes, store as base64."""
- screenshot, scale = take_screenshot(screen)
+ """Annotate the screenshot captured earlier in scan_elements, store as base64."""
+ # Reuse the single frame captured (after extract_all settled) so OCR boxes,
+ # AX boxes, and the displayed screenshot are all consistent. Fall back to a
+ # fresh capture only if it wasn't taken (e.g. called outside scan_elements).
+ if self._screenshot is not None:
+ screenshot, scale = self._screenshot.copy(), self._scale
+ else:
+ screenshot, scale = take_screenshot(screen)
if screenshot is None:
self._annotated_image_base64 = None
self._plain_screenshot = None
@@ -1608,6 +1814,7 @@ def _capture_and_annotate(self, screen):
for item in self.elements_to_draw:
rect = item["rect"]
index = str(item["index"])
+ is_ocr = item.get("source") == "ocr"
# Convert from CG coordinates to pixel coordinates
box = (
@@ -1617,6 +1824,11 @@ def _capture_and_annotate(self, screen):
int((rect.bottom - oy) * scale),
)
+ # OCR boxes get slight padding for breathing room around tight text
+ if is_ocr:
+ pad = int(3 * scale)
+ box = (box[0] - pad, box[1] - pad, box[2] + pad, box[3] + pad)
+
draw.rectangle(box, outline=BOX_COLOR, width=2)
label = f"[{index}]"
@@ -1628,9 +1840,14 @@ def _capture_and_annotate(self, screen):
text_width = len(label) * 8
text_height = 15
- # Position label at top-left inside box
- text_x = box[0] + 4
- text_y = box[1] + 3
+ if is_ocr:
+ # OCR: position label above the box (text usually fills the box)
+ text_x = box[0]
+ text_y = box[1] - text_height - 2
+ else:
+ # Position label at top-left inside box
+ text_x = box[0] + 4
+ text_y = box[1] + 3
# White outline for readability
outline_color = (255, 255, 255)
@@ -1735,6 +1952,14 @@ def _get_tree_text_recursive(self, tree_list, depth):
indent = " " * depth
for item in tree_list:
+ # OCR_TEXT elements use a distinct format (text the AX tree missed)
+ if item.get("source") == "ocr":
+ text = _xml_escape(item['name'])
+ result += f'{indent}[{item["index"]}]\n'
+ if item.get("children"):
+ result += self._get_tree_text_recursive(item["children"], depth + 1)
+ continue
+
name = _xml_escape(item['name'])
visibility = item.get('visibility', 'full')
@@ -1806,6 +2031,14 @@ def _write_tree_recursive(self, file, tree_list, depth):
indent = " " * depth
for item in tree_list:
+ # OCR_TEXT elements use a distinct format (text the AX tree missed)
+ if item.get("source") == "ocr":
+ text = _xml_escape(item['name'])
+ file.write(f'{indent}[{item["index"]}]\n')
+ if item.get("children"):
+ self._write_tree_recursive(file, item["children"], depth + 1)
+ continue
+
name = _xml_escape(item['name'])
visibility = item.get('visibility', 'full')
diff --git a/Auto_Use/macOS_use/tree/ocr.py b/Auto_Use/macOS_use/tree/ocr.py
new file mode 100644
index 0000000..646d7fc
--- /dev/null
+++ b/Auto_Use/macOS_use/tree/ocr.py
@@ -0,0 +1,102 @@
+# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# If you build on this project, please keep this header and credit
+# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works.
+# A small attribution goes a long way toward a healthy open-source
+# community — thank you for contributing.
+
+"""
+OCR Detection — macOS Apple Vision OCR scanner (ocrmac).
+
+Mirrors the Windows OCRScanner (windows_use/tree/ocr_detection.py): captures
+nothing itself — it OCRs a PRE-CAPTURED screenshot so it shares the exact frame
+the AX scan + annotation use — and returns a raw line list. Filtering/merging
+against the AX element tree happens in the caller (element.py).
+
+Designed to run as a parallel thread alongside the AX walk. The returned line
+boxes are already converted to CG/AX POINTS (top-left origin) so they live in
+the same coordinate space as AX element rects.
+"""
+
+from ocrmac import ocrmac
+
+# Drop low-confidence detections (passed straight to ocrmac) and stray
+# single-character / blank lines that add noise without being actionable.
+MIN_CONFIDENCE = 0.4
+MIN_TEXT_LEN = 2
+
+
+class OCRScanner:
+ """Run Apple Vision OCR over a pre-captured PIL screenshot.
+
+ Stores a raw line list in CG/AX POINTS:
+ [{text, left, top, right, bottom, confidence}, ...]
+ Thread-safe for parallel execution — each ocrmac call creates its own
+ Vision request and autorelease pool, with no shared global state.
+ """
+
+ def __init__(self, pil_image, scale, screen_origin, recognition_level="accurate"):
+ self._image = pil_image # PIL image in PIXELS (already captured)
+ self._scale = scale # backing scale factor (retina)
+ self._ox, self._oy = screen_origin # screen["x"], screen["y"] in CG points
+ self._level = recognition_level
+ self.lines = []
+
+ def scan(self):
+ """OCR the image and store results in self.lines. Zero-arg so it can be
+ used directly as a threading.Thread target."""
+ if self._image is None:
+ self.lines = []
+ return
+ try:
+ # px=True returns pixel coords already flipped to top-left origin,
+ # so we only need the pixel -> CG-point conversion below.
+ annotations = ocrmac.OCR(
+ self._image,
+ recognition_level=self._level,
+ confidence_threshold=MIN_CONFIDENCE,
+ ).recognize(px=True)
+ except Exception as e:
+ print(f"OCR error: {e}")
+ self.lines = []
+ return
+
+ s, ox, oy = self._scale, self._ox, self._oy
+ out = []
+ for text, conf, (x1, y1, x2, y2) in annotations:
+ t = (text or "").strip()
+ if len(t) < MIN_TEXT_LEN:
+ continue
+ # pixel -> CG points: inverse of element.py's (rect.left - ox) * scale
+ left = int(round(ox + x1 / s))
+ top = int(round(oy + y1 / s))
+ right = int(round(ox + x2 / s))
+ bottom = int(round(oy + y2 / s))
+ if right <= left or bottom <= top:
+ continue
+ out.append({
+ "text": t,
+ "left": left,
+ "top": top,
+ "right": right,
+ "bottom": bottom,
+ "confidence": float(conf),
+ })
+ self.lines = out
+
+ def get_lines(self):
+ """Return raw line list: [{text, left, top, right, bottom, confidence}, ...]
+ in CG/AX points."""
+ return self.lines
diff --git a/mac_requirements.txt b/mac_requirements.txt
index 903e73f..9c5fb65 100644
--- a/mac_requirements.txt
+++ b/mac_requirements.txt
@@ -10,6 +10,9 @@ Pillow
opencv-python
numpy
scipy
+ocrmac
+pyobjc-framework-Vision
+pyobjc-framework-Quartz
# HTTP Requests & API clients
requests