From 3ce13ba9678503aea35ca4897b7c55bcc812a26c Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 4 Sep 2026 09:37:06 +0200 Subject: [PATCH 1/3] fix: rescale gesture coordinates for compatibility-mode window mismatches XCUICoordinate never rescales a raw points offset when an app's own window size differs from the device's (e.g. an iPhone-only app running in iPad compatibility mode - see appium/appium#16185). Tap, force touch, drag, swipe, and scroll gestures anchored via a raw offset from an element could land on the wrong point as a result. Normalize offsets against the target element's own frame instead, via a new shared FBCoordinateWithAnchorOffset() helper, so XCTest resolves them against whatever frame it reports at gesture-synthesis time. Scroll-to-visible additionally now anchors to a live element resolved from the scroll view's snapshot rather than the application, and surfaces failures instead of silently retrying or falling back to the previous unscaled behavior. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01G67QwebqZCxEYN6XmJDGJs --- .../Categories/XCUIElement+FBForceTouch.m | 7 +- .../Categories/XCUIElement+FBPickerWheel.m | 5 +- .../Categories/XCUIElement+FBScrolling.m | 159 +++++++++++++----- .../Commands/FBElementCommands.m | 38 ++++- .../Utilities/FBBaseActionsSynthesizer.m | 4 +- WebDriverAgentLib/Utilities/FBMathUtils.h | 25 +++ WebDriverAgentLib/Utilities/FBMathUtils.m | 22 +++ .../Utilities/FBW3CActionsSynthesizer.m | 6 +- .../IntegrationTests/FBTapTest.m | 62 +++++++ 9 files changed, 264 insertions(+), 64 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m b/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m index bd5d0bdd37..5d9bd9bdff 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m @@ -11,6 +11,7 @@ #if !TARGET_OS_TV #import "FBErrorBuilder.h" +#import "FBMathUtils.h" #import "XCUICoordinate.h" #import "XCUIDevice.h" @@ -36,8 +37,10 @@ - (BOOL)fb_forceTouchCoordinate:(NSValue *)relativeCoordinate } else { CGVector offset = CGVectorMake(relativeCoordinate.CGPointValue.x, relativeCoordinate.CGPointValue.y); - XCUICoordinate *hitPoint = [[self coordinateWithNormalizedOffset:CGVectorMake(0, 0)] - coordinateWithOffset:offset]; + XCUICoordinate *hitPoint = FBCoordinateWithAnchorOffset(self, CGVectorMake(0, 0), offset, error); + if (nil == hitPoint) { + return NO; + } if (nil == pressure || nil == duration) { [hitPoint forcePress]; } else { diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m b/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m index 3b361e569e..c49bf2559d 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m @@ -25,8 +25,9 @@ - (BOOL)fb_scrollWithOffset:(CGFloat)relativeHeightOffset error:(NSError **)erro { id snapshot = [self fb_standardSnapshot]; NSString *previousValue = snapshot.value; - XCUICoordinate *startCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)]; - XCUICoordinate *endCoord = [startCoord coordinateWithOffset:CGVectorMake(0.0, relativeHeightOffset * snapshot.frame.size.height)]; + // Stay in normalized offsets end-to-end: XCTest never rescales a composed raw + // coordinateWithOffset: for compatibility-mode windows (appium/appium#16185). + XCUICoordinate *endCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5 + relativeHeightOffset)]; // If picker value is reflected in its accessiblity id // then fetching of the next snapshot may fail with StaleElementReferenceError // because we bound elements by their accessbility ids by default. diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 5f470a3e39..6760e4fa1c 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -16,6 +16,8 @@ #import "FBXCElementSnapshotWrapper.h" #import "FBXCElementSnapshotWrapper+Helpers.h" #import "XCUIElement+FBCaching.h" +#import "XCUIElement+FBResolve.h" +#import "XCUIElement+FBUID.h" #import "XCUIApplication.h" #import "XCUICoordinate.h" #import "XCUIElement+FBIsVisible.h" @@ -35,15 +37,37 @@ @interface FBXCElementSnapshotWrapper (FBScrolling) -- (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector inApplication:(XCUIApplication *)application; -- (BOOL)fb_scrollByVector:(CGVector)vector inApplication:(XCUIApplication *)application error:(NSError **)error; +- (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollByVector:(CGVector)vector anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; @end +/** + Resolves a live element for the given snapshot, so gesture coordinates can be anchored + to it (its frame gets rescaled by XCTest for compatibility-mode windows; a raw + XCUIApplication anchor never does - see appium/appium#16185). Returns nil, rather than + falling back to the application, if the snapshot can no longer be located: anchoring to + the application would silently reproduce the very bug this is fixing. + */ +static XCUIElement *FBLiveElementForSnapshot(id snapshot, XCUIApplication *application) +{ + NSString *uid = [FBXCElementSnapshotWrapper wdUIDWithSnapshot:snapshot]; + if (nil == uid) { + return nil; + } + XCUIElement *result; + @autoreleasepool { + NSPredicate *predicate = [NSPredicate predicateWithFormat:@"%K = %@", FBStringify(FBXCElementSnapshotWrapper, fb_uid), uid]; + result = [[application.fb_query descendantsMatchingType:XCUIElementTypeAny] matchingPredicate:predicate].allElementsBoundByIndex.firstObject; + } + result.fb_isResolvedNatively = @NO; + return result; +} + @implementation XCUIElement (FBScrolling) - (BOOL)fb_nativeScrollToVisibleWithError:(NSError **)error @@ -61,28 +85,28 @@ - (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollUpByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollDownByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollLeftByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollRightByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (BOOL)fb_scrollToVisibleWithError:(NSError **)error @@ -176,30 +200,52 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll } } + // The scroll view's own identity is stable across scroll steps, so it only needs + // to be resolved to a live element once, up front; its frame does not, since it can + // change across scroll steps (rotation, keyboard, dynamic layout). + XCUIElement *scrollViewElement = FBLiveElementForSnapshot(scrollView, self.application); + if (nil == scrollViewElement) { + return + [[[FBErrorBuilder builder] + withDescriptionFormat:@"Failed to resolve a live element for the scrollable parent of '%@'", self.description] + buildError:error]; + } + const NSUInteger maxScrollCount = 25; NSUInteger scrollCount = 0; - FBXCElementSnapshotWrapper *scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:scrollView]; + FBXCElementSnapshotWrapper *scrollViewWrapped; // Scrolling till cell is visible and get current value of frames while (![self fb_isEquivalentElementSnapshotVisible:prescrollSnapshot] && scrollCount < maxScrollCount) { + BOOL didScroll; @autoreleasepool { + // Re-snapshotting the scroll view every step keeps its frame from drifting too far + // out of sync with the live anchor element's frame used to resolve touch points. + scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; if (targetCellIndex < visibleCellIndex) { - scrollDirection == FBXCUIElementScrollDirectionVertical ? + didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? [scrollViewWrapped fb_scrollUpByNormalizedDistance:normalizedScrollDistance - inApplication:self.application] : + anchorElement:scrollViewElement] : [scrollViewWrapped fb_scrollLeftByNormalizedDistance:normalizedScrollDistance - inApplication:self.application]; + anchorElement:scrollViewElement]; } else { - scrollDirection == FBXCUIElementScrollDirectionVertical ? + didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? [scrollViewWrapped fb_scrollDownByNormalizedDistance:normalizedScrollDistance - inApplication:self.application] : + anchorElement:scrollViewElement] : [scrollViewWrapped fb_scrollRightByNormalizedDistance:normalizedScrollDistance - inApplication:self.application]; + anchorElement:scrollViewElement]; } scrollCount++; // Wait for scroll animation [self fb_waitUntilStableWithTimeout:FBConfiguration.sharedInstance.animationCoolOffTimeout]; } + // The `error` out-param must not be written from inside the autorelease pool above. + if (!didScroll) { + return + [[[FBErrorBuilder builder] + withDescriptionFormat:@"Failed to scroll '%@': its frame is empty", self.description] + buildError:error]; + } } if (scrollCount >= maxScrollCount) { @@ -215,12 +261,13 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; CGRect visibleFrame = [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame; - + CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, visibleFrame.size.height - targetCellSnapshot.frame.size.height ); + scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; return [scrollViewWrapped fb_scrollByVector:scrollVector - inApplication:self.application + anchorElement:scrollViewElement error:error]; } @@ -254,42 +301,42 @@ - (CGRect)scrollingFrame return self.visibleFrame; } -- (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) anchorElement:anchorElement]; } -- (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) anchorElement:anchorElement]; } -- (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) anchorElement:anchorElement]; } -- (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) anchorElement:anchorElement]; } - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector - inApplication:(XCUIApplication *)application + anchorElement:(XCUIElement *)anchorElement { CGVector scrollVector = CGVectorMake(CGRectGetWidth(self.scrollingFrame) * normalizedScrollVector.dx, CGRectGetHeight(self.scrollingFrame) * normalizedScrollVector.dy ); - return [self fb_scrollByVector:scrollVector inApplication:application error:nil]; + return [self fb_scrollByVector:scrollVector anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollByVector:(CGVector)vector - inApplication:(XCUIApplication *)application - error:(NSError **)error + anchorElement:(XCUIElement *)anchorElement + error:(NSError **)error { CGVector scrollBoundingVector = CGVectorMake( CGRectGetWidth(self.scrollingFrame) * FBScrollTouchProportion, @@ -306,29 +353,49 @@ - (BOOL)fb_scrollByVector:(CGVector)vector fabs(vector.dy) > fabs(scrollBoundingVector.dy) ? scrollBoundingVector.dy : vector.dy); vector = CGVectorMake(vector.dx - scrollVector.dx, vector.dy - scrollVector.dy); shouldFinishScrolling = FBVectorFuzzyEqualToVector(vector, CGZeroVector, 1) || --preciseScrollAttemptsCount <= 0; - if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector inApplication:application error:error]){ + if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector anchorElement:anchorElement error:error]){ return NO; } } return YES; } -- (CGVector)fb_hitPointOffsetForScrollingVector:(CGVector)scrollingVector +// Normalized (0.0-1.0) touch-down offset within the scrolling frame, for the given +// scroll vector's direction. +- (CGVector)fb_normalizedHitPointOffsetForScrollingVector:(CGVector)scrollingVector { - CGFloat x = CGRectGetMinX(self.scrollingFrame) + CGRectGetWidth(self.scrollingFrame) * (scrollingVector.dx < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion)); - CGFloat y = CGRectGetMinY(self.scrollingFrame) + CGRectGetHeight(self.scrollingFrame) * (scrollingVector.dy < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion)); - return CGVectorMake((CGFloat)floor(x), (CGFloat)floor(y)); + CGFloat x = scrollingVector.dx < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion); + CGFloat y = scrollingVector.dy < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion); + return CGVectorMake(x, y); } - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vector - inApplication:(XCUIApplication *)application - error:(NSError **)error + anchorElement:(XCUIElement *)anchorElement + error:(NSError **)error { - CGVector hitpointOffset = [self fb_hitPointOffsetForScrollingVector:vector]; + CGRect scrollingFrame = self.scrollingFrame; + CGRect anchorFrame = anchorElement.frame; + if (CGRectIsEmpty(scrollingFrame) || CGRectIsEmpty(anchorFrame)) { + return [[[FBErrorBuilder builder] + withDescriptionFormat:@"Cannot compute a scroll gesture for '%@': its frame is empty", self.fb_description] + buildError:error]; + } - XCUICoordinate *appCoordinate = [[XCUICoordinate alloc] initWithElement:application normalizedOffset:CGVectorMake(0.0, 0.0)]; - XCUICoordinate *startCoordinate = [[XCUICoordinate alloc] initWithCoordinate:appCoordinate pointsOffset:hitpointOffset]; - XCUICoordinate *endCoordinate = [[XCUICoordinate alloc] initWithCoordinate:startCoordinate pointsOffset:vector]; + // Compute the touch-down/up points within the (possibly clipped) scrolling frame as + // before, then express them as fractions of the anchor element's own frame instead of + // raw points, which XCTest never rescales for compatibility-mode windows + // (appium/appium#16185). When scrollingFrame == anchorFrame this resolves to the exact + // same absolute point as before; it only differs once XCTest itself rescales anchorFrame. + CGVector proportion = [self fb_normalizedHitPointOffsetForScrollingVector:vector]; + CGPoint startPoint = CGPointMake((CGFloat)floor(scrollingFrame.origin.x + scrollingFrame.size.width * proportion.dx), + (CGFloat)floor(scrollingFrame.origin.y + scrollingFrame.size.height * proportion.dy)); + CGPoint endPoint = CGPointMake((CGFloat)floor(startPoint.x + vector.dx), (CGFloat)floor(startPoint.y + vector.dy)); + CGVector startOffset = CGVectorMake((startPoint.x - anchorFrame.origin.x) / anchorFrame.size.width, + (startPoint.y - anchorFrame.origin.y) / anchorFrame.size.height); + CGVector endOffset = CGVectorMake((endPoint.x - anchorFrame.origin.x) / anchorFrame.size.width, + (endPoint.y - anchorFrame.origin.y) / anchorFrame.size.height); + XCUICoordinate *startCoordinate = [anchorElement coordinateWithNormalizedOffset:startOffset]; + XCUICoordinate *endCoordinate = [anchorElement coordinateWithNormalizedOffset:endOffset]; if (FBPointFuzzyEqualToPoint(startCoordinate.screenPoint, endCoordinate.screenPoint, FBFuzzyPointThreshold)) { return YES; diff --git a/WebDriverAgentLib/Commands/FBElementCommands.m b/WebDriverAgentLib/Commands/FBElementCommands.m index 2594eed940..a46958bce4 100644 --- a/WebDriverAgentLib/Commands/FBElementCommands.m +++ b/WebDriverAgentLib/Commands/FBElementCommands.m @@ -353,14 +353,23 @@ + (NSArray *)routes + (id)handlePressAndDragCoordinateWithVelocity:(FBRouteRequest *)request { XCUIApplication *application = request.session.activeApplication; + NSError *error; CGVector startOffset = CGVectorMake((CGFloat)[request.arguments[@"fromX"] doubleValue], (CGFloat)[request.arguments[@"fromY"] doubleValue]); XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset - element:application]; + element:application + error:&error]; + if (nil == startCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } CGVector endOffset = CGVectorMake((CGFloat)[request.arguments[@"toX"] doubleValue], (CGFloat)[request.arguments[@"toY"] doubleValue]); XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset - element:application]; + element:application + error:&error]; + if (nil == endCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } [startCoordinate pressForDuration:[request.arguments[@"pressDuration"] doubleValue] thenDragToCoordinate:endCoordinate withVelocity:[request.arguments[@"velocity"] doubleValue] @@ -433,12 +442,19 @@ + (NSArray *)routes + (id)handleDrag:(FBRouteRequest *)request { XCUIElement *target = [self targetFromRequest:request]; + NSError *error; CGVector startOffset = CGVectorMake([request.arguments[@"fromX"] doubleValue], [request.arguments[@"fromY"] doubleValue]); - XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target]; + XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target error:&error]; + if (nil == startCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } CGVector endOffset = CGVectorMake([request.arguments[@"toX"] doubleValue], [request.arguments[@"toY"] doubleValue]); - XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target]; + XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target error:&error]; + if (nil == endCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } NSTimeInterval duration = [request.arguments[@"duration"] doubleValue]; [startCoordinate pressForDuration:duration thenDragToCoordinate:endCoordinate]; return FBResponseWithOK(); @@ -659,12 +675,15 @@ + (NSArray *)routes @param offset absolute screen offset for the given application @param element the element instance to perform the gesture on - @return translated gesture coordinates ready to be passed to XCUICoordinate methods + @param error Error instance if any + @return translated gesture coordinates ready to be passed to XCUICoordinate methods, or + nil if the element is not visible on the screen */ -+ (XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset - element:(XCUIElement *)element ++ (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset + element:(XCUIElement *)element + error:(NSError **)error { - return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset]; + return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error); } /** @@ -688,7 +707,8 @@ + (nullable id)targetWithXyCoordinatesFromRequest:(FBRouteRequest *)request erro return nil; } return [self gestureCoordinateWithOffset:CGVectorMake(x.doubleValue, y.doubleValue) - element:[self targetFromRequest:request]]; + element:[self targetFromRequest:request] + error:error]; } /** diff --git a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m index a5cd5f07fe..ef8604ce68 100644 --- a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m @@ -74,9 +74,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element return [element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)]; } - CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); // TODO: Shall we throw an exception if hitPoint is out of the element frame? - return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset]; + CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); + return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0, 0), offset, error); } @end diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index df9a116969..558f348ebb 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -9,6 +9,10 @@ #import @class XCUIApplication; +@class XCUICoordinate; +@class XCUIElement; + +NS_ASSUME_NONNULL_BEGIN extern CGFloat FBDefaultFrameFuzzyThreshold; @@ -33,4 +37,25 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold); #if !TARGET_OS_TV && !TARGET_OS_WATCH /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); + +/*! + Builds a coordinate for the given element from a raw points offset measured from a + normalized anchor point within the element's own frame - e.g. (0, 0) for an offset + relative to the top-left corner, (0.5, 0.5) for one relative to the center, as W3C + actions use. The offset is normalized against the element's frame instead of being + passed through as a raw points offset, which XCTest never rescales for + compatibility-mode windows (see appium/appium#16185). + + @param element the element to anchor the coordinate to + @param anchorOffset normalized offset of the anchor point within the element's frame + @param pointsOffset raw points offset from the anchor point + @param error populated if the element's frame is empty (not visible on the screen) + @return the resulting coordinate, or nil if the element's frame is empty + */ +XCUICoordinate * _Nullable FBCoordinateWithAnchorOffset(XCUIElement *element, + CGVector anchorOffset, + CGVector pointsOffset, + NSError **error); #endif + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 807f349055..23bd0aefc0 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -8,7 +8,10 @@ #import "FBMathUtils.h" +#import "FBErrorBuilder.h" #import "FBMacros.h" +#import "XCUICoordinate.h" +#import "XCUIElement.h" CGFloat FBDefaultFrameFuzzyThreshold = 2.0; @@ -60,4 +63,23 @@ This verification is just to make sure the bug is still there (since height is n } return actualSize; } + +XCUICoordinate *FBCoordinateWithAnchorOffset(XCUIElement *element, + CGVector anchorOffset, + CGVector pointsOffset, + NSError **error) +{ + // Read the frame once: checking CGRectIsEmpty and then re-reading element.frame for the + // divide below are two separate live round-trips, which a frame could collapse between. + CGRect frame = element.frame; + if (CGRectIsEmpty(frame)) { + [[[FBErrorBuilder builder] + withDescriptionFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description] + buildError:error]; + return nil; + } + CGVector normalizedOffset = CGVectorMake(anchorOffset.dx + pointsOffset.dx / frame.size.width, + anchorOffset.dy + pointsOffset.dy / frame.size.height); + return [element coordinateWithNormalizedOffset:normalizedOffset]; +} #endif diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 8323c4a3b2..c37f2a443c 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -162,7 +162,7 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element return [super hitpointWithElement:element positionOffset:positionOffset error:error]; } - // An offset relative to the element is defined + // An offset relative to the element is defined. if (CGRectIsEmpty(element.frame)) { [FBLogger log:self.application.fb_descriptionRepresentation]; NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", @@ -174,9 +174,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element } // W3C standard requires that relative element coordinates start at the center of the element's rectangle - CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); // TODO: Shall we throw an exception if hitPoint is out of the element frame? - return [[element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)] coordinateWithOffset:offset]; + CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); + return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0.5, 0.5), offset, error); } @end diff --git a/WebDriverAgentTests/IntegrationTests/FBTapTest.m b/WebDriverAgentTests/IntegrationTests/FBTapTest.m index 2d3d2ba8b6..45df7ec874 100644 --- a/WebDriverAgentTests/IntegrationTests/FBTapTest.m +++ b/WebDriverAgentTests/IntegrationTests/FBTapTest.m @@ -12,8 +12,11 @@ #import "FBElementCache.h" #import "FBTestMacros.h" +#import "XCUIApplication+FBTouchAction.h" +#import "XCUICoordinate.h" #import "XCUIDevice+FBRotation.h" #import "XCUIElement+FBIsVisible.h" +#import "XCUIElement+FBWebDriverAttributes.h" @interface FBTapTest : FBIntegrationTestCase @end @@ -100,4 +103,63 @@ - (void)testTapCoordinatesInPortraitUpsideDown [self verifyTapByCoordinatesWithOrientation:UIDeviceOrientationPortraitUpsideDown]; } +// appium/appium#16185: skips unless the app's window size actually differs from +// SpringBoard's, e.g. an iPhone-only app on iPad (built with TARGETED_DEVICE_FAMILY=1). +- (void)skipUnlessWindowSizeMismatchesDevice +{ + CGSize appSize = self.testedApplication.frame.size; + CGSize deviceSize = self.springboard.frame.size; + if (fabs(appSize.width - deviceSize.width) < 1 && fabs(appSize.height - deviceSize.height) < 1) { + XCTSkip(@"App window size matches SpringBoard's on this build/device, so it does not " + "reproduce the compatibility-mode mismatch from appium/appium#16185"); + } +} + +// Element-less absolute offsets are never rescaled by XCTest's XCUICoordinate (verified by +// disassembling XCUIAutomation.framework), so this still fails under a window-size mismatch. +- (void)testTapAtElementRectCenterUnderWindowSizeMismatch +{ + [self skipUnlessWindowSizeMismatchesDevice]; + + XCUIElement *dstButton = self.testedApplication.buttons[FBShowAlertButtonName]; + CGRect rect = dstButton.wdFrame; + CGPoint center = CGPointMake(CGRectGetMidX(rect), CGRectGetMidY(rect)); + + // Mirrors FBBaseActionsSynthesizer's hitpointWithElement:positionOffset: + // for an absolute (x, y) offset, as used by touch/perform and W3C actions. + XCUICoordinate *appOrigin = [self.testedApplication coordinateWithNormalizedOffset:CGVectorMake(0, 0)]; + XCUICoordinate *tapPoint = [appOrigin coordinateWithOffset:CGVectorMake(center.x, center.y)]; + [tapPoint tap]; + + XCTExpectFailureInBlock(@"element-less absolute offsets are never rescaled by XCTest for a " + "compatibility-mode window (appium/appium#16185); starts failing " + "loudly here the moment XCTest fixes this itself", ^{ + FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0); + }); +} + +// FBW3CActionsSynthesizer normalizes element-relative offsets against the element's own +// frame, so this keeps landing correctly under the same window-size mismatch. +- (void)testTapWithElementOffsetUnderWindowSizeMismatch +{ + [self skipUnlessWindowSizeMismatchesDevice]; + + NSArray *> *gesture = + @[@{ + @"type": @"pointer", + @"id": @"finger1", + @"parameters": @{@"pointerType": @"touch"}, + @"actions": @[ + @{@"type": @"pointerMove", @"duration": @0, @"origin": self.testedApplication.buttons[FBShowAlertButtonName], @"x": @5, @"y": @5}, + @{@"type": @"pointerDown"}, + @{@"type": @"pause", @"duration": @50}, + @{@"type": @"pointerUp"}, + ], + }, + ]; + NSError *error; + XCTAssertTrue([self.testedApplication fb_performW3CActions:gesture elementCache:nil error:&error]); + FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0); +} + @end From 810e9486dc092202c18201a8b77bc807ce8f709a Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 4 Sep 2026 16:20:33 +0200 Subject: [PATCH 2/3] fix: build FBCoordinateWithAnchorOffset for watchOS FBElementCommands.m calls it from gesture handlers that are compiled for watchOS (excluded only for tvOS), but it was declared/defined under a TARGET_OS_TV && TARGET_OS_WATCH guard, breaking the watchOS build. Only FBAdjustDimensionsForApplication (needs UIInterfaceOrientation) stays excluded from watchOS. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01G67QwebqZCxEYN6XmJDGJs --- WebDriverAgentLib/Utilities/FBMathUtils.h | 2 ++ WebDriverAgentLib/Utilities/FBMathUtils.m | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index 558f348ebb..8f78e1e1d4 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -37,7 +37,9 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold); #if !TARGET_OS_TV && !TARGET_OS_WATCH /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); +#endif +#if !TARGET_OS_TV /*! Builds a coordinate for the given element from a raw points offset measured from a normalized anchor point within the element's own frame - e.g. (0, 0) for an offset diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 23bd0aefc0..20a79de2f4 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -54,7 +54,7 @@ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientatio if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) { /* There is an XCTest bug that application.frame property returns exchanged dimensions for landscape mode. - This verification is just to make sure the bug is still there (since height is never greater than width in landscape) + This verification is just to make sure the bug is still there (since height is never greater than width in landscape) and to make it still working properly after XCTest itself starts to respect landscape mode. */ if (actualSize.height > actualSize.width) { @@ -63,7 +63,9 @@ This verification is just to make sure the bug is still there (since height is n } return actualSize; } +#endif +#if !TARGET_OS_TV XCUICoordinate *FBCoordinateWithAnchorOffset(XCUIElement *element, CGVector anchorOffset, CGVector pointsOffset, From 2169f8181f7538a506581c41a6e403f3db516cf8 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 5 Sep 2026 17:52:03 +0200 Subject: [PATCH 3/3] fix: normalize gesture offsets against wdFrame, not the live element frame element.frame can already be scaled for a compatibility-mode window mismatch, while the incoming points offset is measured in the WDA-reported (wdFrame) coordinate space - dividing by element.frame double-applied the scaling and left element-relative taps/drags landing on the wrong point, exactly as before the original fix. Also replaces the alert-opened check in the window-size-mismatch regression test with an exact landing-position assertion, using the Touch page's touchable view (which now records each touch-down location as its accessibility value) as ground truth - a large target could still catch an incorrectly-scaled tap and pass silently. Addresses appium/WebDriverAgent#1249 (review comment 5121824304). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01G67QwebqZCxEYN6XmJDGJs --- WebDriverAgentLib/Utilities/FBMathUtils.h | 7 +-- WebDriverAgentLib/Utilities/FBMathUtils.m | 7 +-- .../IntegrationApp/Classes/TouchableView.m | 8 ++- .../IntegrationTests/FBIntegrationTestCase.h | 7 +++ .../IntegrationTests/FBIntegrationTestCase.m | 10 ++++ .../IntegrationTests/FBTapTest.m | 51 +++++++++++++------ 6 files changed, 68 insertions(+), 22 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index 8f78e1e1d4..d0e1939efc 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -44,13 +44,14 @@ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientatio Builds a coordinate for the given element from a raw points offset measured from a normalized anchor point within the element's own frame - e.g. (0, 0) for an offset relative to the top-left corner, (0.5, 0.5) for one relative to the center, as W3C - actions use. The offset is normalized against the element's frame instead of being + actions use. The offset is normalized against the element's wdFrame (the same + WDA-reported coordinate space pointsOffset itself is measured in) instead of being passed through as a raw points offset, which XCTest never rescales for compatibility-mode windows (see appium/appium#16185). @param element the element to anchor the coordinate to - @param anchorOffset normalized offset of the anchor point within the element's frame - @param pointsOffset raw points offset from the anchor point + @param anchorOffset normalized offset of the anchor point within the element's wdFrame + @param pointsOffset raw points offset from the anchor point, in wdFrame's coordinate space @param error populated if the element's frame is empty (not visible on the screen) @return the resulting coordinate, or nil if the element's frame is empty */ diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 20a79de2f4..dacdc54aaf 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -12,6 +12,7 @@ #import "FBMacros.h" #import "XCUICoordinate.h" #import "XCUIElement.h" +#import "XCUIElement+FBWebDriverAttributes.h" CGFloat FBDefaultFrameFuzzyThreshold = 2.0; @@ -71,9 +72,9 @@ This verification is just to make sure the bug is still there (since height is n CGVector pointsOffset, NSError **error) { - // Read the frame once: checking CGRectIsEmpty and then re-reading element.frame for the - // divide below are two separate live round-trips, which a frame could collapse between. - CGRect frame = element.frame; + // wdFrame matches the coordinate space pointsOffset was measured in; element.frame alone + // can already be pre-scaled for a compatibility-mode window mismatch, double-applying it. + CGRect frame = element.wdFrame; if (CGRectIsEmpty(frame)) { [[[FBErrorBuilder builder] withDescriptionFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description] diff --git a/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m b/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m index 9e7412ab7b..0a9ac7b04c 100644 --- a/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m +++ b/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m @@ -73,9 +73,15 @@ - (void)createViewForTouch:(UITouch *)touch { if (touch) { + CGPoint location = [touch locationInView:self]; + // Exposes the last touch-down location, in this view's own bounds coordinate + // space, for tests to assert on regardless of any window-level scaling. + self.isAccessibilityElement = YES; + self.accessibilityValue = [NSString stringWithFormat:@"%.2f,%.2f", location.x, location.y]; + TouchSpotView *newView = [[TouchSpotView alloc] init]; newView.bounds = CGRectMake(0, 0, 1, 1); - newView.center = [touch locationInView:self]; + newView.center = location; [self addSubview:newView]; [UIView animateWithDuration:0.2 animations:^{ newView.bounds = CGRectMake(0, 0, 100, 100); diff --git a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h index a2b8faf92c..a9a09a5b8c 100644 --- a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h +++ b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h @@ -91,4 +91,11 @@ extern NSArray *const FBMainViewButtonLabels; */ - (void)resetOrientation; +/** + appium/appium#16185: skips the current test unless the app's window size actually + differs from SpringBoard's, e.g. an iPhone-only app on iPad (built with + TARGETED_DEVICE_FAMILY=1). + */ +- (void)skipUnlessWindowSizeMismatchesDevice; + @end diff --git a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m index e89af5ef68..4bb2a8b390 100644 --- a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m +++ b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m @@ -162,4 +162,14 @@ - (void)clearAlert FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count == 0); } +- (void)skipUnlessWindowSizeMismatchesDevice +{ + CGSize appSize = self.testedApplication.frame.size; + CGSize deviceSize = self.springboard.frame.size; + if (fabs(appSize.width - deviceSize.width) < 1 && fabs(appSize.height - deviceSize.height) < 1) { + XCTSkip(@"App window size matches SpringBoard's on this build/device, so it does not " + "reproduce the compatibility-mode mismatch from appium/appium#16185"); + } +} + @end diff --git a/WebDriverAgentTests/IntegrationTests/FBTapTest.m b/WebDriverAgentTests/IntegrationTests/FBTapTest.m index 45df7ec874..96f05f9a23 100644 --- a/WebDriverAgentTests/IntegrationTests/FBTapTest.m +++ b/WebDriverAgentTests/IntegrationTests/FBTapTest.m @@ -11,6 +11,7 @@ #import "FBIntegrationTestCase.h" #import "FBElementCache.h" +#import "FBMathUtils.h" #import "FBTestMacros.h" #import "XCUIApplication+FBTouchAction.h" #import "XCUICoordinate.h" @@ -103,18 +104,6 @@ - (void)testTapCoordinatesInPortraitUpsideDown [self verifyTapByCoordinatesWithOrientation:UIDeviceOrientationPortraitUpsideDown]; } -// appium/appium#16185: skips unless the app's window size actually differs from -// SpringBoard's, e.g. an iPhone-only app on iPad (built with TARGETED_DEVICE_FAMILY=1). -- (void)skipUnlessWindowSizeMismatchesDevice -{ - CGSize appSize = self.testedApplication.frame.size; - CGSize deviceSize = self.springboard.frame.size; - if (fabs(appSize.width - deviceSize.width) < 1 && fabs(appSize.height - deviceSize.height) < 1) { - XCTSkip(@"App window size matches SpringBoard's on this build/device, so it does not " - "reproduce the compatibility-mode mismatch from appium/appium#16185"); - } -} - // Element-less absolute offsets are never rescaled by XCTest's XCUICoordinate (verified by // disassembling XCUIAutomation.framework), so this still fails under a window-size mismatch. - (void)testTapAtElementRectCenterUnderWindowSizeMismatch @@ -138,19 +127,50 @@ - (void)testTapAtElementRectCenterUnderWindowSizeMismatch }); } +@end + +// The Touch page's touchable view records each touch-down's location, in its own bounds +// coordinate space, as its accessibility value - a ground truth unaffected by any +// window-level scaling, letting these tests assert on exact landing position rather than +// just on whether a tap happened to land inside some (possibly large) target. +@interface FBElementOffsetTapTest : FBIntegrationTestCase +@end + +@implementation FBElementOffsetTapTest + +- (void)setUp +{ + [super setUp]; + [self launchApplication]; + [self goToTouchPage]; +} + +- (CGPoint)lastTouchLocationOf:(XCUIElement *)touchable +{ + NSString *value = touchable.value; + NSArray *components = [value componentsSeparatedByString:@","]; + return CGPointMake(components.firstObject.doubleValue, components.lastObject.doubleValue); +} + // FBW3CActionsSynthesizer normalizes element-relative offsets against the element's own -// frame, so this keeps landing correctly under the same window-size mismatch. +// frame, so this keeps landing at the intended point under a window-size mismatch +// (appium/appium#16185), unlike an element-less absolute offset. - (void)testTapWithElementOffsetUnderWindowSizeMismatch { [self skipUnlessWindowSizeMismatchesDevice]; + XCUIElement *touchable = self.testedApplication.otherElements[@"touchableView"]; + CGSize size = touchable.wdFrame.size; + CGVector offset = CGVectorMake(size.width / 4, -size.height / 4); + CGPoint expectedLocation = CGPointMake(size.width / 2 + offset.dx, size.height / 2 + offset.dy); + NSArray *> *gesture = @[@{ @"type": @"pointer", @"id": @"finger1", @"parameters": @{@"pointerType": @"touch"}, @"actions": @[ - @{@"type": @"pointerMove", @"duration": @0, @"origin": self.testedApplication.buttons[FBShowAlertButtonName], @"x": @5, @"y": @5}, + @{@"type": @"pointerMove", @"duration": @0, @"origin": touchable, @"x": @(offset.dx), @"y": @(offset.dy)}, @{@"type": @"pointerDown"}, @{@"type": @"pause", @"duration": @50}, @{@"type": @"pointerUp"}, @@ -159,7 +179,8 @@ - (void)testTapWithElementOffsetUnderWindowSizeMismatch ]; NSError *error; XCTAssertTrue([self.testedApplication fb_performW3CActions:gesture elementCache:nil error:&error]); - FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0); + + FBAssertWaitTillBecomesTrue(FBPointFuzzyEqualToPoint([self lastTouchLocationOf:touchable], expectedLocation, 5.0)); } @end