diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m b/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m index bd5d0bdd3..5d9bd9bdf 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m @@ -11,6 +11,7 @@ #if !TARGET_OS_TV #import "FBErrorBuilder.h" +#import "FBMathUtils.h" #import "XCUICoordinate.h" #import "XCUIDevice.h" @@ -36,8 +37,10 @@ - (BOOL)fb_forceTouchCoordinate:(NSValue *)relativeCoordinate } else { CGVector offset = CGVectorMake(relativeCoordinate.CGPointValue.x, relativeCoordinate.CGPointValue.y); - XCUICoordinate *hitPoint = [[self coordinateWithNormalizedOffset:CGVectorMake(0, 0)] - coordinateWithOffset:offset]; + XCUICoordinate *hitPoint = FBCoordinateWithAnchorOffset(self, CGVectorMake(0, 0), offset, error); + if (nil == hitPoint) { + return NO; + } if (nil == pressure || nil == duration) { [hitPoint forcePress]; } else { diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m b/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m index 3b361e569..c49bf2559 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m @@ -25,8 +25,9 @@ - (BOOL)fb_scrollWithOffset:(CGFloat)relativeHeightOffset error:(NSError **)erro { id snapshot = [self fb_standardSnapshot]; NSString *previousValue = snapshot.value; - XCUICoordinate *startCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)]; - XCUICoordinate *endCoord = [startCoord coordinateWithOffset:CGVectorMake(0.0, relativeHeightOffset * snapshot.frame.size.height)]; + // Stay in normalized offsets end-to-end: XCTest never rescales a composed raw + // coordinateWithOffset: for compatibility-mode windows (appium/appium#16185). + XCUICoordinate *endCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5 + relativeHeightOffset)]; // If picker value is reflected in its accessiblity id // then fetching of the next snapshot may fail with StaleElementReferenceError // because we bound elements by their accessbility ids by default. diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 5f470a3e3..6760e4fa1 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -16,6 +16,8 @@ #import "FBXCElementSnapshotWrapper.h" #import "FBXCElementSnapshotWrapper+Helpers.h" #import "XCUIElement+FBCaching.h" +#import "XCUIElement+FBResolve.h" +#import "XCUIElement+FBUID.h" #import "XCUIApplication.h" #import "XCUICoordinate.h" #import "XCUIElement+FBIsVisible.h" @@ -35,15 +37,37 @@ @interface FBXCElementSnapshotWrapper (FBScrolling) -- (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance inApplication:(XCUIApplication *)application; -- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector inApplication:(XCUIApplication *)application; -- (BOOL)fb_scrollByVector:(CGVector)vector inApplication:(XCUIApplication *)application error:(NSError **)error; +- (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement; +- (BOOL)fb_scrollByVector:(CGVector)vector anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; @end +/** + Resolves a live element for the given snapshot, so gesture coordinates can be anchored + to it (its frame gets rescaled by XCTest for compatibility-mode windows; a raw + XCUIApplication anchor never does - see appium/appium#16185). Returns nil, rather than + falling back to the application, if the snapshot can no longer be located: anchoring to + the application would silently reproduce the very bug this is fixing. + */ +static XCUIElement *FBLiveElementForSnapshot(id snapshot, XCUIApplication *application) +{ + NSString *uid = [FBXCElementSnapshotWrapper wdUIDWithSnapshot:snapshot]; + if (nil == uid) { + return nil; + } + XCUIElement *result; + @autoreleasepool { + NSPredicate *predicate = [NSPredicate predicateWithFormat:@"%K = %@", FBStringify(FBXCElementSnapshotWrapper, fb_uid), uid]; + result = [[application.fb_query descendantsMatchingType:XCUIElementTypeAny] matchingPredicate:predicate].allElementsBoundByIndex.firstObject; + } + result.fb_isResolvedNatively = @NO; + return result; +} + @implementation XCUIElement (FBScrolling) - (BOOL)fb_nativeScrollToVisibleWithError:(NSError **)error @@ -61,28 +85,28 @@ - (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollUpByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollDownByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollLeftByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance { id snapshot = [self fb_customSnapshot]; [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_scrollRightByNormalizedDistance:distance - inApplication:self.application]; + anchorElement:self]; } - (BOOL)fb_scrollToVisibleWithError:(NSError **)error @@ -176,30 +200,52 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll } } + // The scroll view's own identity is stable across scroll steps, so it only needs + // to be resolved to a live element once, up front; its frame does not, since it can + // change across scroll steps (rotation, keyboard, dynamic layout). + XCUIElement *scrollViewElement = FBLiveElementForSnapshot(scrollView, self.application); + if (nil == scrollViewElement) { + return + [[[FBErrorBuilder builder] + withDescriptionFormat:@"Failed to resolve a live element for the scrollable parent of '%@'", self.description] + buildError:error]; + } + const NSUInteger maxScrollCount = 25; NSUInteger scrollCount = 0; - FBXCElementSnapshotWrapper *scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:scrollView]; + FBXCElementSnapshotWrapper *scrollViewWrapped; // Scrolling till cell is visible and get current value of frames while (![self fb_isEquivalentElementSnapshotVisible:prescrollSnapshot] && scrollCount < maxScrollCount) { + BOOL didScroll; @autoreleasepool { + // Re-snapshotting the scroll view every step keeps its frame from drifting too far + // out of sync with the live anchor element's frame used to resolve touch points. + scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; if (targetCellIndex < visibleCellIndex) { - scrollDirection == FBXCUIElementScrollDirectionVertical ? + didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? [scrollViewWrapped fb_scrollUpByNormalizedDistance:normalizedScrollDistance - inApplication:self.application] : + anchorElement:scrollViewElement] : [scrollViewWrapped fb_scrollLeftByNormalizedDistance:normalizedScrollDistance - inApplication:self.application]; + anchorElement:scrollViewElement]; } else { - scrollDirection == FBXCUIElementScrollDirectionVertical ? + didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? [scrollViewWrapped fb_scrollDownByNormalizedDistance:normalizedScrollDistance - inApplication:self.application] : + anchorElement:scrollViewElement] : [scrollViewWrapped fb_scrollRightByNormalizedDistance:normalizedScrollDistance - inApplication:self.application]; + anchorElement:scrollViewElement]; } scrollCount++; // Wait for scroll animation [self fb_waitUntilStableWithTimeout:FBConfiguration.sharedInstance.animationCoolOffTimeout]; } + // The `error` out-param must not be written from inside the autorelease pool above. + if (!didScroll) { + return + [[[FBErrorBuilder builder] + withDescriptionFormat:@"Failed to scroll '%@': its frame is empty", self.description] + buildError:error]; + } } if (scrollCount >= maxScrollCount) { @@ -215,12 +261,13 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; CGRect visibleFrame = [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame; - + CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, visibleFrame.size.height - targetCellSnapshot.frame.size.height ); + scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; return [scrollViewWrapped fb_scrollByVector:scrollVector - inApplication:self.application + anchorElement:scrollViewElement error:error]; } @@ -254,42 +301,42 @@ - (CGRect)scrollingFrame return self.visibleFrame; } -- (void)fb_scrollUpByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) anchorElement:anchorElement]; } -- (void)fb_scrollDownByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) anchorElement:anchorElement]; } -- (void)fb_scrollLeftByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) anchorElement:anchorElement]; } -- (void)fb_scrollRightByNormalizedDistance:(CGFloat)distance - inApplication:(XCUIApplication *)application +- (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance + anchorElement:(XCUIElement *)anchorElement { - [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) inApplication:application]; + return [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) anchorElement:anchorElement]; } - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector - inApplication:(XCUIApplication *)application + anchorElement:(XCUIElement *)anchorElement { CGVector scrollVector = CGVectorMake(CGRectGetWidth(self.scrollingFrame) * normalizedScrollVector.dx, CGRectGetHeight(self.scrollingFrame) * normalizedScrollVector.dy ); - return [self fb_scrollByVector:scrollVector inApplication:application error:nil]; + return [self fb_scrollByVector:scrollVector anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollByVector:(CGVector)vector - inApplication:(XCUIApplication *)application - error:(NSError **)error + anchorElement:(XCUIElement *)anchorElement + error:(NSError **)error { CGVector scrollBoundingVector = CGVectorMake( CGRectGetWidth(self.scrollingFrame) * FBScrollTouchProportion, @@ -306,29 +353,49 @@ - (BOOL)fb_scrollByVector:(CGVector)vector fabs(vector.dy) > fabs(scrollBoundingVector.dy) ? scrollBoundingVector.dy : vector.dy); vector = CGVectorMake(vector.dx - scrollVector.dx, vector.dy - scrollVector.dy); shouldFinishScrolling = FBVectorFuzzyEqualToVector(vector, CGZeroVector, 1) || --preciseScrollAttemptsCount <= 0; - if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector inApplication:application error:error]){ + if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector anchorElement:anchorElement error:error]){ return NO; } } return YES; } -- (CGVector)fb_hitPointOffsetForScrollingVector:(CGVector)scrollingVector +// Normalized (0.0-1.0) touch-down offset within the scrolling frame, for the given +// scroll vector's direction. +- (CGVector)fb_normalizedHitPointOffsetForScrollingVector:(CGVector)scrollingVector { - CGFloat x = CGRectGetMinX(self.scrollingFrame) + CGRectGetWidth(self.scrollingFrame) * (scrollingVector.dx < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion)); - CGFloat y = CGRectGetMinY(self.scrollingFrame) + CGRectGetHeight(self.scrollingFrame) * (scrollingVector.dy < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion)); - return CGVectorMake((CGFloat)floor(x), (CGFloat)floor(y)); + CGFloat x = scrollingVector.dx < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion); + CGFloat y = scrollingVector.dy < 0.0f ? FBScrollTouchProportion : (1 - FBScrollTouchProportion); + return CGVectorMake(x, y); } - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vector - inApplication:(XCUIApplication *)application - error:(NSError **)error + anchorElement:(XCUIElement *)anchorElement + error:(NSError **)error { - CGVector hitpointOffset = [self fb_hitPointOffsetForScrollingVector:vector]; + CGRect scrollingFrame = self.scrollingFrame; + CGRect anchorFrame = anchorElement.frame; + if (CGRectIsEmpty(scrollingFrame) || CGRectIsEmpty(anchorFrame)) { + return [[[FBErrorBuilder builder] + withDescriptionFormat:@"Cannot compute a scroll gesture for '%@': its frame is empty", self.fb_description] + buildError:error]; + } - XCUICoordinate *appCoordinate = [[XCUICoordinate alloc] initWithElement:application normalizedOffset:CGVectorMake(0.0, 0.0)]; - XCUICoordinate *startCoordinate = [[XCUICoordinate alloc] initWithCoordinate:appCoordinate pointsOffset:hitpointOffset]; - XCUICoordinate *endCoordinate = [[XCUICoordinate alloc] initWithCoordinate:startCoordinate pointsOffset:vector]; + // Compute the touch-down/up points within the (possibly clipped) scrolling frame as + // before, then express them as fractions of the anchor element's own frame instead of + // raw points, which XCTest never rescales for compatibility-mode windows + // (appium/appium#16185). When scrollingFrame == anchorFrame this resolves to the exact + // same absolute point as before; it only differs once XCTest itself rescales anchorFrame. + CGVector proportion = [self fb_normalizedHitPointOffsetForScrollingVector:vector]; + CGPoint startPoint = CGPointMake((CGFloat)floor(scrollingFrame.origin.x + scrollingFrame.size.width * proportion.dx), + (CGFloat)floor(scrollingFrame.origin.y + scrollingFrame.size.height * proportion.dy)); + CGPoint endPoint = CGPointMake((CGFloat)floor(startPoint.x + vector.dx), (CGFloat)floor(startPoint.y + vector.dy)); + CGVector startOffset = CGVectorMake((startPoint.x - anchorFrame.origin.x) / anchorFrame.size.width, + (startPoint.y - anchorFrame.origin.y) / anchorFrame.size.height); + CGVector endOffset = CGVectorMake((endPoint.x - anchorFrame.origin.x) / anchorFrame.size.width, + (endPoint.y - anchorFrame.origin.y) / anchorFrame.size.height); + XCUICoordinate *startCoordinate = [anchorElement coordinateWithNormalizedOffset:startOffset]; + XCUICoordinate *endCoordinate = [anchorElement coordinateWithNormalizedOffset:endOffset]; if (FBPointFuzzyEqualToPoint(startCoordinate.screenPoint, endCoordinate.screenPoint, FBFuzzyPointThreshold)) { return YES; diff --git a/WebDriverAgentLib/Commands/FBElementCommands.m b/WebDriverAgentLib/Commands/FBElementCommands.m index 2594eed94..a46958bce 100644 --- a/WebDriverAgentLib/Commands/FBElementCommands.m +++ b/WebDriverAgentLib/Commands/FBElementCommands.m @@ -353,14 +353,23 @@ + (NSArray *)routes + (id)handlePressAndDragCoordinateWithVelocity:(FBRouteRequest *)request { XCUIApplication *application = request.session.activeApplication; + NSError *error; CGVector startOffset = CGVectorMake((CGFloat)[request.arguments[@"fromX"] doubleValue], (CGFloat)[request.arguments[@"fromY"] doubleValue]); XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset - element:application]; + element:application + error:&error]; + if (nil == startCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } CGVector endOffset = CGVectorMake((CGFloat)[request.arguments[@"toX"] doubleValue], (CGFloat)[request.arguments[@"toY"] doubleValue]); XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset - element:application]; + element:application + error:&error]; + if (nil == endCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } [startCoordinate pressForDuration:[request.arguments[@"pressDuration"] doubleValue] thenDragToCoordinate:endCoordinate withVelocity:[request.arguments[@"velocity"] doubleValue] @@ -433,12 +442,19 @@ + (NSArray *)routes + (id)handleDrag:(FBRouteRequest *)request { XCUIElement *target = [self targetFromRequest:request]; + NSError *error; CGVector startOffset = CGVectorMake([request.arguments[@"fromX"] doubleValue], [request.arguments[@"fromY"] doubleValue]); - XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target]; + XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target error:&error]; + if (nil == startCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } CGVector endOffset = CGVectorMake([request.arguments[@"toX"] doubleValue], [request.arguments[@"toY"] doubleValue]); - XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target]; + XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target error:&error]; + if (nil == endCoordinate) { + return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]); + } NSTimeInterval duration = [request.arguments[@"duration"] doubleValue]; [startCoordinate pressForDuration:duration thenDragToCoordinate:endCoordinate]; return FBResponseWithOK(); @@ -659,12 +675,15 @@ + (NSArray *)routes @param offset absolute screen offset for the given application @param element the element instance to perform the gesture on - @return translated gesture coordinates ready to be passed to XCUICoordinate methods + @param error Error instance if any + @return translated gesture coordinates ready to be passed to XCUICoordinate methods, or + nil if the element is not visible on the screen */ -+ (XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset - element:(XCUIElement *)element ++ (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset + element:(XCUIElement *)element + error:(NSError **)error { - return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset]; + return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error); } /** @@ -688,7 +707,8 @@ + (nullable id)targetWithXyCoordinatesFromRequest:(FBRouteRequest *)request erro return nil; } return [self gestureCoordinateWithOffset:CGVectorMake(x.doubleValue, y.doubleValue) - element:[self targetFromRequest:request]]; + element:[self targetFromRequest:request] + error:error]; } /** diff --git a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m index a5cd5f07f..ef8604ce6 100644 --- a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m @@ -74,9 +74,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element return [element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)]; } - CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); // TODO: Shall we throw an exception if hitPoint is out of the element frame? - return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset]; + CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); + return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0, 0), offset, error); } @end diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index df9a11696..d0e1939ef 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -9,6 +9,10 @@ #import @class XCUIApplication; +@class XCUICoordinate; +@class XCUIElement; + +NS_ASSUME_NONNULL_BEGIN extern CGFloat FBDefaultFrameFuzzyThreshold; @@ -34,3 +38,27 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold); /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); #endif + +#if !TARGET_OS_TV +/*! + Builds a coordinate for the given element from a raw points offset measured from a + normalized anchor point within the element's own frame - e.g. (0, 0) for an offset + relative to the top-left corner, (0.5, 0.5) for one relative to the center, as W3C + actions use. The offset is normalized against the element's wdFrame (the same + WDA-reported coordinate space pointsOffset itself is measured in) instead of being + passed through as a raw points offset, which XCTest never rescales for + compatibility-mode windows (see appium/appium#16185). + + @param element the element to anchor the coordinate to + @param anchorOffset normalized offset of the anchor point within the element's wdFrame + @param pointsOffset raw points offset from the anchor point, in wdFrame's coordinate space + @param error populated if the element's frame is empty (not visible on the screen) + @return the resulting coordinate, or nil if the element's frame is empty + */ +XCUICoordinate * _Nullable FBCoordinateWithAnchorOffset(XCUIElement *element, + CGVector anchorOffset, + CGVector pointsOffset, + NSError **error); +#endif + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 807f34905..dacdc54aa 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -8,7 +8,11 @@ #import "FBMathUtils.h" +#import "FBErrorBuilder.h" #import "FBMacros.h" +#import "XCUICoordinate.h" +#import "XCUIElement.h" +#import "XCUIElement+FBWebDriverAttributes.h" CGFloat FBDefaultFrameFuzzyThreshold = 2.0; @@ -51,7 +55,7 @@ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientatio if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) { /* There is an XCTest bug that application.frame property returns exchanged dimensions for landscape mode. - This verification is just to make sure the bug is still there (since height is never greater than width in landscape) + This verification is just to make sure the bug is still there (since height is never greater than width in landscape) and to make it still working properly after XCTest itself starts to respect landscape mode. */ if (actualSize.height > actualSize.width) { @@ -61,3 +65,24 @@ This verification is just to make sure the bug is still there (since height is n return actualSize; } #endif + +#if !TARGET_OS_TV +XCUICoordinate *FBCoordinateWithAnchorOffset(XCUIElement *element, + CGVector anchorOffset, + CGVector pointsOffset, + NSError **error) +{ + // wdFrame matches the coordinate space pointsOffset was measured in; element.frame alone + // can already be pre-scaled for a compatibility-mode window mismatch, double-applying it. + CGRect frame = element.wdFrame; + if (CGRectIsEmpty(frame)) { + [[[FBErrorBuilder builder] + withDescriptionFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description] + buildError:error]; + return nil; + } + CGVector normalizedOffset = CGVectorMake(anchorOffset.dx + pointsOffset.dx / frame.size.width, + anchorOffset.dy + pointsOffset.dy / frame.size.height); + return [element coordinateWithNormalizedOffset:normalizedOffset]; +} +#endif diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 8323c4a3b..c37f2a443 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -162,7 +162,7 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element return [super hitpointWithElement:element positionOffset:positionOffset error:error]; } - // An offset relative to the element is defined + // An offset relative to the element is defined. if (CGRectIsEmpty(element.frame)) { [FBLogger log:self.application.fb_descriptionRepresentation]; NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", @@ -174,9 +174,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element } // W3C standard requires that relative element coordinates start at the center of the element's rectangle - CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); // TODO: Shall we throw an exception if hitPoint is out of the element frame? - return [[element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)] coordinateWithOffset:offset]; + CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y); + return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0.5, 0.5), offset, error); } @end diff --git a/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m b/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m index 9e7412ab7..0a9ac7b04 100644 --- a/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m +++ b/WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m @@ -73,9 +73,15 @@ - (void)createViewForTouch:(UITouch *)touch { if (touch) { + CGPoint location = [touch locationInView:self]; + // Exposes the last touch-down location, in this view's own bounds coordinate + // space, for tests to assert on regardless of any window-level scaling. + self.isAccessibilityElement = YES; + self.accessibilityValue = [NSString stringWithFormat:@"%.2f,%.2f", location.x, location.y]; + TouchSpotView *newView = [[TouchSpotView alloc] init]; newView.bounds = CGRectMake(0, 0, 1, 1); - newView.center = [touch locationInView:self]; + newView.center = location; [self addSubview:newView]; [UIView animateWithDuration:0.2 animations:^{ newView.bounds = CGRectMake(0, 0, 100, 100); diff --git a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h index a2b8faf92..a9a09a5b8 100644 --- a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h +++ b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h @@ -91,4 +91,11 @@ extern NSArray *const FBMainViewButtonLabels; */ - (void)resetOrientation; +/** + appium/appium#16185: skips the current test unless the app's window size actually + differs from SpringBoard's, e.g. an iPhone-only app on iPad (built with + TARGETED_DEVICE_FAMILY=1). + */ +- (void)skipUnlessWindowSizeMismatchesDevice; + @end diff --git a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m index e89af5ef6..4bb2a8b39 100644 --- a/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m +++ b/WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m @@ -162,4 +162,14 @@ - (void)clearAlert FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count == 0); } +- (void)skipUnlessWindowSizeMismatchesDevice +{ + CGSize appSize = self.testedApplication.frame.size; + CGSize deviceSize = self.springboard.frame.size; + if (fabs(appSize.width - deviceSize.width) < 1 && fabs(appSize.height - deviceSize.height) < 1) { + XCTSkip(@"App window size matches SpringBoard's on this build/device, so it does not " + "reproduce the compatibility-mode mismatch from appium/appium#16185"); + } +} + @end diff --git a/WebDriverAgentTests/IntegrationTests/FBTapTest.m b/WebDriverAgentTests/IntegrationTests/FBTapTest.m index 2d3d2ba8b..96f05f9a2 100644 --- a/WebDriverAgentTests/IntegrationTests/FBTapTest.m +++ b/WebDriverAgentTests/IntegrationTests/FBTapTest.m @@ -11,9 +11,13 @@ #import "FBIntegrationTestCase.h" #import "FBElementCache.h" +#import "FBMathUtils.h" #import "FBTestMacros.h" +#import "XCUIApplication+FBTouchAction.h" +#import "XCUICoordinate.h" #import "XCUIDevice+FBRotation.h" #import "XCUIElement+FBIsVisible.h" +#import "XCUIElement+FBWebDriverAttributes.h" @interface FBTapTest : FBIntegrationTestCase @end @@ -100,4 +104,83 @@ - (void)testTapCoordinatesInPortraitUpsideDown [self verifyTapByCoordinatesWithOrientation:UIDeviceOrientationPortraitUpsideDown]; } +// Element-less absolute offsets are never rescaled by XCTest's XCUICoordinate (verified by +// disassembling XCUIAutomation.framework), so this still fails under a window-size mismatch. +- (void)testTapAtElementRectCenterUnderWindowSizeMismatch +{ + [self skipUnlessWindowSizeMismatchesDevice]; + + XCUIElement *dstButton = self.testedApplication.buttons[FBShowAlertButtonName]; + CGRect rect = dstButton.wdFrame; + CGPoint center = CGPointMake(CGRectGetMidX(rect), CGRectGetMidY(rect)); + + // Mirrors FBBaseActionsSynthesizer's hitpointWithElement:positionOffset: + // for an absolute (x, y) offset, as used by touch/perform and W3C actions. + XCUICoordinate *appOrigin = [self.testedApplication coordinateWithNormalizedOffset:CGVectorMake(0, 0)]; + XCUICoordinate *tapPoint = [appOrigin coordinateWithOffset:CGVectorMake(center.x, center.y)]; + [tapPoint tap]; + + XCTExpectFailureInBlock(@"element-less absolute offsets are never rescaled by XCTest for a " + "compatibility-mode window (appium/appium#16185); starts failing " + "loudly here the moment XCTest fixes this itself", ^{ + FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0); + }); +} + +@end + +// The Touch page's touchable view records each touch-down's location, in its own bounds +// coordinate space, as its accessibility value - a ground truth unaffected by any +// window-level scaling, letting these tests assert on exact landing position rather than +// just on whether a tap happened to land inside some (possibly large) target. +@interface FBElementOffsetTapTest : FBIntegrationTestCase +@end + +@implementation FBElementOffsetTapTest + +- (void)setUp +{ + [super setUp]; + [self launchApplication]; + [self goToTouchPage]; +} + +- (CGPoint)lastTouchLocationOf:(XCUIElement *)touchable +{ + NSString *value = touchable.value; + NSArray *components = [value componentsSeparatedByString:@","]; + return CGPointMake(components.firstObject.doubleValue, components.lastObject.doubleValue); +} + +// FBW3CActionsSynthesizer normalizes element-relative offsets against the element's own +// frame, so this keeps landing at the intended point under a window-size mismatch +// (appium/appium#16185), unlike an element-less absolute offset. +- (void)testTapWithElementOffsetUnderWindowSizeMismatch +{ + [self skipUnlessWindowSizeMismatchesDevice]; + + XCUIElement *touchable = self.testedApplication.otherElements[@"touchableView"]; + CGSize size = touchable.wdFrame.size; + CGVector offset = CGVectorMake(size.width / 4, -size.height / 4); + CGPoint expectedLocation = CGPointMake(size.width / 2 + offset.dx, size.height / 2 + offset.dy); + + NSArray *> *gesture = + @[@{ + @"type": @"pointer", + @"id": @"finger1", + @"parameters": @{@"pointerType": @"touch"}, + @"actions": @[ + @{@"type": @"pointerMove", @"duration": @0, @"origin": touchable, @"x": @(offset.dx), @"y": @(offset.dy)}, + @{@"type": @"pointerDown"}, + @{@"type": @"pause", @"duration": @50}, + @{@"type": @"pointerUp"}, + ], + }, + ]; + NSError *error; + XCTAssertTrue([self.testedApplication fb_performW3CActions:gesture elementCache:nil error:&error]); + + FBAssertWaitTillBecomesTrue(FBPointFuzzyEqualToPoint([self lastTouchLocationOf:touchable], expectedLocation, 5.0)); +} + @end