fix: rescale gesture coordinates for compatibility-mode window mismatches

WebDriverAgentLib/Categories/XCUIElement+FBForceTouch.m

@@ -11,6 +11,7 @@
 #if !TARGET_OS_TV
 
 #import "FBErrorBuilder.h"
+#import "FBMathUtils.h"
 #import "XCUICoordinate.h"
 #import "XCUIDevice.h"
 
@@ -36,8 +37,10 @@ - (BOOL)fb_forceTouchCoordinate:(NSValue *)relativeCoordinate
   } else {
     CGVector offset = CGVectorMake(relativeCoordinate.CGPointValue.x,
                                    relativeCoordinate.CGPointValue.y);
-    XCUICoordinate *hitPoint = [[self coordinateWithNormalizedOffset:CGVectorMake(0, 0)]
-                                coordinateWithOffset:offset];
+    XCUICoordinate *hitPoint = FBCoordinateWithAnchorOffset(self, CGVectorMake(0, 0), offset, error);
+    if (nil == hitPoint) {
+      return NO;
+    }
     if (nil == pressure || nil == duration) {
       [hitPoint forcePress];
     } else {

WebDriverAgentLib/Categories/XCUIElement+FBPickerWheel.m

@@ -25,8 +25,9 @@ - (BOOL)fb_scrollWithOffset:(CGFloat)relativeHeightOffset error:(NSError **)erro
 {
   id<FBXCElementSnapshot> snapshot = [self fb_standardSnapshot];
   NSString *previousValue = snapshot.value;
-  XCUICoordinate *startCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)];
-  XCUICoordinate *endCoord = [startCoord coordinateWithOffset:CGVectorMake(0.0, relativeHeightOffset * snapshot.frame.size.height)];
+  // Stay in normalized offsets end-to-end: XCTest never rescales a composed raw
+  // coordinateWithOffset: for compatibility-mode windows (appium/appium#16185).
+  XCUICoordinate *endCoord = [self coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5 + relativeHeightOffset)];
   // If picker value is reflected in its accessiblity id
   // then fetching of the next snapshot may fail with StaleElementReferenceError
   // because we bound elements by their accessbility ids by default.

WebDriverAgentLib/Commands/FBElementCommands.m

@@ -353,14 +353,23 @@ + (NSArray *)routes
 + (id<FBResponsePayload>)handlePressAndDragCoordinateWithVelocity:(FBRouteRequest *)request
 {
   XCUIApplication *application = request.session.activeApplication;
+  NSError *error;
   CGVector startOffset = CGVectorMake((CGFloat)[request.arguments[@"fromX"] doubleValue],
                                      (CGFloat)[request.arguments[@"fromY"] doubleValue]);
   XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset
-                                                                    element:application];
+                                                                    element:application
+                                                                      error:&error];
+  if (nil == startCoordinate) {
+    return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]);
+  }
   CGVector endOffset = CGVectorMake((CGFloat)[request.arguments[@"toX"] doubleValue],
                                     (CGFloat)[request.arguments[@"toY"] doubleValue]);
   XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset
-                                                                  element:application];
+                                                                  element:application
+                                                                    error:&error];
+  if (nil == endCoordinate) {
+    return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]);
+  }
   [startCoordinate pressForDuration:[request.arguments[@"pressDuration"] doubleValue]
                thenDragToCoordinate:endCoordinate
                        withVelocity:[request.arguments[@"velocity"] doubleValue]
@@ -433,12 +442,19 @@ + (NSArray *)routes
 + (id<FBResponsePayload>)handleDrag:(FBRouteRequest *)request
 {
   XCUIElement *target = [self targetFromRequest:request];
+  NSError *error;
   CGVector startOffset = CGVectorMake([request.arguments[@"fromX"] doubleValue],
                                       [request.arguments[@"fromY"] doubleValue]);
-  XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target];
+  XCUICoordinate *startCoordinate = [self.class gestureCoordinateWithOffset:startOffset element:target error:&error];
+  if (nil == startCoordinate) {
+    return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]);
+  }
   CGVector endOffset = CGVectorMake([request.arguments[@"toX"] doubleValue],
                                     [request.arguments[@"toY"] doubleValue]);
-  XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target];
+  XCUICoordinate *endCoordinate = [self.class gestureCoordinateWithOffset:endOffset element:target error:&error];
+  if (nil == endCoordinate) {
+    return FBResponseWithStatus([FBCommandStatus invalidElementStateErrorWithMessage:error.description traceback:nil]);
+  }
   NSTimeInterval duration = [request.arguments[@"duration"] doubleValue];
   [startCoordinate pressForDuration:duration thenDragToCoordinate:endCoordinate];
   return FBResponseWithOK();
@@ -659,12 +675,15 @@ + (NSArray *)routes
 
  @param offset absolute screen offset for the given application
  @param element the element instance to perform the gesture on
- @return translated gesture coordinates ready to be passed to XCUICoordinate methods
+ @param error Error instance if any
+ @return translated gesture coordinates ready to be passed to XCUICoordinate methods, or
+ nil if the element is not visible on the screen
  */
-+ (XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset
-                                        element:(XCUIElement *)element
++ (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset
+                                                  element:(XCUIElement *)element
+                                                    error:(NSError **)error
 {
-  return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset];
+  return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error);
 }
 
 /**
@@ -688,7 +707,8 @@ + (nullable id)targetWithXyCoordinatesFromRequest:(FBRouteRequest *)request erro
     return nil;
   }
   return [self gestureCoordinateWithOffset:CGVectorMake(x.doubleValue, y.doubleValue)
-                                   element:[self targetFromRequest:request]];
+                                   element:[self targetFromRequest:request]
+                                     error:error];
 }
 
 /**

WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m

@@ -74,9 +74,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element
     return [element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)];
   }
 
-  CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y);
   // TODO: Shall we throw an exception if hitPoint is out of the element frame?
-  return [[element coordinateWithNormalizedOffset:CGVectorMake(0, 0)] coordinateWithOffset:offset];
+  CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y);
+  return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0, 0), offset, error);
 }
 
 @end

WebDriverAgentLib/Utilities/FBMathUtils.h

@@ -9,6 +9,10 @@
 #import <UIKit/UIKit.h>
 
 @class XCUIApplication;
+@class XCUICoordinate;
+@class XCUIElement;
+
+NS_ASSUME_NONNULL_BEGIN
 
 extern CGFloat FBDefaultFrameFuzzyThreshold;
 
@@ -34,3 +38,27 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold);
 /*! Inverts size if necessary to match current screen orientation */
 CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation);
 #endif
+
+#if !TARGET_OS_TV
+/*!
+ Builds a coordinate for the given element from a raw points offset measured from a
+ normalized anchor point within the element's own frame - e.g. (0, 0) for an offset
+ relative to the top-left corner, (0.5, 0.5) for one relative to the center, as W3C
+ actions use. The offset is normalized against the element's wdFrame (the same
+ WDA-reported coordinate space pointsOffset itself is measured in) instead of being
+ passed through as a raw points offset, which XCTest never rescales for
+ compatibility-mode windows (see appium/appium#16185).
+
+ @param element the element to anchor the coordinate to
+ @param anchorOffset normalized offset of the anchor point within the element's wdFrame
+ @param pointsOffset raw points offset from the anchor point, in wdFrame's coordinate space
+ @param error populated if the element's frame is empty (not visible on the screen)
+ @return the resulting coordinate, or nil if the element's frame is empty
+ */
+XCUICoordinate * _Nullable FBCoordinateWithAnchorOffset(XCUIElement *element,
+                                                         CGVector anchorOffset,
+                                                         CGVector pointsOffset,
+                                                         NSError **error);
+#endif
+
+NS_ASSUME_NONNULL_END

WebDriverAgentLib/Utilities/FBMathUtils.m

@@ -8,7 +8,11 @@
 
 #import "FBMathUtils.h"
 
+#import "FBErrorBuilder.h"
 #import "FBMacros.h"
+#import "XCUICoordinate.h"
+#import "XCUIElement.h"
+#import "XCUIElement+FBWebDriverAttributes.h"
 
 CGFloat FBDefaultFrameFuzzyThreshold = 2.0;
 
@@ -51,7 +55,7 @@ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientatio
   if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) {
     /*
      There is an XCTest bug that application.frame property returns exchanged dimensions for landscape mode.
-     This verification is just to make sure the bug is still there (since height is never greater than width in landscape) 
+     This verification is just to make sure the bug is still there (since height is never greater than width in landscape)
      and to make it still working properly after XCTest itself starts to respect landscape mode.
      */
     if (actualSize.height > actualSize.width) {
@@ -61,3 +65,24 @@ This verification is just to make sure the bug is still there (since height is n
   return actualSize;
 }
 #endif
+
+#if !TARGET_OS_TV
+XCUICoordinate *FBCoordinateWithAnchorOffset(XCUIElement *element,
+                                              CGVector anchorOffset,
+                                              CGVector pointsOffset,
+                                              NSError **error)
+{
+  // wdFrame matches the coordinate space pointsOffset was measured in; element.frame alone
+  // can already be pre-scaled for a compatibility-mode window mismatch, double-applying it.
+  CGRect frame = element.wdFrame;
+  if (CGRectIsEmpty(frame)) {
+    [[[FBErrorBuilder builder]
+      withDescriptionFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description]
+     buildError:error];
+    return nil;
+  }
+  CGVector normalizedOffset = CGVectorMake(anchorOffset.dx + pointsOffset.dx / frame.size.width,
+                                           anchorOffset.dy + pointsOffset.dy / frame.size.height);
+  return [element coordinateWithNormalizedOffset:normalizedOffset];
+}
+#endif

WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m

@@ -162,7 +162,7 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element
     return [super hitpointWithElement:element positionOffset:positionOffset error:error];
   }
 
-  // An offset relative to the element is defined
+  // An offset relative to the element is defined.
   if (CGRectIsEmpty(element.frame)) {
     [FBLogger log:self.application.fb_descriptionRepresentation];
     NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable",
@@ -174,9 +174,9 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element
   }
 
   // W3C standard requires that relative element coordinates start at the center of the element's rectangle
-  CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y);
   // TODO: Shall we throw an exception if hitPoint is out of the element frame?
-  return [[element coordinateWithNormalizedOffset:CGVectorMake(0.5, 0.5)] coordinateWithOffset:offset];
+  CGVector offset = CGVectorMake(positionOffset.CGPointValue.x, positionOffset.CGPointValue.y);
+  return FBCoordinateWithAnchorOffset((XCUIElement *)element, CGVectorMake(0.5, 0.5), offset, error);
 }
 
 @end

WebDriverAgentTests/IntegrationApp/Classes/TouchableView.m

@@ -73,9 +73,15 @@ - (void)createViewForTouch:(UITouch *)touch
 {
   if (touch)
   {
+    CGPoint location = [touch locationInView:self];
+    // Exposes the last touch-down location, in this view's own bounds coordinate
+    // space, for tests to assert on regardless of any window-level scaling.
+    self.isAccessibilityElement = YES;
+    self.accessibilityValue = [NSString stringWithFormat:@"%.2f,%.2f", location.x, location.y];
+
     TouchSpotView *newView = [[TouchSpotView alloc] init];
     newView.bounds = CGRectMake(0, 0, 1, 1);
-    newView.center = [touch locationInView:self];
+    newView.center = location;
     [self addSubview:newView];
     [UIView animateWithDuration:0.2 animations:^{
       newView.bounds = CGRectMake(0, 0, 100, 100);

WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.h

@@ -91,4 +91,11 @@ extern NSArray<NSString *> *const FBMainViewButtonLabels;
  */
 - (void)resetOrientation;
 
+/**
+ appium/appium#16185: skips the current test unless the app's window size actually
+ differs from SpringBoard's, e.g. an iPhone-only app on iPad (built with
+ TARGETED_DEVICE_FAMILY=1).
+ */
+- (void)skipUnlessWindowSizeMismatchesDevice;
+
 @end

WebDriverAgentTests/IntegrationTests/FBIntegrationTestCase.m

@@ -162,4 +162,14 @@ - (void)clearAlert
   FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count == 0);
 }
 
+- (void)skipUnlessWindowSizeMismatchesDevice
+{
+  CGSize appSize = self.testedApplication.frame.size;
+  CGSize deviceSize = self.springboard.frame.size;
+  if (fabs(appSize.width - deviceSize.width) < 1 && fabs(appSize.height - deviceSize.height) < 1) {
+    XCTSkip(@"App window size matches SpringBoard's on this build/device, so it does not "
+            "reproduce the compatibility-mode mismatch from appium/appium#16185");
+  }
+}
+
 @end

WebDriverAgentTests/IntegrationTests/FBTapTest.m

@@ -11,9 +11,13 @@
 #import "FBIntegrationTestCase.h"
 
 #import "FBElementCache.h"
+#import "FBMathUtils.h"
 #import "FBTestMacros.h"
+#import "XCUIApplication+FBTouchAction.h"
+#import "XCUICoordinate.h"
 #import "XCUIDevice+FBRotation.h"
 #import "XCUIElement+FBIsVisible.h"
+#import "XCUIElement+FBWebDriverAttributes.h"
 
 @interface FBTapTest : FBIntegrationTestCase
 @end
@@ -100,4 +104,83 @@ - (void)testTapCoordinatesInPortraitUpsideDown
   [self verifyTapByCoordinatesWithOrientation:UIDeviceOrientationPortraitUpsideDown];
 }
 
+// Element-less absolute offsets are never rescaled by XCTest's XCUICoordinate (verified by
+// disassembling XCUIAutomation.framework), so this still fails under a window-size mismatch.
+- (void)testTapAtElementRectCenterUnderWindowSizeMismatch
+{
+  [self skipUnlessWindowSizeMismatchesDevice];
+
+  XCUIElement *dstButton = self.testedApplication.buttons[FBShowAlertButtonName];
+  CGRect rect = dstButton.wdFrame;
+  CGPoint center = CGPointMake(CGRectGetMidX(rect), CGRectGetMidY(rect));
+
+  // Mirrors FBBaseActionsSynthesizer's hitpointWithElement:positionOffset:
+  // for an absolute (x, y) offset, as used by touch/perform and W3C actions.
+  XCUICoordinate *appOrigin = [self.testedApplication coordinateWithNormalizedOffset:CGVectorMake(0, 0)];
+  XCUICoordinate *tapPoint = [appOrigin coordinateWithOffset:CGVectorMake(center.x, center.y)];
+  [tapPoint tap];
+
+  XCTExpectFailureInBlock(@"element-less absolute offsets are never rescaled by XCTest for a "
+                           "compatibility-mode window (appium/appium#16185); starts failing "
+                           "loudly here the moment XCTest fixes this itself", ^{
+    FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0);
+  });
+}
+
+@end
+
+// The Touch page's touchable view records each touch-down's location, in its own bounds
+// coordinate space, as its accessibility value - a ground truth unaffected by any
+// window-level scaling, letting these tests assert on exact landing position rather than
+// just on whether a tap happened to land inside some (possibly large) target.
+@interface FBElementOffsetTapTest : FBIntegrationTestCase
+@end
+
+@implementation FBElementOffsetTapTest
+
+- (void)setUp
+{
+  [super setUp];
+  [self launchApplication];
+  [self goToTouchPage];
+}
+
+- (CGPoint)lastTouchLocationOf:(XCUIElement *)touchable
+{
+  NSString *value = touchable.value;
+  NSArray<NSString *> *components = [value componentsSeparatedByString:@","];
+  return CGPointMake(components.firstObject.doubleValue, components.lastObject.doubleValue);
+}
+
+// FBW3CActionsSynthesizer normalizes element-relative offsets against the element's own
+// frame, so this keeps landing at the intended point under a window-size mismatch
+// (appium/appium#16185), unlike an element-less absolute offset.
+- (void)testTapWithElementOffsetUnderWindowSizeMismatch
+{
+  [self skipUnlessWindowSizeMismatchesDevice];
+
+  XCUIElement *touchable = self.testedApplication.otherElements[@"touchableView"];
+  CGSize size = touchable.wdFrame.size;
+  CGVector offset = CGVectorMake(size.width / 4, -size.height / 4);
+  CGPoint expectedLocation = CGPointMake(size.width / 2 + offset.dx, size.height / 2 + offset.dy);
+
+  NSArray<NSDictionary<NSString *, id> *> *gesture =
+  @[@{
+      @"type": @"pointer",
+      @"id": @"finger1",
+      @"parameters": @{@"pointerType": @"touch"},
+      @"actions": @[
+          @{@"type": @"pointerMove", @"duration": @0, @"origin": touchable, @"x": @(offset.dx), @"y": @(offset.dy)},
+          @{@"type": @"pointerDown"},
+          @{@"type": @"pause", @"duration": @50},
+          @{@"type": @"pointerUp"},
+          ],
+      },
+    ];
+  NSError *error;
+  XCTAssertTrue([self.testedApplication fb_performW3CActions:gesture elementCache:nil error:&error]);
+
+  FBAssertWaitTillBecomesTrue(FBPointFuzzyEqualToPoint([self lastTouchLocationOf:touchable], expectedLocation, 5.0));
+}
+
 @end