From 1b2ab2bf4c5a849a70a1c959fbe8c6f50e3df3ec Mon Sep 17 00:00:00 2001 From: lvladika Date: Fri, 6 Nov 2026 11:03:09 +0100 Subject: [PATCH] feat(wda): an element check takes its fields from the find's answer getElementInfo sent four requests after each find (GET /element/{id}/text, /rect, /displayed and /name). WebDriverAgent answers them one at a time, and each resolves the element again, so on a real iPhone a check cost the find plus four round trips; on a long list a one-match check took about 2 s. The session now asks WebDriverAgent to put those four fields in every find's answer (shouldUseCompactResponses: false, elementResponseAttributes "name,text,rect,displayed", sent with snapshotMaxDepth). They are the same values the GETs return, read from the find's own snapshot (FBDictionaryResponseWithElement). The finds that feed getElementInfo take them from the answer and send no further request; when an answer lacks any of them (a WebDriverAgent that ignored the settings), the four GETs run as before. The id is read from ELEMENT or the W3C key and never from a field. Refs #258 --- pkg/driver/wda/client.go | 137 +++++++++++++++++++++----- pkg/driver/wda/client_test.go | 10 +- pkg/driver/wda/driver.go | 107 +++++++++++--------- pkg/driver/wda/element_fields_test.go | 129 ++++++++++++++++++++++++ 4 files changed, 313 insertions(+), 70 deletions(-) create mode 100644 pkg/driver/wda/element_fields_test.go diff --git a/pkg/driver/wda/client.go b/pkg/driver/wda/client.go index 78e931b9..14291b00 100644 --- a/pkg/driver/wda/client.go +++ b/pkg/driver/wda/client.go @@ -169,7 +169,16 @@ func (c *Client) CreateSession(bundleID string, alertAction string) error { // even though stock Maestro's XCTest traversal (no such cap) sees them // (#171). Applied here so every session — the primary one and the one // launchApp recreates — gets it. - _ = c.UpdateSettings(map[string]interface{}{"snapshotMaxDepth": wdaSnapshotMaxDepth()}) + // + // The other two make every find answer with the element's type, text, + // rect and visibility, which getElementInfo would otherwise read with + // four more requests (#258). WebDriverAgent ignores a setting it does not + // know, and keeps all of them process-wide, not per session. + _ = c.UpdateSettings(map[string]interface{}{ + "snapshotMaxDepth": wdaSnapshotMaxDepth(), + "shouldUseCompactResponses": false, + "elementResponseAttributes": elementResponseAttributes, + }) return nil } @@ -662,32 +671,124 @@ func (c *Client) DeepLink(url string) error { // Element finding +// elementResponseAttributes are the fields a find's answer carries for each +// element once CreateSession's settings are in (#258): the same values as +// GET /element/{id}/name, /text, /rect and /displayed, read from the find's +// own snapshot (FBDictionaryResponseWithElement in FBResponsePayload.m). +const elementResponseAttributes = "name,text,rect,displayed" + +// Element is an element as a find answers it. +type Element struct { + ID string + // Fields is nil when the answer did not carry all four fields: a + // WebDriverAgent without the settings, or another session that turned + // them off. getElementInfo then reads them. + Fields *ElementFields +} + +// ElementFields are an element's fields from a find's answer. +type ElementFields struct { + Name string // the element type, as GET /name + Text string // the value, else the label, as GET /text + X, Y, Width, Height int + Displayed bool +} + +// fieldKeys are the keys of a find's answer that are fields, not the id. +var fieldKeys = map[string]bool{"name": true, "text": true, "rect": true, "displayed": true, "error": true} + +// elementFromAnswer reads one element of a find's answer: its id, as +// JSONWP's ELEMENT or the W3C key, and the fields when all four are there. +func elementFromAnswer(m map[string]interface{}) (Element, bool) { + id, ok := m["ELEMENT"].(string) + if !ok { + // W3C format + for k, v := range m { + if str, isStr := v.(string); isStr && !fieldKeys[k] { + id, ok = str, true + break + } + } + } + if !ok { + return Element{}, false + } + return Element{ID: id, Fields: fieldsFromAnswer(m)}, true +} + +func fieldsFromAnswer(m map[string]interface{}) *ElementFields { + name, okName := m["name"].(string) + text, okText := m["text"] + rect, okRect := m["rect"].(map[string]interface{}) + displayed, okDisp := m["displayed"].(bool) + if !okName || !okText || !okRect || !okDisp { + return nil + } + x, y, w, h, okRect := rectFromAnswer(rect) + if !okRect { + return nil + } + f := &ElementFields{Name: name, X: x, Y: y, Width: w, Height: h, Displayed: displayed} + // null when the element has neither value nor label; GET /text says "". + f.Text, _ = text.(string) + return f +} + +// rectFromAnswer reads a rect as WebDriverAgent writes one; ok is false +// when one of its four numbers is missing. +func rectFromAnswer(r map[string]interface{}) (x, y, width, height int, ok bool) { + vals := [4]int{} + for i, k := range [4]string{"x", "y", "width", "height"} { + v, isNum := r[k].(float64) + if !isNum { + return 0, 0, 0, 0, false + } + vals[i] = int(v) + } + return vals[0], vals[1], vals[2], vals[3], true +} + // FindElement finds a single element. func (c *Client) FindElement(using, value string) (string, error) { + el, err := c.FindElementWithFields(using, value) + return el.ID, err +} + +// FindElementWithFields finds a single element, with its fields when the +// answer carries them. +func (c *Client) FindElementWithFields(using, value string) (Element, error) { resp, err := c.post(c.sessionPath("/element"), map[string]interface{}{ "using": using, "value": value, }) if err != nil { - return "", err + return Element{}, err } if val, ok := resp["value"].(map[string]interface{}); ok { - if elem, ok := val["ELEMENT"].(string); ok { - return elem, nil - } - // W3C format - for k, v := range val { - if str, ok := v.(string); ok && k != "error" { - return str, nil - } + if el, ok := elementFromAnswer(val); ok { + return el, nil } } - return "", fmt.Errorf("element not found") + return Element{}, fmt.Errorf("element not found") } // FindElements finds multiple elements. func (c *Client) FindElements(using, value string) ([]string, error) { + els, err := c.FindElementsWithFields(using, value) + if err != nil { + return nil, err + } + var ids []string + for _, el := range els { + ids = append(ids, el.ID) + } + return ids, nil +} + +// FindElementsWithFields finds multiple elements, with their fields when +// the answer carries them. +func (c *Client) FindElementsWithFields(using, value string) ([]Element, error) { resp, err := c.post(c.sessionPath("/elements"), map[string]interface{}{ "using": using, "value": value, @@ -696,20 +797,12 @@ func (c *Client) FindElements(using, value string) ([]string, error) { return nil, err } - var elements []string + var elements []Element if val, ok := resp["value"].([]interface{}); ok { for _, elem := range val { if m, ok := elem.(map[string]interface{}); ok { - if id, ok := m["ELEMENT"].(string); ok { - elements = append(elements, id) - } else { - // W3C format - for _, v := range m { - if str, ok := v.(string); ok { - elements = append(elements, str) - break - } - } + if el, ok := elementFromAnswer(m); ok { + elements = append(elements, el) } } } diff --git a/pkg/driver/wda/client_test.go b/pkg/driver/wda/client_test.go index b3d915ec..849af324 100644 --- a/pkg/driver/wda/client_test.go +++ b/pkg/driver/wda/client_test.go @@ -39,7 +39,7 @@ func TestNewClient(t *testing.T) { // TestCreateSession tests session creation func TestCreateSession(t *testing.T) { - var sawSnapshotDepth bool + var sawSnapshotDepth, sawElementFields bool server := mockWDAServer(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == "POST" && r.URL.Path == "/session": @@ -57,6 +57,11 @@ func TestCreateSession(t *testing.T) { if _, ok := body.Settings["snapshotMaxDepth"]; ok { sawSnapshotDepth = true } + // ...and asks finds for the fields getElementInfo reads (#258). + if body.Settings["shouldUseCompactResponses"] == false && + body.Settings["elementResponseAttributes"] == "name,text,rect,displayed" { + sawElementFields = true + } jsonResponse(w, map[string]interface{}{"value": map[string]interface{}{}}) default: t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) @@ -80,6 +85,9 @@ func TestCreateSession(t *testing.T) { if !sawSnapshotDepth { t.Error("CreateSession should raise snapshotMaxDepth so deep RN trees are not clipped") } + if !sawElementFields { + t.Error("CreateSession should ask WDA for the element fields in each find's answer") + } } // TestCreateSessionAlternateFormat tests alternate session response format diff --git a/pkg/driver/wda/driver.go b/pkg/driver/wda/driver.go index d880fd24..da293dcf 100644 --- a/pkg/driver/wda/driver.go +++ b/pkg/driver/wda/driver.go @@ -760,12 +760,12 @@ func (d *Driver) findElementForTapWithContext(ctx context.Context, sel flow.Sele // name, and Maestro matches ids only through id: (#178). exactPredicate := fmt.Sprintf("(label == '%s' OR value == '%s')%s", sel.Text, sel.Text, stateFilter) - exactElemID, _ := d.client.FindElement("predicate string", exactPredicate) + exact, _ := d.client.FindElementWithFields("predicate string", exactPredicate) // If exact predicate found element, try getElementInfo directly — avoids // a full page source fetch when the element is already identified. - if exactElemID != "" { - if info, err := d.getElementInfo(exactElemID); err == nil { + if exact.ID != "" { + if info, err := d.elementInfo(exact); err == nil { return info, nil } } @@ -810,9 +810,9 @@ func (d *Driver) findElementForTapWithContext(ctx context.Context, sel flow.Sele // findInteractiveElementByWDA tries WDA queries for interactive element types in parallel. func (d *Driver) findInteractiveElementByWDA(sel flow.Selector, stateFilter string) (*core.ElementInfo, error) { type queryResult struct { - elemID string - err error - prio int // lower = higher priority (TextField > SecureTextField > Button > fallback predicate) + elem Element + err error + prio int // lower = higher priority (TextField > SecureTextField > Button > fallback predicate) } textFieldChain := fmt.Sprintf("**/XCUIElementTypeTextField[`(label CONTAINS[c] '%s' OR value CONTAINS[c] '%s' OR placeholderValue CONTAINS[c] '%s')%s`]", sel.Text, sel.Text, sel.Text, stateFilter) @@ -843,27 +843,27 @@ func (d *Driver) findInteractiveElementByWDA(sel flow.Selector, stateFilter stri ch := make(chan queryResult, len(queries)) for _, q := range queries { go func(strategy, value string, prio int) { - elemID, err := d.client.FindElement(strategy, value) - if err != nil || elemID == "" { - ch <- queryResult{"", err, prio} + elem, err := d.client.FindElementWithFields(strategy, value) + if err != nil || elem.ID == "" { + ch <- queryResult{Element{}, err, prio} return } - ch <- queryResult{elemID, nil, prio} + ch <- queryResult{elem, nil, prio} }(q.strategy, q.value, q.prio) } - var bestID string + var best Element bestPrio := len(queries) // higher than any valid prio for i := 0; i < len(queries); i++ { r := <-ch - if r.err == nil && r.elemID != "" && r.prio < bestPrio { - bestID = r.elemID + if r.err == nil && r.elem.ID != "" && r.prio < bestPrio { + best = r.elem bestPrio = r.prio } } - if bestID != "" { - return d.getElementInfo(bestID) + if best.ID != "" { + return d.elementInfo(best) } return nil, fmt.Errorf("no interactive element found via WDA") @@ -1019,8 +1019,8 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) if looksLikeRegex(sel.ID) { // Regex id: match against name via MATCHES. query := fmt.Sprintf("**/XCUIElementTypeAny[`name MATCHES '%s'%s`]", sel.ID, stateFilter) - if elemID, err := d.client.FindElement("class chain", query); err == nil && elemID != "" { - return d.getElementInfo(elemID) + if elem, err := d.client.FindElementWithFields("class chain", query); err == nil && elem.ID != "" { + return d.elementInfo(elem) } } else { // Literal id: prefer an EXACT name match, then fall back to @@ -1046,8 +1046,8 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) predicateBase := fmt.Sprintf("label CONTAINS[c] '%s' OR value CONTAINS[c] '%s'", sel.Text, sel.Text) predicate := "(" + predicateBase + ")" + stateFilter - if elemID, err := d.client.FindElement("predicate string", predicate); err == nil && elemID != "" { - return d.getElementInfo(elemID) + if elem, err := d.client.FindElementWithFields("predicate string", predicate); err == nil && elem.ID != "" { + return d.elementInfo(elem) } } @@ -1067,6 +1067,14 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) // → accept, set MatchNote noting we overrode XCUITest's opinion based on bounds // - displayed=false AND bounds are off-screen → reject (genuinely not visible) func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { + return d.elementInfo(Element{ID: elemID}) +} + +// elementInfo is getElementInfo for an element a find answered. When the +// answer carried the element's fields (#258), it uses them and sends no +// request; otherwise it reads them. +func (d *Driver) elementInfo(elem Element) (*core.ElementInfo, error) { + elemID := elem.ID info := &core.ElementInfo{ ID: elemID, Enabled: true, // WDA doesn't have separate enabled check @@ -1081,24 +1089,29 @@ func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { wg sync.WaitGroup ) - wg.Add(4) - go func() { - defer wg.Done() - text, textErr = d.client.ElementText(elemID) - }() - go func() { - defer wg.Done() - x, y, w, h, rectErr = d.client.ElementRect(elemID) - }() - go func() { - defer wg.Done() - displayed, dispErr = d.client.ElementDisplayed(elemID) - }() - go func() { - defer wg.Done() - elemName, nameErr = d.client.ElementName(elemID) - }() - wg.Wait() + if f := elem.Fields; f != nil { + text, elemName, displayed = f.Text, f.Name, f.Displayed + x, y, w, h = f.X, f.Y, f.Width, f.Height + } else { + wg.Add(4) + go func() { + defer wg.Done() + text, textErr = d.client.ElementText(elemID) + }() + go func() { + defer wg.Done() + x, y, w, h, rectErr = d.client.ElementRect(elemID) + }() + go func() { + defer wg.Done() + displayed, dispErr = d.client.ElementDisplayed(elemID) + }() + go func() { + defer wg.Done() + elemName, nameErr = d.client.ElementName(elemID) + }() + wg.Wait() + } if rectErr != nil { return nil, fmt.Errorf("element bounds unavailable: %w", rectErr) @@ -1141,24 +1154,24 @@ func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { // element when several match (onScreenOf). found is false when nothing // matched. A WDA that cannot list elements gets the single-element query. func (d *Driver) findByIDQuery(query string) (*core.ElementInfo, error, bool) { //nolint:revive,staticcheck // found last reads naturally at call sites - ids, err := d.client.FindElements("class chain", query) + elems, err := d.client.FindElementsWithFields("class chain", query) if err != nil { - elemID, ferr := d.client.FindElement("class chain", query) - if ferr != nil || elemID == "" { + elem, ferr := d.client.FindElementWithFields("class chain", query) + if ferr != nil || elem.ID == "" { return nil, nil, false } - info, ierr := d.getElementInfo(elemID) + info, ierr := d.elementInfo(elem) return info, ierr, true } - if len(ids) == 0 { + if len(elems) == 0 { return nil, nil, false } - info, ierr := d.onScreenOf(ids) + info, ierr := d.onScreenOf(elems) return info, ierr, true } // maxIDCandidates caps how many same-id elements are inspected; each costs -// the element calls in getElementInfo. +// the element calls in getElementInfo when the find's answer has no fields. const maxIDCandidates = 5 // onScreenOf picks among elements sharing an id: the first XCUITest reports @@ -1171,15 +1184,15 @@ const maxIDCandidates = 5 // for the one inside it, as Maestro takes the deepest match: a SwiftUI // Toggle is a row-wide switch holding the real switch, both with the same // id, and the row's centre is its label, where a tap flips nothing. -func (d *Driver) onScreenOf(ids []string) (*core.ElementInfo, error) { +func (d *Driver) onScreenOf(elems []Element) (*core.ElementInfo, error) { var fallback *core.ElementInfo var visible []*core.ElementInfo var lastErr error - for i, id := range ids { + for i, elem := range elems { if i == maxIDCandidates { break } - info, err := d.getElementInfo(id) + info, err := d.elementInfo(elem) if err != nil { lastErr = err continue diff --git a/pkg/driver/wda/element_fields_test.go b/pkg/driver/wda/element_fields_test.go new file mode 100644 index 00000000..c8b3889f --- /dev/null +++ b/pkg/driver/wda/element_fields_test.go @@ -0,0 +1,129 @@ +package wda + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/flow" +) + +// elementReadsServer is a WDA whose finds answer with fields (withFields) or +// with the id alone, and which counts the element GETs getElementInfo sends. +func elementReadsServer(t *testing.T, withFields bool) (*httptest.Server, *int) { + t.Helper() + var mu sync.Mutex + reads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + p := r.URL.Path + el := map[string]interface{}{"ELEMENT": "btn-1"} + if withFields { + el["name"] = "XCUIElementTypeButton" + el["text"] = "Continue" + el["rect"] = map[string]interface{}{"x": 20.0, "y": 600.0, "width": 350.0, "height": 48.0} + el["displayed"] = true + } + switch { + case r.Method == "POST" && strings.HasSuffix(p, "/elements"): + jsonResponse(w, map[string]interface{}{"value": []interface{}{el}}) + case r.Method == "POST" && strings.HasSuffix(p, "/element"): + jsonResponse(w, map[string]interface{}{"value": el}) + case r.Method == "GET" && strings.Contains(p, "/element/btn-1/"): + mu.Lock() + reads++ + mu.Unlock() + switch { + case strings.HasSuffix(p, "/text"): + jsonResponse(w, map[string]interface{}{"value": "Continue"}) + case strings.HasSuffix(p, "/name"): + jsonResponse(w, map[string]interface{}{"value": "XCUIElementTypeButton"}) + case strings.HasSuffix(p, "/displayed"): + jsonResponse(w, map[string]interface{}{"value": true}) + case strings.HasSuffix(p, "/rect"): + jsonResponse(w, map[string]interface{}{"value": map[string]interface{}{"x": 20, "y": 600, "width": 350, "height": 48}}) + default: + w.WriteHeader(http.StatusNotFound) + } + default: + jsonResponse(w, map[string]interface{}{"status": 0}) + } + })) + t.Cleanup(server.Close) + return server, &reads +} + +// A find whose answer carries the element's fields needs no element GETs, +// and gives the same element as the GETs would (#258). +func TestFindAnswerFieldsSkipElementReads(t *testing.T) { + for _, tc := range []struct { + name string + sel flow.Selector + }{ + {"text", flow.Selector{Text: "Continue"}}, + {"id", flow.Selector{ID: "continue-button"}}, + } { + t.Run(tc.name, func(t *testing.T) { + server, reads := elementReadsServer(t, true) + d := createTestDriver(server) + + info, err := d.findElementByWDA(tc.sel) + if err != nil { + t.Fatalf("findElementByWDA: %v", err) + } + if *reads != 0 { + t.Errorf("sent %d element GETs; the find's answer had every field", *reads) + } + if info.Text != "Continue" || info.Class != "XCUIElementTypeButton" || !info.Visible { + t.Errorf("info = text %q, class %q, visible %v", info.Text, info.Class, info.Visible) + } + if info.Bounds.X != 20 || info.Bounds.Y != 600 || info.Bounds.Width != 350 || info.Bounds.Height != 48 { + t.Errorf("bounds = %+v", info.Bounds) + } + }) + } +} + +// Without the fields (a WebDriverAgent that ignored the settings) the driver +// reads them as before. +func TestFindAnswerWithoutFieldsReadsThem(t *testing.T) { + server, reads := elementReadsServer(t, false) + d := createTestDriver(server) + + info, err := d.findElementByWDA(flow.Selector{Text: "Continue"}) + if err != nil { + t.Fatalf("findElementByWDA: %v", err) + } + if *reads != 4 { + t.Errorf("sent %d element GETs, want the four reads", *reads) + } + if info.Text != "Continue" || info.Class != "XCUIElementTypeButton" || !info.Visible || info.Bounds.Width != 350 { + t.Errorf("info = %+v", info) + } +} + +func TestElementFromAnswer(t *testing.T) { + // W3C key only: the id is the element's, never its name or text. + el, ok := elementFromAnswer(map[string]interface{}{ + "element-6066-11e4-a52e-4f735466cecf": "w3c-id", + "name": "XCUIElementTypeStaticText", + "text": nil, + "rect": map[string]interface{}{"x": 0.0, "y": 10.5, "width": 100.0, "height": 20.0}, + "displayed": false, + }) + if !ok || el.ID != "w3c-id" { + t.Fatalf("id = %q, %v", el.ID, ok) + } + if el.Fields == nil || el.Fields.Text != "" || el.Fields.Y != 10 || el.Fields.Displayed { + t.Errorf("fields = %+v; a null text is \"\" as GET /text answers", el.Fields) + } + + // A field missing: no fields, so getElementInfo reads all four. + el, _ = elementFromAnswer(map[string]interface{}{"ELEMENT": "id", "name": "XCUIElementTypeButton", "text": "Go", + "rect": map[string]interface{}{"x": 1.0, "y": 2.0, "width": 3.0}, "displayed": true}) + if el.ID != "id" || el.Fields != nil { + t.Errorf("a rect without height gave fields %+v", el.Fields) + } +}