diff --git a/pkg/driver/wda/client.go b/pkg/driver/wda/client.go index 78e931b9..14291b00 100644 --- a/pkg/driver/wda/client.go +++ b/pkg/driver/wda/client.go @@ -169,7 +169,16 @@ func (c *Client) CreateSession(bundleID string, alertAction string) error { // even though stock Maestro's XCTest traversal (no such cap) sees them // (#171). Applied here so every session — the primary one and the one // launchApp recreates — gets it. - _ = c.UpdateSettings(map[string]interface{}{"snapshotMaxDepth": wdaSnapshotMaxDepth()}) + // + // The other two make every find answer with the element's type, text, + // rect and visibility, which getElementInfo would otherwise read with + // four more requests (#258). WebDriverAgent ignores a setting it does not + // know, and keeps all of them process-wide, not per session. + _ = c.UpdateSettings(map[string]interface{}{ + "snapshotMaxDepth": wdaSnapshotMaxDepth(), + "shouldUseCompactResponses": false, + "elementResponseAttributes": elementResponseAttributes, + }) return nil } @@ -662,32 +671,124 @@ func (c *Client) DeepLink(url string) error { // Element finding +// elementResponseAttributes are the fields a find's answer carries for each +// element once CreateSession's settings are in (#258): the same values as +// GET /element/{id}/name, /text, /rect and /displayed, read from the find's +// own snapshot (FBDictionaryResponseWithElement in FBResponsePayload.m). +const elementResponseAttributes = "name,text,rect,displayed" + +// Element is an element as a find answers it. +type Element struct { + ID string + // Fields is nil when the answer did not carry all four fields: a + // WebDriverAgent without the settings, or another session that turned + // them off. getElementInfo then reads them. + Fields *ElementFields +} + +// ElementFields are an element's fields from a find's answer. +type ElementFields struct { + Name string // the element type, as GET /name + Text string // the value, else the label, as GET /text + X, Y, Width, Height int + Displayed bool +} + +// fieldKeys are the keys of a find's answer that are fields, not the id. +var fieldKeys = map[string]bool{"name": true, "text": true, "rect": true, "displayed": true, "error": true} + +// elementFromAnswer reads one element of a find's answer: its id, as +// JSONWP's ELEMENT or the W3C key, and the fields when all four are there. +func elementFromAnswer(m map[string]interface{}) (Element, bool) { + id, ok := m["ELEMENT"].(string) + if !ok { + // W3C format + for k, v := range m { + if str, isStr := v.(string); isStr && !fieldKeys[k] { + id, ok = str, true + break + } + } + } + if !ok { + return Element{}, false + } + return Element{ID: id, Fields: fieldsFromAnswer(m)}, true +} + +func fieldsFromAnswer(m map[string]interface{}) *ElementFields { + name, okName := m["name"].(string) + text, okText := m["text"] + rect, okRect := m["rect"].(map[string]interface{}) + displayed, okDisp := m["displayed"].(bool) + if !okName || !okText || !okRect || !okDisp { + return nil + } + x, y, w, h, okRect := rectFromAnswer(rect) + if !okRect { + return nil + } + f := &ElementFields{Name: name, X: x, Y: y, Width: w, Height: h, Displayed: displayed} + // null when the element has neither value nor label; GET /text says "". + f.Text, _ = text.(string) + return f +} + +// rectFromAnswer reads a rect as WebDriverAgent writes one; ok is false +// when one of its four numbers is missing. +func rectFromAnswer(r map[string]interface{}) (x, y, width, height int, ok bool) { + vals := [4]int{} + for i, k := range [4]string{"x", "y", "width", "height"} { + v, isNum := r[k].(float64) + if !isNum { + return 0, 0, 0, 0, false + } + vals[i] = int(v) + } + return vals[0], vals[1], vals[2], vals[3], true +} + // FindElement finds a single element. func (c *Client) FindElement(using, value string) (string, error) { + el, err := c.FindElementWithFields(using, value) + return el.ID, err +} + +// FindElementWithFields finds a single element, with its fields when the +// answer carries them. +func (c *Client) FindElementWithFields(using, value string) (Element, error) { resp, err := c.post(c.sessionPath("/element"), map[string]interface{}{ "using": using, "value": value, }) if err != nil { - return "", err + return Element{}, err } if val, ok := resp["value"].(map[string]interface{}); ok { - if elem, ok := val["ELEMENT"].(string); ok { - return elem, nil - } - // W3C format - for k, v := range val { - if str, ok := v.(string); ok && k != "error" { - return str, nil - } + if el, ok := elementFromAnswer(val); ok { + return el, nil } } - return "", fmt.Errorf("element not found") + return Element{}, fmt.Errorf("element not found") } // FindElements finds multiple elements. func (c *Client) FindElements(using, value string) ([]string, error) { + els, err := c.FindElementsWithFields(using, value) + if err != nil { + return nil, err + } + var ids []string + for _, el := range els { + ids = append(ids, el.ID) + } + return ids, nil +} + +// FindElementsWithFields finds multiple elements, with their fields when +// the answer carries them. +func (c *Client) FindElementsWithFields(using, value string) ([]Element, error) { resp, err := c.post(c.sessionPath("/elements"), map[string]interface{}{ "using": using, "value": value, @@ -696,20 +797,12 @@ func (c *Client) FindElements(using, value string) ([]string, error) { return nil, err } - var elements []string + var elements []Element if val, ok := resp["value"].([]interface{}); ok { for _, elem := range val { if m, ok := elem.(map[string]interface{}); ok { - if id, ok := m["ELEMENT"].(string); ok { - elements = append(elements, id) - } else { - // W3C format - for _, v := range m { - if str, ok := v.(string); ok { - elements = append(elements, str) - break - } - } + if el, ok := elementFromAnswer(m); ok { + elements = append(elements, el) } } } diff --git a/pkg/driver/wda/client_test.go b/pkg/driver/wda/client_test.go index b3d915ec..849af324 100644 --- a/pkg/driver/wda/client_test.go +++ b/pkg/driver/wda/client_test.go @@ -39,7 +39,7 @@ func TestNewClient(t *testing.T) { // TestCreateSession tests session creation func TestCreateSession(t *testing.T) { - var sawSnapshotDepth bool + var sawSnapshotDepth, sawElementFields bool server := mockWDAServer(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == "POST" && r.URL.Path == "/session": @@ -57,6 +57,11 @@ func TestCreateSession(t *testing.T) { if _, ok := body.Settings["snapshotMaxDepth"]; ok { sawSnapshotDepth = true } + // ...and asks finds for the fields getElementInfo reads (#258). + if body.Settings["shouldUseCompactResponses"] == false && + body.Settings["elementResponseAttributes"] == "name,text,rect,displayed" { + sawElementFields = true + } jsonResponse(w, map[string]interface{}{"value": map[string]interface{}{}}) default: t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) @@ -80,6 +85,9 @@ func TestCreateSession(t *testing.T) { if !sawSnapshotDepth { t.Error("CreateSession should raise snapshotMaxDepth so deep RN trees are not clipped") } + if !sawElementFields { + t.Error("CreateSession should ask WDA for the element fields in each find's answer") + } } // TestCreateSessionAlternateFormat tests alternate session response format diff --git a/pkg/driver/wda/driver.go b/pkg/driver/wda/driver.go index d880fd24..da293dcf 100644 --- a/pkg/driver/wda/driver.go +++ b/pkg/driver/wda/driver.go @@ -760,12 +760,12 @@ func (d *Driver) findElementForTapWithContext(ctx context.Context, sel flow.Sele // name, and Maestro matches ids only through id: (#178). exactPredicate := fmt.Sprintf("(label == '%s' OR value == '%s')%s", sel.Text, sel.Text, stateFilter) - exactElemID, _ := d.client.FindElement("predicate string", exactPredicate) + exact, _ := d.client.FindElementWithFields("predicate string", exactPredicate) // If exact predicate found element, try getElementInfo directly — avoids // a full page source fetch when the element is already identified. - if exactElemID != "" { - if info, err := d.getElementInfo(exactElemID); err == nil { + if exact.ID != "" { + if info, err := d.elementInfo(exact); err == nil { return info, nil } } @@ -810,9 +810,9 @@ func (d *Driver) findElementForTapWithContext(ctx context.Context, sel flow.Sele // findInteractiveElementByWDA tries WDA queries for interactive element types in parallel. func (d *Driver) findInteractiveElementByWDA(sel flow.Selector, stateFilter string) (*core.ElementInfo, error) { type queryResult struct { - elemID string - err error - prio int // lower = higher priority (TextField > SecureTextField > Button > fallback predicate) + elem Element + err error + prio int // lower = higher priority (TextField > SecureTextField > Button > fallback predicate) } textFieldChain := fmt.Sprintf("**/XCUIElementTypeTextField[`(label CONTAINS[c] '%s' OR value CONTAINS[c] '%s' OR placeholderValue CONTAINS[c] '%s')%s`]", sel.Text, sel.Text, sel.Text, stateFilter) @@ -843,27 +843,27 @@ func (d *Driver) findInteractiveElementByWDA(sel flow.Selector, stateFilter stri ch := make(chan queryResult, len(queries)) for _, q := range queries { go func(strategy, value string, prio int) { - elemID, err := d.client.FindElement(strategy, value) - if err != nil || elemID == "" { - ch <- queryResult{"", err, prio} + elem, err := d.client.FindElementWithFields(strategy, value) + if err != nil || elem.ID == "" { + ch <- queryResult{Element{}, err, prio} return } - ch <- queryResult{elemID, nil, prio} + ch <- queryResult{elem, nil, prio} }(q.strategy, q.value, q.prio) } - var bestID string + var best Element bestPrio := len(queries) // higher than any valid prio for i := 0; i < len(queries); i++ { r := <-ch - if r.err == nil && r.elemID != "" && r.prio < bestPrio { - bestID = r.elemID + if r.err == nil && r.elem.ID != "" && r.prio < bestPrio { + best = r.elem bestPrio = r.prio } } - if bestID != "" { - return d.getElementInfo(bestID) + if best.ID != "" { + return d.elementInfo(best) } return nil, fmt.Errorf("no interactive element found via WDA") @@ -1019,8 +1019,8 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) if looksLikeRegex(sel.ID) { // Regex id: match against name via MATCHES. query := fmt.Sprintf("**/XCUIElementTypeAny[`name MATCHES '%s'%s`]", sel.ID, stateFilter) - if elemID, err := d.client.FindElement("class chain", query); err == nil && elemID != "" { - return d.getElementInfo(elemID) + if elem, err := d.client.FindElementWithFields("class chain", query); err == nil && elem.ID != "" { + return d.elementInfo(elem) } } else { // Literal id: prefer an EXACT name match, then fall back to @@ -1046,8 +1046,8 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) predicateBase := fmt.Sprintf("label CONTAINS[c] '%s' OR value CONTAINS[c] '%s'", sel.Text, sel.Text) predicate := "(" + predicateBase + ")" + stateFilter - if elemID, err := d.client.FindElement("predicate string", predicate); err == nil && elemID != "" { - return d.getElementInfo(elemID) + if elem, err := d.client.FindElementWithFields("predicate string", predicate); err == nil && elem.ID != "" { + return d.elementInfo(elem) } } @@ -1067,6 +1067,14 @@ func (d *Driver) findElementByWDA(sel flow.Selector) (*core.ElementInfo, error) // → accept, set MatchNote noting we overrode XCUITest's opinion based on bounds // - displayed=false AND bounds are off-screen → reject (genuinely not visible) func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { + return d.elementInfo(Element{ID: elemID}) +} + +// elementInfo is getElementInfo for an element a find answered. When the +// answer carried the element's fields (#258), it uses them and sends no +// request; otherwise it reads them. +func (d *Driver) elementInfo(elem Element) (*core.ElementInfo, error) { + elemID := elem.ID info := &core.ElementInfo{ ID: elemID, Enabled: true, // WDA doesn't have separate enabled check @@ -1081,24 +1089,29 @@ func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { wg sync.WaitGroup ) - wg.Add(4) - go func() { - defer wg.Done() - text, textErr = d.client.ElementText(elemID) - }() - go func() { - defer wg.Done() - x, y, w, h, rectErr = d.client.ElementRect(elemID) - }() - go func() { - defer wg.Done() - displayed, dispErr = d.client.ElementDisplayed(elemID) - }() - go func() { - defer wg.Done() - elemName, nameErr = d.client.ElementName(elemID) - }() - wg.Wait() + if f := elem.Fields; f != nil { + text, elemName, displayed = f.Text, f.Name, f.Displayed + x, y, w, h = f.X, f.Y, f.Width, f.Height + } else { + wg.Add(4) + go func() { + defer wg.Done() + text, textErr = d.client.ElementText(elemID) + }() + go func() { + defer wg.Done() + x, y, w, h, rectErr = d.client.ElementRect(elemID) + }() + go func() { + defer wg.Done() + displayed, dispErr = d.client.ElementDisplayed(elemID) + }() + go func() { + defer wg.Done() + elemName, nameErr = d.client.ElementName(elemID) + }() + wg.Wait() + } if rectErr != nil { return nil, fmt.Errorf("element bounds unavailable: %w", rectErr) @@ -1141,24 +1154,24 @@ func (d *Driver) getElementInfo(elemID string) (*core.ElementInfo, error) { // element when several match (onScreenOf). found is false when nothing // matched. A WDA that cannot list elements gets the single-element query. func (d *Driver) findByIDQuery(query string) (*core.ElementInfo, error, bool) { //nolint:revive,staticcheck // found last reads naturally at call sites - ids, err := d.client.FindElements("class chain", query) + elems, err := d.client.FindElementsWithFields("class chain", query) if err != nil { - elemID, ferr := d.client.FindElement("class chain", query) - if ferr != nil || elemID == "" { + elem, ferr := d.client.FindElementWithFields("class chain", query) + if ferr != nil || elem.ID == "" { return nil, nil, false } - info, ierr := d.getElementInfo(elemID) + info, ierr := d.elementInfo(elem) return info, ierr, true } - if len(ids) == 0 { + if len(elems) == 0 { return nil, nil, false } - info, ierr := d.onScreenOf(ids) + info, ierr := d.onScreenOf(elems) return info, ierr, true } // maxIDCandidates caps how many same-id elements are inspected; each costs -// the element calls in getElementInfo. +// the element calls in getElementInfo when the find's answer has no fields. const maxIDCandidates = 5 // onScreenOf picks among elements sharing an id: the first XCUITest reports @@ -1171,15 +1184,15 @@ const maxIDCandidates = 5 // for the one inside it, as Maestro takes the deepest match: a SwiftUI // Toggle is a row-wide switch holding the real switch, both with the same // id, and the row's centre is its label, where a tap flips nothing. -func (d *Driver) onScreenOf(ids []string) (*core.ElementInfo, error) { +func (d *Driver) onScreenOf(elems []Element) (*core.ElementInfo, error) { var fallback *core.ElementInfo var visible []*core.ElementInfo var lastErr error - for i, id := range ids { + for i, elem := range elems { if i == maxIDCandidates { break } - info, err := d.getElementInfo(id) + info, err := d.elementInfo(elem) if err != nil { lastErr = err continue diff --git a/pkg/driver/wda/element_fields_test.go b/pkg/driver/wda/element_fields_test.go new file mode 100644 index 00000000..c8b3889f --- /dev/null +++ b/pkg/driver/wda/element_fields_test.go @@ -0,0 +1,129 @@ +package wda + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/devicelab-dev/maestro-runner/pkg/flow" +) + +// elementReadsServer is a WDA whose finds answer with fields (withFields) or +// with the id alone, and which counts the element GETs getElementInfo sends. +func elementReadsServer(t *testing.T, withFields bool) (*httptest.Server, *int) { + t.Helper() + var mu sync.Mutex + reads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + p := r.URL.Path + el := map[string]interface{}{"ELEMENT": "btn-1"} + if withFields { + el["name"] = "XCUIElementTypeButton" + el["text"] = "Continue" + el["rect"] = map[string]interface{}{"x": 20.0, "y": 600.0, "width": 350.0, "height": 48.0} + el["displayed"] = true + } + switch { + case r.Method == "POST" && strings.HasSuffix(p, "/elements"): + jsonResponse(w, map[string]interface{}{"value": []interface{}{el}}) + case r.Method == "POST" && strings.HasSuffix(p, "/element"): + jsonResponse(w, map[string]interface{}{"value": el}) + case r.Method == "GET" && strings.Contains(p, "/element/btn-1/"): + mu.Lock() + reads++ + mu.Unlock() + switch { + case strings.HasSuffix(p, "/text"): + jsonResponse(w, map[string]interface{}{"value": "Continue"}) + case strings.HasSuffix(p, "/name"): + jsonResponse(w, map[string]interface{}{"value": "XCUIElementTypeButton"}) + case strings.HasSuffix(p, "/displayed"): + jsonResponse(w, map[string]interface{}{"value": true}) + case strings.HasSuffix(p, "/rect"): + jsonResponse(w, map[string]interface{}{"value": map[string]interface{}{"x": 20, "y": 600, "width": 350, "height": 48}}) + default: + w.WriteHeader(http.StatusNotFound) + } + default: + jsonResponse(w, map[string]interface{}{"status": 0}) + } + })) + t.Cleanup(server.Close) + return server, &reads +} + +// A find whose answer carries the element's fields needs no element GETs, +// and gives the same element as the GETs would (#258). +func TestFindAnswerFieldsSkipElementReads(t *testing.T) { + for _, tc := range []struct { + name string + sel flow.Selector + }{ + {"text", flow.Selector{Text: "Continue"}}, + {"id", flow.Selector{ID: "continue-button"}}, + } { + t.Run(tc.name, func(t *testing.T) { + server, reads := elementReadsServer(t, true) + d := createTestDriver(server) + + info, err := d.findElementByWDA(tc.sel) + if err != nil { + t.Fatalf("findElementByWDA: %v", err) + } + if *reads != 0 { + t.Errorf("sent %d element GETs; the find's answer had every field", *reads) + } + if info.Text != "Continue" || info.Class != "XCUIElementTypeButton" || !info.Visible { + t.Errorf("info = text %q, class %q, visible %v", info.Text, info.Class, info.Visible) + } + if info.Bounds.X != 20 || info.Bounds.Y != 600 || info.Bounds.Width != 350 || info.Bounds.Height != 48 { + t.Errorf("bounds = %+v", info.Bounds) + } + }) + } +} + +// Without the fields (a WebDriverAgent that ignored the settings) the driver +// reads them as before. +func TestFindAnswerWithoutFieldsReadsThem(t *testing.T) { + server, reads := elementReadsServer(t, false) + d := createTestDriver(server) + + info, err := d.findElementByWDA(flow.Selector{Text: "Continue"}) + if err != nil { + t.Fatalf("findElementByWDA: %v", err) + } + if *reads != 4 { + t.Errorf("sent %d element GETs, want the four reads", *reads) + } + if info.Text != "Continue" || info.Class != "XCUIElementTypeButton" || !info.Visible || info.Bounds.Width != 350 { + t.Errorf("info = %+v", info) + } +} + +func TestElementFromAnswer(t *testing.T) { + // W3C key only: the id is the element's, never its name or text. + el, ok := elementFromAnswer(map[string]interface{}{ + "element-6066-11e4-a52e-4f735466cecf": "w3c-id", + "name": "XCUIElementTypeStaticText", + "text": nil, + "rect": map[string]interface{}{"x": 0.0, "y": 10.5, "width": 100.0, "height": 20.0}, + "displayed": false, + }) + if !ok || el.ID != "w3c-id" { + t.Fatalf("id = %q, %v", el.ID, ok) + } + if el.Fields == nil || el.Fields.Text != "" || el.Fields.Y != 10 || el.Fields.Displayed { + t.Errorf("fields = %+v; a null text is \"\" as GET /text answers", el.Fields) + } + + // A field missing: no fields, so getElementInfo reads all four. + el, _ = elementFromAnswer(map[string]interface{}{"ELEMENT": "id", "name": "XCUIElementTypeButton", "text": "Go", + "rect": map[string]interface{}{"x": 1.0, "y": 2.0, "width": 3.0}, "displayed": true}) + if el.ID != "id" || el.Fields != nil { + t.Errorf("a rect without height gave fields %+v", el.Fields) + } +}