Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,60 @@ class NativeUIEngineTest {
send(JSONObject().put("action", "release").put("token", token))
assertFalse(send(request).getBoolean("shown"))
main { assertEquals(View.VISIBLE, native.visibility) }
main {
rendererRetired = false
selectionRejected = false
}
val webCapture =
send(
JSONObject()
.put("action", "prepare")
.put("ids", org.json.JSONArray().put("button"))
.put("revision", 7)
.put("viewportWidth", activity.web.width / 2.0)
.put("imageSurface", "web")
)
val webToken = webCapture.getString("token")
val root = activity.window.decorView as android.view.ViewGroup
var before = 0
main { before = root.childCount }
assertTrue(
send(
JSONObject()
.put("action", "cover")
.put("token", webToken)
.put("transient", true)
)
.getBoolean("shown")
)
main {
assertEquals(View.INVISIBLE, native.visibility)
assertEquals(
"Web surface must not install a fixed native entry image",
before,
root.childCount,
)
}
assertTrue(
send(
JSONObject()
.put("action", "restore")
.put("token", webToken)
.put("viewportWidth", activity.web.width / 2.0)
.put("parts", webCapture.getJSONArray("parts"))
)
.getBoolean("shown")
)
main {
assertEquals(
"Web surface must not install a fixed native return image",
before,
root.childCount,
)
snapshots.restoreViews()
assertEquals(View.VISIBLE, native.visibility)
}
send(JSONObject().put("action", "release").put("token", webToken))
} finally {
main {
snapshots.close()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable {
val referenceWidth: Int,
val referenceHeight: Int,
val parts: List<Part>,
var webSurface: Boolean = false,
)

private data class Copy(
Expand Down Expand Up @@ -133,7 +134,11 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable {
initial = null
measured.takeIf { it.keys == expected } ?: emptyMap()
},
complete = respond,
complete = { result ->
captures[result.optString("token")]?.webSurface =
request.optString("imageSurface") == "web"
respond(result)
},
)
}
"cover" -> {
Expand Down Expand Up @@ -541,7 +546,11 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable {
// Composite pixels cover both the live translucent view and its Web replacement.
// Install the entry copy before changing native visibility in the same UI turn.
expiredEntries.remove(token)
if (!restore(token, window, reference, frames = frames, entry = true, clips = clips))
if (captures[token]?.webSurface == true) {
val capture = valid(token, window, reference) ?: return false
if (frames != null && capture.parts.any { frames[it.id] == null }) return false
clearCopy()
} else if (!restore(token, window, reference, frames = frames, entry = true, clips = clips))
return false
if (!transient) coveringToken = token
hide(token, views)
Expand Down Expand Up @@ -603,6 +612,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable {
)
return false
clearCopy()
if (capture.webSurface) return true
val root = window.decorView as? ViewGroup ?: return false
val rootLocation = origin(root)
val sourceLocation = origin(reference)
Expand Down
10 changes: 10 additions & 0 deletions docs/advanced/native-handoff.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,13 @@ On iOS, `NativeUIScrollBindings.replace` groups sources by owner and binds only
An established binding rejected by native loses ownership and may be freshly acquired through the **same** update/ACK path. A failed fresh acquisition stays on the fallback until a relevant layout/scroll-end event. Never add an immediate retry loop or replay user actions. A frame/props change does not recalibrate an established coordinate origin.

This uses WKWebView's internal overflow hierarchy without private class names. Missing/ambiguous views, nested scrolling, document scrolling, unsupported transforms, shaped clipping, or a moving unconfirmed owner cannot be bound. Do not claim support solely because a `UIScrollView` exists. A native view may intercept gestures that begin inside it; routing those gestures and SDK accessibility behavior remain renderer-specific.

### Renderer-only capture on iOS

A `prepare` request may specify `captureMode: 'view'` to draw only the renderer views returned by the view selector, preserving transparent pixels outside their material. The default remains composited window capture. Renderer-only capture can change or omit backdrop-dependent blur and refraction; it does not guarantee transparency inside a glass surface. Missing, hidden or detached renderer views reject preparation. Android and older native binaries keep their existing capture route.

Moving DOM surfaces can provide a fixed viewport transfer layer through `WebProjectionOptions.imageSurface` (`HTMLElement`). The Web projection marks its prepare requests with `imageSurface: 'web'`; these tokens do not install fixed Native image copies. During motion, the decoded DOM images live on their source surface so its transform carries them. Translation does not invalidate their reusable capture; dimensions are compared at display-pixel resolution, while actual size and source changes still invalidate it.

Keep the Native renderer and source ownership alive while DOM images own the pixels. For entry, the Engine temporarily places those same images in the transfer layer at the captured Native coordinates, waits for Web painting, covers Native pixels, and returns the images to their source surface. For return, first lay out Native controls while keeping their pixels concealed. Read their actual frames and call `prepareNativeReturn(frames, revision)`. When the frames are usable, the Engine parks the same DOM images there; otherwise, pass `undefined` and it keeps their current placement. Both paths set `nativeReady` after Web painting. Missing alignment is not a failure of an accepted Native renderer. Reveal Native controls in the following update, then call `updated(token, ids, revision)` on its accepted acknowledgement to retire the DOM images. Only a revision newer than the staged layout can retire them; an acknowledgement for the layout stage must not. `prepareNativeReturn` ignores obsolete measurements. The displayed image remains leased until the reveal acknowledgement or explicit retirement, independently of whether its cached capture is still reusable. No snapshot is captured during either transfer.

On iOS, use `NativeUICoveredSnapshots.concealPixels(in:)` on the main thread when layout must remain measurable. It masks pixels and disables interaction/accessibility without changing `isHidden` or geometry. Its returned restoration closure is idempotent; overlapping leases restore the original mask and flags only after the last lease ends. The default composited handoff retains its existing Native copy and drawing-completion contract.
79 changes: 72 additions & 7 deletions ios/Sources/WebViewNativeUIEngine/CoveredSnapshots.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,21 @@ public final class NativeUICoveredSnapshots {
let size: CGSize
let webFrame: CGRect
let parts: [Part]
var webSurface = false
}
private final class PixelVisibility {
let originalMask: CALayer?
let interaction: Bool
let accessibility: Bool
let concealedMask = CALayer()
var owners = 0
init(_ view: UIView) {
originalMask = view.layer.mask
interaction = view.isUserInteractionEnabled
accessibility = view.accessibilityElementsHidden
}
}
private static let pixelVisibility = NSMapTable<UIView, PixelVisibility>(keyOptions: .weakMemory, valueOptions: .strongMemory)
public init() {}
private let imageContext = CIContext()
private var captures: [String: Capture] = [:]
Expand All @@ -42,12 +56,18 @@ public final class NativeUICoveredSnapshots {
let expected = Set(ids)
let firstFrames = frames(expected, revision)
guard Set(firstFrames.keys) == expected else { complete([:]); return }
let viewCapture = request["captureMode"] as? String == "view"
let selected = viewCapture ? views(expected) : nil
guard !viewCapture || selected?.isEmpty == false else { complete([:]); return }
var initial: [String: CGRect]? = firstFrames
prepare(frames: {
let measured = initial ?? frames(expected, revision)
initial = nil
return Set(measured.keys) == expected ? measured : [:]
}, in: webView, viewportWidth: width, complete: complete)
}, in: webView, viewportWidth: width, views: selected, complete: { result in
if let token = result["token"] as? String { self.captures[token]?.webSurface = request["imageSurface"] as? String == "web" }
complete(result)
})
case "cover":
let ids = identifiers(for: token)
let destination = request["parts"] == nil ? nil : projectionFrames(request, in: webView)
Expand Down Expand Up @@ -107,15 +127,15 @@ public final class NativeUICoveredSnapshots {
// Gesture release can keep changing SwiftUI button bounds after touchesEnded.
// Wait off the presentation path for unchanged rendered geometry, not a guessed delay.
public func prepare(frames: @escaping () -> [String: CGRect], in webView: UIView, viewportWidth: CGFloat,
complete: @escaping ([String: Any]) -> Void) {
views: [UIView]? = nil, complete: @escaping ([String: Any]) -> Void) {
let stamp = generation
let start = CACurrentMediaTime()
func sample(_ previous: [String: CGRect], _ stable: Int) {
guard generation == stamp, !touching, CACurrentMediaTime() - start < 1 else { complete([:]); return }
let current = frames()
let count = !current.isEmpty && current == previous ? stable + 1 : 0
if count >= 3 {
capture(frames: current, in: webView, viewportWidth: viewportWidth, complete: complete)
capture(frames: current, in: webView, viewportWidth: viewportWidth, views: views, complete: complete)
} else {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0 / 60.0) { sample(current, count) }
}
Expand All @@ -124,7 +144,7 @@ public final class NativeUICoveredSnapshots {
}

private func capture(frames: [String: CGRect], in webView: UIView, viewportWidth: CGFloat,
complete: @escaping ([String: Any]) -> Void) {
views: [UIView]?, complete: @escaping ([String: Any]) -> Void) {
guard !touching, let window = webView.window, !frames.isEmpty,
viewportWidth > 0, webView.bounds.width > 0 else { complete([:]); return }
let stamp = generation
Expand Down Expand Up @@ -156,9 +176,25 @@ public final class NativeUICoveredSnapshots {
context.translateBy(x: 0, y: bounds.height * scale)
context.scaleBy(x: scale, y: -scale)
context.translateBy(x: -bounds.minX, y: -bounds.minY)
// Capture the window itself: display-replica views can rasterize as black on devices.
// View capture preserves transparent pixels outside the renderer's own material.
// The default window capture retains composited backdrop effects.
var roots: [UIView]?
if let views {
guard !views.isEmpty, views.allSatisfy({ $0.window === window && !$0.isHidden && $0.alpha > 0 }) else { complete([:]); return }
var seen = Set<ObjectIdentifier>()
roots = views.filter { view in
seen.insert(ObjectIdentifier(view)).inserted && !views.contains { $0 !== view && view.isDescendant(of: $0) }
}
}
UIGraphicsPushContext(context)
let drawn = window.drawHierarchy(in: window.bounds, afterScreenUpdates: true)
var drawn = true
if let roots {
for root in roots {
drawn = root.drawHierarchy(in: root.convert(root.bounds, to: window), afterScreenUpdates: true) && drawn
}
} else {
drawn = window.drawHierarchy(in: window.bounds, afterScreenUpdates: true)
}
UIGraphicsPopContext()
guard drawn, let pixels = context.makeImage() else { complete([:]); return }
let token = UUID().uuidString
Expand Down Expand Up @@ -213,18 +249,46 @@ public final class NativeUICoveredSnapshots {
/// Install prepared pixels before hiding live views, within the same main-thread turn.
/// A transient (moving) surface does not replace background covering ownership.
@discardableResult public func cover(_ token: String, in webView: UIView, views: [UIView], transient: Bool = false, frames: [String: CGRect]? = nil) -> Bool {
guard restore(in: webView, token: token, frames: frames, entry: true) else { return false }
if captures[token]?.webSurface == true {
guard let capture = captures[token], let window = webView.window,
window.bounds.size == capture.size, webView.convert(webView.bounds, to: window) == capture.webFrame,
frames == nil || capture.parts.allSatisfy({ part in part.ids.allSatisfy { frames?[$0] != nil } }) else { return false }
clearRestoration()
} else if !restore(in: webView, token: token, frames: frames, entry: true) { return false }
if !transient { coveringToken = token }
hide(views, for: token)
return true
}

public var hidesNativeControls: Bool { hiddenControls != nil }

/// Conceal only presentation pixels; keep layout and hosted renderer lifetimes intact.
/// The owner restores this lease before applying its next presentation policy.
public static func concealPixels(in view: UIView) -> () -> Void {
let state = pixelVisibility.object(forKey: view) ?? PixelVisibility(view)
pixelVisibility.setObject(state, forKey: view)
state.owners += 1
view.layer.mask = state.concealedMask
view.isUserInteractionEnabled = false
view.accessibilityElementsHidden = true
var restored = false
return { [weak view] in
guard !restored else { return }
restored = true
state.owners -= 1
guard state.owners == 0, let view else { return }
view.layer.mask = state.originalMask
view.isUserInteractionEnabled = state.interaction
view.accessibilityElementsHidden = state.accessibility
pixelVisibility.removeObject(forKey: view)
}
}

private func hide(_ views: [UIView], for token: String) {
generation += 1
restoreControls()
hiddenControls = (token, views.map { view in
if captures[token]?.webSurface == true { return Self.concealPixels(in: view) }
let hidden = view.isHidden
view.isHidden = true
return { [weak view] in view?.isHidden = hidden }
Expand All @@ -246,6 +310,7 @@ public final class NativeUICoveredSnapshots {
window.bounds.size == capture.size, webView.convert(webView.bounds, to: window) == capture.webFrame else { return false }
if let frames, capture.parts.contains(where: { part in !part.ids.allSatisfy { frames[$0] != nil } }) { return false }
clearRestoration()
if capture.webSurface { return true }
let views = capture.parts.map { part in
let view = UIImageView(image: part.image)
view.frame = webView.convert(frames?[part.ids[0]] ?? part.frame, to: window)
Expand Down
Loading
Loading