From 91db098df5077f5141f82c206bfe10b16c178bf9 Mon Sep 17 00:00:00 2001 From: rdlabo Date: Sun, 8 Nov 2026 03:53:27 +0900 Subject: [PATCH 1/3] fix: retain moving-surface snapshots through native handoffs --- .../engine/NativeUIEngineTest.kt | 54 +++++ .../engine/NativeUISnapshots.kt | 14 +- docs/advanced/native-handoff.md | 10 + .../CoveredSnapshots.swift | 79 ++++++- .../CoveredSnapshotsTests.swift | 104 +++++++++ src/projection/web.ts | 197 +++++++++++++----- src/types.ts | 10 +- tests/projection/web.spec.ts | 90 +++++++- 8 files changed, 497 insertions(+), 61 deletions(-) diff --git a/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIEngineTest.kt b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIEngineTest.kt index 84ee638..b8a4114 100644 --- a/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIEngineTest.kt +++ b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIEngineTest.kt @@ -205,6 +205,60 @@ class NativeUIEngineTest { send(JSONObject().put("action", "release").put("token", token)) assertFalse(send(request).getBoolean("shown")) main { assertEquals(View.VISIBLE, native.visibility) } + main { + rendererRetired = false + selectionRejected = false + } + val webCapture = + send( + JSONObject() + .put("action", "prepare") + .put("ids", org.json.JSONArray().put("button")) + .put("revision", 7) + .put("viewportWidth", activity.web.width / 2.0) + .put("imageSurface", "web") + ) + val webToken = webCapture.getString("token") + val root = activity.window.decorView as android.view.ViewGroup + var before = 0 + main { before = root.childCount } + assertTrue( + send( + JSONObject() + .put("action", "cover") + .put("token", webToken) + .put("transient", true) + ) + .getBoolean("shown") + ) + main { + assertEquals(View.INVISIBLE, native.visibility) + assertEquals( + "Web surface must not install a fixed native entry image", + before, + root.childCount, + ) + } + assertTrue( + send( + JSONObject() + .put("action", "restore") + .put("token", webToken) + .put("viewportWidth", activity.web.width / 2.0) + .put("parts", webCapture.getJSONArray("parts")) + ) + .getBoolean("shown") + ) + main { + assertEquals( + "Web surface must not install a fixed native return image", + before, + root.childCount, + ) + snapshots.restoreViews() + assertEquals(View.VISIBLE, native.visibility) + } + send(JSONObject().put("action", "release").put("token", webToken)) } finally { main { snapshots.close() diff --git a/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt b/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt index a3128bb..0bf7f44 100644 --- a/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt +++ b/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt @@ -36,6 +36,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { val referenceWidth: Int, val referenceHeight: Int, val parts: List, + var webSurface: Boolean = false, ) private data class Copy( @@ -133,7 +134,11 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { initial = null measured.takeIf { it.keys == expected } ?: emptyMap() }, - complete = respond, + complete = { result -> + captures[result.optString("token")]?.webSurface = + request.optString("imageSurface") == "web" + respond(result) + }, ) } "cover" -> { @@ -541,7 +546,11 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { // Composite pixels cover both the live translucent view and its Web replacement. // Install the entry copy before changing native visibility in the same UI turn. expiredEntries.remove(token) - if (!restore(token, window, reference, frames = frames, entry = true, clips = clips)) + if (captures[token]?.webSurface == true) { + val capture = valid(token, window, reference) ?: return false + if (frames != null && capture.parts.any { frames[it.id] == null }) return false + clearCopy() + } else if (!restore(token, window, reference, frames = frames, entry = true, clips = clips)) return false if (!transient) coveringToken = token hide(token, views) @@ -603,6 +612,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { ) return false clearCopy() + if (capture.webSurface) return true val root = window.decorView as? ViewGroup ?: return false val rootLocation = origin(root) val sourceLocation = origin(reference) diff --git a/docs/advanced/native-handoff.md b/docs/advanced/native-handoff.md index c97e98e..e6fda3f 100644 --- a/docs/advanced/native-handoff.md +++ b/docs/advanced/native-handoff.md @@ -87,3 +87,13 @@ On iOS, `NativeUIScrollBindings.replace` groups sources by owner and binds only An established binding rejected by native loses ownership and may be freshly acquired through the **same** update/ACK path. A failed fresh acquisition stays on the fallback until a relevant layout/scroll-end event. Never add an immediate retry loop or replay user actions. A frame/props change does not recalibrate an established coordinate origin. This uses WKWebView's internal overflow hierarchy without private class names. Missing/ambiguous views, nested scrolling, document scrolling, unsupported transforms, shaped clipping, or a moving unconfirmed owner cannot be bound. Do not claim support solely because a `UIScrollView` exists. A native view may intercept gestures that begin inside it; routing those gestures and SDK accessibility behavior remain renderer-specific. + +### Renderer-only capture on iOS + +A `prepare` request may specify `captureMode: 'view'` to draw only the renderer views returned by the view selector, preserving transparent pixels outside their material. The default remains composited window capture. Renderer-only capture can change or omit backdrop-dependent blur and refraction; it does not guarantee transparency inside a glass surface. Missing, hidden or detached renderer views reject preparation. Android and older native binaries keep their existing capture route. + +Moving DOM surfaces can provide a fixed viewport transfer layer through `WebProjectionOptions.imageSurface` (`HTMLElement`). The Web projection marks its prepare requests with `imageSurface: 'web'`; these tokens do not install fixed Native image copies. During motion, the decoded DOM images live on their source surface so its transform carries them. + +Keep the Native renderer and source ownership alive while DOM images own the pixels. For entry, the Engine temporarily places those same images in the transfer layer at the captured Native coordinates, waits for Web painting, covers Native pixels, and returns the images to their source surface. For return, first lay out Native controls while keeping their pixels concealed. Read their actual frames and call `prepareNativeReturn(frames, revision)`. The Engine parks the same DOM images at those frames and sets `nativeReady` after Web painting. Reveal Native controls in the following update, then call `updated(token, ids, revision)` on its accepted acknowledgement to retire the DOM images. Only a revision newer than the staged layout can retire them; an acknowledgement for the layout stage must not. `prepareNativeReturn` ignores obsolete measurements and returns `false` only when the current return geometry is unusable. No snapshot is captured during either transfer. + +On iOS, use `NativeUICoveredSnapshots.concealPixels(in:)` on the main thread when layout must remain measurable. It masks pixels and disables interaction/accessibility without changing `isHidden` or geometry. Its returned restoration closure is idempotent; overlapping leases restore the original mask and flags only after the last lease ends. The default composited handoff retains its existing Native copy and drawing-completion contract. diff --git a/ios/Sources/WebViewNativeUIEngine/CoveredSnapshots.swift b/ios/Sources/WebViewNativeUIEngine/CoveredSnapshots.swift index 313855f..ee62003 100644 --- a/ios/Sources/WebViewNativeUIEngine/CoveredSnapshots.swift +++ b/ios/Sources/WebViewNativeUIEngine/CoveredSnapshots.swift @@ -15,7 +15,21 @@ public final class NativeUICoveredSnapshots { let size: CGSize let webFrame: CGRect let parts: [Part] + var webSurface = false } + private final class PixelVisibility { + let originalMask: CALayer? + let interaction: Bool + let accessibility: Bool + let concealedMask = CALayer() + var owners = 0 + init(_ view: UIView) { + originalMask = view.layer.mask + interaction = view.isUserInteractionEnabled + accessibility = view.accessibilityElementsHidden + } + } + private static let pixelVisibility = NSMapTable(keyOptions: .weakMemory, valueOptions: .strongMemory) public init() {} private let imageContext = CIContext() private var captures: [String: Capture] = [:] @@ -42,12 +56,18 @@ public final class NativeUICoveredSnapshots { let expected = Set(ids) let firstFrames = frames(expected, revision) guard Set(firstFrames.keys) == expected else { complete([:]); return } + let viewCapture = request["captureMode"] as? String == "view" + let selected = viewCapture ? views(expected) : nil + guard !viewCapture || selected?.isEmpty == false else { complete([:]); return } var initial: [String: CGRect]? = firstFrames prepare(frames: { let measured = initial ?? frames(expected, revision) initial = nil return Set(measured.keys) == expected ? measured : [:] - }, in: webView, viewportWidth: width, complete: complete) + }, in: webView, viewportWidth: width, views: selected, complete: { result in + if let token = result["token"] as? String { self.captures[token]?.webSurface = request["imageSurface"] as? String == "web" } + complete(result) + }) case "cover": let ids = identifiers(for: token) let destination = request["parts"] == nil ? nil : projectionFrames(request, in: webView) @@ -107,7 +127,7 @@ public final class NativeUICoveredSnapshots { // Gesture release can keep changing SwiftUI button bounds after touchesEnded. // Wait off the presentation path for unchanged rendered geometry, not a guessed delay. public func prepare(frames: @escaping () -> [String: CGRect], in webView: UIView, viewportWidth: CGFloat, - complete: @escaping ([String: Any]) -> Void) { + views: [UIView]? = nil, complete: @escaping ([String: Any]) -> Void) { let stamp = generation let start = CACurrentMediaTime() func sample(_ previous: [String: CGRect], _ stable: Int) { @@ -115,7 +135,7 @@ public final class NativeUICoveredSnapshots { let current = frames() let count = !current.isEmpty && current == previous ? stable + 1 : 0 if count >= 3 { - capture(frames: current, in: webView, viewportWidth: viewportWidth, complete: complete) + capture(frames: current, in: webView, viewportWidth: viewportWidth, views: views, complete: complete) } else { DispatchQueue.main.asyncAfter(deadline: .now() + 1.0 / 60.0) { sample(current, count) } } @@ -124,7 +144,7 @@ public final class NativeUICoveredSnapshots { } private func capture(frames: [String: CGRect], in webView: UIView, viewportWidth: CGFloat, - complete: @escaping ([String: Any]) -> Void) { + views: [UIView]?, complete: @escaping ([String: Any]) -> Void) { guard !touching, let window = webView.window, !frames.isEmpty, viewportWidth > 0, webView.bounds.width > 0 else { complete([:]); return } let stamp = generation @@ -156,9 +176,25 @@ public final class NativeUICoveredSnapshots { context.translateBy(x: 0, y: bounds.height * scale) context.scaleBy(x: scale, y: -scale) context.translateBy(x: -bounds.minX, y: -bounds.minY) - // Capture the window itself: display-replica views can rasterize as black on devices. + // View capture preserves transparent pixels outside the renderer's own material. + // The default window capture retains composited backdrop effects. + var roots: [UIView]? + if let views { + guard !views.isEmpty, views.allSatisfy({ $0.window === window && !$0.isHidden && $0.alpha > 0 }) else { complete([:]); return } + var seen = Set() + roots = views.filter { view in + seen.insert(ObjectIdentifier(view)).inserted && !views.contains { $0 !== view && view.isDescendant(of: $0) } + } + } UIGraphicsPushContext(context) - let drawn = window.drawHierarchy(in: window.bounds, afterScreenUpdates: true) + var drawn = true + if let roots { + for root in roots { + drawn = root.drawHierarchy(in: root.convert(root.bounds, to: window), afterScreenUpdates: true) && drawn + } + } else { + drawn = window.drawHierarchy(in: window.bounds, afterScreenUpdates: true) + } UIGraphicsPopContext() guard drawn, let pixels = context.makeImage() else { complete([:]); return } let token = UUID().uuidString @@ -213,7 +249,12 @@ public final class NativeUICoveredSnapshots { /// Install prepared pixels before hiding live views, within the same main-thread turn. /// A transient (moving) surface does not replace background covering ownership. @discardableResult public func cover(_ token: String, in webView: UIView, views: [UIView], transient: Bool = false, frames: [String: CGRect]? = nil) -> Bool { - guard restore(in: webView, token: token, frames: frames, entry: true) else { return false } + if captures[token]?.webSurface == true { + guard let capture = captures[token], let window = webView.window, + window.bounds.size == capture.size, webView.convert(webView.bounds, to: window) == capture.webFrame, + frames == nil || capture.parts.allSatisfy({ part in part.ids.allSatisfy { frames?[$0] != nil } }) else { return false } + clearRestoration() + } else if !restore(in: webView, token: token, frames: frames, entry: true) { return false } if !transient { coveringToken = token } hide(views, for: token) return true @@ -221,10 +262,33 @@ public final class NativeUICoveredSnapshots { public var hidesNativeControls: Bool { hiddenControls != nil } + /// Conceal only presentation pixels; keep layout and hosted renderer lifetimes intact. + /// The owner restores this lease before applying its next presentation policy. + public static func concealPixels(in view: UIView) -> () -> Void { + let state = pixelVisibility.object(forKey: view) ?? PixelVisibility(view) + pixelVisibility.setObject(state, forKey: view) + state.owners += 1 + view.layer.mask = state.concealedMask + view.isUserInteractionEnabled = false + view.accessibilityElementsHidden = true + var restored = false + return { [weak view] in + guard !restored else { return } + restored = true + state.owners -= 1 + guard state.owners == 0, let view else { return } + view.layer.mask = state.originalMask + view.isUserInteractionEnabled = state.interaction + view.accessibilityElementsHidden = state.accessibility + pixelVisibility.removeObject(forKey: view) + } + } + private func hide(_ views: [UIView], for token: String) { generation += 1 restoreControls() hiddenControls = (token, views.map { view in + if captures[token]?.webSurface == true { return Self.concealPixels(in: view) } let hidden = view.isHidden view.isHidden = true return { [weak view] in view?.isHidden = hidden } @@ -246,6 +310,7 @@ public final class NativeUICoveredSnapshots { window.bounds.size == capture.size, webView.convert(webView.bounds, to: window) == capture.webFrame else { return false } if let frames, capture.parts.contains(where: { part in !part.ids.allSatisfy { frames[$0] != nil } }) { return false } clearRestoration() + if capture.webSurface { return true } let views = capture.parts.map { part in let view = UIImageView(image: part.image) view.frame = webView.convert(frames?[part.ids[0]] ?? part.frame, to: window) diff --git a/ios/Tests/WebViewNativeUIEngineTests/CoveredSnapshotsTests.swift b/ios/Tests/WebViewNativeUIEngineTests/CoveredSnapshotsTests.swift index ce58609..f3389ad 100644 --- a/ios/Tests/WebViewNativeUIEngineTests/CoveredSnapshotsTests.swift +++ b/ios/Tests/WebViewNativeUIEngineTests/CoveredSnapshotsTests.swift @@ -5,6 +5,60 @@ import UIKit @testable import WebViewNativeUIEngine final class NativeUICoveredSnapshotsTests: XCTestCase { + @MainActor func testPixelVisibilityLeasesPreserveLayoutAndReleaseInEitherOrder() { + for reverse in [false, true] { + let view = UIView(frame: CGRect(x: 20, y: 80, width: 48, height: 48)) + let mask = CALayer() + view.layer.mask = mask + let first = NativeUICoveredSnapshots.concealPixels(in: view) + let second = NativeUICoveredSnapshots.concealPixels(in: view) + XCTAssertFalse(view.isHidden) + XCTAssertEqual(view.frame, CGRect(x: 20, y: 80, width: 48, height: 48)) + (reverse ? second : first)() + XCTAssertFalse(view.layer.mask === mask) + XCTAssertFalse(view.isUserInteractionEnabled) + (reverse ? first : second)() + XCTAssertTrue(view.layer.mask === mask) + XCTAssertTrue(view.isUserInteractionEnabled) + XCTAssertFalse(view.accessibilityElementsHidden) + first(); second() + XCTAssertTrue(view.layer.mask === mask) + } + } + + @MainActor func testWebSurfaceHandoffUsesDOMPixelsWithoutFixedNativeCopies() async throws { + let window = SnapshotTestWindow(frame: UIScreen.main.bounds) + window.rootViewController = UIViewController() + window.makeKeyAndVisible() + let web = window.rootViewController!.view! + web.frame = window.bounds + window.layoutIfNeeded() + let snapshots = NativeUICoveredSnapshots() + defer { snapshots.reset(); window.isHidden = true } + let control = UIView() + let result: [String: Any] = await withCheckedContinuation { continuation in + snapshots.handle(["action": "prepare", "ids": ["button"], "revision": 1, + "viewportWidth": Double(web.bounds.width), "imageSurface": "web"], in: web, + frames: { _, _ in ["button": CGRect(x: 20, y: 80, width: 48, height: 48)] }, + views: { _ in [control] }, complete: { continuation.resume(returning: $0) }) + } + let token = try XCTUnwrap(result["token"] as? String) + XCTAssertTrue(snapshots.cover(token, in: web, views: [control], transient: true)) + XCTAssertFalse(control.isHidden) + XCTAssertNotNil(control.layer.mask) + XCTAssertFalse(control.isUserInteractionEnabled) + XCTAssertTrue(control.accessibilityElementsHidden) + XCTAssertTrue(window.subviews.compactMap { $0 as? UIImageView }.isEmpty) + XCTAssertTrue(snapshots.restore(in: web, token: token)) + XCTAssertTrue(window.subviews.compactMap { $0 as? UIImageView }.isEmpty) + snapshots.restoreControls() + XCTAssertFalse(control.isHidden) + XCTAssertNil(control.layer.mask) + XCTAssertTrue(control.isUserInteractionEnabled) + XCTAssertFalse(control.accessibilityElementsHidden) + XCTAssertTrue(window.subviews.compactMap { $0 as? UIImageView }.isEmpty) + } + @MainActor func testRequestHandoffValidatesDestinationsAndRetainsRendererVisibility() async throws { let window = SnapshotTestWindow(frame: UIScreen.main.bounds) window.rootViewController = UIViewController() @@ -138,6 +192,48 @@ final class NativeUICoveredSnapshotsTests: XCTestCase { "A successful preparation must contain the white window pixels, not a black display replica") } + @MainActor func testViewCapturePreservesAlphaWithoutWindowBackgroundOrDuplicateViews() async throws { + let window = SnapshotTestWindow(frame: UIScreen.main.bounds) + window.rootViewController = UIViewController() + window.makeKeyAndVisible() + let web = window.rootViewController!.view! + web.frame = window.bounds + let control = TransparentSnapshotTestView(frame: CGRect(x: 20, y: 80, width: 60, height: 44)) + web.addSubview(control) + window.layoutIfNeeded() + let snapshots = NativeUICoveredSnapshots() + defer { snapshots.reset(); window.isHidden = true } + let result: [String: Any] = await withCheckedContinuation { continuation in + snapshots.handle(["action": "prepare", "ids": ["button"], "revision": 1, + "viewportWidth": Double(web.bounds.width), "captureMode": "view"], in: web, + frames: { _, _ in ["button": control.frame] }, + views: { _ in [control, control] }, complete: { continuation.resume(returning: $0) }) + } + let parts = try XCTUnwrap(result["parts"] as? [[String: Any]]) + let url = try XCTUnwrap(URL(string: try XCTUnwrap(parts.first?["url"] as? String))) + let image = try XCTUnwrap(UIImage(contentsOfFile: url.path)?.cgImage) + XCTAssertEqual(image.bitsPerComponent, 16) + for (fraction, expectedAlpha) in [(0.01, 0), (0.5, 128)] { + let sample = try XCTUnwrap(image.cropping(to: CGRect(x: Int(Double(image.width) * fraction), y: image.height / 2, width: 1, height: 1))) + var pixel = [UInt8](repeating: 0, count: 4) + pixel.withUnsafeMutableBytes { bytes in + let context = CGContext(data: bytes.baseAddress, width: 1, height: 1, bitsPerComponent: 8, + bytesPerRow: 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.draw(sample, in: CGRect(x: 0, y: 0, width: 1, height: 1)) + } + XCTAssertEqual(Double(pixel[3]), Double(expectedAlpha), accuracy: 1, "Capture must preserve renderer alpha instead of including the white window or drawing the renderer twice") + } + control.removeFromSuperview() + let rejected: [String: Any] = await withCheckedContinuation { continuation in + snapshots.handle(["action": "prepare", "ids": ["button"], "revision": 1, + "viewportWidth": Double(web.bounds.width), "captureMode": "view"], in: web, + frames: { _, _ in ["button": control.frame] }, + views: { _ in [control] }, complete: { continuation.resume(returning: $0) }) + } + XCTAssertTrue(rejected.isEmpty, "A detached renderer must not produce a valid empty image") + } + @MainActor func testCapturePreservesCropPositionAndOrientation() async throws { let owner = UIViewController() let window = SnapshotTestWindow(frame: UIScreen.main.bounds) @@ -342,3 +438,11 @@ final class SnapshotTestWindow: UIWindow { } // swiftlint:enable force_unwrapping force_cast + +private final class TransparentSnapshotTestView: UIView { + override func drawHierarchy(in rect: CGRect, afterScreenUpdates: Bool) -> Bool { + UIColor.red.withAlphaComponent(0.5).setFill() + UIRectFillUsingBlendMode(rect, .normal) + return true + } +} diff --git a/src/projection/web.ts b/src/projection/web.ts index 65f75fd..047de82 100644 --- a/src/projection/web.ts +++ b/src/projection/web.ts @@ -30,6 +30,8 @@ export interface WebProjectionOptions { current?(sources: HTMLElement[]): boolean; /** Temporary movement handoff: rebase return pixels to the current display DOM. */ transient?: boolean; + /** Keep moving-surface handoff pixels in their DOM, without fixed native image copies. */ + imageSurface?: HTMLElement; /** Opaque full-view pixels map to the host border box and may paint beneath native. */ opaqueUnderlay?: boolean; } @@ -49,7 +51,8 @@ interface Prepared { /** One pixel handoff; requests and renderer source ownership remain independent. */ type ImageHandoff = - { phase: 'covering' | 'image'; image: Prepared } | { phase: 'restoring'; image: Prepared; visible: boolean; revision: number }; + | { phase: 'covering' | 'image'; image: Prepared } + | { phase: 'restoring'; image: Prepared; visible: boolean; revision: number; nativeReady?: boolean; layoutRevision?: number }; const layout = ({ control: c }: Candidate, positionIndependent = false) => [ c.id, @@ -268,6 +271,7 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP let expired = false; const capture = call({ action: 'prepare', + ...(callbacks.imageSurface ? { imageSurface: 'web' as const } : {}), revision: callbacks.captureRevision?.() ?? revision, viewportWidth: win.innerWidth, ids: selected.map((item) => item.control.id), @@ -475,6 +479,7 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP presentation++; win.cancelAnimationFrame(paintFrame); for (const element of offsets.keys()) element.style.setProperty('opacity', '0', 'important'); + unparkImages(); offsets.clear(); paintFrame = 0; handoffPending = false; @@ -513,6 +518,10 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP handoffPending = false; rendered.forEach((target) => target.removeAttribute('data-native-ui-snapshot-source')); showImages(image, !!callbacks.opaqueUnderlay); + if (callbacks.imageSurface) { + unparkImages(); + updateOffsets(); + } handoff = undefined; win.clearTimeout(restoreTimeout); // Retire only the temporary return copy. Keep the pre-interaction image @@ -542,10 +551,32 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP }; const preparedKeys = new Map(); const offsets = new Map(); + const parked = new Map(); + const parkImages = (image: Prepared, frames: Frame[]) => { + const layer = callbacks.imageSurface; + if (!layer?.isConnected) return; + image.images.forEach((element, index) => { + if (!parked.has(element) && element.parentNode) parked.set(element, { parent: element.parentNode, next: element.nextSibling }); + layer.append(element); + const frame = frames[index]; + for (const [property, value] of Object.entries({ left: frame.x, top: frame.y, width: frame.width, height: frame.height })) + element.style.setProperty(property, `${value}px`, 'important'); + }); + showImages(image, true); + }; + const unparkImages = () => { + for (const [element, saved] of parked) { + if (saved.parent.isConnected) saved.parent.insertBefore(element, saved.next?.parentNode === saved.parent ? saved.next : null); + element.style.setProperty('left', '0px', 'important'); + element.style.setProperty('top', '0px', 'important'); + } + parked.clear(); + }; const updateOffsets = () => { const writes: { element: HTMLImageElement; left: number; top: number; width: number; height: number }[] = []; // Read all geometry before writing positions; never reset a painted image to zero to measure it. for (const [element, placement] of offsets) { + if (parked.has(element)) continue; const { target, frame } = placement; if (!target.isConnected) { element.remove(); @@ -666,7 +697,12 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP }); const present = (force = false) => { - retireRestoration(); + if (callbacks.imageSurface && restoring()) { + presentation++; + win.cancelAnimationFrame(paintFrame); + win.clearTimeout(restoreTimeout); + handoff = undefined; + } else retireRestoration(); if (stopped || destination === 'none' || (handoff && !force)) return; const image = handoff?.image ?? cached; if ( @@ -695,54 +731,70 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP showImages(image, imageVisible()); // Reconciliation may continue after one frame even if the optional bridge reply // is delayed. The ordered native transport processes cover before that update. - paintFrame = win.requestAnimationFrame(() => { - paintFrame = 0; - if (!isCurrent()) return; - handoffPending = false; - callbacks.wake(); - }); - void bounded( - call({ - action: 'cover', - token: image.data.token, - ...(callbacks.transient ? { transient: true, viewportWidth: win.innerWidth, parts: currentParts(image) } : {}), - }), - ) - .then((result) => { + if (!callbacks.imageSurface) + paintFrame = win.requestAnimationFrame(() => { + paintFrame = 0; if (!isCurrent()) return; handoffPending = false; - if (result?.shown !== true) { - abandon(image, false); - return; - } - // Live translucent Native must be hidden before its composited Web image - // becomes visible. Native's prepared entry copy spans this bridge interval. - if (!restoring()) { - handoff = { phase: 'image', image }; - render(); - updateOffsets(); - } - win.cancelAnimationFrame(paintFrame); - // rAF callbacks run before paint. Cross a paint boundary before retiring - // Native's entry copy, keeping it over WebKit's compositing interval. - paintFrame = win.requestAnimationFrame(() => { + callbacks.wake(); + }); + const cover = () => + bounded( + call({ + action: 'cover', + token: image.data.token, + ...(callbacks.transient ? { transient: true, viewportWidth: win.innerWidth, parts: currentParts(image) } : {}), + }), + ) + .then((result) => { + if (!isCurrent()) return; + handoffPending = false; + if (result?.shown !== true) { + abandon(image, false); + return; + } + // Live translucent Native must be hidden before its composited Web image + // becomes visible. Native's prepared entry copy spans this bridge interval. + if (!restoring()) { + unparkImages(); + handoff = { phase: 'image', image }; + render(); + updateOffsets(); + } + if (callbacks.imageSurface) { + callbacks.wake(); + return; + } + win.cancelAnimationFrame(paintFrame); + // rAF callbacks run before paint. Cross a paint boundary before retiring + // Native's entry copy, keeping it over WebKit's compositing interval. paintFrame = win.requestAnimationFrame(() => { - paintFrame = 0; - if (!isCurrent()) return; - void call({ action: 'painted', token: image.data.token }) - .then((result) => { - if (isCurrent() && result?.shown === false) abandon(image, false); - }) - .catch(() => { - if (isCurrent()) abandon(image, false); - }); + paintFrame = win.requestAnimationFrame(() => { + paintFrame = 0; + if (!isCurrent()) return; + void call({ action: 'painted', token: image.data.token }) + .then((result) => { + if (isCurrent() && result?.shown === false) abandon(image, false); + }) + .catch(() => { + if (isCurrent()) abandon(image, false); + }); + }); }); + callbacks.wake(); + }) + .catch(() => { + if (isCurrent()) abandon(image, false); + }); + if (callbacks.imageSurface) { + parkImages(image, image.data.parts); + paintFrame = win.requestAnimationFrame(() => { + paintFrame = win.requestAnimationFrame(() => { + paintFrame = 0; + if (isCurrent()) void cover(); }); - callbacks.wake(); - }) - .catch(() => { - if (isCurrent()) abandon(image, false); }); + } else void cover(); }; const restore = () => { if (!handoff || stopped || restoring()) return; @@ -783,7 +835,8 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP image.data = { ...image.data, parts }; else cached = undefined; // Both requests use the same ordered bridge. Reprojection need not await this optional copy. - void call({ action: 'restore', token: image.data.token, viewportWidth: win.innerWidth, parts }).catch(() => {}); + if (!callbacks.imageSurface) + void call({ action: 'restore', token: image.data.token, viewportWidth: win.innerWidth, parts }).catch(() => {}); } handoff = { phase: 'restoring', @@ -791,13 +844,53 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP visible: handoff.phase === 'image', revision: callbacks.revision?.() ?? latest?.revision ?? 0, }; - restoreTimeout = win.setTimeout(() => { - if (handoff?.phase === 'restoring') abandon(handoff.image, false); - }, 250); + restoreTimeout = win.setTimeout( + () => { + if (handoff?.phase === 'restoring') abandon(handoff.image, false); + }, + callbacks.imageSurface ? 2000 : 250, + ); callbacks.wake(); }; + const prepareNativeReturn = (frames: Map, revision: number) => { + // A delayed measurement belongs to its submitted layout, never a later return. + // Obsolete work is a successful no-op; false means the current layout is unusable. + if (!callbacks.imageSurface || handoff?.phase !== 'restoring' || handoff.nativeReady || revision <= handoff.revision) return true; + const image = handoff.image; + const parts: Frame[] = []; + for (const part of image.data.parts) { + const rects = part.ids.map((id) => frames.get(id)); + if ( + rects.some( + (rect) => !rect || ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) || rect.width <= 0 || rect.height <= 0, + ) + ) + return false; + const values = rects as Frame[]; + const x = Math.min(...values.map((rect) => rect.x)); + const y = Math.min(...values.map((rect) => rect.y)); + const width = Math.max(...values.map((rect) => rect.x + rect.width)) - x; + const height = Math.max(...values.map((rect) => rect.y + rect.height)) - y; + parts.push({ x: x - (part.width - width) / 2, y: y - (part.height - height) / 2, width: part.width, height: part.height }); + } + const returning = handoff; + returning.layoutRevision = revision; + parkImages(image, parts); + image.data = { ...image.data, parts: image.data.parts.map((part, index) => ({ ...part, ...parts[index] })) }; + win.cancelAnimationFrame(paintFrame); + paintFrame = win.requestAnimationFrame(() => { + paintFrame = win.requestAnimationFrame(() => { + paintFrame = 0; + if (handoff !== returning) return; + returning.nativeReady = true; + callbacks.wake(); + }); + }); + return true; + }; const updated = (token: string | undefined, restoredIDs?: string[], revision?: number) => { if (handoff?.phase !== 'restoring' || revision === undefined || revision <= handoff.revision) return; + if (callbacks.imageSurface && (!handoff.nativeReady || revision <= (handoff.layoutRevision ?? Infinity))) return; if (!callbacks.transient && token) return; if (!handoff.image.data.parts.every((part) => part.ids.every((id) => restoredIDs?.includes(id)))) return; // Retire the Web image at the native restore acknowledgement, not after @@ -911,11 +1004,15 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP get restoring() { return restoring(); }, + get nativeReady() { + return handoff?.phase === 'restoring' && !!handoff.nativeReady; + }, get handoffPending() { return handoffPending; }, retireRestoration, end, + prepareNativeReturn, updated, invalidate, async destroy() { @@ -979,6 +1076,9 @@ export const createWebProjectionBatch = async ( get restoring() { return engine?.restoring ?? false; }, + get nativeReady() { + return engine?.nativeReady ?? false; + }, get handoffPending() { return engine?.handoffPending ?? false; }, @@ -1027,6 +1127,7 @@ export const createWebProjectionBatch = async ( pendingRequest = false; engine?.end(); }, + prepareNativeReturn: (frames, revision) => engine?.prepareNativeReturn(frames, revision) ?? false, updated: (token, ids, revision) => engine?.updated(token, ids, revision), retireRestoration: () => engine?.retireRestoration(), invalidate: (nativeHidden?: boolean) => engine?.invalidate(nativeHidden), diff --git a/src/types.ts b/src/types.ts index aca2ca6..d63fa62 100644 --- a/src/types.ts +++ b/src/types.ts @@ -36,8 +36,14 @@ export interface PreparedWebProjection { } /** Each action carries only the fields required by that phase of handoff. */ export type WebProjectionRequest = - | { action: 'prepare'; revision: number; viewportWidth: number; ids: string[] } - | { action: 'cover'; token: string; transient?: boolean; viewportWidth?: number; parts?: (Frame & { ids: string[] })[] } + | { action: 'prepare'; revision: number; viewportWidth: number; ids: string[]; captureMode?: 'view'; imageSurface?: 'web' } + | { + action: 'cover'; + token: string; + transient?: boolean; + viewportWidth?: number; + parts?: (Frame & { ids: string[] })[]; + } | { action: 'restore'; token: string; viewportWidth: number; parts: (Frame & { ids: string[] })[] } | { action: 'prepared' | 'painted' | 'restored' | 'release'; token: string } | { action: 'reset' }; diff --git a/tests/projection/web.spec.ts b/tests/projection/web.spec.ts index b755030..38cdb74 100644 --- a/tests/projection/web.spec.ts +++ b/tests/projection/web.spec.ts @@ -34,6 +34,7 @@ const prepare = async ( events = false, captureDelayMs?: number, transient = false, + webSurface = false, ) => { await boot(page); if (shadow) @@ -44,7 +45,7 @@ const prepare = async ( (window as any).sourceRoot = host.shadowRoot; }); await page.evaluate( - async ({ proxy, shared, events, captureDelayMs, transient }) => { + async ({ proxy, shared, events, captureDelayMs, transient, webSurface }) => { const w = window as any, source = (w.sourceRoot ?? document).querySelector('#source') as HTMLElement, owner = (w.sourceRoot ?? document).querySelector('#scroll') as HTMLElement; @@ -118,6 +119,7 @@ const prepare = async ( { captureDelayMs, transient, + imageSurface: webSurface ? document.body : undefined, wake() { w.onWake?.(); }, @@ -138,7 +140,7 @@ const prepare = async ( ); w.manager.consider(1, [w.source]); }, - { proxy, shared, events, captureDelayMs, transient }, + { proxy, shared, events, captureDelayMs, transient, webSurface }, ); await expect(page.locator(shared ? '#projection-host' : proxy ? '#proxy' : '#source')).toHaveAttribute('data-native-ui-snapshot', /.+/); }; @@ -1584,3 +1586,87 @@ for (const localFirst of [true, false]) { expect(await page.evaluate(() => (window as any).manager.visibleOwners.length)).toBe(0); }); } + +test('moving DOM handoff retains pixels through layout staging and retires after the accepted reveal', async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => (window as any).manager.end()); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => { + const w = window as any; + w.manager.prepareNativeReturn(new Map(), 2); + w.manager.updated(undefined, ['counter'], 2); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => + (window as any).manager.prepareNativeReturn(new Map([['counter', { x: 8, y: 108, width: 120, height: 40 }]]), 2), + ); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 2)); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 3)); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); + expect(await page.evaluate(() => (window as any).calls.filter((c: any) => ['restore', 'painted'].includes(c.action)))).toEqual([]); + await destroyProjection(page); +}); + +test('moving DOM handoff ignores an old native acknowledgement after immediate reversal', async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => { + const w = window as any; + w.manager.end(); + w.manager.prepareNativeReturn(new Map([['counter', { x: 8, y: 108, width: 120, height: 40 }]]), 2); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 2)); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); + await destroyProjection(page); +}); + +test('a delayed Native layout measurement cannot fail or complete a later return', async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => { + const w = window as any; + w.manager.end(); + w.revision = 3; + w.manager.begin(); + }); + await expect.poll(() => page.evaluate(() => (window as any).manager.restoring)).toBe(false); + await page.evaluate(() => (window as any).manager.end()); + expect(await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map(), 2))).toBe(true); + expect(await page.evaluate(() => (window as any).manager.nativeReady)).toBe(false); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map(), 4))).toBe(false); + await page.evaluate(() => + (window as any).manager.prepareNativeReturn(new Map([['counter', { x: 8, y: 108, width: 120, height: 40 }]]), 4), + ); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 4)); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 5)); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); + await destroyProjection(page); +}); From 4db41f4fef79d0ef897d5d17cdbf03039a490fe7 Mon Sep 17 00:00:00 2001 From: rdlabo Date: Sun, 8 Nov 2026 10:35:42 +0900 Subject: [PATCH 2/3] fix: separate image leases from optional alignment and cache reuse --- docs/advanced/native-handoff.md | 2 +- src/projection/web.ts | 45 +++++++++------- tests/projection/web.spec.ts | 96 +++++++++++++++++++++++++++++++-- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/docs/advanced/native-handoff.md b/docs/advanced/native-handoff.md index e6fda3f..d4aa9f8 100644 --- a/docs/advanced/native-handoff.md +++ b/docs/advanced/native-handoff.md @@ -94,6 +94,6 @@ A `prepare` request may specify `captureMode: 'view'` to draw only the renderer Moving DOM surfaces can provide a fixed viewport transfer layer through `WebProjectionOptions.imageSurface` (`HTMLElement`). The Web projection marks its prepare requests with `imageSurface: 'web'`; these tokens do not install fixed Native image copies. During motion, the decoded DOM images live on their source surface so its transform carries them. -Keep the Native renderer and source ownership alive while DOM images own the pixels. For entry, the Engine temporarily places those same images in the transfer layer at the captured Native coordinates, waits for Web painting, covers Native pixels, and returns the images to their source surface. For return, first lay out Native controls while keeping their pixels concealed. Read their actual frames and call `prepareNativeReturn(frames, revision)`. The Engine parks the same DOM images at those frames and sets `nativeReady` after Web painting. Reveal Native controls in the following update, then call `updated(token, ids, revision)` on its accepted acknowledgement to retire the DOM images. Only a revision newer than the staged layout can retire them; an acknowledgement for the layout stage must not. `prepareNativeReturn` ignores obsolete measurements and returns `false` only when the current return geometry is unusable. No snapshot is captured during either transfer. +Keep the Native renderer and source ownership alive while DOM images own the pixels. For entry, the Engine temporarily places those same images in the transfer layer at the captured Native coordinates, waits for Web painting, covers Native pixels, and returns the images to their source surface. For return, first lay out Native controls while keeping their pixels concealed. Read their actual frames and call `prepareNativeReturn(frames, revision)`. When the frames are usable, the Engine parks the same DOM images there; otherwise, pass `undefined` and it keeps their current placement. Both paths set `nativeReady` after Web painting. Missing alignment is not a failure of an accepted Native renderer. Reveal Native controls in the following update, then call `updated(token, ids, revision)` on its accepted acknowledgement to retire the DOM images. Only a revision newer than the staged layout can retire them; an acknowledgement for the layout stage must not. `prepareNativeReturn` ignores obsolete measurements. The displayed image remains leased until the reveal acknowledgement or explicit retirement, independently of whether its cached capture is still reusable. No snapshot is captured during either transfer. On iOS, use `NativeUICoveredSnapshots.concealPixels(in:)` on the main thread when layout must remain measurable. It masks pixels and disables interaction/accessibility without changing `isHidden` or geometry. Its returned restoration closure is idempotent; overlapping leases restore the original mask and flags only after the last lease ends. The default composited handoff retains its existing Native copy and drawing-completion contract. diff --git a/src/projection/web.ts b/src/projection/web.ts index 047de82..d00b6dd 100644 --- a/src/projection/web.ts +++ b/src/projection/web.ts @@ -697,13 +697,10 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP }); const present = (force = false) => { - if (callbacks.imageSurface && restoring()) { - presentation++; - win.cancelAnimationFrame(paintFrame); - win.clearTimeout(restoreTimeout); - handoff = undefined; - } else retireRestoration(); - if (stopped || destination === 'none' || (handoff && !force)) return; + const returning = !!callbacks.imageSurface && restoring(); + if (!returning) retireRestoration(); + if (stopped || destination === 'none' || (handoff && !force && !returning)) return; + // The active pixel lease outlives cache reuse eligibility. const image = handoff?.image ?? cached; if ( !image || @@ -713,8 +710,14 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP image.targets.some((source) => !preparedKeys.has(source)) || targetsFor(image).some((target) => !target?.isConnected) || [...rendered].some((target) => !target.isConnected) - ) + ) { + if (returning) abandon(handoff?.image, false); return; + } + if (returning) { + win.cancelAnimationFrame(paintFrame); + win.clearTimeout(restoreTimeout); + } // Reparenting is renderer-owned. Only that case needs a fresh coordinate conversion; // ordinary presentation keeps the prepared layout off the critical path. if (image.scopes.some((scope, index) => scope.element.parentNode !== image.parents[index])) render(); @@ -852,11 +855,9 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP ); callbacks.wake(); }; - const prepareNativeReturn = (frames: Map, revision: number) => { - // A delayed measurement belongs to its submitted layout, never a later return. - // Obsolete work is a successful no-op; false means the current layout is unusable. - if (!callbacks.imageSurface || handoff?.phase !== 'restoring' || handoff.nativeReady || revision <= handoff.revision) return true; - const image = handoff.image; + // Alignment improves the visual transfer; it is not proof of Native ownership. + const returnParts = (image: Prepared, frames: Map | undefined): Frame[] | undefined => { + if (!frames) return; const parts: Frame[] = []; for (const part of image.data.parts) { const rects = part.ids.map((id) => frames.get(id)); @@ -865,7 +866,7 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP (rect) => !rect || ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) || rect.width <= 0 || rect.height <= 0, ) ) - return false; + return; const values = rects as Frame[]; const x = Math.min(...values.map((rect) => rect.x)); const y = Math.min(...values.map((rect) => rect.y)); @@ -873,10 +874,19 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP const height = Math.max(...values.map((rect) => rect.y + rect.height)) - y; parts.push({ x: x - (part.width - width) / 2, y: y - (part.height - height) / 2, width: part.width, height: part.height }); } + return parts; + }; + const prepareNativeReturn = (frames: Map | undefined, revision: number) => { + // A delayed measurement belongs to its submitted layout, never a later return. + if (!callbacks.imageSurface || handoff?.phase !== 'restoring' || handoff.nativeReady || revision <= handoff.revision) return; + const image = handoff.image; const returning = handoff; returning.layoutRevision = revision; - parkImages(image, parts); - image.data = { ...image.data, parts: image.data.parts.map((part, index) => ({ ...part, ...parts[index] })) }; + const parts = returnParts(image, frames); + if (parts) { + parkImages(image, parts); + image.data = { ...image.data, parts: image.data.parts.map((part, index) => ({ ...part, ...parts[index] })) }; + } win.cancelAnimationFrame(paintFrame); paintFrame = win.requestAnimationFrame(() => { paintFrame = win.requestAnimationFrame(() => { @@ -886,7 +896,6 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP callbacks.wake(); }); }); - return true; }; const updated = (token: string | undefined, restoredIDs?: string[], revision?: number) => { if (handoff?.phase !== 'restoring' || revision === undefined || revision <= handoff.revision) return; @@ -1127,7 +1136,7 @@ export const createWebProjectionBatch = async ( pendingRequest = false; engine?.end(); }, - prepareNativeReturn: (frames, revision) => engine?.prepareNativeReturn(frames, revision) ?? false, + prepareNativeReturn: (frames, revision) => engine?.prepareNativeReturn(frames, revision), updated: (token, ids, revision) => engine?.updated(token, ids, revision), retireRestoration: () => engine?.retireRestoration(), invalidate: (nativeHidden?: boolean) => engine?.invalidate(nativeHidden), diff --git a/tests/projection/web.spec.ts b/tests/projection/web.spec.ts index 38cdb74..1376844 100644 --- a/tests/projection/web.spec.ts +++ b/tests/projection/web.spec.ts @@ -1599,7 +1599,6 @@ test('moving DOM handoff retains pixels through layout staging and retires after await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); await page.evaluate(() => { const w = window as any; - w.manager.prepareNativeReturn(new Map(), 2); w.manager.updated(undefined, ['counter'], 2); }); await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); @@ -1655,11 +1654,10 @@ test('a delayed Native layout measurement cannot fail or complete a later return }); await expect.poll(() => page.evaluate(() => (window as any).manager.restoring)).toBe(false); await page.evaluate(() => (window as any).manager.end()); - expect(await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map(), 2))).toBe(true); + await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map(), 2)); expect(await page.evaluate(() => (window as any).manager.nativeReady)).toBe(false); expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); - expect(await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map(), 4))).toBe(false); await page.evaluate(() => (window as any).manager.prepareNativeReturn(new Map([['counter', { x: 8, y: 108, width: 120, height: 40 }]]), 4), ); @@ -1670,3 +1668,95 @@ test('a delayed Native layout measurement cannot fail or complete a later return expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); await destroyProjection(page); }); + +test('a resized active image survives immediate reversal after its reusable cache is invalidated', async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await expect.poll(() => page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'cover').length)).toBe(1); + await page.evaluate(() => { + const w = window as any; + document.getElementById('source')!.style.transform = 'scale(2)'; + w.activeImage = document.querySelector('[data-native-ui-projection-image]'); + w.manager.end(); + w.manager.begin(); + }); + await expect.poll(() => page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'cover').length)).toBe(2); + expect(await page.evaluate(() => (window as any).manager.owners.length)).toBe(1); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); + expect(await page.evaluate(() => document.querySelector('[data-native-ui-projection-image]') === (window as any).activeImage)).toBe(true); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => { + const w = window as any; + w.manager.end(); + w.manager.prepareNativeReturn(undefined, 2); + }); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 3)); + expect(await page.evaluate(() => (window as any).manager.owners.length)).toBe(0); + await expect(page.locator('#source')).not.toHaveAttribute('data-native-ui-snapshot-visible', ''); + await expect.poll(() => page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'release').length)).toBe(1); + await destroyProjection(page); +}); + +test('an invalid source ends both its visible image and pixel lease on immediate reversal', async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await expect.poll(() => page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'cover').length)).toBe(1); + const result = await page.evaluate(() => { + const w = window as any; + const source = document.getElementById('source')!; + const image = document.querySelector('[data-native-ui-projection-image]')!; + w.manager.end(); + source.remove(); + w.manager.begin(); + return { + owners: w.manager.owners.length, + restoring: w.manager.restoring, + visible: source.hasAttribute('data-native-ui-snapshot-visible'), + opacity: (image as HTMLElement).style.opacity, + covers: w.calls.filter((call: any) => call.action === 'cover').length, + releases: w.calls.filter((call: any) => call.action === 'release').length, + }; + }); + expect(result).toEqual({ owners: 0, restoring: false, visible: false, opacity: '0', covers: 1, releases: 1 }); + await destroyProjection(page); +}); + +for (const geometry of ['missing', 'incomplete'] as const) + test(`a ${geometry} alignment measurement cannot strand the accepted Native return`, async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate((geometry) => { + const w = window as any; + w.manager.end(); + w.manager.prepareNativeReturn(geometry === 'missing' ? undefined : new Map(), 2); + w.manager.updated(undefined, ['counter'], 3); + }, geometry); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 2)); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(true); + await page.evaluate(() => (window as any).manager.updated(undefined, ['counter'], 3)); + expect(await page.evaluate(() => (window as any).manager.owners.length)).toBe(0); + expect(await page.evaluate(() => (window as any).manager.restoring)).toBe(false); + await expect(page.locator('#source')).not.toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).calls.filter((call: any) => ['restore', 'painted'].includes(call.action)))).toEqual( + [], + ); + await destroyProjection(page); + }); From fc80c746454afe73a8dbe37b5274d7203461340f Mon Sep 17 00:00:00 2001 From: rdlabo Date: Sun, 8 Nov 2026 15:01:50 +0900 Subject: [PATCH 3/3] fix: retain moving-surface cache across translated native returns --- docs/advanced/native-handoff.md | 2 +- src/projection/web.ts | 20 +++++++++++-- tests/projection/web.spec.ts | 53 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/docs/advanced/native-handoff.md b/docs/advanced/native-handoff.md index d4aa9f8..d630a23 100644 --- a/docs/advanced/native-handoff.md +++ b/docs/advanced/native-handoff.md @@ -92,7 +92,7 @@ This uses WKWebView's internal overflow hierarchy without private class names. M A `prepare` request may specify `captureMode: 'view'` to draw only the renderer views returned by the view selector, preserving transparent pixels outside their material. The default remains composited window capture. Renderer-only capture can change or omit backdrop-dependent blur and refraction; it does not guarantee transparency inside a glass surface. Missing, hidden or detached renderer views reject preparation. Android and older native binaries keep their existing capture route. -Moving DOM surfaces can provide a fixed viewport transfer layer through `WebProjectionOptions.imageSurface` (`HTMLElement`). The Web projection marks its prepare requests with `imageSurface: 'web'`; these tokens do not install fixed Native image copies. During motion, the decoded DOM images live on their source surface so its transform carries them. +Moving DOM surfaces can provide a fixed viewport transfer layer through `WebProjectionOptions.imageSurface` (`HTMLElement`). The Web projection marks its prepare requests with `imageSurface: 'web'`; these tokens do not install fixed Native image copies. During motion, the decoded DOM images live on their source surface so its transform carries them. Translation does not invalidate their reusable capture; dimensions are compared at display-pixel resolution, while actual size and source changes still invalidate it. Keep the Native renderer and source ownership alive while DOM images own the pixels. For entry, the Engine temporarily places those same images in the transfer layer at the captured Native coordinates, waits for Web painting, covers Native pixels, and returns the images to their source surface. For return, first lay out Native controls while keeping their pixels concealed. Read their actual frames and call `prepareNativeReturn(frames, revision)`. When the frames are usable, the Engine parks the same DOM images there; otherwise, pass `undefined` and it keeps their current placement. Both paths set `nativeReady` after Web painting. Missing alignment is not a failure of an accepted Native renderer. Reveal Native controls in the following update, then call `updated(token, ids, revision)` on its accepted acknowledgement to retire the DOM images. Only a revision newer than the staged layout can retire them; an acknowledgement for the layout stage must not. `prepareNativeReturn` ignores obsolete measurements. The displayed image remains leased until the reveal acknowledgement or explicit retirement, independently of whether its cached capture is still reusable. No snapshot is captured during either transfer. diff --git a/src/projection/web.ts b/src/projection/web.ts index d00b6dd..4ece5af 100644 --- a/src/projection/web.ts +++ b/src/projection/web.ts @@ -107,13 +107,29 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP JSON.stringify( candidates.map((candidate) => { if (!scopeIDs.has(candidate.scope)) scopeIDs.set(candidate.scope, ++scopeSerial); - const origin = callbacks.transient && !callbacks.opaqueUnderlay ? candidate.scope.element.getBoundingClientRect() : undefined; + const origin = + callbacks.transient && !callbacks.opaqueUnderlay && !callbacks.imageSurface + ? candidate.scope.element.getBoundingClientRect() + : undefined; const relative = origin ? { ...candidate, control: { ...candidate.control, x: candidate.control.x - origin.x, y: candidate.control.y - origin.y } } : candidate; + // A moving DOM surface owns placement. Its translated coordinates and + // subpixel measurement noise do not change the captured artwork. + const scale = win.devicePixelRatio || 1; + const reusable = callbacks.imageSurface + ? { + ...relative, + control: { + ...relative.control, + width: Math.round(relative.control.width * scale) / scale, + height: Math.round(relative.control.height * scale) / scale, + }, + } + : relative; return [ scopeIDs.get(candidate.scope), - layout(relative, !!callbacks.opaqueUnderlay), + layout(reusable, !!callbacks.opaqueUnderlay || !!callbacks.imageSurface), (candidate.sources ?? [candidate.element]).map(sourceIdentity), ]; }), diff --git a/tests/projection/web.spec.ts b/tests/projection/web.spec.ts index 1376844..02dd788 100644 --- a/tests/projection/web.spec.ts +++ b/tests/projection/web.spec.ts @@ -1760,3 +1760,56 @@ for (const geometry of ['missing', 'incomplete'] as const) ); await destroyProjection(page); }); + +for (const change of ['translation', 'subpixel-noise'] as const) { + test(`a completed moving-surface return retains its cache through ${change} during the same gesture`, async ({ page }) => { + await prepare(page, false, false, false, false, 50, true, true); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + w.cachedImage = document.querySelector('[data-native-ui-projection-image]'); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await page.evaluate(() => { + const w = window as any; + w.manager.end(); + w.manager.prepareNativeReturn(new Map([['counter', { x: 8, y: 108, width: 120, height: 40 }]]), 2); + }); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await page.evaluate((change) => { + const w = window as any; + w.manager.updated(undefined, ['counter'], 3); + w.source.control = { + ...w.source.control, + ...(change === 'translation' ? { y: w.source.control.y + 2.3431243896484375 } : { height: 40.00000762939453 }), + }; + w.manager.consider(4, [w.source]); + w.manager.begin(); + }, change); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).manager.owners.length)).toBe(1); + expect(await page.evaluate(() => document.querySelector('[data-native-ui-projection-image]') === (window as any).cachedImage)).toBe( + true, + ); + expect(await page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'prepare').length)).toBe(1); + expect(await page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'release'))).toBe(false); + // A real raster size change still invalidates the idle cache. + await page.evaluate(() => { + const w = window as any; + w.manager.end(); + w.manager.prepareNativeReturn(undefined, 5); + }); + await expect.poll(() => page.evaluate(() => (window as any).manager.nativeReady)).toBe(true); + await page.evaluate(() => { + const w = window as any; + w.manager.updated(undefined, ['counter'], 6); + w.source.control.width += 8; + w.manager.consider(7, [w.source]); + w.manager.begin(); + }); + expect(await page.evaluate(() => (window as any).manager.owners.length)).toBe(0); + await expect(page.locator('#source')).not.toHaveAttribute('data-native-ui-snapshot-visible', ''); + await destroyProjection(page); + }); +}