From defa3d87968139ff2c2f478d75ff09ebbacecc68 Mon Sep 17 00:00:00 2001 From: Siubeng Date: Sun, 15 Nov 2026 13:27:07 +0800 Subject: [PATCH] feat: add opt-in WebGL rendering for Canvas video --- common/src/main/assets/web/app.js | 181 +++++++++++++++--- common/src/main/assets/web/index.html | 2 + common/src/main/assets/web/style.css | 2 +- .../shilapi/xcertplay/web/BrowserStrings.kt | 2 + docs/ARCHITECTURE.md | 10 +- docs/VALIDATION.md | 9 + docs/WEBRTC_RECOVERY.md | 2 +- package.json | 1 + scripts/browser-webgl-checks.mjs | 117 +++++++++++ scripts/test-media-presentation.mjs | 19 +- shared/src/main/res/values-zh-rCN/strings.xml | 2 + shared/src/main/res/values-zh-rTW/strings.xml | 2 + shared/src/main/res/values/strings.xml | 2 + test/canvas-frame-painter.test.js | 111 +++++++++++ test/stream-client.test.js | 73 ++++++- test/track-canvas.test.js | 4 +- test/video-canvas.test.js | 25 ++- 17 files changed, 519 insertions(+), 45 deletions(-) create mode 100644 scripts/browser-webgl-checks.mjs create mode 100644 test/canvas-frame-painter.test.js diff --git a/common/src/main/assets/web/app.js b/common/src/main/assets/web/app.js index 6d880276..e5e91c0a 100644 --- a/common/src/main/assets/web/app.js +++ b/common/src/main/assets/web/app.js @@ -390,15 +390,111 @@ export function createControlOutbox(socket, { }; } +// Both Canvas modes must use the same context type: a canvas cannot switch +// between WebGL and 2D, even after its dimensions are reset on disconnect. +const restorableCanvases = new WeakSet(); +export function createCanvasFramePainter(canvas, { webgl = false, onError = () => {} } = {}) { + const gl = webgl && canvas.getContext('webgl', { + alpha: false, antialias: false, depth: false, stencil: false, + // Recovery copies the last picture in a later task, before clearing it. + preserveDrawingBuffer: true, + }); + if (!gl) { + const context = canvas.getContext('2d', { alpha: false }); + if (!context) throw Object.assign(new Error('Canvas rendering unavailable'), { name: 'NotSupportedError' }); + return { backend: '2d', draw: (source, width, height) => context.drawImage(source, 0, 0, width, height), dispose() {} }; + } + if (!restorableCanvases.has(canvas)) { + // Keep restoration enabled even if draw notices loss and disposes the + // painter before the browser dispatches its asynchronous loss event. + canvas.addEventListener('webglcontextlost', event => event.preventDefault()); + restorableCanvases.add(canvas); + } + let program = null, buffer = null, texture = null, disposed = false, width = 0, height = 0; + const shaders = []; + const lost = event => { + event.preventDefault(); // Permit restoration for a subsequent receiver retry. + if (!disposed) onError(new Error('WebGL context lost')); + }; + function dispose() { + if (disposed) return; + disposed = true; + canvas.removeEventListener('webglcontextlost', lost); + if (texture) gl.deleteTexture(texture); + if (buffer) gl.deleteBuffer(buffer); + if (program) gl.deleteProgram(program); + for (const shader of shaders) gl.deleteShader(shader); + } + try { + if (gl.isContextLost()) throw new Error('WebGL context lost'); + const compile = (type, source) => { + const shader = gl.createShader(type); + if (!shader) throw new Error('WebGL shader allocation failed'); + shaders.push(shader); gl.shaderSource(shader, source); gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error('WebGL shader compilation failed'); + return shader; + }; + program = gl.createProgram(); + if (!program) throw new Error('WebGL program allocation failed'); + // Keep texel coordinates precise for high-resolution dashboard text. + const precision = gl.getShaderPrecisionFormat(gl.FRAGMENT_SHADER, gl.HIGH_FLOAT)?.precision > 0 ? 'highp' : 'mediump'; + gl.attachShader(program, compile(gl.VERTEX_SHADER, ` + attribute vec2 position; + varying ${precision} vec2 uv; + void main() { + uv = position * vec2(0.5, -0.5) + 0.5; + gl_Position = vec4(position, 0.0, 1.0); + }`)); + gl.attachShader(program, compile(gl.FRAGMENT_SHADER, ` + precision ${precision} float; + varying ${precision} vec2 uv; + uniform sampler2D frame; + void main() { gl_FragColor = vec4(texture2D(frame, uv).rgb, 1.0); }`)); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) throw new Error('WebGL program linking failed'); + buffer = gl.createBuffer(); texture = gl.createTexture(); + if (!buffer || !texture) throw new Error('WebGL buffer allocation failed'); + gl.useProgram(program); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), gl.STATIC_DRAW); + const position = gl.getAttribLocation(program, 'position'); + gl.enableVertexAttribArray(position); gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0); + gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.uniform1i(gl.getUniformLocation(program, 'frame'), 0); + const maxSize = gl.getParameter(gl.MAX_TEXTURE_SIZE); + canvas.addEventListener('webglcontextlost', lost); + return { + backend: 'webgl', dispose, + draw(source, w, h) { + if (disposed || gl.isContextLost()) throw new Error('WebGL context lost'); + const resized = width !== w || height !== h; + if (resized && (w > maxSize || h > maxSize)) throw Object.assign(new Error('Video exceeds WebGL texture size'), { name: 'NotSupportedError' }); + gl.viewport(0, 0, w, h); + // Upload the browser-owned source directly, without JS pixel readback. + // texImage2D also permits the browser's optimized video upload path. + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + // Check the first upload and size changes, not every steady-state frame. + if (resized && gl.getError() !== gl.NO_ERROR) throw new Error('WebGL frame upload failed'); + width = w; height = h; + }, + }; + } catch (error) { dispose(); throw error; } +} + // Keep the decoder in the DOM, outside the viewport. display:none/hidden can -// suppress video frame callbacks on embedded browsers. drawImage copies decoded -// frames directly; it does not read pixels back through getImageData. +// suppress video frame callbacks on embedded browsers. export function createVideoCanvasRenderer(video, canvas, { onFrame = () => {}, onDraw = () => {}, onError = () => {}, + webgl = false, requestFrame = callback => requestAnimationFrame(callback), cancelFrame = id => cancelAnimationFrame(id), now = () => performance.now(), } = {}) { - let context = null, running = false, generation = 0, videoCallback = null, animationCallback = null; + let painter = null, running = false, generation = 0, videoCallback = null, animationCallback = null; let lastVideoCallbackAt = 0, lastFrameKey = null; function prepare() { video.classList.toggle('offscreen-video', true); @@ -408,11 +504,15 @@ export function createVideoCanvasRenderer(video, canvas, { running = false; generation++; if (videoCallback !== null && video.cancelVideoFrameCallback) video.cancelVideoFrameCallback(videoCallback); if (animationCallback !== null) cancelFrame(animationCallback); - videoCallback = animationCallback = null; context = null; lastFrameKey = null; + videoCallback = animationCallback = null; painter?.dispose(); painter = null; lastFrameKey = null; video.hidden = true; video.classList.toggle('offscreen-video', false); canvas.hidden = true; canvas.width = canvas.height = 0; } - function fail(error) { stop(); onError(error); } + function fail(error) { + const token = generation; + try { onError(error); } + finally { if (generation === token) stop(); } + } function draw(metadata = null, stalled = false) { if (!running || video.readyState < 2 || !video.videoWidth || !video.videoHeight) return; try { @@ -425,7 +525,7 @@ export function createVideoCanvasRenderer(video, canvas, { const key = !stalled && frames > 0 ? `frame:${frames}` : Number.isFinite(time) ? `time:${time}` : null; if (!resized && key !== null && key === lastFrameKey) return; if (resized) { canvas.width = width; canvas.height = height; } - context.drawImage(video, 0, 0, width, height); + painter.draw(video, width, height); onDraw(); lastFrameKey = key; const first = canvas.hidden; @@ -436,8 +536,7 @@ export function createVideoCanvasRenderer(video, canvas, { function start() { stop(); prepare(); try { - context = canvas.getContext('2d', { alpha: false }); - if (!context) throw Object.assign(new Error('Canvas 2D unavailable'), { name: 'NotSupportedError' }); + painter = createCanvasFramePainter(canvas, { webgl, onError: fail }); running = true; lastVideoCallbackAt = now(); const token = generation, current = () => running && token === generation; const observe = (_, metadata) => { @@ -459,7 +558,7 @@ export function createVideoCanvasRenderer(video, canvas, { animationCallback = requestFrame(refresh); } catch (error) { fail(error); } } - return { prepare, start, stop }; + return { prepare, start, stop, get backend() { return painter?.backend ?? null; } }; } function isRtcErrorRetryable(error) { @@ -473,16 +572,24 @@ export function videoRendererPreference(value, legacy) { // A reader owns each VideoFrame until it is drawn or discarded, including after stop. export function createTrackCanvasRenderer(canvas, { onReady = () => {}, onResize = () => {}, onError = () => {}, + webgl = false, createProcessor = init => new MediaStreamTrackProcessor(init), } = {}) { - let generation = 0, reader = null, running = false, width = 0, height = 0, renderedFrames = 0; + let generation = 0, reader = null, painter = null, running = false, width = 0, height = 0, renderedFrames = 0; function stop() { generation++; running = false; const old = reader; reader = null; if (old) { try { Promise.resolve(old.cancel()).catch(() => {}); } catch (_) {} } + painter?.dispose(); painter = null; canvas.hidden = true; canvas.width = canvas.height = 0; } - async function consume(activeReader, context, token) { + function fail(error, token) { + if (!running || generation !== token) return; + // Let the receiver freeze the last picture before stop clears the canvas. + try { onError(error.message === 'track-ended' ? 'track-ended' : 'renderer-error', isRtcErrorRetryable(error)); } + finally { if (generation === token) stop(); } + } + async function consume(activeReader, activePainter, token) { const current = () => running && generation === token; try { while (current()) { @@ -493,7 +600,7 @@ export function createTrackCanvasRenderer(canvas, { if (!frame || !(frame.displayWidth > 0) || !(frame.displayHeight > 0)) continue; const resized = width !== frame.displayWidth || height !== frame.displayHeight; if (resized) { canvas.width = frame.displayWidth; canvas.height = frame.displayHeight; } - context.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight); + activePainter.draw(frame, frame.displayWidth, frame.displayHeight); width = frame.displayWidth; height = frame.displayHeight; renderedFrames++; canvas.hidden = false; if (renderedFrames === 1) onReady({ width, height }); @@ -501,11 +608,7 @@ export function createTrackCanvasRenderer(canvas, { } finally { frame?.close(); } } } catch (error) { - if (current()) { - // Let the receiver freeze the last picture before stop clears the canvas. - try { onError(error.message === 'track-ended' ? 'track-ended' : 'renderer-error', isRtcErrorRetryable(error)); } - finally { if (generation === token) stop(); } - } + fail(error, token); } finally { try { activeReader.releaseLock(); } catch (_) {} if (reader === activeReader) reader = null; @@ -515,21 +618,22 @@ export function createTrackCanvasRenderer(canvas, { start(track) { stop(); width = height = renderedFrames = 0; try { - const context = canvas.getContext('2d', { alpha: false }); - if (!context) throw Object.assign(new Error('Canvas 2D unavailable'), { name: 'NotSupportedError' }); + const token = generation; + painter = createCanvasFramePainter(canvas, { webgl, onError: error => fail(error, token) }); reader = createProcessor({ track, maxBufferSize: 1 }).readable.getReader(); running = true; - void consume(reader, context, generation); + void consume(reader, painter, token); } catch (error) { stop(); onError('renderer-error', isRtcErrorRetryable(error)); } }, stop, - snapshot: () => ({ width, height, renderedFrames, presentedFrames: null }), + snapshot: () => ({ width, height, renderedFrames, presentedFrames: null, renderBackend: painter?.backend ?? null }), }; } // Video elements exist only for the two legacy renderers, and belong to one offer. export function createVideoRenderer(mode, canvas, { createVideo, onReady = () => {}, onResize = () => {}, onError = () => {}, + webgl = false, } = {}) { let video = null, copy = null, frameCallback = null, generation = 0, ready = false, started = false; let width = 0, height = 0, presentedFrames = null, renderedFrames = mode === 'video-canvas' ? 0 : null; @@ -559,6 +663,7 @@ export function createVideoRenderer(mode, canvas, { const token = generation, current = () => video !== null && token === generation; if (mode === 'video-canvas') { copy = createVideoCanvasRenderer(video, canvas, { + webgl, onDraw: () => { renderedFrames++; }, onFrame: report, onError: error => { if (current()) onError('renderer-error', isRtcErrorRetryable(error)); }, }); @@ -589,7 +694,7 @@ export function createVideoRenderer(mode, canvas, { } catch (error) { onError('renderer-error', isRtcErrorRetryable(error)); } }, stop, - snapshot: () => ({ width, height, presentedFrames, renderedFrames }), + snapshot: () => ({ width, height, presentedFrames, renderedFrames, renderBackend: copy?.backend ?? (mode === 'native' ? 'native' : null) }), }; } @@ -683,7 +788,7 @@ export function createRtcReceiver(createRenderer, { receivedFps: rate('framesReceived'), decodedFps: rate('framesDecoded'), presentedFps: presented !== null && previous?.presented !== null && elapsed ? Math.max(0, presented - previous.presented) * 1000 / elapsed : null, renderedFps: rendered !== null && previous?.rendered !== null && elapsed ? Math.max(0, rendered - previous.rendered) * 1000 / elapsed : null, - renderedFrames: rendered, + renderedFrames: rendered, renderBackend: output.renderBackend ?? null, presentedFrames: presented, decodeMs: averageMs('totalDecodeTime', 'framesDecoded'), jitterBufferMs: averageMs('jitterBufferDelay', 'jitterBufferEmittedCount'), @@ -884,12 +989,15 @@ if (typeof document !== 'undefined') { const screens = [screen, $('screen-next'), $('screen-third')]; const heldFrame = $('held-frame'); let video = null; - const canvas = $('video-canvas'); + // Context types cannot change on a canvas. Keep a separate, initially empty + // surface so users can toggle the experiment between display sessions. + const canvas2d = $('video-canvas'), canvasWebgl = $('video-canvas-webgl'); + let canvas = canvas2d; const display = $('display'); const settingsDialog = $('settings-dialog'); const touchSurface = $('touch-surface'); // Read-only diagnostic snapshots; counters never include pairing credentials or video payloads. - const rendererStats = { selectedMode: 'native', activeMode: 'waiting', renderedFps: null, renderedFrames: null, fallbackReason: null }; + const rendererStats = { selectedMode: 'native', activeMode: 'waiting', renderBackend: null, renderedFps: null, renderedFrames: null, fallbackReason: null }; const performanceStats = { source: null, transport: null, receiver: null, image: null, presentation: rendererStats }; const imageDebugStats = createImageDebugStats(); let debugRevision = null; @@ -969,12 +1077,20 @@ if (typeof document !== 'undefined') { }, }); const rendererPreference = 'wheelplay.videoRenderer'; + const webglPreference = 'wheelplay.canvasWebgl'; + const webglOption = $('canvas-webgl'); + webglOption.checked = false; + try { webglOption.checked = window.localStorage.getItem(webglPreference) === 'true'; } catch (_) {} + webglOption.addEventListener('change', () => { + try { window.localStorage.setItem(webglPreference, String(webglOption.checked)); } catch (_) {} + }); let selectedMode = 'native'; try { selectedMode = videoRendererPreference(window.localStorage.getItem(rendererPreference), window.localStorage.getItem('wheelplay.canvasVideo')); window.localStorage.setItem(rendererPreference, selectedMode); } catch (_) {} let launchMode = selectedMode; + let launchWebgl = false; const processorSupported = () => typeof MediaStreamTrackProcessor === 'function'; function updateRendererOptions() { for (const mode of ['native', 'video-canvas', 'track-canvas']) { @@ -982,6 +1098,7 @@ if (typeof document !== 'undefined') { $(`renderer-${mode}-card`).classList.toggle('selected', selectedMode === mode); } $('renderer-hint').hidden = selectedMode !== 'track-canvas' || processorSupported(); + webglOption.disabled = selectedMode === 'native'; } for (const mode of ['native', 'video-canvas', 'track-canvas']) { $(`renderer-${mode}`).addEventListener('change', () => { @@ -1330,6 +1447,7 @@ if (typeof document !== 'undefined') { performanceStats.source = performanceStats.transport = performanceStats.receiver = performanceStats.image = null; imageDebugStats.reset(); rendererStats.activeMode = 'waiting'; + rendererStats.renderBackend = null; } function exitLaunchFullscreen() { try { Promise.resolve(document.exitFullscreen()).catch(() => {}); } catch (_) {} @@ -1380,6 +1498,7 @@ if (typeof document !== 'undefined') { const ws = new WebSocket(`${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}/stream?${auth}&media=2&images=3&nowPlaying=1`); ws.binaryType = 'arraybuffer'; const mode = launchMode; + const webgl = launchWebgl; socket = ws; const connectTimer = setTimeout(() => ws.close(), 8000); ws.onopen = () => { @@ -1398,14 +1517,14 @@ if (typeof document !== 'undefined') { webpSupport.then(supported => { if (supported && socket === ws && ws.readyState === WebSocket.OPEN) send({ type: 'image-format', format: 'webp' }); }); - Object.assign(rendererStats, { selectedMode: mode, activeMode: 'waiting', renderedFps: null, renderedFrames: null, + Object.assign(rendererStats, { selectedMode: mode, activeMode: 'waiting', renderBackend: null, renderedFps: null, renderedFrames: null, fallbackReason: mode === 'track-canvas' && !processorSupported() ? 'processor-unsupported' : null }); rtc = createRtcReceiver(callbacks => { if (mode === 'track-canvas') { if (!processorSupported()) throw new Error('processor-unsupported'); - return createTrackCanvasRenderer(canvas, callbacks); + return createTrackCanvasRenderer(canvas, { ...callbacks, webgl }); } - return createVideoRenderer(mode, canvas, { ...callbacks, createVideo: createLegacyVideo }); + return createVideoRenderer(mode, canvas, { ...callbacks, webgl, createVideo: createLegacyVideo }); }, { send, onReset: () => { holdRtcFrame(); presentation.invalidate('rtc'); }, onResize: size => { if (socket === ws) showRtcSize(size); }, @@ -1419,12 +1538,14 @@ if (typeof document !== 'undefined') { if (socket !== ws) return; video = null; rtcFps = null; rendererStats.fallbackReason = reason; rendererStats.renderedFps = null; performanceStats.receiver = null; + rendererStats.renderBackend = null; presentation.invalidate('rtc'); }, onStats: (fps, metrics) => { const outputFps = mode === 'track-canvas' ? metrics.renderedFps : fps; rtcFps = outputFps === null ? null : Math.round(outputFps); performanceStats.receiver = metrics; rendererStats.renderedFps = metrics.renderedFps; rendererStats.renderedFrames = metrics.renderedFrames; + rendererStats.renderBackend = metrics.renderBackend; presentation.refresh(); }, }); @@ -1566,6 +1687,8 @@ if (typeof document !== 'undefined') { } closePinKeypad(); launchMode = selectedMode; + launchWebgl = selectedMode !== 'native' && webglOption.checked; + canvas = launchWebgl ? canvasWebgl : canvas2d; const generation = ++authGeneration; prepareLaunchPresentation(); stopQr(); $('connect').disabled = true; $('message').textContent = ''; @@ -1671,7 +1794,7 @@ if (typeof document !== 'undefined') { surface.addEventListener('pointercancel', cancel); surface.addEventListener('lostpointercapture', cancel); } - for (const surface of [...screens, canvas, touchSurface]) bindSurface(surface); + for (const surface of [...screens, canvas2d, canvasWebgl, touchSurface]) bindSurface(surface); window.addEventListener('blur', release); document.addEventListener('visibilitychange', () => { if (document.hidden) release(); presentation.refresh(); }); window.addEventListener('pagehide', () => disconnect(false)); diff --git a/common/src/main/assets/web/index.html b/common/src/main/assets/web/index.html index bd153be8..062c9e15 100644 --- a/common/src/main/assets/web/index.html +++ b/common/src/main/assets/web/index.html @@ -44,6 +44,7 @@

{{ui_settings}}

+

@@ -86,6 +87,7 @@

{{web_debug_title}}

+ diff --git a/common/src/main/assets/web/style.css b/common/src/main/assets/web/style.css index 0a05823f..47166d68 100644 --- a/common/src/main/assets/web/style.css +++ b/common/src/main/assets/web/style.css @@ -50,7 +50,7 @@ @media(max-width:959px){#pairing{max-width:620px;padding:28px 24px}#pairing h1{margin-bottom:24px}.setup-panel{padding-top:28px}.qr-image{width:224px;height:224px;margin-bottom:20px}.qr-pairing img{width:208px;height:208px}#connect-form{margin-top:24px}} @media(max-width:380px){#pairing{padding:24px 16px}.pin-inputs{gap:6px}#code-entry .pin-digit{height:52px;font-size:24px}} #touch-surface{position:absolute;inset:0;touch-action:none;user-select:none;-webkit-user-select:none;z-index:1}#touch-surface.stale{pointer-events:none} -#video-canvas{width:100%;height:100%;object-fit:contain;touch-action:none;user-select:none;-webkit-user-select:none;-webkit-touch-callout:none}#video-canvas.stale{opacity:.2;pointer-events:none} +#video-canvas,#video-canvas-webgl{width:100%;height:100%;object-fit:contain;touch-action:none;user-select:none;-webkit-user-select:none;-webkit-touch-callout:none}#video-canvas.stale,#video-canvas-webgl.stale{opacity:.2;pointer-events:none} /* Keep a layout box for decoding without placing a native video surface on screen. */ #video.offscreen-video{position:fixed;left:-10000px;top:0;width:1px;height:1px;pointer-events:none} .launch-option{display:flex;align-items:flex-start;gap:14px;width:fit-content;margin-top:24px;cursor:pointer}.launch-option input{appearance:none;flex:0 0 auto;width:24px;height:24px;margin:1px 0 0;padding:0;border-radius:6px;background:#1c2428;letter-spacing:0}.launch-option input:checked{border-color:#b4d9b1;background:#b4d9b1 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'%3E%3Cpath fill='%2317251b' d='m9.2 16.4-4.3-4.3 1.4-1.4 2.9 2.9 8.5-8.5 1.4 1.4z'/%3E%3C/svg%3E") center/20px no-repeat}.launch-option span{display:grid;gap:4px}.launch-option strong{font-size:15px;font-weight:600}.launch-option small{color:#98a59f;font-size:13px;line-height:1.5} diff --git a/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt b/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt index b59df082..9b289760 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt @@ -153,6 +153,8 @@ internal object BrowserStrings { UiR.string.web_display_video_with_canvas_experimental, "web_decode_video_offscreen_then_draw_it_on_a_canvas" to UiR.string.web_decode_video_offscreen_then_draw_it_on_a_canvas, + "web_canvas_webgl_title" to UiR.string.web_canvas_webgl_title, + "web_canvas_webgl_help" to UiR.string.web_canvas_webgl_help, "web_video_renderer_title" to UiR.string.web_video_renderer_title, "web_video_renderer_native" to UiR.string.web_video_renderer_native, "web_video_renderer_native_description" to UiR.string.web_video_renderer_native_description, diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fd14c827..427b6888 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,7 +39,11 @@ v3 可使用 WebP,最多两帧在途,并以媒体版本和帧序号精确 AC ## 浏览器三种渲染方式 -`native` 为默认原生 Video;`video-canvas` 使用离屏 Video 与可见 Canvas;实验性 `track-canvas` 使用主线程 `MediaStreamTrackProcessor({ track, maxBufferSize: 1 })` → `VideoFrame` → Canvas 2D。不新增网络消息、解码器或依赖。两个旧模式按需创建自己的 Video 元素,停止时移除;直接 Canvas 及其 JPEG / WebP 回退期间不创建 Video。 +`native` 为默认原生 Video;`video-canvas` 使用离屏 Video 与可见 Canvas;实验性 `track-canvas` 使用主线程 `MediaStreamTrackProcessor({ track, maxBufferSize: 1 })` → `VideoFrame` → Canvas。两种 Canvas 默认使用 Canvas 2D;连接页提供默认关闭的「Canvas 使用 WebGL(实验性)」开关。开启后直接上传浏览器的 Video / VideoFrame 至纹理,以全屏四边形绘制;WebGL 上下文不可用时使用 Canvas 2D。不新增网络消息、解码器或依赖。两个旧模式按需创建自己的 Video 元素,停止时移除;直接 Canvas 及其 JPEG / WebP 回退期间不创建 Video。 + +WebGL 每次启动创建一组纹理、顶点缓冲和着色器程序,逐帧复用对象,停止时删除资源;尺寸来自实际帧,旋转时更新画布和 viewport。保持视频新帧去重及 250 ms 回调停滞补偿;不在 JS 中读取像素或每帧创建 GPU 对象。首次上传及尺寸变化时检查 GL 错误,避免逐帧查询错误带来的同步开销。画布上下文类型在其生命周期内固定,因此页面为默认 2D 和 WebGL 实验各保留一个初始为空的画布,两种 Canvas 模式使用所选后端对应的画布。取得 WebGL 后的初始化或绘制失败沿用 JPEG / WebP 回退;此时同一画布无法改为 2D。上下文丢失会触发可恢复错误,允许恢复后的接收器重试。 + +`preserveDrawingBuffer=true` 保留跨任务复制 held-frame 的能力,带来额外缓冲开销;GPU 上下文丢失时旧 GPU 画面可能无法保留。上传和颜色转换的实际执行路径由浏览器决定。原有 Canvas 2D 也可能使用 GPU,WebGL 的帧率收益应在目标设备、相同输入下测量,尤其需区分解码速度与屏幕呈现速度。 接收器通过渲染器工厂创建适配器,调用 `start(track, streams)`、`stop()` 和 `snapshot()`,由适配器报告首次就绪、尺寸变化及错误。接收器持有 PeerConnection、远端轨道及恢复计时器;适配器持有 reader、绘制上下文、元素及绘制回调。关闭先使代次失效,再停止适配器、移除监听、停止远端轨道并关闭 PeerConnection;关闭前先复制当前画面至上游 held-frame,按媒体版本与恢复期限管理显示和触控;取消读取不阻塞重连,读取结束后释放锁。每个读出的 VideoFrame 均在 finally 中 close,包含错误和过期帧。 @@ -47,7 +51,9 @@ v3 可使用 WebP,最多两帧在途,并以媒体版本和帧序号精确 AC 偏好键 `wheelplay.videoRenderer` 保存三种模式;有效新值优先,否则旧 `wheelplay.canvasVideo=true` 迁移为 `video-canvas`,其余默认 native。只写新键并保留旧键;连接时固定模式,自动重连沿用。缺少 Processor 时跳过 rtc-start 并记录本地原因,运行失败使用现有 rtc-fallback,不改变用户选择。 -`wheelplayStats.presentation` 包含 selectedMode、activeMode(含 waiting / jpeg)、renderedFps、renderedFrames、fallbackReason。绘制指标为 Canvas 的成功提交次数,不代表屏幕刷新率;兼容 Canvas 的补偿重绘可能重复同一解码帧。receiver 保留现有统计字段并新增 renderedFps / renderedFrames,rtc-feedback 仍报告 WebRTC framesDecoded。 +WebGL 偏好键 `wheelplay.canvasWebgl` 仅在值为 `true` 时启用,无法读取存储时默认关闭;原生 Video 下禁用开关但保留选择。选择在用户启动显示时固定,自动重连沿用,修改在下一次手动连接生效。关闭后使用独立的 2D 画布,可在同一页面反复开关。 + +`wheelplayStats.presentation` 包含 selectedMode、activeMode(含 waiting / jpeg)、renderBackend、renderedFps、renderedFrames、fallbackReason。统计采样后 renderBackend 为 webgl、2d 或 native,未建立或已停止时为 null;receiver 统计也包含 renderBackend。绘制指标为 Canvas 的成功提交次数,不代表屏幕刷新率;兼容 Canvas 的补偿重绘可能重复同一解码帧。receiver 保留现有统计字段并新增 renderedFps / renderedFrames,rtc-feedback 仍报告 WebRTC framesDecoded。 ## 生命周期与资源约束 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 31e2a953..f1f81c18 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -557,3 +557,12 @@ run-1 捕获的第 12 个访问单元(索引 11)为: - `npm run test:browser:media`:真实 Chromium 在生产 CSP 下通过 seek 能力切换、最后一次拖动位置(42.123 秒 → 42123 毫秒)、暂停状态保持、切歌后的旧 seek 回调隔离,以及既有画面/触控/重连检查。回调由测试脚本触发,未验证车机的实体按键或系统进度条 UI。 - `:mobile:lintDebug`:0 错误、18 警告;`git diff --check` 通过。 - 未安装或操作真实车机/iPhone。手机音乐应用是否执行跳转、进度回报及本地/浏览器输出的声音连续性仍需实机验收。本次复用现有 CarPlay 音频链路,未根据猜测注入 AirPlay 缓冲清空命令。 + +## Canvas WebGL 绘制(2026-11-15) + +兼容 Canvas 与直接 Canvas 默认使用 Canvas 2D;连接页提供默认关闭的 WebGL 实验开关,偏好在当前浏览器保存,下次手动连接生效,自动重连沿用本次选择。原生 Video 下禁用开关。WebGL 不可用时使用 Canvas 2D,已取得 WebGL 后的失败沿用图片回退。为避免上下文类型冲突,2D 和 WebGL 实验使用独立的空画布。纹理、顶点缓冲及程序在会话内复用,停止时释放;保留新帧去重、回调停滞补绘、VideoFrame 释放和会话代次隔离。`wheelplayStats.presentation.renderBackend` 及 receiver 同名字段在采样后区分 webgl / 2d / native。 + +- `npm run check`、`npm test`:同步至最新 main 后通过,232 项 Node 测试。包含 GL 资源复用/释放、旋转、着色器失败、纹理尺寸限制、首次上传错误、上下文丢失与恢复许可、2D 回退,以及错误回调保帧和重启隔离;覆盖默认关闭、原生模式禁用、偏好存储失败、两个 Canvas 模式启用、同页开关和自动重连的选择隔离。 +- `npm run test:browser:webgl`:真实 Chromium / ANGLE SwiftShader 验证四象限颜色与方向、横竖屏、跨动画帧保留画面、HTMLVideoElement 上传、真实 VideoFrame 上传与 close、同一画布切换模式、上下文丢失与恢复及真实 2D 回退;验证页面两个画布上的 2D → WebGL → 2D 切换、开关默认状态和启用/关闭后刷新页面的偏好恢复。输入为合成色块,轨道消费使用测试 reader;未验证手机 WebRTC 吞吐或硬件 GPU 帧率。 +- `npm run test:browser:media`:同步至最新 main 后通过,包含图片回退、触控、重连、配对键盘、设置弹窗与 Media Session 回归。原生全屏未在当前浏览器环境进入,对应 OEM 全屏切换检查未执行。 +- 未操作车机/iPhone;未测量目标设备帧率、功耗或上传耗时。`preserveDrawingBuffer=true` 为沿用断线保帧契约,额外缓冲开销应纳入设备对比;此前 Canvas 2D 的耗时数据不代表 WebGL 性能。 diff --git a/docs/WEBRTC_RECOVERY.md b/docs/WEBRTC_RECOVERY.md index 3030ccb4..7d92f6fb 100644 --- a/docs/WEBRTC_RECOVERY.md +++ b/docs/WEBRTC_RECOVERY.md @@ -68,4 +68,4 @@ WebSocket 控制连接的心跳与重连规则独立于视频策略;整个网 首次成功绘制之前保持 JPEG,首帧等待 `max(10000ms, recoveryGraceMs)`。持续发送时分别检查解码和绘制进展,即使 framesDecoded 增长而画布不再更新,也会在所选容忍时间后回退。没有新输入的静态画面不触发渲染停滞;恢复输入时重新计时。连接短暂断开时仍保留最后一帧。 -`presentation.fallbackReason` 区分 processor-unsupported、renderer-error、track-ended、first-frame-timeout、decode-stalled、render-stalled、stats-unavailable、connection-failed、codec-unsupported 等原因;server-stop 表示服务端停止本次媒体。选择保留;可恢复错误沿用上游有界退避重试,缺少 Processor、Canvas 2D 或明确的平台不支持错误停止当前配置的重试,重新连接或配置变化后再尝试。直接 Canvas 状态栏使用 renderedFps,原生 Video 与兼容 Canvas 继续使用 presentedFps / decodedFps。 +`presentation.fallbackReason` 区分 processor-unsupported、renderer-error、track-ended、first-frame-timeout、decode-stalled、render-stalled、stats-unavailable、connection-failed、codec-unsupported 等原因;server-stop 表示服务端停止本次媒体。选择保留;可恢复错误沿用上游有界退避重试,缺少 Processor、可用 Canvas 绘制上下文或明确的平台不支持错误停止当前配置的重试,重新连接或配置变化后再尝试。直接 Canvas 状态栏使用 renderedFps,原生 Video 与兼容 Canvas 继续使用 presentedFps / decodedFps。 diff --git a/package.json b/package.json index 664799cd..08c5b66a 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "demo": "node scripts/preview.mjs --demo", "test": "node --test test/*.test.js", "test:browser:media": "node scripts/test-media-presentation.mjs", + "test:browser:webgl": "node scripts/test-media-presentation.mjs --webgl-only", "check": "node --check common/src/main/assets/web/app.js", "android:build": "./gradlew :mobile:assembleDebug", "android:test": "./gradlew :common:testDebugUnitTest" diff --git a/scripts/browser-webgl-checks.mjs b/scripts/browser-webgl-checks.mjs new file mode 100644 index 00000000..02e153dc --- /dev/null +++ b/scripts/browser-webgl-checks.mjs @@ -0,0 +1,117 @@ +// Runs inside Chromium via CDP. Synthetic color sources verify rendering, not +// phone/WebRTC throughput. Software ANGLE makes it reproducible without a GPU. +export async function verifyCanvasWebgl() { + const { createCanvasFramePainter, createVideoCanvasRenderer, createTrackCanvasRenderer } = await import('/app.js'); + const check = (condition, message) => { if (!condition) throw new Error(message); }; + const option = document.getElementById('canvas-webgl'); + check(!option.checked && option.disabled, 'WebGL must default off and be disabled for native Video'); + document.getElementById('renderer-video-canvas').click(); + check(!option.disabled, 'Canvas mode must enable the experiment option'); + option.click(); + check(option.checked && localStorage.getItem('wheelplay.canvasWebgl') === 'true', 'WebGL choice must persist'); + const tick = () => new Promise(resolve => requestAnimationFrame(resolve)); + const within = async (promise, stage) => { + let timer; + try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('WebGL check timed out: ' + stage)), 5000); })]); } + finally { clearTimeout(timer); } + }; + const source = document.createElement('canvas'), output = document.getElementById('video-canvas-webgl'); + const defaultCanvas = document.getElementById('video-canvas'); + const context = source.getContext('2d'); + function paint(width, height) { + source.width = width; source.height = height; + context.fillStyle = '#ff0000'; context.fillRect(0, 0, width / 2, height / 2); + context.fillStyle = '#00ff00'; context.fillRect(width / 2, 0, width / 2, height / 2); + context.fillStyle = '#0000ff'; context.fillRect(0, height / 2, width / 2, height / 2); + context.fillStyle = '#ffffff'; context.fillRect(width / 2, height / 2, width / 2, height / 2); + } + function colors(canvas) { + const copy = document.createElement('canvas'); copy.width = canvas.width; copy.height = canvas.height; + const ctx = copy.getContext('2d'); ctx.drawImage(canvas, 0, 0); + return [[.25, .25], [.75, .25], [.25, .75], [.75, .75]].map(([x, y]) => + [...ctx.getImageData(Math.floor(canvas.width * x), Math.floor(canvas.height * y), 1, 1).data]); + } + function verifyColors(canvas) { + const expected = [[255, 0, 0, 255], [0, 255, 0, 255], [0, 0, 255, 255], [255, 255, 255, 255]]; + const actual = colors(canvas); + check(actual.every((pixel, i) => pixel.every((v, channel) => Math.abs(v - expected[i][channel]) <= 8)), + 'Incorrect color, orientation or retained frame: ' + JSON.stringify(actual)); + } + let painter, videoRenderer, trackRenderer, stream, video; + try { + paint(64, 32); defaultCanvas.width = 64; defaultCanvas.height = 32; + const initial = createCanvasFramePainter(defaultCanvas); + check(initial.backend === '2d', 'Default must use Canvas 2D'); + initial.draw(source, 64, 32); verifyColors(defaultCanvas); initial.dispose(); + painter = createCanvasFramePainter(output, { webgl: true }); check(painter.backend === 'webgl', 'WebGL is required for this check'); + for (const [width, height] of [[64, 32], [32, 64]]) { + paint(width, height); output.width = width; output.height = height; + painter.draw(source, width, height); + await tick(); await tick(); verifyColors(output); // Recovery copies in a later task. + const frame = new VideoFrame(source, { timestamp: 0 }); + try { painter.draw(frame, width, height); verifyColors(output); } finally { frame.close(); } + } + painter.dispose(); + video = document.createElement('video'); video.muted = true; video.playsInline = true; document.body.append(video); + stream = source.captureStream(0); video.srcObject = stream; + const playing = video.play(); paint(source.width, source.height); stream.getVideoTracks()[0].requestFrame(); + await within(playing, 'video playback'); + const videoErrors = []; + videoRenderer = createVideoCanvasRenderer(video, output, { webgl: true, onError: error => videoErrors.push(error.message) }); + videoRenderer.start(); check(videoRenderer.backend === 'webgl', 'Video renderer must use WebGL'); + await tick(); await tick(); verifyColors(output); check(!output.hidden, 'Video must produce a visible frame'); + videoRenderer.stop(); check(videoErrors.length === 0, JSON.stringify(videoErrors)); + // The same visible canvas can change modes after stop. VideoFrame uploads + // use actual browser VideoFrames; this reader is a bounded test source. + const frame = new VideoFrame(source, { timestamp: 1 }); + let delivered = false, canceled = false, release; + let resolveReady; + const ready = new Promise(resolve => { resolveReady = resolve; }); + const trackErrors = []; + trackRenderer = createTrackCanvasRenderer(output, { + webgl: true, + onReady: resolveReady, onError: reason => trackErrors.push(reason), + createProcessor: () => ({ readable: { getReader: () => ({ + read() { + if (!delivered) { delivered = true; return Promise.resolve({ value: frame, done: false }); } + return new Promise(resolve => { release = resolve; }); + }, + cancel() { canceled = true; release?.({ done: true }); }, releaseLock() {}, + }) } }), + }); + trackRenderer.start({}); await within(ready, 'track readiness'); await tick(); await tick(); verifyColors(output); + check(trackRenderer.snapshot().renderBackend === 'webgl', 'Track renderer must use WebGL'); + check(frame.displayWidth === 0, 'Consumed VideoFrame must be closed'); + trackRenderer.stop(); check(canceled && trackErrors.length === 0, 'Track must stop cleanly'); + // A context loss follows the same stop/fallback signal as drawing errors, + // and restoration permits a new painter on the original canvas. + const lossErrors = []; + painter = createCanvasFramePainter(output, { webgl: true, onError: error => { lossErrors.push(error.message); painter.dispose(); } }); + const gl = output.getContext('webgl'), extension = gl.getExtension('WEBGL_lose_context'); + check(extension, 'Context loss extension required'); + const lost = new Promise(resolve => output.addEventListener('webglcontextlost', resolve, { once: true })); + extension.loseContext(); await within(lost, 'context loss'); check(lossErrors.length === 1, 'Loss must report once'); + // Restoration is legal after the cancelable loss event finishes dispatching. + await tick(); + const restored = new Promise(resolve => output.addEventListener('webglcontextrestored', resolve, { once: true })); + extension.restoreContext(); await within(restored, 'context restoration'); + painter = createCanvasFramePainter(output, { webgl: true }); output.width = source.width; output.height = source.height; + painter.draw(source, source.width, source.height); verifyColors(output); + // Force acquisition failure on a fresh canvas to exercise the browser's + // real 2D fallback, rather than accepting a skipped WebGL check. + const fallback = document.createElement('canvas'), getContext = fallback.getContext.bind(fallback); + fallback.getContext = (type, options) => type === 'webgl' ? null : getContext(type, options); + const cpu = createCanvasFramePainter(fallback, { webgl: true }); fallback.width = source.width; fallback.height = source.height; + cpu.draw(source, source.width, source.height); verifyColors(fallback); cpu.dispose(); + const disabled = createCanvasFramePainter(defaultCanvas); + defaultCanvas.width = source.width; defaultCanvas.height = source.height; + disabled.draw(source, source.width, source.height); verifyColors(defaultCanvas); disabled.dispose(); + return { backend: 'webgl', checks: ['quadrant colors and orientation', 'landscape and portrait', 'retained recovery frame', + 'HTMLVideoElement upload', 'VideoFrame upload and close', 'Canvas mode switching', 'context loss and restoration', 'Canvas 2D fallback', + 'default 2D, opt-in WebGL and return to 2D on page surfaces', 'experiment UI defaults and persistence'] }; + } finally { + painter?.dispose(); videoRenderer?.stop(); trackRenderer?.stop(); + stream?.getTracks().forEach(track => track.stop()); + if (video) { video.srcObject = null; video.remove(); } + } +} diff --git a/scripts/test-media-presentation.mjs b/scripts/test-media-presentation.mjs index b9e55470..9ab7c90d 100644 --- a/scripts/test-media-presentation.mjs +++ b/scripts/test-media-presentation.mjs @@ -8,6 +8,7 @@ import { spawn, spawnSync } from 'node:child_process'; import WebSocket, { WebSocketServer } from 'ws'; import ffmpeg from 'ffmpeg-static'; import { resourceStrings, renderBrowserTemplate } from './resource-strings.mjs'; +import { verifyCanvasWebgl } from './browser-webgl-checks.mjs'; const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); const messages = resourceStrings('values-zh-rCN'); @@ -15,6 +16,7 @@ const root = new URL('../common/src/main/assets/web/', import.meta.url); const androidServer = await readFile(new URL('../common/src/main/java/com/shilapi/xcertplay/web/LanWebServer.kt', import.meta.url), 'utf8'); const csp = androidServer.match(/"(default-src 'self';[^"\n]+)"/)[1]; const profile = await mkdtemp(join(tmpdir(), 'wheelplay-media-')); +const webglOnly = process.argv.includes('--webgl-only'); function image(width, height, codec = 'mjpeg') { const result = spawnSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', `testsrc=size=${width}x${height}:rate=1`, '-frames:v', '1', '-f', 'image2pipe', '-vcodec', codec, 'pipe:1']); @@ -97,7 +99,8 @@ wss.on('connection', ws => { }); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); -const browser = spawn(process.env.BROWSER_BIN || 'chromium', [...(process.env.WHEELPLAY_BROWSER_HEADFUL === '1' ? [] : ['--headless']), '--no-sandbox', '--disable-gpu', +const browser = spawn(process.env.BROWSER_BIN || 'chromium', [...(process.env.WHEELPLAY_BROWSER_HEADFUL === '1' ? [] : ['--headless']), '--no-sandbox', + ...(webglOnly ? ['--use-gl=angle', '--use-angle=swiftshader', '--enable-unsafe-swiftshader'] : ['--disable-gpu']), '--disable-background-networking', '--no-first-run', '--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank'], { stdio: ['ignore', 'ignore', 'ignore'] }); let devtools; @@ -150,6 +153,19 @@ try { await command('Runtime.enable'); await command('Page.navigate', { url: `http://127.0.0.1:${server.address().port}/` }); await wait(`!!window.wheelplayStats && !!document.getElementById('connect') && !document.getElementById('connect').disabled`); + if (webglOnly) { + const webgl = await evaluate(`(${verifyCanvasWebgl.toString()})()`); + await command('Page.reload'); + await wait(`!!window.wheelplayStats && document.getElementById('renderer-video-canvas').checked && + document.getElementById('canvas-webgl').checked && !document.getElementById('canvas-webgl').disabled`); + await evaluate(`document.getElementById('canvas-webgl').click(); document.getElementById('renderer-native').click()`); + await command('Page.reload'); + await wait(`!!window.wheelplayStats && document.getElementById('renderer-native').checked && + !document.getElementById('canvas-webgl').checked && document.getElementById('canvas-webgl').disabled`); + webgl.checks.push('saved opt-in and opt-out after page reload'); + assert.deepEqual(errors, []); + console.log(JSON.stringify({ passed: true, webgl }, null, 2)); + } else { await wait(`!document.getElementById('pair-qr').hidden && document.getElementById('pair-qr').naturalWidth > 0`); assert.equal(await evaluate(`document.getElementById('hide-toolbar') === null && document.getElementById('fullscreen-on-start') === null`), true); const screenshot = async name => { @@ -490,6 +506,7 @@ try { 'decode without overlay flashes', 'gap releases touch', 'deadline without status', 'cancelled deadline', 'same-size rebuild', 'cancelled mouse drag cannot cross video epochs', 'old frame rejection', 'rotation', 'real socket reconnect', 'native browser Media Session metadata and cover', 'media action roundtrip', 'desktop right drag, scroll and versioned keyboard', 'desktop blur releases modifiers', 'phone-authoritative pause', 'capability-gated seek and final drag position', 'media cleanup and retired callback isolation'] }, null, 2)); + } } finally { devtools?.close(); browser.kill('SIGTERM'); for (const client of wss.clients) client.terminate(); diff --git a/shared/src/main/res/values-zh-rCN/strings.xml b/shared/src/main/res/values-zh-rCN/strings.xml index 084f0c33..44dc3c40 100644 --- a/shared/src/main/res/values-zh-rCN/strings.xml +++ b/shared/src/main/res/values-zh-rCN/strings.xml @@ -602,4 +602,6 @@ 连接设置 正在取消连接 正在释放连接资源,请稍候。 + Canvas 使用 WebGL(实验性) + 默认关闭。适用于两种 Canvas 模式,下次连接生效。性能因设备而异,播放效果变差时请关闭。 diff --git a/shared/src/main/res/values-zh-rTW/strings.xml b/shared/src/main/res/values-zh-rTW/strings.xml index d557355f..3ab3e12b 100644 --- a/shared/src/main/res/values-zh-rTW/strings.xml +++ b/shared/src/main/res/values-zh-rTW/strings.xml @@ -602,4 +602,6 @@ 連線設定 正在取消連線 正在釋放連線資源,請稍候。 + Canvas 使用 WebGL(實驗性) + 預設關閉。適用於兩種 Canvas 模式,下次連線生效。效能因裝置而異,播放效果變差時請關閉。 diff --git a/shared/src/main/res/values/strings.xml b/shared/src/main/res/values/strings.xml index a2f40bd0..9bc9ebb5 100644 --- a/shared/src/main/res/values/strings.xml +++ b/shared/src/main/res/values/strings.xml @@ -602,4 +602,6 @@ Connection settings Cancelling connection Releasing the connection. Please wait. + Use WebGL for Canvas (experimental) + Off by default. Applies to both Canvas modes on your next connection. Performance depends on your device; turn off if playback worsens. diff --git a/test/canvas-frame-painter.test.js b/test/canvas-frame-painter.test.js new file mode 100644 index 00000000..f20123fe --- /dev/null +++ b/test/canvas-frame-painter.test.js @@ -0,0 +1,111 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createCanvasFramePainter } from '../common/src/main/assets/web/app.js'; + +test('default painter only requests Canvas 2D even when WebGL is available', () => { + const requests = [], source = {}, draws = []; + const canvas = { getContext(type) { + requests.push(type); assert.equal(type, '2d'); + return { drawImage: (...args) => draws.push(args) }; + } }; + const painter = createCanvasFramePainter(canvas); + painter.draw(source, 1280, 720); painter.dispose(); + assert.equal(painter.backend, '2d'); assert.deepEqual(requests, ['2d']); + assert.deepEqual(draws, [[source, 0, 0, 1280, 720]]); +}); + +function fixture({ compile = true, link = true } = {}) { + const calls = [], listeners = new Map(); + let lost = false, error = 0, id = 0; + const gl = Object.fromEntries(['VERTEX_SHADER', 'FRAGMENT_SHADER', 'COMPILE_STATUS', 'LINK_STATUS', 'ARRAY_BUFFER', + 'STATIC_DRAW', 'FLOAT', 'TEXTURE0', 'TEXTURE_2D', 'TEXTURE_MIN_FILTER', 'TEXTURE_MAG_FILTER', 'LINEAR', + 'TEXTURE_WRAP_S', 'TEXTURE_WRAP_T', 'CLAMP_TO_EDGE', 'MAX_TEXTURE_SIZE', 'RGBA', 'UNSIGNED_BYTE', 'TRIANGLE_STRIP', 'HIGH_FLOAT'] + .map((key, index) => [key, index + 1])); + gl.NO_ERROR = 0; + for (const name of ['shaderSource', 'compileShader', 'attachShader', 'linkProgram', 'useProgram', 'bindBuffer', + 'bufferData', 'enableVertexAttribArray', 'vertexAttribPointer', 'activeTexture', 'bindTexture', 'texParameteri', + 'uniform1i', 'viewport', 'texImage2D', 'drawArrays', 'deleteTexture', 'deleteBuffer', 'deleteProgram', 'deleteShader']) { + gl[name] = (...args) => calls.push([name, ...args]); + } + for (const name of ['createProgram', 'createShader', 'createBuffer', 'createTexture']) { + gl[name] = () => { const object = { id: ++id }; calls.push([name, object]); return object; }; + } + Object.assign(gl, { + isContextLost: () => lost, getShaderParameter: () => compile, getProgramParameter: () => link, + getAttribLocation: () => 0, getUniformLocation: () => ({}), getParameter: () => 4096, + getShaderPrecisionFormat: () => ({ precision: 23 }), + getError() { calls.push(['getError']); return error; }, + }); + const canvas = { + getContext(type, options) { calls.push(['getContext', type, options]); return type === 'webgl' ? gl : null; }, + addEventListener(type, fn) { if (!listeners.has(type)) listeners.set(type, new Set()); listeners.get(type).add(fn); }, + removeEventListener(type, fn) { listeners.get(type)?.delete(fn); }, + }; + return { gl, canvas, calls, listeners, setLost: value => { lost = value; }, setError: value => { error = value; }, + lose() { + lost = true; + const event = { prevented: false, preventDefault() { this.prevented = true; } }; + for (const fn of [...(listeners.get('webglcontextlost') || [])]) fn(event); + return event; + }, + count: name => calls.filter(call => call[0] === name).length, + }; +} + +test('WebGL uploads sources directly and reuses GPU objects across frames and rotations', () => { + const f = fixture(), painter = createCanvasFramePainter(f.canvas, { webgl: true }), first = {}, second = {}; + assert.equal(painter.backend, 'webgl'); + const options = f.calls.find(call => call[0] === 'getContext')[2]; + assert.equal(options.antialias, false); assert.equal(options.depth, false); assert.equal(options.preserveDrawingBuffer, true); + painter.draw(first, 1280, 720); painter.draw(second, 1280, 720); painter.draw(second, 720, 1280); + assert.equal(f.count('createTexture'), 1); assert.equal(f.count('createBuffer'), 1); assert.equal(f.count('createProgram'), 1); + assert.deepEqual(f.calls.filter(call => call[0] === 'texImage2D').map(call => call.at(-1)), [first, second, second]); + assert.equal(f.count('drawArrays'), 3); assert.equal(f.count('getError'), 2); + assert.deepEqual(f.calls.filter(call => call[0] === 'viewport').at(-1), ['viewport', 0, 0, 720, 1280]); + painter.dispose(); painter.dispose(); + assert.equal(f.count('deleteTexture'), 1); assert.equal(f.count('deleteBuffer'), 1); + assert.equal(f.count('deleteProgram'), 1); assert.equal(f.count('deleteShader'), 2); + assert.throws(() => painter.draw(first, 1280, 720), /context lost/); + const restarted = createCanvasFramePainter(f.canvas, { webgl: true }); restarted.draw(first, 1280, 720); restarted.dispose(); + assert.equal(f.count('drawArrays'), 4); +}); + +test('shader and link failures release partially allocated GPU resources', () => { + for (const options of [{ compile: false }, { link: false }]) { + const f = fixture(options); + assert.throws(() => createCanvasFramePainter(f.canvas, { webgl: true }), /WebGL/); + assert.equal(f.count('deleteProgram'), 1); + assert.equal(f.count('deleteShader'), options.compile === false ? 1 : 2); + } +}); + +test('loss reports once, disposes safely and keeps restoration enabled after stop', () => { + const f = fixture(), errors = []; + const painter = createCanvasFramePainter(f.canvas, { webgl: true, onError: error => { errors.push(error); painter.dispose(); } }); + assert.equal(f.lose().prevented, true); assert.equal(errors.length, 1); + f.lose(); assert.equal(errors.length, 1); + f.setLost(false); + const next = createCanvasFramePainter(f.canvas, { webgl: true }); next.draw({}, 1280, 720); + // draw can detect loss before its asynchronous event arrives. + f.setLost(true); assert.throws(() => next.draw({}, 1280, 720), /context lost/); next.dispose(); + assert.equal(f.lose().prevented, true); +}); + +test('unsupported texture sizes and GL upload errors cannot count as successful frames', () => { + const f = fixture(), painter = createCanvasFramePainter(f.canvas, { webgl: true }); + assert.throws(() => painter.draw({}, 8192, 720), { name: 'NotSupportedError' }); + assert.equal(f.count('texImage2D'), 0); + f.setError(1282); assert.throws(() => painter.draw({}, 1280, 720), /upload failed/); + painter.dispose(); +}); + +test('WebGL absence uses opaque Canvas 2D without pixel readback', () => { + const calls = [], source = {}; + const canvas = { getContext(type, options) { + calls.push([type, options]); + return type === '2d' ? { drawImage: (...args) => calls.push(args) } : null; + } }; + const painter = createCanvasFramePainter(canvas, { webgl: true }); painter.draw(source, 720, 1280); painter.dispose(); + assert.equal(painter.backend, '2d'); assert.deepEqual(calls.at(-1), [source, 0, 0, 720, 1280]); + assert.deepEqual(calls[1], ['2d', { alpha: false }]); +}); diff --git a/test/stream-client.test.js b/test/stream-client.test.js index 2a70772f..13eb80b9 100644 --- a/test/stream-client.test.js +++ b/test/stream-client.test.js @@ -8,15 +8,17 @@ const messages = resourceStrings('values-zh-rCN'); const source = ['text.js', 'desktop-input.js', 'browser-audio.js', 'browser-location.js', 'browser-media-session.js', 'app.js'] .map(name => readFileSync(new URL(`../common/src/main/assets/web/${name}`, import.meta.url), 'utf8') .replace(/^import .*;$/gm, '').replaceAll('export function', 'function')).join('\n'); -function fixture({ storedCanvas = null, storedMode = null, processorAvailable = true, storedIgnoreZoom = null, storageDisabled = false, canvasAvailable = true, supportsWebp = false, +function fixture({ storedCanvas = null, storedMode = null, storedWebgl = null, processorAvailable = true, storedIgnoreZoom = null, storageDisabled = false, canvasAvailable = true, supportsWebp = false, geometry = null, visual = null, mediaSession = null, locationBrowser = null, storedDebug = null, rtcStats = null, fullscreen = 'supported', codeDenied = false, resume = () => ({ token: 'session-token', name: 'Chrome' }) } = {}) { + const contextRequests = []; const elements = new Map(), timers = new Map(), raf = new Map(), sockets = [], peers = [], draws = []; const preferences = new Map(storedCanvas === null ? [] : [['wheelplay.canvasVideo', storedCanvas]]); if (storedIgnoreZoom !== null) preferences.set('wheelplay.ignoreBrowserZoom', storedIgnoreZoom); if (storedDebug !== null) preferences.set('wheelplay.debugOverlay', storedDebug); const videoCallbacks = new Map(), createdVideos = [], processors = []; if (storedMode !== null) preferences.set('wheelplay.videoRenderer', storedMode); + if (storedWebgl !== null) preferences.set('wheelplay.canvasWebgl', storedWebgl); const documentEvents = {}, fullscreenCalls = []; const visualEvents = {}; const visualViewport = visual ? { ...visual, addEventListener(type, fn) { visualEvents[type] = fn; } } : undefined; @@ -25,11 +27,14 @@ function fixture({ storedCanvas = null, storedMode = null, processorAvailable = let nextTimer = 0, nextFrame = 0, clockNow = 1000; const element = id => { const classes = new Set(); - if (!elements.has(id)) elements.set(id, { hidden: id === 'video-canvas', complete: true, value: '123456', handlers: {}, naturalWidth: 1280, naturalHeight: 720, + if (!elements.has(id)) elements.set(id, { hidden: ['video-canvas', 'video-canvas-webgl'].includes(id), complete: true, value: '123456', handlers: {}, naturalWidth: 1280, naturalHeight: 720, checked: false, videoWidth: 1280, videoHeight: 720, readyState: 2, currentTime: 0, style: { removeProperty(name) { delete this[name]; } }, parentElement: { getBoundingClientRect: () => mainRect }, - getContext: () => canvasAvailable ? { drawImage: (...args) => draws.push(args) } : null, + getContext(type) { + contextRequests.push([id, type]); + return type === '2d' && canvasAvailable ? { drawImage: (...args) => draws.push(args) } : null; + }, play: async () => {}, requestVideoFrameCallback(fn) { const frame = ++nextFrame; videoCallbacks.set(frame, fn); return frame; }, cancelVideoFrameCallback(frame) { videoCallbacks.delete(frame); }, @@ -117,7 +122,7 @@ function fixture({ storedCanvas = null, storedMode = null, processorAvailable = }; context.wheelplayMessages = messages; vm.runInNewContext(source, context); - return { element, sockets, timers, peers, draws, preferences, raf, videoCallbacks, document, fullscreenCalls, createdVideos, processors, visualViewport, visualEvents, + return { element, sockets, timers, peers, draws, preferences, contextRequests, raf, videoCallbacks, document, fullscreenCalls, createdVideos, processors, visualViewport, visualEvents, reportViewport() { const [id, timer] = [...timers].find(([, t]) => t.delay === 180); timers.delete(id); timer.fn(); @@ -423,6 +428,62 @@ test('default WebRTC presents native video without allocating a Canvas', async ( assert.equal(ws.sent.at(-1).type, 'rtc-ready'); }); +test('WebGL defaults off and is available only for Canvas modes', async () => { + const f = fixture(), option = f.element('canvas-webgl'); + assert.equal(option.checked, false); assert.equal(option.disabled, true); + f.select('video-canvas'); assert.equal(option.disabled, false); + f.select('track-canvas'); assert.equal(option.disabled, false); + const ws = await f.connect(); await f.offer(ws); await f.decoded(); + assert.equal(f.element('video-canvas').hidden, false); + assert.equal(f.element('video-canvas-webgl').hidden, true); + assert.ok(f.contextRequests.every(([, type]) => type === '2d')); +}); + +test('WebGL preference is explicit, persists and safely defaults off when storage is denied', async () => { + for (const mode of ['video-canvas', 'track-canvas']) { + const f = fixture({ storedMode: mode, storedWebgl: 'true' }); + assert.equal(f.element('canvas-webgl').checked, true); + const ws = await f.connect(); await f.offer(ws); + if (mode === 'track-canvas') await f.decoded(); + assert.ok(f.contextRequests.some(([id, type]) => id === 'video-canvas-webgl' && type === 'webgl')); + assert.equal(f.element('video-canvas').hidden, true); + assert.equal(f.element('video-canvas-webgl').hidden, false); // Unsupported WebGL falls back to 2D. + const option = f.element('canvas-webgl'); option.checked = false; option.handlers.change(); + assert.equal(f.preferences.get('wheelplay.canvasWebgl'), 'false'); + option.checked = true; option.handlers.change(); assert.equal(f.preferences.get('wheelplay.canvasWebgl'), 'true'); + } + assert.equal(fixture({ storedWebgl: 'invalid' }).element('canvas-webgl').checked, false); + const denied = fixture({ storedWebgl: 'true', storageDisabled: true }); + assert.equal(denied.element('canvas-webgl').checked, false); + denied.element('canvas-webgl').checked = true; assert.doesNotThrow(() => denied.element('canvas-webgl').handlers.change()); +}); + +test('a saved WebGL preference does not change native Video rendering', async () => { + const f = fixture({ storedWebgl: 'true' }), ws = await f.connect(); await f.offer(ws); + assert.equal(f.element('canvas-webgl').disabled, true); + assert.equal(f.createdVideos.length, 1); assert.equal(f.contextRequests.length, 0); + assert.equal(f.element('video-canvas-webgl').hidden, true); +}); + +test('changing WebGL takes effect at manual launch while reconnects keep the active selection', async () => { + const f = fixture({ storedMode: 'track-canvas' }); + let ws = await f.connect(); await f.offer(ws); await f.decoded(); + const cpu = f.element('video-canvas'), gpu = f.element('video-canvas-webgl'), option = f.element('canvas-webgl'); + option.checked = true; option.handlers.change(); + ws.close(); f.advance(1000); await new Promise(resolve => setImmediate(resolve)); + ws = f.sockets.at(-1); ws.readyState = 1; ws.onopen(); await f.offer(ws); await f.decoded(); + assert.equal(cpu.hidden, false); assert.equal(gpu.hidden, true); + assert.ok(f.contextRequests.every(([, type]) => type === '2d')); + f.element('settings-disconnect').handlers.click(); + ws = await f.connect(); await f.offer(ws); await f.decoded(); + assert.equal(cpu.hidden, true); assert.equal(gpu.hidden, false); + f.element('settings-disconnect').handlers.click(); option.checked = false; option.handlers.change(); + ws = await f.connect(); await f.offer(ws); await f.decoded(); + assert.equal(cpu.hidden, false); assert.equal(gpu.hidden, true); + f.pointer('pointerdown', 100, 'video-canvas'); f.pointer('pointerup', 100, 'video-canvas'); + assert.equal(ws.sent.filter(message => message.type === 'touch').length, 2); +}); + test('a recoverable WebRTC disconnect preserves video and touch and reports recovery', async () => { const f = fixture(), ws = await f.connect(); await f.offer(ws); const peer = f.peers.at(-1); @@ -669,7 +730,7 @@ test('automatic RTC retry preserves JPEG and touch until the new video frame is test('a transient Canvas draw error permits a later retry to restore video', async () => { const f = fixture({ storedCanvas: 'true' }), ws = await f.connect(); mediaStatus(ws); const canvas = f.element('video-canvas'), getContext = canvas.getContext; - canvas.getContext = () => ({ drawImage() { throw new Error('temporary surface loss'); } }); + canvas.getContext = type => type === '2d' ? { drawImage() { throw new Error('temporary surface loss'); } } : null; await f.offer(ws, 1, { sessionId: 7, videoEpoch: 1 }); assert.equal(ws.sent.at(-1).type, 'rtc-fallback'); assert.notEqual(ws.sent.at(-1).retryable, false); @@ -906,7 +967,7 @@ test('Direct Canvas failures freeze the last frame until the fallback image is d const f = fixture({ storedMode: 'track-canvas' }), ws = await f.connect(); const canvas = f.element('video-canvas'), held = f.element('held-frame'); let failDraw = false, copied = false; - canvas.getContext = () => ({ drawImage() { if (failDraw) throw new Error('draw failed'); } }); + canvas.getContext = type => type === '2d' ? { drawImage() { if (failDraw) throw new Error('draw failed'); } } : null; held.getContext = () => ({ drawImage(source) { assert.equal(source, canvas); assert.equal(source.hidden, false); assert.equal(source.width, 1280); assert.equal(source.height, 720); diff --git a/test/track-canvas.test.js b/test/track-canvas.test.js index ce114d51..28c60af6 100644 --- a/test/track-canvas.test.js +++ b/test/track-canvas.test.js @@ -6,7 +6,7 @@ const flush = () => new Promise(resolve => setImmediate(resolve)); function fixture({ contextAvailable = true, constructError = false, onError = () => {} } = {}) { const draws = [], ready = [], sizes = [], errors = [], readers = [], configurations = []; const context = { drawImage: (...args) => draws.push(args) }; - const canvas = { hidden: true, width: 0, height: 0, getContext: () => contextAvailable ? context : null }; + const canvas = { hidden: true, width: 0, height: 0, getContext: type => type === '2d' && contextAvailable ? context : null }; const renderer = createTrackCanvasRenderer(canvas, { onReady: size => ready.push(size), onResize: size => sizes.push(size), onError: reason => { errors.push(reason); onError(reason); }, @@ -41,7 +41,7 @@ test('processor bounds its queue, waits for valid output, and reports first draw assert.equal(f.ready.length, 1); assert.equal(f.canvas.hidden, false); await f.frame(720, 1280); assert.deepEqual(f.sizes, [{ width: 720, height: 1280 }]); - assert.deepEqual(f.renderer.snapshot(), { width: 720, height: 1280, renderedFrames: 2, presentedFrames: null }); + assert.deepEqual(f.renderer.snapshot(), { width: 720, height: 1280, renderedFrames: 2, presentedFrames: null, renderBackend: '2d' }); }); test('stop cancels without waiting; a late frame is released without drawing into a new session', async () => { diff --git a/test/video-canvas.test.js b/test/video-canvas.test.js index dd6ce794..69a29067 100644 --- a/test/video-canvas.test.js +++ b/test/video-canvas.test.js @@ -2,7 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { createVideoCanvasRenderer } from '../common/src/main/assets/web/app.js'; -function fixture({ frameCallbacks = true, quality = true } = {}) { +function fixture({ frameCallbacks = true, quality = true, onError = () => {} } = {}) { const animation = new Map(), callbacks = new Map(), draws = [], sizes = [], errors = [], classes = new Set(); let sequence = 0, clock = 0, frames = 1; const video = { hidden: true, readyState: 2, videoWidth: 1280, videoHeight: 720, currentTime: 0, @@ -14,9 +14,9 @@ function fixture({ frameCallbacks = true, quality = true } = {}) { } if (quality) video.getVideoPlaybackQuality = () => ({ totalVideoFrames: frames }); const context = { drawImage: (...args) => draws.push(args) }; - const canvas = { width: 0, height: 0, hidden: true, getContext: () => context }; + const canvas = { width: 0, height: 0, hidden: true, getContext: type => type === '2d' ? context : null }; const renderer = createVideoCanvasRenderer(video, canvas, { - onFrame: (width, height) => sizes.push([width, height]), onError: error => errors.push(error), now: () => clock, + onFrame: (width, height) => sizes.push([width, height]), onError: error => { errors.push(error); onError(error); }, now: () => clock, requestFrame(callback) { const id = ++sequence; animation.set(id, callback); return id; }, cancelFrame: id => animation.delete(id), }); @@ -92,3 +92,22 @@ test('context failures and later draw errors stop both callback loops and report assert.equal(f.errors.length, 1); assert.equal(f.animation.size, 0); assert.equal(f.callbacks.size, 0); assert.equal(f.canvas.hidden, true); assert.equal(f.classes.has('offscreen-video'), false); }); + +test('drawing failures allow the receiver to freeze the previous picture before clearing it', () => { + let captured; + const f = fixture({ onError() { captured = { hidden: f.canvas.hidden, width: f.canvas.width, height: f.canvas.height }; } }); + f.renderer.start(); f.context.drawImage = () => { throw new Error('draw failed'); }; f.frame(33); + assert.deepEqual(captured, { hidden: false, width: 1280, height: 720 }); + assert.equal(f.canvas.width, 0); assert.equal(f.canvas.hidden, true); +}); + +test('error cleanup cannot stop a renderer restarted by the error callback', () => { + let draw; + const f = fixture({ onError() { f.context.drawImage = draw; f.renderer.start(); } }); + f.renderer.start(); draw = f.context.drawImage; + f.context.drawImage = () => { throw new Error('draw failed'); }; f.frame(33); + assert.equal(f.errors.length, 1); assert.equal(f.canvas.hidden, false); + assert.equal(f.callbacks.size, 1); assert.equal(f.animation.size, 1); + const count = f.draws.length; f.frame(66); assert.equal(f.draws.length, count + 1); + f.renderer.stop(); +});