diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..5effb49ef --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,239 @@ +# AGENTS.md + +Operational notes for working with Recordly from source. Learned the hard way during a +build-from-source + dev-run session on Windows. Kept practical. + +## Project shape + +- Electron app (screen recorder + editor). Renderer is React + Vite; `electron/main.ts` + compiles to `dist-electron/main.cjs`. +- Versioning is semantic-ish; lots of prerelease bets. Real cadence: fire-hose of small + point releases Nov 2025–Apr 2026, then batched big-feature releases (v1.3.x). There is + NO CHANGELOG file — derive user-facing diffs from `git log ..HEAD`. +- Release process (RELEASING.md) is heavy: signed + notarized macOS (x64+arm64), + Authenticode Windows NSIS, Linux AppImage, Homebrew tap automation. Don't expect fast + stables; they batch fixes into bigger cuts. + +## Build / run from source + +``` +npm ci # runs postinstall which builds native helper C/C++ modules +npm run dev # vite-plugin-electron: starts Vite + launches Electron (dev) +``` + +Key scripts (package.json): `dev`, `build:platform-native-helpers`, `build:whisper-runtime`, +`build:windows-capture`, `build:windows-gpu-export`, `build:nvidia-cuda-compositor`, +`build:cursor-monitor`, `test` (vitest), `lint`/`format` (biome). + +### Windows gotchas (real, hit in this repo) + +1. Native helper compilation needs the MSVC toolchain. On this machine: + `C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\...` with MSVC 19.44. +2. CMake is NOT on PATH. It ships inside VS Build Tools at: + `C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin` + Add it to PATH before any cmake-using build: + `export PATH="/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin:$PATH"` +3. `scripts/build-whisper-runtime.mjs` `findCmake()` only probes + `C:\Program Files\Microsoft Visual Studio\...` — it never checks the `(x86)` path used by + Build Tools-only installs. Pre-existing bug: on such systems it reports "CMake not found". + Workaround: put the (x86) cmake dir on PATH. (Also accepts `WHISPER_RUNTIME_ALLOW_MISSING=1` + to skip auto-caption support.) +4. The same script downloads whisper.cpp, then calls `tar -xzf` — but `execFileSync("tar",...)` + resolves to Windows' system `bsdtar`, which fails on the project path ("Cannot connect to C:"). + Workaround: pre-extract into the expected dir so `ensureSourceTree()` short-circuits: + ``` + cd .tmp/whisper-runtime && tar -xzf v1.8.4.tar.gz -C src-v1.8.4 + ``` + (git-bash tar reads the archive fine; the Windows bsdtar path handling is the problem.) +5. The dev-mode app uses an ISOLATED data dir: `C:\Users\nmz\AppData\Roaming\Recordly-dev` + (note the `-dev` suffix), separate from the installed/released app's + `C:\Users\nmz\AppData\Roaming\Recordly`. Running from source will not touch released-app + recordings/settings. + +### Uninstalling the released app (Windows) + +- Per-user NSIS install lives at `C:\Users\nmz\AppData\Local\Programs\Recordly` + (case-insensitive = also listed as `recordly`). +- Run the bundled `Uninstall Recordly.exe` silently: + `powershell -NoProfile -Command "Start-Process -FilePath '.\Uninstall Recordly.exe' -ArgumentList '/S' -Wait"` +- The uninstaller deletes the install dir + start-menu shortcut but preserves user data in + `C:\Users\nmz\AppData\Roaming\Recordly` (recordings, settings) — call that out to users. +- After uninstalling, an empty `Programs\Recordly` dir may remain; `rmdir` it. + +## Dev-run lifecycle + +- `npm run dev` blocks and is tied to its shell. To background it, use nohup + log file; + on next launch it reconnects to the existing window/vite process. +- Kill the process tree to stop; relaunch with `npm run dev` (CMake PATH only needed again + if rebuilding whisper). +- No auto-update in dev (that's installer-only via electron-updater). + +## Export codec & bitrate (durable rules) + +Recordly supports persistent custom MP4 bitrate control and opt-in H.265/HEVC export with +Auto/Hardware/CPU encoder policies. H.264 + Auto remains the default/unchanged compatibility +path. These rules are durable � preserved across refactors: + +- H.264 + Auto is the compatibility/default path. It preserves the existing + WebCodecs/native static-layout/Breeze routing and the automatic bitrate heuristic. Do not + disturb it. +- Explicit H.264 Hardware/CPU requests continue to use the native FFmpeg `rawvideo` route. + This does not change the H.264 + Auto compatibility path. +- Eligible HEVC Auto/Hardware native-layout jobs may use the dedicated codec-aware NVIDIA CUDA + compositor. It performs GPU decode, compose, and encode with NV12 surfaces and `hevc_nvenc`. + HEVC must never enter the existing H.264-only D3D11, `gpu-export-probe`, or legacy + `nvidia-cuda-compositor` assumptions; the generalized codec-aware compositor is a distinct + safe route. +- The native-layout CUDA source contract remains a decodable H.264/AVC source in a supported + MP4, M4V, or MOV container. Other source codecs or containers are normalized to a validated + H.264/yuv420p proxy before the native graph. Source preparation constraints do not change + the requested output codec. +- The codec-aware CUDA route requires a successful Windows/NVIDIA runtime probe and a helper + built with CMake 3.24+, MSVC/C++17, CUDA Toolkit with CUDA language support, and the + NVIDIA Video Codec SDK samples (`NvCodec`, including `nvcuvid.lib`). Build it with + `npm run build:nvidia-cuda-compositor`; set `RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT` when + the SDK is not at `.tmp/video-sdk-samples`. Missing requirements make this route + unavailable; do not substitute an H.264-only GPU probe or compositor. +- After any required source-proxy preparation, the native FFmpeg/CUDA graph keeps video pixels + on the GPU through decode, NV12 composition, and encode. It must not require renderer + readback of video pixels. +- HEVC CPU remains native FFmpeg `rawvideo` with `libx265`. Unsupported canvas-only effects, + an ineligible source/layout, or an unavailable CUDA route use renderer raw frames. HEVC + Auto may try codec-aware CUDA hardware (`hevc_nvenc`), then native rawvideo hardware, then + CPU according to the encoder resolution rules. Hardware never silently falls back to CPU; + CPU uses the software encoder only. +- On Windows, `hevc_nvenc` is the preferred HEVC hardware encoder for the CUDA route and when + native raw probing succeeds; other raw hardware candidates are `hevc_qsv`, `hevc_amf`, and + `hevc_mf`. macOS uses `hevc_videotoolbox`. Linux uses `hevc_nvenc`/`hevc_qsv`. +- `libx265` is the CPU HEVC fallback; `libx264` is the H.264 CPU encoder. +- Encoder resolution: Auto = try eligible GPU hardware, then native raw hardware candidates, + then CPU; Hardware = hardware candidates only and MUST NOT silently fall back to CPU (fails + with an actionable error); CPU = software encoder only. +- Persist only high-level user preferences (codec enum, encoder preference, bitrate mode, + custom Mbps), NEVER derived bitrate-in-bps, NEVER WebCodecs codec strings (avc1.*), NEVER + FFmpeg encoder names (libx265/hevc_nvenc). +- HEVC output is export-only; playback/source decoding/preview are intentionally unchanged. Do + not route HEVC through the WebCodecs AVC muxer, the H.264 stream-copy path, or any legacy + H.264-only native-layout route. +- Key integration points (optional but helpful): src/lib/exporter/types.ts + (ExportSettings/ExportConfig contract + EXPORT_BITRATE_* constants), + src/lib/exporter/exportBitrate.ts (resolveExportBitrate), + src/components/video-editor/mp4ExportRouting.ts (codec-aware route and needsNativeRawFrame), + src/lib/exporter/modernVideoExporter.ts (native-layout/CUDA selection and raw fallback), + electron/ipc/nativeVideoExport.ts (getNativeEncoderCandidates / + buildNativeVideoExportArgs), electron/ipc/export/nativeStaticLayoutRoutePlan.ts + (codec-aware native-layout route selection), and electron/ipc/export/native-video.ts + (source proxy preparation, CUDA route, and resolveNativeVideoEncoder). + +## Native raw-frame transport (durable rules) + +- H.264 + Auto remains the compatibility path: WebCodecs/Annex-B transport and its + existing routing are unchanged. +- HEVC is not universally a renderer-raw job. Eligible Auto/Hardware native-layout jobs may + use the distinct codec-aware NVIDIA CUDA compositor. Renderer raw frames are used for HEVC + CPU, native raw hardware fallback, unsupported canvas-only effects, and unavailable or + ineligible CUDA routes. HEVC must not use the H.264 WebCodecs/Annex-B path or any legacy + H.264-only native static-layout compositor. +- The preferred internal renderer-raw transport is a negotiated native-frame `MessagePort` + stream. The stream has an explicit protocol/version, sequence ordering, fixed-size frame + validation, acknowledgements, and settlement for cancellation, errors, and port closure. + Byte and frame queues are bounded to provide backpressure; producers must not grow either + queue without limit. +- Electron 43 may fail to deliver transferable `ArrayBuffer` values through + `MessagePortMain`. The cloned `ipcRenderer.send` path remains the safe fallback when + transferable delivery is unavailable or fails. +- Transport metrics must distinguish transferable-stream traffic from cloned-IPC traffic. + They are diagnostic only and must not claim physical zero-copy unless that behavior has been + measured. +- Renderer raw RGBA frames are canonical top-down. Renderer and FFmpeg integration must not + add duplicate vertical flips. The CUDA compositor's internal NV12 surfaces are not a direct + canvas transport contract. +- Transport mode, transport bytes, buffer counts, and selected encoder names are diagnostic + values only. Never persist them as user settings; transport details are not persisted. +- The upstream target is a generic Electron transferable-`ArrayBuffer` + renderer-to-main binary IPC / `MessagePortMain` transferable-resource fix. Use + `scripts/benchmark-native-frame-transport.mjs` and + `docs/upstream/electron-transferable-frame-ipc.md` to support that target. +- Direct canvas-to-NV12 transport is intentionally not assumed until runtime support is proven; + renderer raw RGBA remains the canonical fallback contract. + +## Native overlay-layer composition (durable rules) + +- The FFmpeg CUDA static-layout route owns source-video decode, layout composition, and output + encoding. Browser code must not send source-video RGBA frames for this route. The current + bundled FFmpeg build cannot alpha-compose RGBA/YUVA overlays with `overlay_cuda`; effectful + overlay jobs therefore use FFmpeg's CPU alpha overlay after CUDA decode/scale and before + NVENC, while effect-free jobs retain the existing all-GPU graph. +- Browser export code may prepare temporary transparent RGBA overlay sidecars for cursor, + captions, annotations, webcam, and frame visuals. Overlay sidecars are export-session data, + are never persisted, and must be deleted after the native export attempt. +- Overlay layers are ordered explicitly and must match the output dimensions, frame rate, and + duration. Alpha and z-order must be preserved; a native command must never omit a requested + layer silently. +- The generalized NVIDIA CUDA compositor (run-mp4-pipeline.mjs -> main.cu) ALSO accepts the + renderer-prepared RGBA overlay sidecar through an `--overlay-manifest` JSON (layers with + path/x/y/width/height/frameCount). It streams the raw RGBA frames with a double-buffered + prefetch (bounded: two device buffers + one pinned host buffer per layer) and alpha-blends + them top-down ON TOP of the composed, blurred video — so cursor/captions/annotations stay + sharp. When overlay layers are present the renderer bakes cursor+webcam into the sidecar, + so the CUDA route must NOT also draw the native cursor atlas for that export. +- Temporal zoom motion blur is implemented in the CUDA compositor from the renderer-resolved + plan (`--temporal-blur-sample-count N --temporal-blur-shutter-fraction F + --temporal-blur-weight-power P`, mirroring src/lib/exporter/temporalMotionBlur.ts). Each + output frame is re-composited at the symmetric cos-tapered shutter sample offsets with the + camera transform interpolated from the zoom telemetry, then accumulated into one bounded + scratch NV12 buffer. Temporal blur replaces the spatial blur for that frame (the renderer's + temporal path also disables the velocity filters). If the CUDA route is unavailable, + temporal blur stays an explicit non-duplicated skip (`unsupported-temporal-motion-blur`) + rather than being silently dropped by the FFmpeg overlay route. +- Blur annotations, extension render hooks, and timeline mappings remain raw-fallback cases + until their native representation is implemented and validated. A failed or incomplete + overlay preparation must return to the renderer raw-frame route. +- The custom SDK NVENC compositor is optional for effectful exports. The validated FFmpeg CUDA + path remains authoritative when the SDK wrapper cannot initialize on a supported NVIDIA + system. H.264 + Auto compatibility routing remains unchanged. + +## NVIDIA CUDA compositor stream sync + NVENC capability rules (durable) + +- All composite/zoom-blur/overlay kernels run on a single non-blocking compositor stream + (`cudaStreamNonBlocking`). There is exactly ONE per-frame synchronization + (`cudaStreamSynchronize(copyStream)`) before NVENC's synchronous input copy; there must + never be a per-frame global `cudaDeviceSynchronize` in the encode loop (the only remaining + `cudaDeviceSynchronize` is the one-time prewarm). The CUDA primary context + (`cuDevicePrimaryCtxRetain`) is used so the CUDA runtime allocations and NVENC share one + context; it is released with `cuDevicePrimaryCtxRelease`, never `cuCtxDestroy`. +- NVENC is configured minimal-first from a capability probe, never from the (often empty) + preset config returned by `nvEncGetEncodePresetConfig`: explicit encodeGUID/presetGUID, + tuningInfo (P1/P4/P6 presets are required on Blackwell and must pair with + `NV_ENC_TUNING_INFO_HIGH_QUALITY`), chromaFormatIDC=1 for NV12, VBR when the device lists it + (CBR fallback), custom VBV/AQ only when the caps report them. The build uses the FFmpeg + nv-codec-headers nvEncodeAPI.h 13.x (pinned tag n13.0.19.1), NOT the legacy 8.1 header in + the Video Codec SDK samples checkout, which fails with NV_ENC_ERR_INVALID_PARAM (error 8) + on current drivers; scripts/build-nvidia-cuda-compositor.mjs stages the headers and patches + the samples NvEncoder.cpp for API-13 compatibility. +- Capability/version diagnostics (deviceName, driver/CUDA/SDK versions, compute capability, + per-codec support, RC modes, custom VBV, async, temporal AQ, WxH/MB-per-sec caps) are + reported in the summary `nvencDiagnostics` and in the failure JSON (`noCpuFallback:true`). + The compositor never claims a codec, rate-control mode, AQ, or VBV feature the probe or + live-encoder caps did not confirm (`rcModeUsed`/`customVbvUsed`/`aqUsed` show what was + applied). A failed CUDA helper must not silently route zoom-blur-with-overlay or temporal + blur through CPU FFmpeg; the renderer rejects non-CUDA result routes and native-video.ts + throws when those effects are requested but CUDA cannot run. +- Stage metrics (decode, overlay upload, composite GPU, zoom blur GPU, overlay blend GPU, + NVENC) are reported in the summary and PROGRESS intervals so the bottleneck can be proven + rather than assumed. Renderer-reported FPS is only labeled native when the helper measured + it; the stale native FPS is cleared when a static-layout attempt falls back to raw frames. +- Strict HEVC Hardware policy (durable): when `exportEncoderPreference === "hardware"` with + HEVC, the generalized NVIDIA CUDA compositor is the ONLY acceptable route. Any static-layout + skip reason, CUDA capability/IPC failure, route mismatch, helper failure, or post-validation + failure MUST hard-fail the export with an actionable error (including the first skip reason + and `noCpuFallback:true`) and MUST NOT fall back to the renderer raw frame path + (WebGPU/WebGL -> FFmpeg hevc_nvenc), Breeze, or CPU. The cursor atlas is only required when + the cursor is NOT baked into the transparent overlay sidecar; a missing atlas must never + skip the CUDA route for overlay exports (this previously forced the slow ~17 FPS renderer + raw path). HEVC Auto and H.264 Auto keep their existing fallback behavior. +- Decision observability: every static-layout decision logs `[VideoExporter] Native + static-layout decision` (codec/preference/experimental flags, canUseNativeGpuStaticLayout, + shouldTry, shouldDefer), `[native-export] NVIDIA CUDA availability` (available/skipReason/ + hasNvidiaGpu/hasWrapper), `[VideoExporter] Native static layout skipped|selected` + (reason + route + flags), and the strict-policy guards log the skip reasons before throwing. diff --git a/docs/upstream/electron-transferable-frame-ipc.md b/docs/upstream/electron-transferable-frame-ipc.md new file mode 100644 index 000000000..f89801646 --- /dev/null +++ b/docs/upstream/electron-transferable-frame-ipc.md @@ -0,0 +1,142 @@ +# Electron transferable ArrayBuffer renderer-to-main IPC + +Status: benchmark and narrow upstream proposal. This document is intentionally not a +Recordly product or frame-pipeline design. + +## Reproducer + +Run from the repository root: + +```text +node scripts/benchmark-native-frame-transport.mjs +``` + +The default run sends four payloads per route at each size, with two unacknowledged +payloads allowed at once: + +```text +RECORDLY_IPC_BENCH_ITERATIONS=4 +RECORDLY_IPC_BENCH_WINDOW=2 +RECORDLY_IPC_BENCH_TIMEOUT_MS=120000 +``` + +The script always exercises 1 MiB and 33 MiB payloads. `RECORDLY_IPC_BENCH_ITERATIONS` +and `RECORDLY_IPC_BENCH_WINDOW` can be lowered for a quick smoke run. The fixture is +created under the operating system temporary directory, sets Electron `userData` and +`sessionData` inside that fixture, and is removed on exit. It does not load Recordly, +write project files, or use Recordly application data. Set +`RECORDLY_IPC_BENCH_KEEP_TEMP=1` only when inspecting the temporary fixture. + +The fixture uses a hidden, disposable BrowserWindow. Main creates a +`MessageChannelMain`, transfers one endpoint with `webContents.postMessage`, and the +renderer sends a two-byte transferable probe before the benchmark. If Electron cannot +create the channel, transfer the port, detach the probe ArrayBuffer, or deliver the +probe to `MessagePortMain`, the script exits with an explanatory unavailable message +instead of silently substituting another transport. + +Each payload is checked in main and acknowledged. The report includes: + +| Field | Meaning | +| --- | --- | +| Electron version, Node version, platform, arch | Runtime identity for comparison | +| throughput | Total logical payload bytes divided by elapsed time, in MiB/s | +| ACK latency | Average, median, and p95 time from post to main ACK receipt, in ms | +| buffer ownership | Whether `ArrayBuffer.byteLength` became zero immediately after a transfer-list post | +| peak in-flight payload bytes | Largest sum of unacknowledged logical payload sizes; with defaults this is 2 MiB or 66 MiB | +| physicalZeroCopy | Explicitly reports that physical zero-copy was not measured or guaranteed | + +The legacy route calls `ipcRenderer.send("legacy-frame", { payload })` without a +transfer list. The stream route calls `MessagePort.postMessage(message, [payload])`. +Both routes receive an ACK after main validates the payload. Ownership transfer is +reported separately from physical copying: a detached sender buffer proves an +ownership handoff, not that the OS, Chromium, Electron, or V8 avoided every physical +copy. + +A quick invocation is useful for launch/resource diagnostics, but it is not a stable +performance sample: + +```text +RECORDLY_IPC_BENCH_ITERATIONS=1 RECORDLY_IPC_BENCH_WINDOW=1 node scripts/benchmark-native-frame-transport.mjs +``` + +On shells without an Electron display/runtime, or on a version that cannot carry the +probe across `MessagePortMain`, the expected result is a nonzero exit with the reason +on stderr. That is a capability failure, not a claim that the legacy route is faster. + +## Relevant Electron API facts + +These facts are based on the Electron APIs used by the reproducer and the Electron +43.1.0 declarations installed in this checkout: + +1. `ipcRenderer.send(channel, ...args)` serializes arguments with the Structured Clone + Algorithm. It has no transfer-list parameter. Passing an ArrayBuffer this way is a + normal IPC serialization path; the renderer retains its ArrayBuffer ownership. +2. `ipcRenderer.postMessage(channel, message, transfer)` is the Electron API for + transferring `MessagePort` objects to main. Its documented transfer resources are + MessagePorts, not a general renderer-to-main raw ArrayBuffer streaming primitive. +3. `webContents.postMessage(channel, message, transfer)` can transfer + `MessagePortMain` objects to a renderer. The received endpoint is a native DOM + `MessagePort` and must be started before queued messages are delivered. +4. `MessageChannelMain` creates `MessagePortMain` endpoints. `MessagePortMain` exposes + `on("message")`, `start()`, `close()`, and `postMessage()`. The type contract for + its transfer argument is `MessagePortMain[]`; support for an ArrayBuffer transfer + from a renderer endpoint through this Electron boundary must therefore be tested, + not assumed from the browser MessagePort API alone. +5. Browser/DOM MessagePort APIs define ownership transfer for transferable objects in + a transfer list. Electron's process boundary still has to preserve that resource; + an API that accepts the call is not by itself proof that the bytes travelled + physically zero-copy. + +References: + +- Electron IPC renderer API: https://www.electronjs.org/docs/latest/api/ipc-renderer +- Electron webContents API: https://www.electronjs.org/docs/latest/api/web-contents +- Electron MessageChannelMain API: https://www.electronjs.org/docs/latest/api/message-channel-main +- Electron MessagePortMain API: https://www.electronjs.org/docs/latest/api/message-port-main +- Electron IPC tutorial and Structured Clone notes: https://www.electronjs.org/docs/latest/tutorial/ipc +- MDN `MessagePort.postMessage`: https://developer.mozilla.org/en-US/docs/Web/API/MessagePort/postMessage + +## Narrow upstream proposal + +Electron should provide one of the following supported, documented capabilities: + +A. A high-throughput renderer-to-main binary IPC primitive that accepts transferable + ArrayBuffers (or an equivalent explicitly transferable binary resource). It should + define ownership after submission, ordering, lifecycle/error behavior, capability + detection, and a usable backpressure signal. It should not require an application to + encode binary payloads as JSON or rely on undocumented structured-clone behavior. + +B. Fix `MessagePortMain` transferable resources so a renderer DOM MessagePort can send + an ArrayBuffer in its transfer list and the paired `MessagePortMain` receives the + intact ArrayBuffer with the expected detached sender state. The fix should include + cross-platform tests, declarations, and API documentation that state exactly which + resources are supported. + +Either option should document that ownership transfer and physical zero-copy are +separate guarantees. Ownership transfer can eliminate a sender-side usable reference +while the implementation still copies bytes internally. Physical zero-copy would need +an explicit implementation guarantee and measurement; it must not be inferred from a +successful `postMessage` or a detached ArrayBuffer. + +The proposal is deliberately limited to Electron's generic IPC/resource contract. It +does not prescribe application frame formats, video codecs, editor behavior, or any +other Recordly-specific business logic. + +## Current limitations + +This benchmark measures end-to-end renderer-post-to-main-ACK behavior, not allocator +copies, RSS, page faults, GPU mappings, or kernel transfers. Its peak in-flight metric +is a logical unacknowledged-payload watermark, not total process memory. Payloads use +sentinel bytes for integrity and do not model a particular media format. Results are +sensitive to Electron build, OS, scheduler, renderer process state, payload window, and +iteration count. + +In this checkout, the quick smoke command reached the `MessagePortMain` probe on +Electron 43.1.0 / Windows but the main side received no data for the transferred +ArrayBuffer. The script exited nonzero with an explanatory capability error and did not +publish partial throughput numbers. This is the unsupported-resource case the +reproducer is intended to expose. + +If the transferable probe fails, the benchmark intentionally stops before presenting a +partial legacy-versus-MessagePort comparison. This keeps unsupported runtime behavior +visible and avoids silently degrading the experiment to a copied path. diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..b61ec21e8 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -136,6 +136,37 @@ interface RendererNativeStaticLayoutChunkMetric { outputBytes: number; fallbackReason?: string; windowsGpuSummary?: RendererWindowsGpuExportSummary; + nvidiaCudaSummary?: { + success?: boolean; + outputCodec?: "h264" | "hevc"; + }; +} + +interface RendererNativeTiledOverlayTileRecord { + tileIndex: number; + byteOffset: number; + byteLength: number; +} + +interface RendererNativeTiledOverlayLayerDescriptor { + id: string; + order: number; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + tileSize: 128; + pixelFormat: "rgba"; + payloadPath: string; + payloadByteLength: number; + staticTiles: readonly RendererNativeTiledOverlayTileRecord[]; + frameDeltas: readonly { + frameIndex: number; + changedTiles: readonly RendererNativeTiledOverlayTileRecord[]; + }[]; } interface RendererNativeStaticLayoutMetrics extends RendererFfmpegAudioMuxMetrics { @@ -147,6 +178,15 @@ interface RendererNativeStaticLayoutMetrics extends RendererFfmpegAudioMuxMetric fallbackChunkCount: number; videoOnlyBytes?: number; chunks: RendererNativeStaticLayoutChunkMetric[]; + tiledOverlayLayers?: number; + tiledOverlayBlendFrames?: number; + changedTileCount?: number; + uploadedTileBytes?: number; + cachedTileCount?: number; + rawFallbackReason?: string; + overlayHostReadMs?: number; + overlayH2DEnqueueMs?: number; + overlayCacheHits?: number; } interface RendererNativeStaticLayoutProgress { @@ -156,6 +196,8 @@ interface RendererNativeStaticLayoutProgress { elapsedMs?: number; averageFps?: number; instantFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; intervalMs?: number; intervalFrames?: number; intervalDecodeWallMs?: number; @@ -352,12 +394,28 @@ interface Window { nativeStaticLayoutExport: (options: { sessionId?: string; inputPath: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; width: number; height: number; frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; durationSec: number; + overlayLayers?: Array<{ + id: string; + order: number; + path: string; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + pixelFormat: "rgba"; + }>; + tiledOverlayLayers?: RendererNativeTiledOverlayLayerDescriptor[]; contentWidth: number; contentHeight: number; offsetX: number; @@ -399,7 +457,20 @@ interface Window { anchorY: number; aspectRatio: number; }>; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; + temporalBlur?: { + sampleCount: number; + shutterFraction: number; + weightCurvePower: number; + } | null; timelineSegments?: Array<{ sourceStartMs: number; sourceEndMs: number; @@ -425,6 +496,14 @@ interface Window { }) => Promise<{ success: boolean; tempPath?: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; + route?: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; encoderName?: string; error?: string; metrics?: RendererNativeStaticLayoutMetrics; @@ -442,20 +521,35 @@ interface Window { bitrate: number; encodingMode: "fast" | "balanced" | "quality"; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; }) => Promise<{ success: boolean; sessionId?: string; encoderName?: string; error?: string; }>; + nativeVideoExportOpenFrameChannel: (sessionId: string) => Promise<{ + success: boolean; + error?: string; + fallbackAvailable?: boolean; + }>; + nativeVideoExportWriteFrameViaChannel: ( + sessionId: string, + frameData: Uint8Array, + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; + nativeVideoExportWriteFramesViaChannel: ( + sessionId: string, + frameDataList: Uint8Array[], + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportWriteFrame: ( sessionId: string, frameData: Uint8Array, - ) => Promise<{ success: boolean; error?: string }>; + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportWriteFrames: ( sessionId: string, frameDataList: Uint8Array[], - ) => Promise<{ success: boolean; error?: string }>; + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportFinish: ( sessionId: string, options?: { @@ -735,7 +829,7 @@ interface Window { deleteRecordingFile: (filePath: string) => Promise<{ success: boolean; error?: string }>; getLocalMediaUrl: ( filePath: string, - ) => Promise<{ success: true; url: string } | { success: false }>; + ) => Promise<{ success: true; url: string; pending?: boolean } | { success: false }>; saveProjectFile: ( projectData: unknown, suggestedName?: string, diff --git a/electron/ipc/export/exportStream.test.ts b/electron/ipc/export/exportStream.test.ts index d8889437d..d9dd1ce9c 100644 --- a/electron/ipc/export/exportStream.test.ts +++ b/electron/ipc/export/exportStream.test.ts @@ -125,6 +125,21 @@ describe("exportStream", () => { ); }); + it("allows the lossless tiled payload extension and persists its stream", async () => { + const { streamId, tempPath } = await openExportStream({ extension: "tiledrgba" }); + openedTempPaths.push(tempPath); + + expect(tempPath.endsWith(".tiledrgba")).toBe(true); + expect(hasExportStream(streamId)).toBe(true); + + const tileBytes = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]); + await writeToExportStream(streamId, 0, tileBytes); + const result = await closeExportStream(streamId); + expect(result.tempPath).toBe(tempPath); + expect(result.bytesWritten).toBe(tileBytes.byteLength); + expect(Array.from(await readBytes(tempPath))).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + }); + it("rejects an extension that would escape the temp directory", async () => { await expect(openExportStream({ extension: "mp4/../etc/passwd" })).rejects.toThrow( /Invalid export stream extension/, @@ -135,5 +150,13 @@ describe("exportStream", () => { await expect(openExportStream({ extension: "MP4" })).rejects.toThrow( /Invalid export stream extension/, ); + // Arbitrary/long non-alphanumeric extensions must stay rejected even though + // the length cap was raised to accommodate descriptive payload names. + await expect(openExportStream({ extension: "tiled.rgba" })).rejects.toThrow( + /Invalid export stream extension/, + ); + await expect(openExportStream({ extension: "tiledrgba.exe" })).rejects.toThrow( + /Invalid export stream extension/, + ); }); }); diff --git a/electron/ipc/export/exportStream.ts b/electron/ipc/export/exportStream.ts index d0aac4067..e9225b966 100644 --- a/electron/ipc/export/exportStream.ts +++ b/electron/ipc/export/exportStream.ts @@ -17,7 +17,11 @@ type ExportStreamSession = { const exportStreamSessions = new Map(); -const EXTENSION_ALLOWLIST = /^[a-z0-9]{1,8}$/; +// Strict lowercase-alphanumeric allowlist. The charset (no dots, slashes, +// path separators, or uppercase) is the security control: it defeats path +// traversal and arbitrary extension injection. The length cap is generous +// enough for descriptive lossless payload extensions (e.g. "tiledrgba"). +const EXTENSION_ALLOWLIST = /^[a-z0-9]{1,16}$/; const SESSION_DIR_PREFIX = "recordly-export-"; // Paths that the export pipeline itself produced (stream temp files plus any diff --git a/electron/ipc/export/native-video.test.ts b/electron/ipc/export/native-video.test.ts index ee31dcdd7..8465b7d2a 100644 --- a/electron/ipc/export/native-video.test.ts +++ b/electron/ipc/export/native-video.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("electron", () => ({ app: { @@ -26,7 +26,11 @@ const fsMocks = vi.hoisted(() => ({ writeFile: vi.fn(async () => undefined), readFile: vi.fn(), stat: vi.fn(async () => ({ size: 5_000_000_000 })), + realpath: vi.fn(async (pathValue: string) => pathValue), unlink: vi.fn(async () => undefined), + mkdir: vi.fn(async () => undefined), + rm: vi.fn(async () => undefined), + copyFile: vi.fn(async () => undefined), })); vi.mock("node:fs/promises", () => ({ @@ -46,17 +50,29 @@ vi.mock("node:child_process", () => ({ spawn: vi.fn(), })); +import { spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; import { app } from "electron"; +import type { NativeTiledOverlayLayerDescriptor } from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; import { buildExperimentalNvidiaCudaStaticLayoutArgs, buildExperimentalWindowsGpuStaticLayoutArgs, + buildNativeStaticLayoutOverlayManifest, buildNativeStaticLayoutSourceProxyArgs, + buildNativeStaticLayoutTiledOverlayManifest, buildNativeStaticLayoutTimelineSegments, buildNativeVideoAudioMuxArgs, + buildNvidiaCudaPrepareProgress, canCopyAudioCodecIntoMp4, + cancelInFlightCapabilityOnlyPrewarms, + canReuseNativeStaticLayoutSourceProbe, + exportNativeStaticLayoutVideo, + formatNativeStaticLayoutZoomTelemetryLines, getExperimentalNvidiaCudaExportSkipReason, getNativeExportCapabilities, getNativeGpuCompositorStallTimeoutMs, + getNativeStaticLayoutOverlayExpectedSidecarBytes, + getNativeStaticLayoutRawFrameFallbackReason, getNativeStaticLayoutSourceProxyBitrate, getNvidiaCudaAudioExportSkipReason, getNvidiaCudaAutoStallTimeoutMs, @@ -64,7 +80,9 @@ import { hasNvidiaGpuDeviceInGpuInfo, mapNvidiaCudaWrapperProgressPercentage, muxExportedVideoAudioBuffer, + muxNativeVideoExportAudio, type NativeStaticLayoutExportOptions, + type NativeStaticLayoutOverlayLayer, normalizeNativeStaticLayoutBackground, parseFfmpegDurationSeconds, parseFfmpegFrameRate, @@ -74,11 +92,23 @@ import { parseNvidiaCudaExportSummary, parseWindowsGpuExportProgressLine, parseWindowsGpuExportSummary, + prewarmNativeExportCaches, + registerCapabilityOnlyPrewarmChild, + resetNativeStaticLayoutSourceProbeCache, + resetNvidiaCudaAvailabilityCache, resolveExperimentalNvidiaCudaExportScriptPath, + resolveNativeStaticLayoutFpsFields, + resolveNvidiaCudaCursorAssets, + resolveNvidiaCudaNativeFps, + resolveNvidiaCudaNativeSummaryMetrics, + resolveNvidiaCudaOverlaySidecarSummaryMetrics, + resolveNvidiaCudaStrictHevcHardFail, + resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics, shouldCreateNativeStaticLayoutSourceProxy, validateNativeStaticLayoutSourceProxyMetadata, validateNativeVideoStreamStats, validateNvidiaCudaExportSummary, + validateNvidiaCudaStageMetricInvariants, validateWindowsGpuExportSummary, } from "./native-video"; @@ -114,6 +144,13 @@ function resetFsAccessMock() { }); } +// The NVIDIA CUDA availability cache is session-scoped; reset it after every +// test so the mocked wrapper/GPU probes from one test never leak into another. +afterEach(() => { + resetNvidiaCudaAvailabilityCache(); + resetNativeStaticLayoutSourceProbeCache(); +}); + function createNvidiaCudaSkipOptions( overrides: Partial = {}, ): NativeStaticLayoutExportOptions { @@ -268,6 +305,145 @@ describe("native static-layout source proxy", () => { }); }); +describe("native static-layout source probe cache", () => { + const baseMetadata = { + width: 1920, + height: 1080, + duration: 45, + frameRate: 30, + codec: "h264 (High)", + hasAudio: true, + audioCodec: "aac", + }; + + function baseEntry() { + return { + identity: { + canonicalPath: "C:\\recordings\\session.mp4", + device: 1, + inode: 987654, + size: 5_000_000_000, + mtimeMs: 1_700_000_000_000, + ctimeMs: 1_700_000_000_000, + }, + requestedCodec: "hevc", + encodingMode: "quality", + encoderPreference: "hardware", + metadata: { ...baseMetadata }, + }; + } + + function baseCurrent() { + return { + canonicalPath: "C:\\recordings\\session.mp4", + device: 1, + inode: 987654, + size: 5_000_000_000, + mtimeMs: 1_700_000_000_000, + ctimeMs: 1_700_000_000_000, + requestedCodec: "hevc", + encodingMode: "quality", + encoderPreference: "hardware", + }; + } + + it("reuses a probe only on an exact identity and route match", () => { + expect(canReuseNativeStaticLayoutSourceProbe(baseEntry(), baseCurrent())).toBe(true); + }); + + it("never reuses when there is no cached entry", () => { + expect(canReuseNativeStaticLayoutSourceProbe(undefined, baseCurrent())).toBe(false); + }); + + it("invalidates on canonical path mismatch (never path-only reuse from another file)", () => { + const entry = baseEntry(); + entry.identity.canonicalPath = "C:\\recordings\\other.mp4"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on device mismatch", () => { + const entry = baseEntry(); + entry.identity.device = 2; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on inode mismatch", () => { + const entry = baseEntry(); + entry.identity.inode = 111; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on size change (mutation)", () => { + const entry = baseEntry(); + entry.identity.size = 5_000_000_001; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on mtime change (mutation)", () => { + const entry = baseEntry(); + entry.identity.mtimeMs = 1_700_000_100_000; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on ctime change (mutation)", () => { + const entry = baseEntry(); + entry.identity.ctimeMs = 1_700_000_100_000; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the requested output codec changes (changed settings)", () => { + const entry = baseEntry(); + entry.requestedCodec = "h264"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the encoding mode changes (changed settings)", () => { + const entry = baseEntry(); + entry.encodingMode = "balanced"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the encoder preference changes (changed settings)", () => { + const entry = baseEntry(); + entry.encoderPreference = "cpu"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("never reuses a probe with an unknown source codec (re-probe or fail closed)", () => { + const entry = baseEntry(); + entry.metadata = { ...baseMetadata, codec: "unknown" }; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("never reuses a probe with an empty source codec", () => { + const entry = baseEntry(); + entry.metadata = { ...baseMetadata, codec: "" }; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on any current route mismatch even when identity matches", () => { + const entry = baseEntry(); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + encodingMode: "speed", + }), + ).toBe(false); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + requestedCodec: "h264", + }), + ).toBe(false); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + encoderPreference: "cpu", + }), + ).toBe(false); + }); +}); + describe("getNvidiaCudaAudioExportSkipReason", () => { it("allows video-only CUDA exports by default", () => { withNvidiaCudaAudioOverride(undefined, () => { @@ -410,6 +586,38 @@ describe("getNativeExportCapabilities", () => { process.platform === "win32" ? true : null, ); }); + + it("keeps CUDA available when the GPU probe is inconclusive so the live helper decides", async () => { + const envName = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + const originalEnv = process.env[envName]; + const originalIsPackaged = electronAppMock.isPackaged; + electronAppMock.isPackaged = true; + electronAppMock.getGPUInfo.mockRejectedValue(new Error("GPU info unavailable")); + fsMocks.access.mockResolvedValue(undefined); + delete process.env[envName]; + + try { + const capabilities = await getNativeExportCapabilities(); + if (process.platform === "win32") { + expect(capabilities.nvidiaCuda.available).toBe(true); + expect(capabilities.nvidiaCuda.hasNvidiaGpu).toBeNull(); + expect(capabilities.nvidiaCuda.skipReason).toBeNull(); + } else { + expect(capabilities.nvidiaCuda.available).toBe(false); + expect(capabilities.nvidiaCuda.skipReason).toBe("not-windows"); + } + } finally { + if (originalEnv === undefined) { + delete process.env[envName]; + } else { + process.env[envName] = originalEnv; + } + electronAppMock.isPackaged = originalIsPackaged; + electronAppMock.getGPUInfo.mockReset(); + electronAppMock.getGPUInfo.mockResolvedValue({ gpuDevice: [] }); + resetFsAccessMock(); + } + }); }); describe("getExperimentalNvidiaCudaExportSkipReason", () => { @@ -552,6 +760,119 @@ describe("getExperimentalNvidiaCudaExportSkipReason", () => { }); }); +describe("NVIDIA CUDA availability cache", () => { + afterEach(() => { + resetNvidiaCudaAvailabilityCache(); + }); + + it("reuses the wrapper-path and GPU probes across capability and route queries", async () => { + await withPackagedCudaCandidate( + { gpuDevice: [{ vendorId: 0x10de, deviceString: "NVIDIA GeForce GTX 1650" }] }, + async () => { + const first = await getNativeExportCapabilities(); + const second = await getNativeExportCapabilities(); + const route = await getExperimentalNvidiaCudaExportSkipReason( + createNvidiaCudaSkipOptions({ experimentalNvidiaCudaExport: true }), + ); + + if (process.platform === "win32") { + // Two capabilities + a route decision share a single GPU probe. + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(1); + expect(route).toBeNull(); + } else { + expect(route).toBe("not-windows"); + } + expect(second.nvidiaCuda).toEqual(first.nvidiaCuda); + }, + ); + }); + + it("invalidates the cache when a relevant environment override changes the helper path", async () => { + const scriptEnv = "RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT"; + const exportEnv = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + const originalScript = process.env[scriptEnv]; + const originalExport = process.env[exportEnv]; + const originalIsPackaged = electronAppMock.isPackaged; + const customScript = "C:\\custom\\nvidia\\run-mp4-pipeline.mjs"; + electronAppMock.isPackaged = true; + electronAppMock.getGPUInfo.mockResolvedValue({ + gpuDevice: [{ vendorId: 0x10de, deviceString: "NVIDIA GeForce GTX 1650" }], + }); + delete process.env[scriptEnv]; + delete process.env[exportEnv]; + + try { + fsMocks.access.mockResolvedValue(undefined); + const first = await getNativeExportCapabilities(); + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(1); + + process.env[scriptEnv] = customScript; + fsMocks.access.mockImplementation(async (candidate: string) => { + if (candidate === customScript) { + return; + } + throw new Error(`missing ${candidate}`); + }); + + const second = await getNativeExportCapabilities(); + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(2); + if (process.platform === "win32") { + expect(second.nvidiaCuda.hasWrapper).toBe(true); + } else { + expect(second.nvidiaCuda).toBe(first.nvidiaCuda); + } + } finally { + if (originalScript === undefined) { + delete process.env[scriptEnv]; + } else { + process.env[scriptEnv] = originalScript; + } + if (originalExport === undefined) { + delete process.env[exportEnv]; + } else { + process.env[exportEnv] = originalExport; + } + electronAppMock.isPackaged = originalIsPackaged; + electronAppMock.getGPUInfo.mockReset(); + electronAppMock.getGPUInfo.mockResolvedValue({ gpuDevice: [] }); + resetFsAccessMock(); + resetNvidiaCudaAvailabilityCache(); + } + }); + + it("reports an unavailable GPU consistently for capability and route queries", async () => { + await withPackagedCudaCandidate( + { gpuDevice: [{ vendorId: 0x8086, deviceString: "Intel UHD Graphics" }] }, + async () => { + const capabilities = await getNativeExportCapabilities(); + const route = await getExperimentalNvidiaCudaExportSkipReason( + createNvidiaCudaSkipOptions({ experimentalNvidiaCudaExport: true }), + ); + if (process.platform === "win32") { + expect(capabilities.nvidiaCuda.available).toBe(false); + expect(capabilities.nvidiaCuda.skipReason).toBe("nvidia-gpu-unavailable"); + expect(capabilities.nvidiaCuda.hasNvidiaGpu).toBe(false); + expect(route).toBe("nvidia-gpu-unavailable"); + } else { + expect(capabilities.nvidiaCuda.skipReason).toBe("not-windows"); + expect(route).toBe("not-windows"); + } + }, + ); + }); + + it("keeps strict HEVC Hardware CUDA-only hard-fail behavior unchanged", () => { + expect(resolveNvidiaCudaStrictHevcHardFail(true, false, "nvidia-gpu-unavailable")).toBe( + "HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (nvidia-gpu-unavailable) (noCpuFallback:true)", + ); + expect(resolveNvidiaCudaStrictHevcHardFail(true, false, null)).toBe( + "HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (cursor-atlas-unavailable) (noCpuFallback:true)", + ); + expect(resolveNvidiaCudaStrictHevcHardFail(true, true, "env-disabled")).toBeNull(); + expect(resolveNvidiaCudaStrictHevcHardFail(false, false, "env-disabled")).toBeNull(); + }); +}); + describe("resolveExperimentalNvidiaCudaExportScriptPath", () => { it("prefers the packaged app.asar.unpacked CUDA wrapper over the virtual app.asar copy", async () => { const envName = "RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT"; @@ -601,6 +922,296 @@ describe("resolveExperimentalNvidiaCudaExportScriptPath", () => { }); }); +describe("buildNativeStaticLayoutOverlayManifest", () => { + it("serializes sorted overlay layers into the CUDA manifest contract", () => { + expect( + buildNativeStaticLayoutOverlayManifest([ + { + id: "captions", + order: 2, + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]), + ).toEqual({ + layers: [ + { + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + }, + { + id: "captions", + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameCount: 60, + }, + ], + }); + }); + + it("passes effectiveFrameCount through while preserving the logical frameCount", () => { + expect( + buildNativeStaticLayoutOverlayManifest([ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }, + { + id: "captions", + order: 2, + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]), + ).toEqual({ + layers: [ + { + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + effectiveFrameCount: 41, + }, + { + id: "captions", + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameCount: 60, + }, + ], + }); + }); + + it("omits effectiveFrameCount for fully dynamic layers", () => { + const manifest = buildNativeStaticLayoutOverlayManifest([ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]); + + expect(manifest.layers[0]).toEqual({ + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + }); + expect("effectiveFrameCount" in manifest.layers[0]).toBe(false); + }); + + it("serializes a cursor-sprite layer with its additive positions sidecar fields", () => { + const manifest = buildNativeStaticLayoutOverlayManifest([ + { + id: "cursor-sprite", + order: 1, + kind: "cursor-sprite", + path: "cursor.sprite", + positionsPath: "cursor.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameRate: 30, + durationSec: 2, + frameCount: 60, + positions: Array.from({ length: 60 }, () => ({ x: 0, y: 0 })), + pixelFormat: "rgba", + }, + ]); + + expect(manifest.layers[0]).toEqual({ + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + path: "cursor.sprite", + positionsPath: "cursor.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameCount: 60, + }); + expect("effectiveFrameCount" in manifest.layers[0]).toBe(false); + }); +}); + +describe("getNativeStaticLayoutOverlayExpectedSidecarBytes", () => { + const frameByteSize = 1920 * 1080 * 4; + + it("sizes deduped sidecars from effectiveFrameCount, not the logical frameCount", () => { + expect( + getNativeStaticLayoutOverlayExpectedSidecarBytes({ + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }), + ).toBe(frameByteSize * 41); + }); + + it("falls back to the logical frameCount for fully dynamic layers", () => { + expect( + getNativeStaticLayoutOverlayExpectedSidecarBytes({ + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }), + ).toBe(frameByteSize * 60); + }); +}); + +describe("native static-layout overlay sidecar validation", () => { + const frameByteSize = 1920 * 1080 * 4; + + function createOverlayLayer( + overrides: Partial = {}, + ): NativeStaticLayoutOverlayLayer { + return { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + ...overrides, + }; + } + + afterEach(() => { + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + }); + + it("rejects deduped sidecars truncated below effectiveFrameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 204 }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [ + createOverlayLayer({ frameCount: 300, effectiveFrameCount: 205 }), + ], + }), + ), + ).rejects.toThrow( + `Native overlay layer effects is truncated: expected ${frameByteSize * 205} bytes, received ${frameByteSize * 204}`, + ); + }); + + it("accepts deduped sidecars whose physical bytes match effectiveFrameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 205 }); + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createOverlayLayer({ frameCount: 300, effectiveFrameCount: 205 })], + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch(/truncated/i); + }); + + it("keeps validating fully dynamic layers against the logical frameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 299 }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createOverlayLayer({ frameCount: 300 })], + }), + ), + ).rejects.toThrow( + `Native overlay layer effects is truncated: expected ${frameByteSize * 300} bytes, received ${frameByteSize * 299}`, + ); + }); +}); + describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { it("passes output canvas dimensions to the CUDA wrapper", () => { const args = buildExperimentalNvidiaCudaStaticLayoutArgs( @@ -612,6 +1223,22 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { expect(args).toEqual(expect.arrayContaining(["--width", "1020", "--height", "572"])); }); + it("passes the high-level HEVC codec to the generalized CUDA compositor", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + "output.mp4", + "work", + ); + + expect(args).toEqual( + expect.arrayContaining(["--output-codec", "hevc", "--encoding-mode", "quality"]), + ); + expect(args).not.toContain("h264"); + }); + it("keeps explicit copy-source CUDA audio inline by default", () => { const args = buildExperimentalNvidiaCudaStaticLayoutArgs( createNvidiaCudaSkipOptions({ @@ -761,58 +1388,246 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { ]), ); }); -}); -describe("buildExperimentalWindowsGpuStaticLayoutArgs", () => { - it("passes background blur to the D3D11 compositor", () => { - const args = buildExperimentalWindowsGpuStaticLayoutArgs( + it("passes the overlay sidecar manifest to the CUDA wrapper", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( createNvidiaCudaSkipOptions({ - backgroundImagePath: "wallpaper.jpg", - backgroundBlurPx: 36, + overlayManifestPath: "overlay-manifest.json", }), "output.mp4", + "work", ); - expect(args).toEqual(expect.arrayContaining(["--background-blur", "36"])); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); }); - it("passes source crop coordinates to the D3D11 compositor", () => { - const args = buildExperimentalWindowsGpuStaticLayoutArgs( + it("passes the resolved temporal zoom blur plan to the CUDA wrapper", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( createNvidiaCudaSkipOptions({ - sourceCropX: 192, - sourceCropY: 108, - sourceCropWidth: 1536, - sourceCropHeight: 864, + temporalBlur: { + sampleCount: 13, + shutterFraction: 0.62, + weightCurvePower: 1.5, + }, }), "output.mp4", + "work", ); expect(args).toEqual( expect.arrayContaining([ - "--source-crop-x", - "192", - "--source-crop-y", - "108", - "--source-crop-width", - "1536", - "--source-crop-height", - "864", + "--temporal-blur-sample-count", + "13", + "--temporal-blur-shutter-fraction", + "0.62", + "--temporal-blur-weight-power", + "1.5", ]), ); }); -}); -describe("buildNativeStaticLayoutTimelineSegments", () => { - it("derives contiguous output timeline ranges from edited-track source segments", () => { - expect( - buildNativeStaticLayoutTimelineSegments([ - { startMs: 0, endMs: 2_000, speed: 1 }, - { startMs: 2_000, endMs: 8_000, speed: 1.5 }, - { startMs: 8_000, endMs: 10_000, speed: 0.5 }, - ]), - ).toEqual([ - { - sourceStartMs: 0, + it("omits temporal blur args when no plan is configured", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({}), + "output.mp4", + "work", + ); + + expect(args).not.toContain("--temporal-blur-sample-count"); + }); + + it("rejects temporal blur plans below the minimum sample count instead of silently dropping the effect", () => { + expect(() => + buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + temporalBlur: { + sampleCount: 2, + shutterFraction: 0.62, + weightCurvePower: 1.5, + }, + }), + "output.mp4", + "work", + ), + ).toThrow(/unsupported-temporal-motion-blur/); + }); + + it("keeps the D3D11 builder free of overlay manifest args", () => { + const args = buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + overlayManifestPath: "overlay-manifest.json", + }), + "output.mp4", + "work", + ); + + expect(args).not.toContain("--overlay-manifest"); + }); +}); + +describe("resolveNvidiaCudaCursorAssets", () => { + it("strips cursor asset paths when the CUDA route must not draw the cursor", () => { + const options = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + }); + + expect(resolveNvidiaCudaCursorAssets(options, true)).toEqual({ + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }); + }); + + it("preserves cursor asset paths when native cursor drawing is allowed", () => { + const options = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }); + + expect(resolveNvidiaCudaCursorAssets(options, false)).toEqual({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }); + }); + + it("normalizes missing cursor asset paths to null", () => { + const options = createNvidiaCudaSkipOptions({}); + expect(resolveNvidiaCudaCursorAssets(options, false)).toEqual({ + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }); + }); +}); + +describe("CUDA cursor telemetry contract", () => { + it("never emits --cursor-json for overlay exports after cursor assets are stripped", () => { + // Mirrors the reported failure: the Windows GPU prep leaves CSV telemetry + // paths on the options, overlay layers are present, and the CUDA wrapper + // must not receive the CSV file as --cursor-json (it JSON.parses the path). + const csvOptions = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + overlayManifestPath: "overlay-manifest.json", + }); + const cudaOptions = { + ...csvOptions, + ...resolveNvidiaCudaCursorAssets(csvOptions, true), + }; + + const args = buildExperimentalNvidiaCudaStaticLayoutArgs(cudaOptions, "output.mp4", "work"); + + expect(args).not.toContain("--cursor-json"); + expect(args).not.toContain("cursor-telemetry.csv"); + expect(args).not.toContain("--cursor-atlas-png"); + expect(args).not.toContain("--cursor-atlas-metadata"); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("emits --cursor-json only for non-overlay CUDA exports with prepared JSON telemetry", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorSize: 96, + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }), + "output.mp4", + "work", + ); + + expect(args).toEqual(expect.arrayContaining(["--cursor-json", "cursor-telemetry.json"])); + expect(args).not.toContain("cursor-telemetry.csv"); + }); +}); + +describe("native static-layout encoder preference", () => { + it("requires rawvideo for CPU preference", () => { + expect( + getNativeStaticLayoutRawFrameFallbackReason({ + videoCodec: "hevc", + encoderPreference: "cpu", + }), + ).toBe("encoder-preference-cpu-requires-native-rawvideo"); + }); + + it("keeps HEVC hardware eligible for the CUDA route", () => { + expect( + getNativeStaticLayoutRawFrameFallbackReason({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ).toBeNull(); + }); +}); + +describe("buildExperimentalWindowsGpuStaticLayoutArgs", () => { + it("rejects HEVC instead of constructing an H.264 GPU command", () => { + expect(() => + buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ videoCodec: "hevc" }), + "output.mp4", + ), + ).toThrow(/generalized NVIDIA CUDA compositor/i); + }); + it("passes background blur to the D3D11 compositor", () => { + const args = buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + backgroundImagePath: "wallpaper.jpg", + backgroundBlurPx: 36, + }), + "output.mp4", + ); + + expect(args).toEqual(expect.arrayContaining(["--background-blur", "36"])); + }); + + it("passes source crop coordinates to the D3D11 compositor", () => { + const args = buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + sourceCropX: 192, + sourceCropY: 108, + sourceCropWidth: 1536, + sourceCropHeight: 864, + }), + "output.mp4", + ); + + expect(args).toEqual( + expect.arrayContaining([ + "--source-crop-x", + "192", + "--source-crop-y", + "108", + "--source-crop-width", + "1536", + "--source-crop-height", + "864", + ]), + ); + }); +}); + +describe("buildNativeStaticLayoutTimelineSegments", () => { + it("derives contiguous output timeline ranges from edited-track source segments", () => { + expect( + buildNativeStaticLayoutTimelineSegments([ + { startMs: 0, endMs: 2_000, speed: 1 }, + { startMs: 2_000, endMs: 8_000, speed: 1.5 }, + { startMs: 8_000, endMs: 10_000, speed: 0.5 }, + ]), + ).toEqual([ + { + sourceStartMs: 0, sourceEndMs: 2_000, outputStartMs: 0, outputEndMs: 2_000, @@ -854,6 +1669,58 @@ describe("muxExportedVideoAudioBuffer", () => { expect(result.outputPath).toMatch(/recordly-export-video-/); }); + + it("swallows the child-process error from a terminating kill of an already-exited child", async () => { + const spawnMock = vi.mocked(spawn); + spawnMock.mockClear(); + const child = new EventEmitter() as unknown as { + stdout: { on: (event: string, cb: (chunk: Buffer) => void) => void }; + stderr: { on: (event: string, cb: (chunk: Buffer) => void) => void }; + pid: number; + killed: boolean; + kill: (signal?: string) => boolean; + } & ReturnType; + (child as unknown as { stdout: unknown }).stdout = { on: vi.fn() }; + (child as unknown as { stderr: unknown }).stderr = { on: vi.fn() }; + (child as unknown as { pid: number }).pid = 4242; + let killedWith: string[] = []; + (child as unknown as { killed: boolean }).killed = false; + (child as unknown as { kill: unknown }).kill = (signal?: string) => { + if (signal) { + killedWith.push(signal); + } + // Killing an already-exited child on Windows emits an 'error' event + // asynchronously. With the immediate no-op listener attached (Module 3 + // fix) this is handled, not an uncaught exception; the dedicated + // once-handler (attached right after) then settles the promise. + queueMicrotask(() => { + child.emit("error", new Error('The process "4242" not found.')); + }); + return false; + }; + spawnMock.mockReturnValue(child); + + const session = { + terminating: true, + currentProcess: null as unknown, + }; + const error = await muxNativeVideoExportAudio( + "video.mp4", + { + audioMode: "copy-source", + audioSourceCodec: "aac", + audioSourcePath: "source-audio.mp4", + } as never, + undefined, + session as never, + ).catch((caught: unknown) => caught); + + // The child was terminated (SIGKILL) and its error event was handled + // rather than surfacing as an uncaught "process not found" noise line. + expect(killedWith).toContain("SIGKILL"); + expect(error).toBeInstanceOf(Error); + spawnMock.mockReset(); + }); }); describe("buildNativeVideoAudioMuxArgs", () => { @@ -1101,11 +1968,214 @@ describe("parseNvidiaCudaExportSummary", () => { expect(summary?.nativeSummary?.fps).toBe(326.1); }); + it("passes through the extended additive compositor counters", () => { + const summary = parseNvidiaCudaExportSummary( + JSON.stringify({ + success: true, + fps: 30, + nativeSummary: { + success: true, + frames: 300, + totalMs: 920.5, + measuredFps: 326.1, + temporalBlurSampleCount: 13, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + overlayFileLoads: 4, + overlayCacheHits: 596, + compositeGpuMs: 240.25, + zoomBlurGpuMs: 12.5, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayBlendFrames: 300, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + compositeMs: 262.9, + flushMs: 2.4, + realtimeMultiplier: 32.6, + outputBytes: 1843200, + }, + }), + ); + + expect(summary?.nativeSummary).toMatchObject({ + totalMs: 920.5, + measuredFps: 326.1, + temporalBlurSampleCount: 13, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + overlayFileLoads: 4, + overlayCacheHits: 596, + compositeGpuMs: 240.25, + zoomBlurGpuMs: 12.5, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayBlendFrames: 300, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + compositeMs: 262.9, + flushMs: 2.4, + realtimeMultiplier: 32.6, + outputBytes: 1843200, + }); + }); + it("returns null when the wrapper output has no JSON object", () => { expect(parseNvidiaCudaExportSummary("native helper failed before summary")).toBeNull(); }); }); +describe("resolveNvidiaCudaNativeSummaryMetrics", () => { + it("maps only finite additive counters the helper actually reported", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + success: true, + totalMs: 920.5, + nvencMs: 512.4, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayFileLoads: 4, + overlayCacheHits: 596, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + // Legacy counters stay out of the metric mapping; the completion + // log surfaces them through explicit keys. + roiCompositeFrames: 300, + }), + ).toEqual({ + totalMs: 920.5, + nvencMs: 512.4, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayFileLoads: 4, + overlayCacheHits: 596, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + }); + }); + + it("omits absent, non-finite, and non-numeric counters", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + temporalBlurSampleCount: Number.NaN, + temporalBlurFrames: Number.POSITIVE_INFINITY, + nvencMs: "512" as unknown as number, + overlayFileLoads: 0, + }), + ).toEqual({ overlayFileLoads: 0 }); + }); + + it("returns an empty object for missing native summaries", () => { + expect(resolveNvidiaCudaNativeSummaryMetrics(undefined)).toEqual({}); + }); + + it("surfaces the separated overlay host-read and H2D enqueue stage fields", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + overlayHostReadMs: 6.2, + overlayH2DEnqueueMs: 9.4, + overlayUploadMs: 18.1, + }), + ).toEqual({ + overlayHostReadMs: 6.2, + overlayH2DEnqueueMs: 9.4, + overlayUploadMs: 18.1, + }); + }); +}); + +describe("resolveNvidiaCudaNativeFps", () => { + it("prefers the helper measured flush-span FPS over the encode-loop fps", () => { + expect( + resolveNvidiaCudaNativeFps({ + fps: 30, + nativeSummary: { fps: 320.5, measuredFps: 326.1 }, + }), + ).toBe(326.1); + }); + + it("falls back to the encode-loop fps and preserves the legacy nativeFps contract", () => { + expect(resolveNvidiaCudaNativeFps({ nativeSummary: { fps: 320.5 } })).toBe(320.5); + expect(resolveNvidiaCudaNativeFps({ nativeSummary: { fps: 0 } })).toBeUndefined(); + expect(resolveNvidiaCudaNativeFps(undefined)).toBeUndefined(); + }); + + it("never falls back to the configured output fps and labels it measured speed", () => { + // summary.fps is the configured stream FPS (30), not a measured encode + // rate. When the helper reports no measured FPS the resolver must return + // undefined rather than surfacing 30 as nativeFps. + expect(resolveNvidiaCudaNativeFps({ fps: 30, nativeSummary: {} })).toBeUndefined(); + expect( + resolveNvidiaCudaNativeFps({ fps: 30, nativeSummary: { measuredFps: 0 } }), + ).toBeUndefined(); + }); +}); + +describe("validateNvidiaCudaStageMetricInvariants", () => { + it("accepts coherent additive stage metrics", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + totalMs: 920.5, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + }, + }), + ).toEqual([]); + }); + + it("rejects additive counters that exceed the helper wall time", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + totalMs: 100, + nvencMs: 512.4, + overlayUploadMs: 18.1, + }, + }), + ).toEqual(["nvencMs 512.4ms exceeds helper wall time 100ms"]); + }); + + it("rejects temporal blur counters that contradict the frame count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 20, + }, + }), + ).toEqual(["temporalBlurSamplesTotal 20 below temporalBlurFrames 40"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 41, + }, + }), + ).toEqual(["temporalBlurBgPrecomposedFrames 41 exceeds temporalBlurFrames 40"]); + }); + + it("reports no issues when the native summary is absent", () => { + expect(validateNvidiaCudaStageMetricInvariants({})).toEqual([]); + }); +}); + describe("validateNvidiaCudaExportSummary", () => { it("accepts CUDA output when frames and stream durations match the export target", () => { const issues = validateNvidiaCudaExportSummary( @@ -1270,7 +2340,7 @@ describe("parseWindowsGpuExportProgressLine", () => { it("parses bounded helper progress lines", () => { expect( parseWindowsGpuExportProgressLine( - 'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0,"stage":"finalizing"}', + 'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0,"intervalZoomBlurFrames":13,"stage":"finalizing"}', ), ).toEqual({ currentFrame: 30, @@ -1284,6 +2354,7 @@ describe("parseWindowsGpuExportProgressLine", () => { intervalEncodeMs: 120.2, intervalPipelineWaitMs: 46.2, intervalMonolithicCompositeFrames: 0, + intervalZoomBlurFrames: 13, }); }); @@ -1309,6 +2380,137 @@ describe("parseWindowsGpuExportProgressLine", () => { ), ).toBeNull(); }); + describe("resolveNativeStaticLayoutFpsFields", () => { + it("prefers native measured FPS over the preparation-inclusive estimate", () => { + const progress = { + currentFrame: 300, + totalFrames: 600, + percentage: 50, + averageFps: 17.2, + instantFps: 19.1, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 45_000)).toEqual({ + averageFps: 17.2, + estimatedFps: undefined, + fpsSource: "native", + }); + }); + + it("never labels the preparation-inclusive estimate as measured encode speed", () => { + const progress = { + currentFrame: 120, + totalFrames: 600, + percentage: 20, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 30_000)).toEqual({ + averageFps: undefined, + estimatedFps: 4, + fpsSource: "estimated", + }); + }); + + it("reports no FPS fields before any frame has been encoded", () => { + expect( + resolveNativeStaticLayoutFpsFields( + { currentFrame: 0, totalFrames: 600, percentage: 0 }, + 12_000, + ), + ).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: undefined, + }); + }); + + it("keeps the estimate when only interval FPS is present but no helper average", () => { + const progress = { + currentFrame: 60, + totalFrames: 600, + percentage: 10, + instantFps: 240.5, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 10_000)).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: "native", + }); + }); + + it("never emits a preparation-inclusive estimate for finalizing progress", () => { + // During finalizing the helper has already reported measured encode + // FPS; a frames/since-spawn estimate at this stage would include source + // preparation and misrepresent the display as measured speed. + expect( + resolveNativeStaticLayoutFpsFields( + { currentFrame: 300, totalFrames: 300, percentage: 100, stage: "finalizing" }, + 3_900, + ), + ).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: undefined, + }); + }); + + it("still reports the helper measured encode FPS on finalizing progress", () => { + expect( + resolveNativeStaticLayoutFpsFields( + { + currentFrame: 300, + totalFrames: 300, + percentage: 100, + stage: "finalizing", + averageFps: 135.4, + }, + 3_900, + ), + ).toEqual({ + averageFps: 135.4, + estimatedFps: undefined, + fpsSource: "native", + }); + }); + }); + describe("formatNativeStaticLayoutZoomTelemetryLines", () => { + it("writes renderer zoom-blur columns for the CUDA compositor", () => { + const lines = formatNativeStaticLayoutZoomTelemetryLines([ + { + timeMs: 0, + scale: 1, + x: 0, + y: 0, + blurStrength: 0, + blurCenterX: 960, + blurCenterY: 540, + }, + { + timeMs: 33.333, + scale: 1.0123, + x: -11.8, + y: -6.6, + blurStrength: 0.00345, + blurCenterX: 960, + blurCenterY: 540, + }, + ]); + + expect(lines).toEqual([ + "0,1,0,0,0,960,540", + "33.333,1.0123,-11.8,-6.6,0.00345,960,540", + ]); + }); + + it("keeps 4-column telemetry backward compatible with blur defaults", () => { + const lines = formatNativeStaticLayoutZoomTelemetryLines([ + { timeMs: 0, scale: 1, x: 0, y: 0 }, + ]); + + expect(lines).toEqual(["0,1,0,0,0,0,0"]); + }); + }); }); describe("mapNvidiaCudaWrapperProgressPercentage", () => { @@ -1352,6 +2554,63 @@ describe("mapNvidiaCudaWrapperProgressPercentage", () => { }); }); +describe("buildNvidiaCudaPrepareProgress", () => { + it.each([ + ["encoder-probe"] as const, + ["source-validation"] as const, + ["wrapper-launch"] as const, + ["cuda-nvenc-init"] as const, + ["first-frame"] as const, + ])("labels every preparation substate (%s) as NVIDIA CUDA compositor", (substate) => { + const progress = buildNvidiaCudaPrepareProgress("sess-1", substate, 600, 1_234); + expect(progress).toMatchObject({ + sessionId: "sess-1", + stage: "preparing", + substate, + backend: "nvidia-cuda-compositor", + currentFrame: 0, + totalFrames: 600, + elapsedMs: 1_234, + }); + // Preparation substates must never fabricate FPS or claim encode speed. + expect(progress.averageFps).toBeUndefined(); + expect(progress.instantFps).toBeUndefined(); + expect(progress.estimatedFps).toBeUndefined(); + expect(progress.fpsSource).toBeUndefined(); + }); + + it("stays within the preparing window and never claims a rendered frame", () => { + const orderedSubstates = [ + "encoder-probe", + "source-validation", + "wrapper-launch", + "cuda-nvenc-init", + "first-frame", + ] as const; + const percentages = orderedSubstates.map( + (substate) => buildNvidiaCudaPrepareProgress(undefined, substate, 600, 100).percentage, + ); + // Additive progression, all display-only within the preparing window. + for (let index = 0; index < percentages.length; index += 1) { + expect(percentages[index]).toBeGreaterThan(0); + expect(percentages[index]).toBeLessThanOrEqual(3); + if (index > 0) { + expect(percentages[index]).toBeGreaterThan(percentages[index - 1]); + } + } + expect(buildNvidiaCudaPrepareProgress(undefined, "encoder-probe", 600, 100)).toHaveProperty( + "currentFrame", + 0, + ); + }); + + it("clamps the total frames to a positive integer", () => { + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 0, 10).totalFrames).toBe(1); + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 2.9, 10).totalFrames).toBe(2); + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 600, -5).elapsedMs).toBe(0); + }); +}); + describe("hasNativeStaticLayoutProgressAdvanced", () => { it("treats repeated preparation heartbeats as stalled until real progress arrives", () => { const previous = { currentFrame: 0, percentage: 2.5 }; @@ -1451,3 +2710,1320 @@ describe("parseFfmpegFrameRate", () => { expect(parseFfmpegFrameRate("Video: h264")).toBeNull(); }); }); + +describe("native cursor atlas ownership", () => { + function createCursorOverlayLayer( + overrides: Partial = {}, + ): NativeStaticLayoutOverlayLayer { + return { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + ...overrides, + }; + } + + it("passes cursor assets and the overlay manifest through when the atlas owns the cursor", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorSize: 96, + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + overlayManifestPath: "overlay-manifest.json", + cursorAtlasOwned: true, + }), + "output.mp4", + "work", + ); + + // The sidecar excluded cursor pixels, so the CUDA wrapper must receive both + // the overlay manifest and the cursor atlas: dropping either would lose the + // cursor and baking it again would double-render. + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + expect(args).toEqual( + expect.arrayContaining([ + "--cursor-json", + "cursor-telemetry.json", + "--cursor-atlas-png", + "cursor-atlas.png", + "--cursor-atlas-metadata", + "cursor-atlas.tsv", + ]), + ); + }); + + it("keeps stripping baked-cursor assets even when a manifest path is present", () => { + const csvOptions = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + overlayManifestPath: "overlay-manifest.json", + }); + const cudaOptions = { + ...csvOptions, + ...resolveNvidiaCudaCursorAssets(csvOptions, true), + }; + + const args = buildExperimentalNvidiaCudaStaticLayoutArgs(cudaOptions, "output.mp4", "work"); + + expect(args).not.toContain("--cursor-json"); + expect(args).not.toContain("--cursor-atlas-png"); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("refuses a native-owned cursor when the CUDA route cannot run", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createCursorOverlayLayer()], + cursorAtlasOwned: true, + experimentalWindowsGpuCompositor: false, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor/i, + ); + }); + + it("allows a native-owned cursor when the CUDA route is explicitly opted in on Windows", async () => { + if (process.platform !== "win32") { + return; + } + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createCursorOverlayLayer()], + cursorAtlasOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + // The preflight guard must not reject the CUDA-opt-in route; any later + // failure is a runtime/skip error, not a cursor-ownership refusal. + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch( + /Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor/i, + ); + }); +}); + +describe("native webcam ownership", () => { + function createWebcamOverlayLayer(): NativeStaticLayoutOverlayLayer { + return { + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + path: "cursor-sprite.rgba", + positionsPath: "cursor-sprite.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + positions: Array.from({ length: 300 }, () => ({ x: 1, y: 1 })), + }; + } + + it("passes webcam args and the overlay manifest through when the CUDA compositor owns the webcam", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamLeft: 32, + webcamTop: 48, + webcamSize: 240, + webcamRadius: 18, + webcamMirror: false, + webcamNativeOwned: true, + overlayManifestPath: "overlay-manifest.json", + overlayLayers: [createWebcamOverlayLayer()], + }), + "output.mp4", + "work", + ); + + // The sidecar excluded webcam pixels, so the CUDA wrapper must receive + // BOTH the webcam input and the overlay manifest (cursor sprite); + // dropping either would lose the webcam or cursor and baking the webcam + // again would double-render. + expect(args).toEqual(expect.arrayContaining(["--webcam-input", "webcam.mp4"])); + expect(args).toEqual(expect.arrayContaining(["--webcam-x", "32"])); + expect(args).toEqual(expect.arrayContaining(["--webcam-size", "240"])); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("refuses a native-owned webcam when the CUDA route cannot run", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamSize: 240, + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: false, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor/i, + ); + }); + + it("refuses a native-owned webcam without an input path", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Native webcam ownership requires a webcam input path/i, + ); + }); + + it("refuses a native-owned webcam without a positive size", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Native webcam ownership requires a positive webcam size/i, + ); + }); + + it("allows a native-owned webcam when the CUDA route is explicitly opted in on Windows", async () => { + if (process.platform !== "win32") { + return; + } + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamSize: 240, + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + // The preflight guard must not reject the CUDA-opt-in route; any later + // failure is a runtime/skip error, not a webcam-ownership refusal. + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch( + /Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor/i, + ); + }); +}); + +describe("resolveNvidiaCudaOverlaySidecarSummaryMetrics", () => { + it("surfaces dimensions and physical/effective frame counts for deduped sidecars", () => { + expect( + resolveNvidiaCudaOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }, + ], + }), + ), + ).toEqual({ + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + }); + }); + + it("falls back to the logical frame count for fully dynamic sidecars", () => { + const metrics = resolveNvidiaCudaOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + }, + ], + }), + ); + + expect(metrics).toEqual({ + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 300, + }); + expect("overlayEffectiveFrames" in metrics).toBe(false); + }); + + it("returns no metrics when overlay layers are absent", () => { + expect( + resolveNvidiaCudaOverlaySidecarSummaryMetrics(createNvidiaCudaSkipOptions({})), + ).toEqual({}); + }); +}); + +describe("native summary metric mapping (module 3 surface)", () => { + it("maps the additive temporal/overlay counters the helper emitted", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + temporalBlurStationaryFrames: 12, + temporalBgCacheBuilds: 14, + temporalBgCacheHits: 26, + overlayStaticRegionBlends: 598, + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + overlayBlendFrames: 300, + }), + ).toEqual({ + temporalBlurStationaryFrames: 12, + temporalBgCacheBuilds: 14, + temporalBgCacheHits: 26, + overlayStaticRegionBlends: 598, + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + overlayBlendFrames: 300, + }); + }); + + it("keeps absent host-read/H2D fields out of the mapping until the helper emits them", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + overlayHostReadMs: Number.NaN, + overlayH2DEnqueueMs: 4.5, + }), + ).toEqual({ overlayH2DEnqueueMs: 4.5 }); + }); +}); + +describe("native summary metric invariants (module 3)", () => { + it("accepts coherent temporal cache and stationary counters", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurStationaryFrames: 10, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 8, + temporalBgCacheHits: 22, + overlayBlendFrames: 300, + overlayStaticRegionBlends: 297, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + }, + }), + ).toEqual([]); + }); + + it("rejects stationary frames beyond the temporal frame count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurStationaryFrames: 41, + }, + }), + ).toEqual(["temporalBlurStationaryFrames 41 exceeds temporalBlurFrames 40"]); + }); + + it("does not count cache builds against the precomposed frame budget", () => { + // Mirrors the reported false positive: builds count cache allocations/ + // segments, hits and precomposedFrames count per-frame reuse. builds must + // NOT be added to the frame budget. + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 4, + temporalBgCacheBuilds: 1, + temporalBgCacheHits: 4, + }, + }), + ).toEqual([]); + }); + + it("rejects temporal cache hits that exceed the precomposed frame budget", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 20, + temporalBgCacheHits: 31, + }, + }), + ).toEqual(["temporalBgCacheHits 31 exceeds temporalBlurBgPrecomposedFrames 30"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 40, + temporalBgCacheHits: 30, + }, + }), + ).toEqual([]); + }); + + it("binds temporal cache hits to the temporal frame budget when precomposed is absent", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBgCacheHits: 41, + temporalBgCacheBuilds: 10, + }, + }), + ).toEqual(["temporalBgCacheHits 41 exceeds temporalBlurFrames 40"]); + }); + + it("rejects negative temporal cache counters", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBgCacheBuilds: -1, + temporalBgCacheHits: 4, + }, + }), + ).toEqual(["temporalBgCacheBuilds -1 must be non-negative"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBgCacheBuilds: 1, + temporalBgCacheHits: -4, + }, + }), + ).toEqual(["temporalBgCacheHits -4 must be non-negative"]); + }); + + it("rejects overlay static-region blends beyond overlay blend frames", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayBlendFrames: 300, + overlayStaticRegionBlends: 301, + }, + }), + ).toEqual(["overlayStaticRegionBlends 301 exceeds overlayBlendFrames 300"]); + }); + + it("rejects overlay sidecar frame counts that contradict the logical count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayFrameCount: 300, + overlayPhysicalFrames: 301, + }, + }), + ).toEqual(["overlayPhysicalFrames 301 exceeds overlayFrameCount 300"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayFrameCount: 300, + overlayEffectiveFrames: 0, + }, + }), + ).toEqual(["overlayEffectiveFrames 0 out of range for overlayFrameCount 300"]); + }); +}); + +describe("CUDA progress interval fields (module 3 surface)", () => { + it("parses the additive temporal/overlay interval counters from PROGRESS lines", () => { + expect( + parseWindowsGpuExportProgressLine( + 'PROGRESS {"currentFrame":120,"totalFrames":300,"percentage":40,"intervalTemporalBlurStationaryFrames":3,"intervalTemporalBgCacheBuilds":4,"intervalTemporalBgCacheHits":9,"intervalOverlayStaticRegionBlends":11}', + ), + ).toMatchObject({ + currentFrame: 120, + totalFrames: 300, + percentage: 40, + intervalTemporalBlurStationaryFrames: 3, + intervalTemporalBgCacheBuilds: 4, + intervalTemporalBgCacheHits: 9, + intervalOverlayStaticRegionBlends: 11, + }); + }); +}); + +const TILED_TILE_BYTE_SIZE = 128 * 128 * 4; +const TILED_LAYER_WIDTH = 384; +const TILED_LAYER_HEIGHT = 256; +const TILED_LAYER_TILE_COUNT = 3 * 2; + +function tiledTileRecord( + tileIndex: number, + byteOffset: number, + overrides: Partial<{ byteLength: number; byteOffset: number }> = {}, +) { + return { tileIndex, byteOffset, byteLength: TILED_TILE_BYTE_SIZE, ...overrides }; +} + +function staticTilesFor(tileCount = TILED_LAYER_TILE_COUNT) { + return Array.from({ length: tileCount }, (_, tileIndex) => + tiledTileRecord(tileIndex, tileIndex * TILED_TILE_BYTE_SIZE), + ); +} + +function tiledLayer( + overrides: Partial = {}, +): NativeTiledOverlayLayerDescriptor { + return { + id: "tiled-effects", + order: 0, + x: 0, + y: 0, + width: TILED_LAYER_WIDTH, + height: TILED_LAYER_HEIGHT, + frameRate: 30, + durationSec: 2, + frameCount: 60, + tileSize: 128, + pixelFormat: "rgba", + payloadPath: "C:/Temp/tiled-overlay.bin", + payloadByteLength: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE, + staticTiles: staticTilesFor(), + frameDeltas: [], + ...overrides, + }; +} + +describe("tiled overlay integration (native-video module)", () => { + it("passes the tiled overlay manifest only when tiled layers are present", () => { + const noTiled = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + overlayManifestPath: "overlay-manifest.json", + }), + "output.mp4", + "work", + ); + expect(noTiled).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + expect(noTiled).not.toContain("--tiled-overlay-manifest"); + + const withTiled = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + overlayManifestPath: "overlay-manifest.json", + tiledOverlayManifestPath: "tiled-overlay-manifest.json", + }), + "output.mp4", + "work", + ); + expect(withTiled).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + expect(withTiled).toEqual( + expect.arrayContaining(["--tiled-overlay-manifest", "tiled-overlay-manifest.json"]), + ); + }); + + it("keeps the D3D11 builder free of tiled overlay args", () => { + const args = buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + tiledOverlayManifestPath: "tiled-overlay-manifest.json", + }), + "output.mp4", + ); + expect(args).not.toContain("--tiled-overlay-manifest"); + expect(args).not.toContain("--overlay-manifest"); + }); + + it("builds the versioned tiled storage descriptor sorted by order then id", () => { + const manifest = buildNativeStaticLayoutTiledOverlayManifest( + { + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + }, + [ + tiledLayer({ id: "captions", order: 2 }), + tiledLayer({ id: "effects", order: 0 }), + tiledLayer({ id: "annotations", order: 0 }), + ], + ); + + expect(manifest.version).toBe(1); + expect(manifest.outputWidth).toBe(1920); + expect(manifest.outputHeight).toBe(1080); + expect(manifest.layers.map((layer) => layer.id)).toEqual([ + "annotations", + "effects", + "captions", + ]); + }); + + it("rejects a malformed tiled descriptor through the export preflight", async () => { + fsMocks.stat.mockResolvedValue({ size: 1_000_000 }); + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + durationSec: 2, + experimentalWindowsGpuCompositor: false, + tiledOverlayLayers: [ + tiledLayer({ + id: "", + payloadPath: "", + }), + ], + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/Invalid tiled overlay descriptor/i); + }); + + it("rejects a truncated tiled payload through the export preflight", async () => { + fsMocks.stat.mockResolvedValue({ size: 1 }); + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + durationSec: 2, + experimentalWindowsGpuCompositor: false, + tiledOverlayLayers: [tiledLayer()], + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/Tiled overlay layer .* payload is truncated/i); + }); + + it("resolves additive tiled overlay summary metrics without touching helper output", () => { + const nextOffset = TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE; + const firstLayer = tiledLayer({ + payloadByteLength: nextOffset + 4 * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { frameIndex: 10, changedTiles: [tiledTileRecord(1, nextOffset)] }, + { + frameIndex: 20, + changedTiles: [ + tiledTileRecord(4, nextOffset + TILED_TILE_BYTE_SIZE), + tiledTileRecord(5, nextOffset + 2 * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 30, + changedTiles: [tiledTileRecord(2, nextOffset + 3 * TILED_TILE_BYTE_SIZE)], + }, + ], + }); + + expect( + resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + tiledOverlayLayers: [firstLayer, tiledLayer({ id: "captions", order: 1 })], + }), + ), + ).toEqual({ + tiledOverlayLayers: 2, + changedTileCount: 4, + uploadedTileBytes: + (TILED_LAYER_TILE_COUNT + 4 + TILED_LAYER_TILE_COUNT) * TILED_TILE_BYTE_SIZE, + cachedTileCount: + TILED_LAYER_TILE_COUNT * 60 - + (TILED_LAYER_TILE_COUNT + 4) + + (TILED_LAYER_TILE_COUNT * 60 - TILED_LAYER_TILE_COUNT), + }); + }); + + it("preserves raw overlay sidecar metrics and does not mix them with tiled metrics", () => { + const rawLayer: NativeStaticLayoutOverlayLayer = { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + }; + const tiled = tiledLayer(); + + const bothMetrics = resolveNvidiaCudaOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [rawLayer], + tiledOverlayLayers: [tiled], + }), + ); + const tiledMetrics = resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [rawLayer], + tiledOverlayLayers: [tiled], + }), + ); + + expect(bothMetrics).toEqual({ + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 300, + }); + expect(tiledMetrics).toEqual({ + tiledOverlayLayers: 1, + changedTileCount: 0, + uploadedTileBytes: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE, + cachedTileCount: TILED_LAYER_TILE_COUNT * 60 - TILED_LAYER_TILE_COUNT, + }); + }); + + it("reports the conservative raw fallback reason for ineligible tiled layers", () => { + const denseLayer = tiledLayer({ + frameCount: 2, + payloadByteLength: (TILED_LAYER_TILE_COUNT + 3) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [0, 1, 2].map((tileIndex, index) => + tiledTileRecord( + tileIndex, + (TILED_LAYER_TILE_COUNT + index) * TILED_TILE_BYTE_SIZE, + ), + ), + }, + ], + }); + + expect( + resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + tiledOverlayLayers: [denseLayer], + }), + ), + ).toMatchObject({ + rawFallbackReason: "payload-bytes-exceed-raw", + }); + }); + + it("adds the additive tiled counters into the native summary mapping", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + success: true, + changedTileCount: 8, + uploadedTileBytes: 1024, + cachedTileCount: 200, + rawFallbackReason: "small-layer", + overlayHostReadMs: 12.5, + overlayH2DEnqueueMs: 4.5, + overlayCacheHits: 96, + totalMs: 920.5, + nativeFps: 240.5, + }), + ).toEqual({ + changedTileCount: 8, + uploadedTileBytes: 1024, + cachedTileCount: 200, + rawFallbackReason: "small-layer", + overlayHostReadMs: 12.5, + overlayH2DEnqueueMs: 4.5, + overlayCacheHits: 96, + totalMs: 920.5, + }); + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + nativeFps: 240.5, + }), + ).toEqual({}); + }); + + it("validates tiled overlay metric invariants for additive counters", () => { + const tileCount = TILED_LAYER_TILE_COUNT; + const frameCount = 60; + const uploadedBytes = (tileCount + 1) * TILED_TILE_BYTE_SIZE; + const cached = tileCount * frameCount - (tileCount + 1); + const issues = validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + totalMs: 100, + changedTileCount: -1, + uploadedTileBytes: uploadedBytes, + cachedTileCount: cached, + overlayCacheHits: cached + 1, + overlayHostReadMs: 20, + overlayH2DEnqueueMs: 40, + }, + }); + + expect(issues).toEqual([]); + }); + + it("keeps uploaded tile bytes bounded by the payload size and consistent with tile counts", () => { + const layer = tiledLayer(); + const metrics = resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + tiledOverlayLayers: [layer], + }), + ); + + expect(metrics.uploadedTileBytes).toBe(TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE); + expect(metrics.uploadedTileBytes).toBeLessThanOrEqual(layer.payloadByteLength); + expect(metrics.cachedTileCount).toBeGreaterThanOrEqual(0); + expect(metrics.cachedTileCount).toBeLessThanOrEqual(TILED_LAYER_TILE_COUNT * 60); + expect(metrics.changedTileCount).toBe(0); + }); + + it("does not confuse logical frame count with effective state versions", () => { + const layer = tiledLayer({ + frameCount: 60, + frameDeltas: [ + { + frameIndex: 10, + changedTiles: [ + tiledTileRecord(1, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + + expect(layer.frameCount).toBe(60); + expect(layer.frameDeltas.length + 1).toBe(2); + expect( + resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + tiledOverlayLayers: [layer], + }), + ), + ).toMatchObject({ + changedTileCount: 1, + cachedTileCount: TILED_LAYER_TILE_COUNT * 60 - (TILED_LAYER_TILE_COUNT + 1), + }); + }); + + it("keeps HEVC Hardware eligible for the CUDA route despite tiled overlay layers", () => { + expect( + getNativeStaticLayoutRawFrameFallbackReason({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ).toBeNull(); + }); +}); + +describe("prewarmNativeExportCaches", () => { + const nvidiaIdentityStat = { + dev: 123, + ino: 456, + size: 1_000_000, + mtimeMs: 1_000, + ctimeMs: 2_000, + }; + const cacheableMetadataProbe = + "Duration: 00:00:02.00, start: 0.000000\n Stream #0:0: Video: h264 (High), yuv420p, 1920x1080, 30 fps"; + + function countMetadataProbeCalls(): number { + return execFileMock.mock.calls.filter( + (call) => Array.isArray(call[1]) && call[1].includes("-i"), + ).length; + } + + function restoreExecFileMock(): void { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null); + return { stdout: "", stderr: "" } as unknown; + }) as never); + } + + beforeEach(() => { + resetNativeStaticLayoutSourceProbeCache(); + resetNvidiaCudaAvailabilityCache(); + fsMocks.realpath.mockResolvedValue("canonical.mp4"); + fsMocks.stat.mockResolvedValue(nvidiaIdentityStat as never); + execFileMock.mockClear(); + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null); + return { stdout: "", stderr: "" } as unknown; + }) as never); + }); + + afterEach(() => { + restoreExecFileMock(); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + }); + + it("warms source metadata and reuses exact cache keys (codec + encoder preference)", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + + const first = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(countMetadataProbeCalls()).toBe(1); + expect(parseNativeVideoMetadataProbeOutput(cacheableMetadataProbe)).not.toBeNull(); + expect( + parseNativeVideoMetadataProbeOutput("\n" + cacheableMetadataProbe)?.codec, + "probe-result-codec", + ).toBe("h264 (High)"); + expect(first.sourceMetadataCached, `skip=${JSON.stringify(first.skipReasons)}`).toBe(true); + + // Exact same route reuses the cache: no second probe. + const second = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(second.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A different encoder preference is a distinct exact key: it re-probes. + const changedPref = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "hardware", + encodingMode: "balanced", + }); + expect(changedPref.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(2); + }); + + it("treats encoding mode as part of the exact cache key (prewarm/export alignment)", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + + // Warm with the persisted/export default (balanced) once. + const first = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(first.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A real export resolving the same route with balanced hits the cache. + const hit = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(hit.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A different encoding mode is a distinct exact key and re-probes. + const otherMode = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "quality", + }); + expect(otherMode.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(2); + }); + + it("reports source metadata as uncached when the probe fails without throwing", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + try { + cb(new Error("probe failed") as never); + } catch { + // swallow + } + return { stdout: "", stderr: "" } as unknown; + }) as never); + + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(outcome.sourceMetadataCached).toBe(false); + expect(outcome.skipReasons).toContain("source-metadata-unavailable"); + expect(outcome.resolvedEncoders).toContain("h264_nvenc"); + }); + + it("reports CUDA availability when the runtime probe succeeds", async () => { + await withPackagedCudaCandidate( + { + gpuDevice: [{ vendorId: 0x10de, deviceString: "RTX 4090" }], + }, + async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(outcome.cudaAvailabilityResolved).toBe(true); + }, + ); + }); + + it("does not fabricate CUDA readiness from an unavailable runtime probe", async () => { + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + encodingMode: "balanced", + }); + expect(outcome.cudaAvailabilityResolved).toBe(false); + expect(outcome.skipReasons.some((reason) => reason.startsWith("cuda-unavailable"))).toBe( + true, + ); + }); +}); + +describe("capability-only prewarm cancellation (native-video)", () => { + function fakeCapabilityChild() { + const child = new EventEmitter() as { + stdout: unknown; + stderr: unknown; + killedWith: string[]; + kill: (signal?: string) => void; + }; + child.stdout = {}; + child.stderr = {}; + child.killedWith = []; + child.kill = (signal?: string) => { + if (signal) { + child.killedWith.push(signal); + } + }; + return child; + } + + it("cancels every in-flight capability-only prewarm child before a real NVENC session opens", () => { + const childA = fakeCapabilityChild(); + const childB = fakeCapabilityChild(); + const unregisterA = registerCapabilityOnlyPrewarmChild(childA); + const unregisterB = registerCapabilityOnlyPrewarmChild(childB); + + const cancelled = cancelInFlightCapabilityOnlyPrewarms(); + + expect(cancelled).toBe(2); + expect(childA.killedWith).toContain("SIGKILL"); + expect(childB.killedWith).toContain("SIGKILL"); + + // Unregister after settle is a no-op (set already cleared). + unregisterA(); + unregisterB(); + }); + + it("is harmless when no capability-only prewarm child is running", () => { + expect(cancelInFlightCapabilityOnlyPrewarms()).toBe(0); + }); + + it("does not cancel a child that already finished (unregistered)", () => { + const child = fakeCapabilityChild(); + const unregister = registerCapabilityOnlyPrewarmChild(child); + unregister(); + expect(cancelInFlightCapabilityOnlyPrewarms()).toBe(0); + expect(child.killedWith).toHaveLength(0); + }); +}); + +describe("native static-layout lazy FFmpeg encoder resolution", () => { + const METADATA_H264 = + "Duration: 00:00:02.00, start: 0.000000\n Stream #0:0: Video: h264 (High), yuv420p, 1920x1080, 30 fps"; + const FFPROBE_STATS = JSON.stringify({ + streams: [ + { + duration: "1.9999", + nb_read_frames: "10", + avg_frame_rate: "5/1", + r_frame_rate: "5/1", + }, + ], + }); + const CUDA_SUMMARY = JSON.stringify({ + success: true, + outputCodec: "h264", + targetFrames: 10, + durationSec: 2, + nativeSummary: { success: true, frames: 10 }, + outputVideo: { duration: "1.999900", nb_frames: "10" }, + }); + + function makeOptions( + overrides: Partial = {}, + ): NativeStaticLayoutExportOptions { + return { + inputPath: "input.mp4", + width: 1920, + height: 1080, + frameRate: 5, + bitrate: 8_000_000, + encodingMode: "balanced", + durationSec: 2, + contentWidth: 1920, + contentHeight: 1080, + offsetX: 0, + offsetY: 0, + backgroundColor: "#101010", + audioOptions: { audioMode: "none" }, + ...overrides, + }; + } + + function routeExecFileTo(encodersStdout: string) { + execFileMock.mockImplementation((( + _cmd: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, res?: { stdout: string; stderr: string }) => void, + ) => { + if (Array.isArray(args) && args.includes("-encoders")) { + cb(null, { stdout: encodersStdout, stderr: "" }); + return { stdout: encodersStdout, stderr: "" } as unknown; + } + if (Array.isArray(args) && args.includes("-select_streams")) { + cb(null, { stdout: FFPROBE_STATS, stderr: "" }); + return { stdout: FFPROBE_STATS, stderr: "" } as unknown; + } + // FFmpeg metadata probe (-i source) -> valid H.264 source (no proxy). + cb(null, { stdout: "", stderr: METADATA_H264 }); + return { stdout: "", stderr: METADATA_H264 } as unknown; + }) as never); + } + + function fakeSpawnChild() { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + stdin: { end: () => void }; + kill: (signal?: string) => boolean; + }; + child.stdout = stdout; + child.stderr = stderr; + child.stdin = { end: () => undefined }; + child.kill = () => true; + return child; + } + + function countEncodersProbes(): number { + return execFileMock.mock.calls.filter( + (call) => Array.isArray(call[1]) && call[1].includes("-encoders"), + ).length; + } + + beforeEach(() => { + resetNativeStaticLayoutSourceProbeCache(); + resetNvidiaCudaAvailabilityCache(); + vi.mocked(spawn).mockClear(); + fsMocks.access.mockResolvedValue(undefined); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 } as never); + }); + + afterEach(() => { + fsMocks.access.mockImplementation(async () => { + throw new Error("missing"); + }); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + delete process.env.RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT; + delete process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT; + }); + + it("does not invoke the FFmpeg encoder probe when the NVIDIA CUDA route is selected", async () => { + routeExecFileTo(""); + const cudaEnv = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + process.env[cudaEnv] = "1"; + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT = "run-mp4-pipeline.mjs"; + + const spawnMock = vi.mocked(spawn); + const child = fakeSpawnChild(); + spawnMock.mockImplementation((cmd: string) => { + // Only the CUDA compositor wrapper (node -- run-mp4-pipeline.mjs) may be + // spawned; no ffmpeg encoder probe may happen before/on the CUDA route. + expect(cmd).toBe(process.execPath); + return child; + }); + + const pending = exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ experimentalWindowsGpuCompositor: true }), + ); + await vi.waitFor(() => { + expect(spawnMock).toHaveBeenCalled(); + }); + // CUDA path reached the wrapper spawn without every probing the encoder. + expect(countEncodersProbes()).toBe(0); + child.stdout.emit("data", Buffer.from(CUDA_SUMMARY)); + child.emit("close", 0, null); + + const result = await pending; + expect(result.route).toBe("nvidia-cuda-compositor"); + expect(result.encoderName).toBe("nvidia-cuda-compositor"); + // The CUDA route succeeded with zero FFmpeg encoder listing probes. + expect(countEncodersProbes()).toBe(0); + }); + + it("still probes/resolves the FFmpeg encoder on the raw/FFmpeg fallback branch", async () => { + // Only libx264 is available, so exactly one probe spawn happens for it. + routeExecFileTo(" V....D libx264"); + const spawnMock = vi.mocked(spawn); + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + const probeChild = fakeSpawnChild(); + if (Array.isArray(args) && args.includes("pipe:0")) { + // Encoder probe succeeds -> resolveNativeVideoEncoder returns libx264. + queueMicrotask(() => probeChild.emit("close", 0, null)); + } else { + // FFmpeg render spawns fail, so the export rejects after resolution. + queueMicrotask(() => probeChild.emit("close", 1, null)); + } + return probeChild; + }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ experimentalWindowsGpuCompositor: false }), + ), + ).rejects.toThrow(); + + // The FFmpeg/raw fallback probed and resolved the encoder (-encoders), + // unlike the CUDA route above. + expect(countEncodersProbes()).toBeGreaterThan(0); + }); + + it("strict HEVC Hardware hard-fails on CUDA runtime failure without webcam/zoom/timeline (no FFmpeg fallback/probe)", async () => { + // Strict HEVC Hardware: the generalized NVIDIA CUDA compositor is the ONLY + // acceptable route. A CUDA runtime failure with no webcam, zoom telemetry, + // or native timeline must NOT be swallowed by the outer GPU-block catch and + // turned into a full FFmpeg hevc_nvenc fallback; the original actionable + // CUDA/noCpuFallback error must reach the caller and no FFmpeg encoder probe + // or fallback spawn may happen. + routeExecFileTo(""); + process.env.RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT = "1"; + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT = "run-mp4-pipeline.mjs"; + const probesBefore = countEncodersProbes(); + + const spawnMock = vi.mocked(spawn); + const child = fakeSpawnChild(); + spawnMock.mockImplementation((cmd: string) => { + // Only the CUDA compositor wrapper may spawn; no Windows-GPU or FFmpeg + // fallback child may ever be launched on the strict route. + expect(cmd).toBe(process.execPath); + return child; + }); + + const pending = exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ + experimentalWindowsGpuCompositor: true, + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ); + await vi.waitFor(() => { + expect(spawnMock).toHaveBeenCalled(); + }); + // CUDA runtime failure: the helper exits nonzero without a success summary. + child.emit("close", 1, null); + + await expect(pending).rejects.toThrow(/noCpuFallback:true/); + // The strict route reached only the single CUDA wrapper spawn; no FFmpeg + // encoder probe (-encoders) and no FFmpeg/GPU fallback spawn occurred. + expect(spawnMock).toHaveBeenCalledTimes(1); + // No FFmpeg encoder probe (-encoders) may have occurred for this export. + // countEncodersProbes() accumulates across the describe, so compare against + // the count captured before this export ran. + expect(countEncodersProbes()).toBe(probesBefore); + }); +}); diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index 16b2993e8..de96e52fe 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -6,10 +6,32 @@ import path from "node:path"; import { performance } from "node:perf_hooks"; import type { Readable, Writable } from "node:stream"; import { promisify } from "node:util"; -import type { WebContents } from "electron"; +import type { MessagePortMain, WebContents } from "electron"; import { app, powerSaveBlocker } from "electron"; +import type { + NativeCursorSpriteOverlayLayer, + NativeStaticLayoutOverlayLayer, + NativeTiledOverlayLayerDescriptor, + NativeTiledOverlayStorageDescriptor, +} from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; +import { + getNativeStaticLayoutOverlayFrameByteSize, + NATIVE_CURSOR_SPRITE_LAYER_KIND, + NATIVE_TILED_OVERLAY_STORAGE_VERSION, + resolveNativeTiledOverlayMetrics, + resolveNativeTiledOverlayRawFallbackReason, + sortNativeStaticLayoutOverlayLayers, + sortNativeTiledOverlayLayers, + validateNativeCursorSpriteOverlayLayer, + validateNativeStaticLayoutOverlayLayer, + validateNativeTiledOverlayStorageDescriptor, +} from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; +import { TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT } from "../../../src/lib/exporter/temporalMotionBlur"; import { getFfmpegBinaryPath, getFfprobeBinaryPath } from "../ffmpeg/binary"; +import { formatLogTs } from "../log"; import type { + ExportEncoderPreference, + ExportVideoCodec, NativeExportEncodingMode, NativeStaticLayoutBackend, NativeStaticLayoutExportArgsConfig, @@ -30,8 +52,8 @@ import { buildTrimmedSourceAudioFilter, createNativeSquircleMaskPgmBuffer, getEditedAudioExtension, + getNativeEncoderCandidates, getNativeVideoInputByteSize, - getPreferredNativeVideoEncoders, isNativeCudaOutOfMemory, parseAvailableFfmpegEncoders, } from "../nativeVideoExport"; @@ -42,6 +64,10 @@ const getNowMs = () => performance.now(); const formatFfmpegSeconds = (milliseconds: number) => (milliseconds / 1000).toFixed(3); const MISSING_NATIVE_STATIC_BACKGROUND_COLOR = "#ffffff"; const NATIVE_EXPORT_HIGH_PRIORITY = os.constants.priority.PRIORITY_HIGH; +// Dummy frame size used to probe whether an encoder can initialize. Must be +// large enough to satisfy hardware encoder minimums (NVENC rejects frames +// smaller than ~192x192 on recent NVIDIA drivers), while staying cheap. +const NATIVE_ENCODER_PROBE_DIMENSION = 256; const NVIDIA_PCI_VENDOR_ID = 0x10de; const NVIDIA_CUDA_EXPORT_ENV = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; const NVIDIA_CUDA_ALLOW_AUDIO_EXPORT_ENV = "RECORDLY_NVIDIA_CUDA_ALLOW_AUDIO_EXPORT"; @@ -65,6 +91,51 @@ type ElectronGpuInfoLike = { gpuDevice?: ElectronGpuDeviceLike[]; }; +export const NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION = 1; + +export type NativeVideoExportFramePortMessage = + | { + type: "hello"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + capabilityProbe: ArrayBuffer; + } + | { + type: "frame"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId: number; + sequence: number; + frame: ArrayBuffer; + }; + +export type NativeVideoExportFramePortResponse = + | { + type: "ready"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + transferable: true; + transferProbe: ArrayBuffer; + } + | { + type: "ack"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId: number; + sequence: number; + success: true; + } + | { + type: "error"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId?: number; + sequence?: number; + success: false; + error: string; + fallbackAvailable: boolean; + }; + export type NativeVideoExportSession = { ffmpegProcess: ChildProcessByStdio; outputPath: string; @@ -80,10 +151,441 @@ export type NativeVideoExportSession = { completionPromise: Promise; sender: WebContents | null; pendingWriteRequestIds: Set; + framePort: MessagePortMain | null; + framePortReady: boolean; + nextFrameSequence: number; + pendingFrameRequests: Map; + /** + * Monotonic watermark of the highest accepted frame request id. Request ids + * are allocated strictly monotonically by the renderer + * (nextNativeVideoExportWriteRequestId++), and the strict sequence check + * below already rejects replays, so a bounded O(1) watermark replaces the + * per-session completed-id Set (which grew to ~1 entry per exported frame + * for the whole session). Reset together with nextFrameSequence whenever a + * new frame port is attached. + */ + highestAcceptedFrameRequestId: number; }; export const nativeVideoExportSessions = new Map(); +// In-flight NVIDIA CUDA/NVENC capability-only prewarm children, tracked here +// (not in native-prewarm) so a real native export can cancel them without the +// coordinator importing back into this module (avoiding a circular dependency). +// Each child opens a brief NVENC capability probe; cancelling them before a +// real export opens its own NVENC session prevents concurrent NVENC session +// contention on the GPU. Fire-and-forget: the coordinator never awaits these. +const activeCapabilityOnlyPrewarmChildren = new Set>(); + +/** + * Registers an in-flight capability-only prewarm child process. Returns a + * cleanup that removes it from the tracked set when it settles. Diagnostic/data + * only; never awaited by the coordinator (fire-and-forget). + */ +export function registerCapabilityOnlyPrewarmChild(child: ReturnType): () => void { + activeCapabilityOnlyPrewarmChildren.add(child); + return () => { + activeCapabilityOnlyPrewarmChildren.delete(child); + }; +} + +/** + * Terminates every in-flight capability-only prewarm child. Called when a real + * native export opens its NVENC session so the brief capability probe never + * contends with the real encode. Harmless if none are running. Returns how many + * children were killed (diagnostic only; never used for control flow). + */ +export function cancelInFlightCapabilityOnlyPrewarms(): number { + let cancelled = 0; + for (const child of activeCapabilityOnlyPrewarmChildren) { + try { + child.kill("SIGKILL"); + cancelled += 1; + } catch { + /* process may already be exited */ + } + } + activeCapabilityOnlyPrewarmChildren.clear(); + if (cancelled > 0) { + console.info( + formatLogTs(), + "[native-export] Cancelled in-flight CUDA capability-only prewarm children", + { + cancelled, + }, + ); + } + return cancelled; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isArrayBuffer(value: unknown): value is ArrayBuffer { + return value instanceof ArrayBuffer; +} + +function isValidRequestNumber(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function sendNativeVideoExportFramePortMessage( + port: MessagePortMain, + message: NativeVideoExportFramePortResponse, + transfer?: ArrayBuffer, +) { + try { + if (transfer) { + // Electron 43 types MessagePortMain's transfer list as MessagePortMain[], + // although Chromium's structured clone implementation also accepts + // ArrayBuffers. Verify detachment at runtime instead of assuming zero-copy. + port.postMessage(message, [transfer as unknown as MessagePortMain]); + return transfer.byteLength === 0; + } + port.postMessage(message); + return true; + } catch { + return false; + } +} + +export function sendNativeVideoExportFramePortError( + port: MessagePortMain, + sessionId: string, + error: string, + options: { + requestId?: number; + sequence?: number; + fallbackAvailable: boolean; + }, +) { + return sendNativeVideoExportFramePortMessage(port, { + type: "error", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + success: false, + error, + ...options, + }); +} + +function clearNativeVideoExportFramePort(session: NativeVideoExportSession) { + const port = session.framePort; + session.framePort = null; + session.framePortReady = false; + session.pendingFrameRequests.clear(); + if (port) { + try { + port.close(); + } catch { + // The renderer may already have closed the port. + } + } +} + +export function flushNativeVideoExportFramePortPendingRequests( + sessionId: string, + session: NativeVideoExportSession, + error: string, +) { + const port = session.framePort; + if (port && session.framePortReady) { + for (const [requestId, pendingRequest] of session.pendingFrameRequests) { + sendNativeVideoExportFramePortError(port, sessionId, error, { + requestId, + sequence: pendingRequest.sequence, + fallbackAvailable: false, + }); + } + } + session.pendingFrameRequests.clear(); +} + +function handleNativeVideoExportFramePortMessage( + sessionId: string, + session: NativeVideoExportSession, + port: MessagePortMain, + value: unknown, +) { + if (!isRecord(value)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Invalid native export frame message", + { + fallbackAvailable: false, + }, + ); + return; + } + + if (value.protocol !== NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Unsupported native export frame protocol", + { + fallbackAvailable: true, + }, + ); + return; + } + if (value.sessionId !== sessionId) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame session mismatch", + { + fallbackAvailable: false, + }, + ); + return; + } + + if (value.type === "hello") { + if (session.framePortReady) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel handshake was already completed", + { fallbackAvailable: false }, + ); + return; + } + if (!isArrayBuffer(value.capabilityProbe) || value.capabilityProbe.byteLength !== 1) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel transferable probe was invalid", + { fallbackAvailable: true }, + ); + clearNativeVideoExportFramePort(session); + return; + } + + if ( + !sendNativeVideoExportFramePortMessage( + port, + { + type: "ready", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + transferable: true, + transferProbe: value.capabilityProbe, + }, + value.capabilityProbe, + ) + ) { + clearNativeVideoExportFramePort(session); + return; + } + session.framePortReady = true; + return; + } + + if (value.type !== "frame") { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Unknown native export frame message", + { + fallbackAvailable: false, + }, + ); + return; + } + if (!session.framePortReady) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel handshake is incomplete", + { fallbackAvailable: true }, + ); + return; + } + + const requestId = value.requestId; + const sequence = value.sequence; + if (!isValidRequestNumber(requestId) || !isValidRequestNumber(sequence)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame request and sequence must be non-negative safe integers", + { fallbackAvailable: false }, + ); + return; + } + if ( + requestId <= session.highestAcceptedFrameRequestId || + session.pendingFrameRequests.has(requestId) + ) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Duplicate native export frame request", + { + requestId, + sequence, + fallbackAvailable: false, + }, + ); + return; + } + if (sequence !== session.nextFrameSequence) { + sendNativeVideoExportFramePortError( + port, + sessionId, + sequence < session.nextFrameSequence + ? "Duplicate native export frame sequence" + : `Out-of-order native export frame sequence; expected ${session.nextFrameSequence}`, + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (!isArrayBuffer(value.frame) || value.frame.byteLength === 0) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame payload must be a non-empty ArrayBuffer", + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (session.inputMode !== "h264-stream" && value.frame.byteLength !== session.inputByteSize) { + sendNativeVideoExportFramePortError( + port, + sessionId, + `Native video export expected ${session.inputByteSize} bytes per frame but received ${value.frame.byteLength}`, + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (session.terminating) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session was cancelled", + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + + session.pendingFrameRequests.set(requestId, { sequence }); + session.highestAcceptedFrameRequestId = requestId; + session.nextFrameSequence += 1; + void enqueueNativeVideoExportFrameWrite(session, value.frame) + .then(() => { + const pendingRequest = session.pendingFrameRequests.get(requestId); + if (!pendingRequest) { + return; + } + session.pendingFrameRequests.delete(requestId); + if ( + !sendNativeVideoExportFramePortMessage(port, { + type: "ack", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + requestId, + sequence: pendingRequest.sequence, + success: true, + }) + ) { + clearNativeVideoExportFramePort(session); + } + }) + .catch((error: unknown) => { + const nativeError = error instanceof Error ? error : new Error(String(error)); + session.stdinError = nativeError; + const pendingRequest = session.pendingFrameRequests.get(requestId); + if (!pendingRequest) { + return; + } + session.pendingFrameRequests.delete(requestId); + if ( + !sendNativeVideoExportFramePortError(port, sessionId, nativeError.message, { + requestId, + sequence: pendingRequest.sequence, + fallbackAvailable: false, + }) + ) { + clearNativeVideoExportFramePort(session); + } + }); +} + +export function attachNativeVideoExportFramePort( + sessionId: string, + session: NativeVideoExportSession, + port: MessagePortMain, + sender: WebContents, +) { + if (session.terminating) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session was cancelled", + { + fallbackAvailable: false, + }, + ); + port.close(); + return false; + } + + if (session.framePort) { + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + "Native export frame channel was replaced", + ); + clearNativeVideoExportFramePort(session); + } + + session.sender = sender; + session.framePort = port; + session.framePortReady = false; + session.nextFrameSequence = 0; + session.pendingFrameRequests.clear(); + session.highestAcceptedFrameRequestId = -1; + port.on("message", (event) => { + if (session.framePort !== port) { + return; + } + handleNativeVideoExportFramePortMessage(sessionId, session, port, event.data); + }); + port.on("close", () => { + if (session.framePort !== port) { + return; + } + session.framePort = null; + session.framePortReady = false; + session.pendingFrameRequests.clear(); + }); + sender.once("destroyed", () => { + if (session.framePort !== port) { + return; + } + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + "Native export renderer was reloaded before frame acknowledgements settled", + ); + clearNativeVideoExportFramePort(session); + }); + port.start(); + return true; +} + +export function closeNativeVideoExportFramePort( + sessionId: string, + session: NativeVideoExportSession, + error: string, +) { + flushNativeVideoExportFramePortPendingRequests(sessionId, session, error); + clearNativeVideoExportFramePort(session); +} + export interface NativeStaticLayoutTimelineSegment { sourceStartMs: number; sourceEndMs: number; @@ -92,15 +594,38 @@ export interface NativeStaticLayoutTimelineSegment { speed: number; } +/** Fixed-position rgba raw overlay layer or packed cursor-sprite layer. */ +export type NativeStaticLayoutOverlayLayerUnion = + | NativeStaticLayoutOverlayLayer + | NativeCursorSpriteOverlayLayer; + +/** Discriminates a cursor-sprite layer from a fixed-position rgba layer. */ +export function isCursorSpriteOverlayLayer( + layer: NativeStaticLayoutOverlayLayerUnion, +): layer is NativeCursorSpriteOverlayLayer { + return (layer as { kind?: string }).kind === NATIVE_CURSOR_SPRITE_LAYER_KIND; +} + export interface NativeStaticLayoutExportOptions { sessionId?: string; inputPath: string; + /** High-level output contract; encoder names never cross this boundary. */ + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; width: number; height: number; frameRate: number; bitrate: number; encodingMode: NativeExportEncodingMode; durationSec: number; + overlayLayers?: NativeStaticLayoutOverlayLayerUnion[]; + /** + * Optional tiled/delta overlay layers (sparse overlay optimization). The + * renderer emits this instead of raw overlayLayers for sparse content; the + * descriptor is versioned, bounded, and independently validated by + * validateNativeTiledOverlayLayerDescriptor. Additive and never persisted. + */ + tiledOverlayLayers?: NativeTiledOverlayLayerDescriptor[]; contentWidth: number; contentHeight: number; offsetX: number; @@ -122,6 +647,18 @@ export interface NativeStaticLayoutExportOptions { webcamShadowIntensity?: number; webcamMirror?: boolean; webcamTimeOffsetMs?: number; + /** + * True when the renderer excluded webcam pixels from the overlay sidecars + * and the generalized NVIDIA CUDA compositor owns the webcam overlay natively + * (--webcam-input contract). The CUDA compositor must draw the webcam on top + * of the composed video; stripping the webcam args while the sidecar already + * baked them would double-render, and passing them while the sidecar still + * contained the webcam would double-render too. Absent/false keeps the + * baked-webcam contract. Only the strict HEVC Hardware CUDA route sets this + * (that route hard-fails instead of falling back), so a fallback that cannot + * draw a native webcam can never silently drop it. + */ + webcamNativeOwned?: boolean; cursorTelemetry?: Array<{ timeMs: number; cx: number; @@ -145,8 +682,34 @@ export interface NativeStaticLayoutExportOptions { aspectRatio: number; }>; cursorAtlasMetadataPath?: string | null; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; zoomTelemetryPath?: string | null; + /** Resolved temporal zoom motion blur plan (temporalMotionBlur.ts config). */ + temporalBlur?: { + sampleCount: number; + shutterFraction: number; + weightCurvePower: number; + } | null; + /** JSON manifest describing renderer-prepared RGBA overlay sidecars. */ + overlayManifestPath?: string | null; + /** JSON manifest describing the versioned tiled/delta overlay descriptor. */ + tiledOverlayManifestPath?: string | null; + /** + * True when the renderer excluded cursor pixels from the overlay sidecars + * and the native cursor atlas owns them. The CUDA compositor must draw the + * atlas on top of the sidecars; stripping the cursor assets would drop the + * cursor, and passing them while the sidecar still baked them would + * double-render. Absent/false keeps the baked-sidecar contract. + */ + cursorAtlasOwned?: boolean; timelineSegments?: NativeStaticLayoutTimelineSegment[]; timelineMapPath?: string | null; chunkDurationSec?: number; @@ -156,13 +719,37 @@ export interface NativeStaticLayoutExportOptions { nvidiaCudaForceVideoOnly?: boolean; } +/** + * Additive CUDA-preparation sub-stage. Emitted as display-only progress during + * the native NVIDIA CUDA compositor startup so the renderer can show what the + * main/native side is doing (encoder probe -> source validation -> wrapper + * launch -> CUDA/NVENC initialization -> first-frame readiness). These are + * preparation substates, never encode throughput: they must not fabricate FPS. + */ +export type NativeStaticLayoutPrepareSubstate = + | "encoder-probe" + | "source-validation" + | "wrapper-launch" + | "cuda-nvenc-init" + | "first-frame"; + export interface NativeStaticLayoutExportProgress { sessionId?: string; backend?: NativeStaticLayoutBackend; stage?: "preparing" | "finalizing"; + /** Additive preparation sub-stage (see NativeStaticLayoutPrepareSubstate). */ + substate?: NativeStaticLayoutPrepareSubstate; elapsedMs?: number; averageFps?: number; instantFps?: number; + /** + * End-to-end estimate (frames / wall-clock since process spawn). This + * includes preparation time, so it is NOT a native encode speed; it must + * never be presented as measured encode FPS. + */ + estimatedFps?: number; + /** Where the reported FPS values came from. */ + fpsSource?: "native" | "estimated"; intervalMs?: number; intervalFrames?: number; intervalDecodeWallMs?: number; @@ -176,6 +763,11 @@ export interface NativeStaticLayoutExportProgress { intervalRoiCompositeFrames?: number; intervalMonolithicCompositeFrames?: number; intervalCopyCompositeFrames?: number; + intervalZoomBlurFrames?: number; + intervalTemporalBlurStationaryFrames?: number; + intervalTemporalBgCacheBuilds?: number; + intervalTemporalBgCacheHits?: number; + intervalOverlayStaticRegionBlends?: number; currentFrame: number; totalFrames: number; percentage: number; @@ -231,9 +823,98 @@ export interface WindowsGpuExportSummary { realtimeMultiplier?: number; } +export interface NvidiaCudaNativeSummary { + success?: boolean; + selectionStage?: string; + sourceTimestampMode?: string; + timelineMode?: string; + frames?: number; + totalMs?: number; + fps?: number; + measuredFps?: number; + mappedDisplayFrames?: number; + selectedDisplayFrames?: number; + skippedDisplayFrames?: number; + roiCompositeFrames?: number; + monolithicCompositeFrames?: number; + copyCompositeFrames?: number; + cursorAtlas?: boolean; + webcamOverlay?: boolean; + zoomOverlay?: boolean; + zoomSamples?: number; + overlayLayers?: number; + overlayBlendFrames?: number; + /** Tiled overlay layers in the export (renderer-derived, additive). */ + tiledOverlayLayers?: number; + /** Tiles whose payload changed across all frame deltas (additive). */ + changedTileCount?: number; + /** Tile payload bytes uploaded once across the stream (additive). */ + uploadedTileBytes?: number; + /** Tile-state references served from previously uploaded payloads (additive). */ + cachedTileCount?: number; + /** + * Observable reason the layer used the raw full-frame fallback instead of a + * tiled stream (small-layer | dense-frame-delta | payload-bytes-exceed-raw). + */ + rawFallbackReason?: string; + /** Renderer-resolved temporal zoom motion blur sample count (config, not + * measured: the helper does not yet echo it in its summary). */ + temporalBlurSampleCount?: number; + /** Temporal blur output frames (additive compositor counter). */ + temporalBlurFrames?: number; + /** Total temporal blur sample composites across all output frames. */ + temporalBlurSamplesTotal?: number; + /** Temporal blur frames that reused the precomposed invariant background. */ + temporalBlurBgPrecomposedFrames?: number; + /** Overlay sidecar frames read from disk (additive). */ + overlayFileLoads?: number; + /** Overlay sidecar frames served from the resident ring slot (additive). */ + overlayCacheHits?: number; + /** Overlay blends confined to static alpha bounds (additive). */ + overlayStaticRegionBlends?: number; + /** Overlay sidecar width in pixels (renderer-derived, additive). */ + overlayWidth?: number; + /** Overlay sidecar height in pixels (renderer-derived, additive). */ + overlayHeight?: number; + /** Overlay sidecar logical frame count (output duration frames). */ + overlayFrameCount?: number; + /** Overlay sidecar physical frame count (effectiveFrameCount when deduped). */ + overlayPhysicalFrames?: number; + /** Overlay sidecar stored frame count after identical-suffix dedup. */ + overlayEffectiveFrames?: number; + /** Temporal blur frames composited by the stationary fused kernel. */ + temporalBlurStationaryFrames?: number; + /** Temporal blur invariant-background cache builds (additive). */ + temporalBgCacheBuilds?: number; + /** Temporal blur invariant-background cache hits (additive). */ + temporalBgCacheHits?: number; + /** + * Overlay host-read wall time in ms. main.cu reports the separated + * host-read and H2D enqueue spans; these fields are emitted when the helper + * measured them (additive over the encode run). + */ + overlayHostReadMs?: number; + overlayH2DEnqueueMs?: number; + /** Stage wall/GPU timings in ms, additive over the encode run. */ + compositeMs?: number; + compositeGpuMs?: number; + zoomBlurGpuMs?: number; + overlayBlendGpuMs?: number; + overlayUploadMs?: number; + nvencMs?: number; + packetWriteMs?: number; + decodeMs?: number; + decodeWallMs?: number; + encodeMs?: number; + flushMs?: number; + realtimeMultiplier?: number; + outputBytes?: number; +} + export interface NvidiaCudaExportSummary { success?: boolean; inputPath?: string; + outputCodec?: ExportVideoCodec; outputPath?: string; fps?: number; bitrateMbps?: number; @@ -252,26 +933,7 @@ export interface NvidiaCudaExportSummary { mux?: number; endToEnd?: number; }; - nativeSummary?: { - success?: boolean; - selectionStage?: string; - sourceTimestampMode?: string; - timelineMode?: string; - frames?: number; - totalMs?: number; - fps?: number; - measuredFps?: number; - mappedDisplayFrames?: number; - selectedDisplayFrames?: number; - skippedDisplayFrames?: number; - roiCompositeFrames?: number; - monolithicCompositeFrames?: number; - copyCompositeFrames?: number; - cursorAtlas?: boolean; - webcamOverlay?: boolean; - zoomOverlay?: boolean; - zoomSamples?: number; - }; + nativeSummary?: NvidiaCudaNativeSummary; nativeProcessPriorityBoosted?: boolean; appRuntimeGuard?: { powerGuardStarted?: boolean; @@ -327,6 +989,15 @@ export interface NativeStaticLayoutExportMetrics extends NativeVideoAudioMuxMetr chunks: NativeStaticLayoutChunkMetric[]; } +export interface NativeStaticLayoutExportResult { + outputPath: string; + metrics: NativeStaticLayoutExportMetrics; + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + encoderName: string; + route: NativeStaticLayoutBackend; +} + export interface NativeStaticLayoutExportSession { terminating: boolean; currentProcess: ReturnType | null; @@ -335,6 +1006,11 @@ export interface NativeStaticLayoutExportSession { export function cleanupNativeVideoExportSessions() { for (const [sessionId, session] of nativeVideoExportSessions) { session.terminating = true; + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export sessions were cleaned up", + ); try { if (!session.ffmpegProcess.stdin.destroyed) { session.ffmpegProcess.stdin.destroy(); @@ -397,6 +1073,347 @@ export function parseNvidiaCudaExportSummary(stdout: string): NvidiaCudaExportSu } } +const NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS = [ + "temporalBlurSampleCount", + "temporalBlurSamplesTotal", + "temporalBlurBgPrecomposedFrames", + "temporalBlurFrames", + "temporalBlurStationaryFrames", + "temporalBgCacheBuilds", + "temporalBgCacheHits", + "compositeGpuMs", + "zoomBlurGpuMs", + "overlayBlendGpuMs", + "overlayUploadMs", + "overlayFileLoads", + "overlayCacheHits", + "overlayStaticRegionBlends", + "overlayBlendFrames", + "overlayWidth", + "overlayHeight", + "overlayFrameCount", + "overlayPhysicalFrames", + "overlayEffectiveFrames", + "overlayHostReadMs", + "overlayH2DEnqueueMs", + "changedTileCount", + "uploadedTileBytes", + "cachedTileCount", + "nvencMs", + "packetWriteMs", + "totalMs", + "encodeMs", + "decodeMs", + "decodeWallMs", + "compositeMs", + "flushMs", + "realtimeMultiplier", + "outputBytes", +] as const satisfies ReadonlyArray; + +export type NvidiaCudaNativeSummaryMetricField = + (typeof NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS)[number]; + +/** + * Maps the additive native compositor counters into a flat metric object for + * the completion log. Only fields the helper actually reported as finite + * numbers are included, so absent counters never appear as undefined noise and + * non-finite payloads are never surfaced as measured values. The values are + * cumulative over the whole helper run (decode/compose/overlay/NVENC stages), + * never interval deltas. The tiled rawFallbackReason rides along as a string + * when the helper or wrapper reported it. + */ +export function resolveNvidiaCudaNativeSummaryMetrics( + nativeSummary: NvidiaCudaNativeSummary | undefined, +): Partial> { + const metrics: Partial< + Record + > = {}; + if (!nativeSummary) { + return metrics; + } + + for (const field of NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS) { + const value = nativeSummary[field]; + if (typeof value === "number" && Number.isFinite(value)) { + metrics[field] = value; + } + } + if (typeof nativeSummary.rawFallbackReason === "string" && nativeSummary.rawFallbackReason) { + metrics.rawFallbackReason = nativeSummary.rawFallbackReason; + } + return metrics; +} + +export type NvidiaCudaOverlaySidecarSummaryMetric = Pick< + NvidiaCudaNativeSummary, + | "overlayWidth" + | "overlayHeight" + | "overlayFrameCount" + | "overlayPhysicalFrames" + | "overlayEffectiveFrames" +>; + +/** + * Derives the overlay sidecar dimensions and physical/effective frame counts + * from the renderer-prepared overlay layers for CUDA summary surfacing. All + * fields are additive and only present when overlay layers exist. The physical + * count is the sidecar's stored frame count (effectiveFrameCount after + * identical-suffix dedup); the logical count is the output duration. + */ +export function resolveNvidiaCudaOverlaySidecarSummaryMetrics( + options: NativeStaticLayoutExportOptions, +): Partial { + const layer = options.overlayLayers?.[0]; + if ( + !layer || + !Number.isFinite(layer.width) || + !Number.isFinite(layer.height) || + !Number.isFinite(layer.frameCount) + ) { + return {}; + } + + // A cursor-sprite layer is a packed strip (no effectiveFrameCount dedup); its + // physical frame count equals the logical output duration. + const effectiveFrameCount = (layer as { effectiveFrameCount?: number }).effectiveFrameCount; + const metrics: Partial = { + overlayWidth: Math.max(1, Math.round(layer.width)), + overlayHeight: Math.max(1, Math.round(layer.height)), + overlayFrameCount: Math.max(1, Math.round(layer.frameCount)), + overlayPhysicalFrames: Math.max(1, Math.round(effectiveFrameCount ?? layer.frameCount)), + }; + if (effectiveFrameCount !== undefined) { + metrics.overlayEffectiveFrames = Math.max(1, Math.round(effectiveFrameCount)); + } + return metrics; +} + +export type NvidiaCudaTiledOverlaySummaryMetric = Pick< + NvidiaCudaNativeSummary, + | "tiledOverlayLayers" + | "changedTileCount" + | "uploadedTileBytes" + | "cachedTileCount" + | "rawFallbackReason" + | "overlayHostReadMs" + | "overlayH2DEnqueueMs" + | "overlayCacheHits" +>; + +/** + * Derives the additive tiled/delta overlay metrics from the renderer-prepared + * tiled overlay layers for CUDA summary surfacing. The values are diagnostic + * only (changedTileCount, uploadedTileBytes, cachedTileCount are renderer + * bookkeeping, never a zero-copy claim) and only present when tiled layers + * exist. rawFallbackReason is the conservative eligibility decision of the + * first layer that needed the raw full-frame fallback. + */ +export function resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( + options: NativeStaticLayoutExportOptions, +): Partial { + const layers = options.tiledOverlayLayers; + if (!layers?.length) { + return {}; + } + + const metrics: Partial = { + tiledOverlayLayers: layers.length, + }; + let changedTileCount = 0; + let uploadedTileBytes = 0; + let cachedTileCount = 0; + let fallbackReason: string | null = null; + for (const layer of layers) { + const layerMetrics = resolveNativeTiledOverlayMetrics(layer); + changedTileCount += layerMetrics.changedTileCount; + uploadedTileBytes += layerMetrics.uploadedTileBytes; + cachedTileCount += layerMetrics.cachedTileCount; + fallbackReason ??= resolveNativeTiledOverlayRawFallbackReason(layer); + } + metrics.changedTileCount = changedTileCount; + metrics.uploadedTileBytes = uploadedTileBytes; + metrics.cachedTileCount = cachedTileCount; + if (fallbackReason !== null) { + metrics.rawFallbackReason = fallbackReason; + } + return metrics; +} + +/** + * Resolves the measured native encode FPS reported by the helper: the flush + * span measuredFps is authoritative, with the helper's encode-loop fps as the + * backward-compatible fallback. Never derives FPS from wall-clock estimates + * and never falls back to the configured output stream fps (summary.fps) and + * labels it as measured encode speed; if the helper reported no measured FPS + * this returns undefined. + */ +export function resolveNvidiaCudaNativeFps( + summary: NvidiaCudaExportSummary | undefined, +): number | undefined { + const nativeSummary = summary?.nativeSummary; + const measuredFps = nativeSummary?.measuredFps; + if (typeof measuredFps === "number" && Number.isFinite(measuredFps) && measuredFps > 0) { + return measuredFps; + } + const fallbackFps = nativeSummary?.fps; + if (typeof fallbackFps === "number" && Number.isFinite(fallbackFps) && fallbackFps > 0) { + return fallbackFps; + } + return undefined; +} + +/** + * Verifies the additive compositor counters against each other so the + * completion diagnostics can prove (rather than assume) that the reported + * stage timings are sane. Returns human-readable issue strings; an empty array + * means the metrics are internally consistent. These are diagnostics only and + * never gate the export result. + */ +export function validateNvidiaCudaStageMetricInvariants( + summary: NvidiaCudaExportSummary, +): string[] { + const issues: string[] = []; + const native = summary.nativeSummary; + if (!native) { + return issues; + } + + const totalMs = native.totalMs; + if (typeof totalMs === "number" && Number.isFinite(totalMs) && totalMs >= 0) { + const stageFields: ReadonlyArray< + readonly [NvidiaCudaNativeSummaryMetricField, number | undefined] + > = [ + ["compositeGpuMs", native.compositeGpuMs], + ["overlayBlendGpuMs", native.overlayBlendGpuMs], + ["overlayUploadMs", native.overlayUploadMs], + ["nvencMs", native.nvencMs], + ["packetWriteMs", native.packetWriteMs], + ["compositeMs", native.compositeMs], + ["zoomBlurGpuMs", native.zoomBlurGpuMs], + ["encodeMs", native.encodeMs], + ["decodeWallMs", native.decodeWallMs], + ]; + for (const [field, value] of stageFields) { + if (typeof value === "number" && Number.isFinite(value) && value > totalMs + 0.5) { + issues.push(`${field} ${value}ms exceeds helper wall time ${totalMs}ms`); + } + } + } + + const temporalBlurFrames = native.temporalBlurFrames; + if (typeof temporalBlurFrames === "number" && Number.isFinite(temporalBlurFrames)) { + const samplesTotal = native.temporalBlurSamplesTotal; + if ( + typeof samplesTotal === "number" && + Number.isFinite(samplesTotal) && + samplesTotal < temporalBlurFrames + ) { + issues.push( + `temporalBlurSamplesTotal ${samplesTotal} below temporalBlurFrames ${temporalBlurFrames}`, + ); + } + const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; + if ( + typeof bgPrecomposedFrames === "number" && + Number.isFinite(bgPrecomposedFrames) && + bgPrecomposedFrames > temporalBlurFrames + ) { + issues.push( + `temporalBlurBgPrecomposedFrames ${bgPrecomposedFrames} exceeds temporalBlurFrames ${temporalBlurFrames}`, + ); + } + } + + const stationaryFrames = native.temporalBlurStationaryFrames; + if ( + typeof stationaryFrames === "number" && + Number.isFinite(stationaryFrames) && + typeof temporalBlurFrames === "number" && + Number.isFinite(temporalBlurFrames) && + stationaryFrames > temporalBlurFrames + ) { + issues.push( + `temporalBlurStationaryFrames ${stationaryFrames} exceeds temporalBlurFrames ${temporalBlurFrames}`, + ); + } + // temporalBlurBgCacheBuilds count cache allocations/segments, while hits and + // temporalBlurBgPrecomposedFrames count per-frame background reuse. Builds are + // therefore NOT part of the frame budget and must never be summed into it; + // adding them produced a false positive (e.g. builds 1 + hits 4 vs. 4 + // precomposed frames). Each counter must stay finite/non-negative, and hits + // must not exceed the precomposed (or total temporal) frame budget. + const bgCacheBuilds = native.temporalBgCacheBuilds; + if (typeof bgCacheBuilds === "number" && Number.isFinite(bgCacheBuilds) && bgCacheBuilds < 0) { + issues.push(`temporalBgCacheBuilds ${bgCacheBuilds} must be non-negative`); + } + + const bgCacheHits = native.temporalBgCacheHits; + if (typeof bgCacheHits === "number" && Number.isFinite(bgCacheHits)) { + if (bgCacheHits < 0) { + issues.push(`temporalBgCacheHits ${bgCacheHits} must be non-negative`); + } else { + const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; + const hasBgPrecomposedFrames = + typeof bgPrecomposedFrames === "number" && Number.isFinite(bgPrecomposedFrames); + const cacheBudget: number | null = hasBgPrecomposedFrames + ? bgPrecomposedFrames + : typeof temporalBlurFrames === "number" && Number.isFinite(temporalBlurFrames) + ? temporalBlurFrames + : null; + if (cacheBudget !== null && bgCacheHits > cacheBudget) { + issues.push( + `temporalBgCacheHits ${bgCacheHits} exceeds ${ + hasBgPrecomposedFrames + ? "temporalBlurBgPrecomposedFrames" + : "temporalBlurFrames" + } ${cacheBudget}`, + ); + } + } + } + + const overlayBlendFrames = native.overlayBlendFrames; + const overlayStaticRegionBlends = native.overlayStaticRegionBlends; + if ( + typeof overlayStaticRegionBlends === "number" && + Number.isFinite(overlayStaticRegionBlends) && + typeof overlayBlendFrames === "number" && + Number.isFinite(overlayBlendFrames) && + overlayStaticRegionBlends > overlayBlendFrames + ) { + issues.push( + `overlayStaticRegionBlends ${overlayStaticRegionBlends} exceeds overlayBlendFrames ${overlayBlendFrames}`, + ); + } + + const overlayFrameCount = native.overlayFrameCount; + if (typeof overlayFrameCount === "number" && Number.isFinite(overlayFrameCount)) { + const overlayPhysicalFrames = native.overlayPhysicalFrames; + if ( + typeof overlayPhysicalFrames === "number" && + Number.isFinite(overlayPhysicalFrames) && + overlayPhysicalFrames > overlayFrameCount + ) { + issues.push( + `overlayPhysicalFrames ${overlayPhysicalFrames} exceeds overlayFrameCount ${overlayFrameCount}`, + ); + } + const overlayEffectiveFrames = native.overlayEffectiveFrames; + if ( + typeof overlayEffectiveFrames === "number" && + Number.isFinite(overlayEffectiveFrames) && + (overlayEffectiveFrames < 1 || overlayEffectiveFrames > overlayFrameCount) + ) { + issues.push( + `overlayEffectiveFrames ${overlayEffectiveFrames} out of range for overlayFrameCount ${overlayFrameCount}`, + ); + } + } + + return issues; +} + function getFiniteNumber(value: unknown) { const numberValue = typeof value === "string" ? Number(value) : value; return typeof numberValue === "number" && Number.isFinite(numberValue) ? numberValue : null; @@ -416,6 +1433,7 @@ export function validateNvidiaCudaExportSummary( durationSec: number; targetFrames: number; requiresTimelineSync?: boolean; + videoCodec?: ExportVideoCodec; }, ) { const issues: string[] = []; @@ -428,6 +1446,11 @@ export function validateNvidiaCudaExportSummary( const outputVideoDurationSec = getNvidiaCudaOutputStreamNumber(summary.outputVideo, "duration"); const outputAudioDurationSec = getNvidiaCudaOutputStreamNumber(summary.outputAudio, "duration"); + if (expected.videoCodec && summary.outputCodec !== expected.videoCodec) { + issues.push( + `CUDA output codec ${summary.outputCodec ?? "unknown"} does not match expected ${expected.videoCodec}`, + ); + } if (!summary.outputVideo) { issues.push("missing output video probe"); } @@ -807,6 +1830,11 @@ export function parseWindowsGpuExportProgressLine( intervalRoiCompositeFrames?: unknown; intervalMonolithicCompositeFrames?: unknown; intervalCopyCompositeFrames?: unknown; + intervalZoomBlurFrames?: unknown; + intervalTemporalBlurStationaryFrames?: unknown; + intervalTemporalBgCacheBuilds?: unknown; + intervalTemporalBgCacheHits?: unknown; + intervalOverlayStaticRegionBlends?: unknown; stage?: unknown; }; const currentFrame = Number(parsed.currentFrame); @@ -845,6 +1873,11 @@ export function parseWindowsGpuExportProgressLine( "intervalRoiCompositeFrames", "intervalMonolithicCompositeFrames", "intervalCopyCompositeFrames", + "intervalZoomBlurFrames", + "intervalTemporalBlurStationaryFrames", + "intervalTemporalBgCacheBuilds", + "intervalTemporalBgCacheHits", + "intervalOverlayStaticRegionBlends", ] as const; for (const field of optionalNumberFields) { const value = Number(parsed[field]); @@ -870,6 +1903,51 @@ export function mapNvidiaCudaWrapperProgressPercentage(progress: NativeStaticLay return progress.percentage; } +// Distinguish native measured encode FPS from the end-to-end (preparation- +// inclusive) estimate. Only averageFps/instantFps reported by the native helper +// count as measured encode speed; the frames/wall-clock estimate since process +// spawn must be surfaced separately so callers never mistake it for encode +// throughput. Finalizing-stage progress must never emit the preparation- +// inclusive estimate: by then the helper has already measured encode speed on +// earlier progress lines, and the flush/mux span has no frame rate of its own, +// so an estimate there would only misrepresent the display. +export function resolveNativeStaticLayoutFpsFields( + progress: NativeStaticLayoutExportProgress, + elapsedMs: number, +): { + averageFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; +} { + const nativeAverageFps = + typeof progress.averageFps === "number" && + Number.isFinite(progress.averageFps) && + progress.averageFps > 0 + ? progress.averageFps + : undefined; + const nativeInstantFps = + typeof progress.instantFps === "number" && + Number.isFinite(progress.instantFps) && + progress.instantFps > 0 + ? progress.instantFps + : undefined; + const hasNativeMeasured = nativeInstantFps !== undefined || nativeAverageFps !== undefined; + const finalizing = progress.stage === "finalizing"; + const estimatedFps = + !hasNativeMeasured && !finalizing && elapsedMs > 0 && progress.currentFrame > 0 + ? (progress.currentFrame * 1000) / elapsedMs + : undefined; + return { + averageFps: nativeAverageFps, + estimatedFps, + fpsSource: hasNativeMeasured + ? "native" + : estimatedFps !== undefined + ? "estimated" + : undefined, + }; +} + export function hasNativeStaticLayoutProgressAdvanced( progress: { currentFrame: number; percentage: number; stage?: string }, previous: { currentFrame: number; percentage: number; stage?: string }, @@ -888,6 +1966,57 @@ export function hasNativeStaticLayoutProgressAdvanced( return progress.stage === "finalizing" && previous.stage !== "finalizing"; } +// Display-only preparation percentages for the NVIDIA CUDA startup substates. +// All stay inside the preparing window (<=3) so the renderer treats every +// event as non-rendering preparation and never derives an encode FPS from it. +// They are additive progression markers only; encode speed is reported later +// by the measured helper PROGRESS lines (nativeFps) and never here. +const NATIVE_STATIC_LAYOUT_PREPARE_SUBSTATE_PERCENTAGE: Readonly< + Record +> = { + "encoder-probe": 0.4, + "source-validation": 1.0, + "wrapper-launch": 1.6, + "cuda-nvenc-init": 2.2, + "first-frame": 2.8, +}; + +/** + * Builds an additive CUDA-preparation progress payload for a selected NVIDIA + * CUDA compositor route. The backend is labelled "nvidia-cuda-compositor" + * from the very first preparation event so the UI names the correct encoder + * before any renderer frame is produced. currentFrame stays 0 and percentage + * stays within the preparing window: this payload carries no FPS fields and + * must never be mistaken for measured encode speed (it is display-only). + */ +export function buildNvidiaCudaPrepareProgress( + sessionId: string | undefined, + substate: NativeStaticLayoutPrepareSubstate, + totalFrames: number, + elapsedMs: number, +): NativeStaticLayoutExportProgress { + return { + sessionId, + stage: "preparing", + substate, + backend: "nvidia-cuda-compositor", + currentFrame: 0, + totalFrames: Math.max(1, Math.floor(totalFrames)), + percentage: NATIVE_STATIC_LAYOUT_PREPARE_SUBSTATE_PERCENTAGE[substate], + elapsedMs: Math.max(0, Math.round(elapsedMs)), + }; +} + +function emitNvidiaCudaPrepareProgress( + onProgress: ((progress: NativeStaticLayoutExportProgress) => void) | undefined, + sessionId: string | undefined, + substate: NativeStaticLayoutPrepareSubstate, + totalFrames: number, + elapsedMs: number, +) { + onProgress?.(buildNvidiaCudaPrepareProgress(sessionId, substate, totalFrames, elapsedMs)); +} + function startNativeStaticLayoutExportPowerGuard() { try { const blockerId = powerSaveBlocker.start("prevent-app-suspension"); @@ -900,7 +2029,11 @@ function startNativeStaticLayoutExportPowerGuard() { }, }; } catch (error) { - console.warn("[native-static-layout-export] Failed to start power guard", error); + console.warn( + formatLogTs(), + "[native-static-layout-export] Failed to start power guard", + error, + ); return { started: false, release: () => undefined, @@ -917,7 +2050,11 @@ function setNativeStaticLayoutExportProcessPriority(pid: number | undefined, lab os.setPriority(pid, NATIVE_EXPORT_HIGH_PRIORITY); return true; } catch (error) { - console.warn(`[native-static-layout-export] Failed to raise ${label} priority`, error); + console.warn( + formatLogTs(), + `[native-static-layout-export] Failed to raise ${label} priority`, + error, + ); return false; } } @@ -1223,6 +2360,12 @@ async function runFfmpegWithMetrics( }); if (session) { session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below (killing an already-exited child on Windows emits an unhandled + // 'error' event when no listener is attached yet). + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -1368,6 +2511,11 @@ async function runFfmpegAudioMux( }); if (session) { session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below; see the CUDA wrapper spawn for the rationale. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -1491,6 +2639,7 @@ export function flushNativeVideoExportPendingWriteRequests( session: NativeVideoExportSession, error: string, ) { + flushNativeVideoExportFramePortPendingRequests(sessionId, session, error); for (const requestId of session.pendingWriteRequestIds) { sendNativeVideoExportWriteFrameResult(session.sender, sessionId, requestId, { success: false, @@ -1869,11 +3018,6 @@ export function hasNvidiaGpuDeviceInGpuInfo(gpuInfo: unknown) { return Array.isArray(devices) && devices.some(isNvidiaGpuDevice); } -async function hasNvidiaGpuForCudaExportCandidate() { - const hasNvidiaGpu = await probeNvidiaGpuForCudaExportCandidate(); - return hasNvidiaGpu ?? true; -} - async function probeNvidiaGpuForCudaExportCandidate(): Promise { const getGPUInfo = ( app as typeof app & { @@ -2022,6 +3166,143 @@ export function getNativeGpuCompositorStallTimeoutMs() { return DEFAULT_NATIVE_GPU_STALL_TIMEOUT_MS; } +/** + * Session-scoped cache for the expensive NVIDIA CUDA availability probes + * (enumerating the helper wrapper candidates via fs.access and inspecting GPU + * info via Electron's getGPUInfo). Both the capabilities query + * (getNativeExportCapabilities) and the export route decision + * (getExperimentalNvidiaCudaExportSkipReason) share these values so the probes + * run once per session instead of once per call. The cache is keyed on a + * signature of the environment overrides and resolved app paths; whenever a + * relevant override or the resolved helper path changes the entry is rebuilt. + * A runtime helper failure is never promoted into availability: this cache only + * records the pre-flight probe results and is bypassed by the actual runtime + * wrapper invocation (runExperimentalNvidiaCudaStaticLayoutExport), so strict + * HEVC Hardware CUDA-only hard-fail behavior is unchanged. + */ +type NvidiaCudaAvailabilityCache = { + signature: string; + wrapperPath: string | null; + gpuAvailability: boolean | null; + capability: NativeExportCapabilities["nvidiaCuda"]; +}; + +let nvidiaCudaAvailabilityCache: NvidiaCudaAvailabilityCache | null = null; + +/** + * Resets the NVIDIA CUDA availability cache. Exposed for tests; the cache is + * session-scoped so resetting it forces the next capability/route query to + * re-probe the wrapper and GPU. + */ +export function resetNvidiaCudaAvailabilityCache() { + nvidiaCudaAvailabilityCache = null; +} + +/** + * Strict HEVC Hardware policy: when HEVC Hardware is requested the generalized + * NVIDIA CUDA compositor is the ONLY acceptable route. A non-zero + * shouldTryNvidiaCuda here hard-fails rather than falling back to the renderer + * raw path, Breeze, or CPU. Returns the error message to throw, or null when + * the strict guard does not apply. + */ +export function resolveNvidiaCudaStrictHevcHardFail( + requiresStrictHevcCuda: boolean, + shouldTryNvidiaCuda: boolean, + nvidiaCudaSkipReason: string | null, +): string | null { + if (!requiresStrictHevcCuda || shouldTryNvidiaCuda) { + return null; + } + return `HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (${nvidiaCudaSkipReason ?? "cursor-atlas-unavailable"}) (noCpuFallback:true)`; +} + +function getNvidiaCudaAvailabilityEnvSignature() { + const resourcesPath = ( + process as NodeJS.Process & { + resourcesPath?: string; + } + ).resourcesPath; + return JSON.stringify([ + process.platform, + process.env[NVIDIA_CUDA_EXPORT_ENV] ?? null, + process.env[NVIDIA_CUDA_ALLOW_AUDIO_EXPORT_ENV] ?? null, + process.env[NVIDIA_CUDA_FORCE_VIDEO_ONLY_ENV] ?? null, + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT ?? null, + process.env.RECORDLY_NVIDIA_CUDA_NODE_EXE ?? null, + resourcesPath ?? null, + process.cwd(), + app.getAppPath(), + ]); +} + +function resolveNvidiaCudaCapability( + wrapperPath: string | null, + gpuAvailability: boolean | null, +): NativeExportCapabilities["nvidiaCuda"] { + const explicitEnabled = isExplicitNvidiaCudaExportEnabled(); + const explicitDisabled = isExplicitNvidiaCudaExportDisabled(); + // An inconclusive GPU probe (null) must NOT report the CUDA route + // unavailable: Electron's getGPUInfo can fail in dev/packaged runs while the + // live CUDA helper builds and initializes fine. The runtime attempt is the + // authoritative check (the helper fails with noCpuFallback on non-NVIDIA + // hardware), matching getExperimentalNvidiaCudaExportSkipReason which also + // treats an inconclusive probe as "let the helper decide". + const skipReason = explicitDisabled + ? "env-disabled" + : !wrapperPath + ? "cuda-wrapper-unavailable" + : gpuAvailability === false + ? "nvidia-gpu-unavailable" + : null; + if (gpuAvailability === null && !explicitDisabled && wrapperPath) { + console.info( + formatLogTs(), + "[native-export] NVIDIA CUDA GPU probe was inconclusive; letting the live helper decide at runtime", + { wrapperPath, reason: skipReason }, + ); + } else if (skipReason) { + console.info(formatLogTs(), "[native-export] NVIDIA CUDA availability", { + available: false, + skipReason, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + }); + } else { + console.info(formatLogTs(), "[native-export] NVIDIA CUDA availability", { + available: true, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + }); + } + + return { + available: skipReason === null, + skipReason, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + explicitEnabled, + explicitDisabled, + userOptInRequired: !explicitEnabled, + }; +} + +async function ensureNvidiaCudaAvailabilityResolved(): Promise { + const signature = getNvidiaCudaAvailabilityEnvSignature(); + if (nvidiaCudaAvailabilityCache?.signature === signature) { + return nvidiaCudaAvailabilityCache; + } + + const wrapperPath = await resolveExperimentalNvidiaCudaExportScriptPath(); + const gpuAvailability = await probeNvidiaGpuForCudaExportCandidate(); + nvidiaCudaAvailabilityCache = { + signature, + wrapperPath, + gpuAvailability, + capability: resolveNvidiaCudaCapability(wrapperPath, gpuAvailability), + }; + return nvidiaCudaAvailabilityCache; +} + export async function getExperimentalNvidiaCudaExportSkipReason( options: NativeStaticLayoutExportOptions, ) { @@ -2041,10 +3322,11 @@ export async function getExperimentalNvidiaCudaExportSkipReason( } if (userOptIn) { - if (!(await resolveExperimentalNvidiaCudaExportScriptPath())) { + const cache = await ensureNvidiaCudaAvailabilityResolved(); + if (!cache.wrapperPath) { return "cuda-wrapper-unavailable"; } - if (!(await hasNvidiaGpuForCudaExportCandidate())) { + if ((cache.gpuAvailability ?? true) === false) { return "nvidia-gpu-unavailable"; } } @@ -2071,34 +3353,120 @@ export async function getNativeExportCapabilities(): Promise boolean; +} + +export interface NativeExportPrewarmOutcome { + sourceMetadataCached: boolean; + cudaAvailabilityResolved: boolean; + resolvedEncoders: string[]; + skipReasons: string[]; +} + +/** + * Deterministic, side-effect-free prewarming of the session-scoped caches used + * by native static-layout export: the validated source metadata probe cache and + * the NVIDIA CUDA availability cache. Encoder capability resolution is derived + * purely from the requested high-level codec/preference via + * getNativeEncoderCandidates. This never creates source proxies, output files, + * helper exports, or persists derived encoder names/settings. Any failure is + * diagnostics-only and must never poison availability: normal export probing and + * fallback stay unchanged. Strict HEVC Hardware CUDA-only hard-fail behavior is + * untouched because the availability cache only records pre-flight probe results + * and the live runtime export still validates its own route. + */ +export async function prewarmNativeExportCaches( + context: NativeExportPrewarmContext, +): Promise { + const skipReasons: string[] = []; + const outcome: NativeExportPrewarmOutcome = { + sourceMetadataCached: false, + cudaAvailabilityResolved: false, + resolvedEncoders: [], + skipReasons, + }; + + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + + // Canonical source stat/identity + validated source metadata probe, reusing + // the existing exact-keyed bounded probe cache (identity + codec + encoding + // mode + encoder preference). Best-effort warm with the persisted export + // encoding mode (the same key a real export resolves for its source + // metadata), so a later export with the matching route hits the cache + // instead of re-probing. A later export with a different mode still re-probes + // because the exact cache key will not match. + try { + const ffmpegPath = getFfmpegBinaryPath(); + const metadata = await resolveNativeStaticLayoutSourceMetadata( + ffmpegPath, + { inputPath: context.inputPath, encodingMode: context.encodingMode }, + context.videoCodec, + context.encoderPreference, + ); + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + outcome.sourceMetadataCached = isNativeStaticLayoutSourceProbeCacheable(metadata); + skipReasons.push( + outcome.sourceMetadataCached + ? `source-metadata-cached:${metadata.codec}` + : "source-metadata-uncacheable", + ); + } catch { + // Diagnostics-only: a failed metadata probe never poisons availability and + // never blocks the response. Normal export probing is unchanged. + skipReasons.push("source-metadata-unavailable"); + } + + // Encoder capability resolution for the persisted/high-level codec and + // preference. Pure deterministic derivation (no I/O), so this is cheap. + outcome.resolvedEncoders = getNativeEncoderCandidates( + context.videoCodec, + context.encoderPreference, + process.platform, + ); + + // NVIDIA CUDA availability cache (only resolvable on platforms that ship the + // helper). The availability cache records pre-flight probe results only; a + // prewarm failure is never promoted into availability. + try { + const capabilities = await getNativeExportCapabilities(); + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + if (capabilities.nvidiaCuda.available) { + outcome.cudaAvailabilityResolved = true; + } else { + skipReasons.push(`cuda-unavailable:${capabilities.nvidiaCuda.skipReason ?? "unknown"}`); + } + } catch { + skipReasons.push("cuda-probe-failed"); + } + + return outcome; +} + export async function resolveExperimentalNvidiaCudaExportScriptPath() { if (process.platform !== "win32") { return null; @@ -2188,6 +3556,20 @@ function convertHexColorToNv12(color: string) { }; } +export function getNativeStaticLayoutRawFrameFallbackReason( + options: Pick, +) { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + if (encoderPreference === "cpu") { + return "encoder-preference-cpu-requires-native-rawvideo"; + } + if (encoderPreference === "hardware" && videoCodec === "h264") { + return "encoder-preference-hardware-requires-native-rawvideo"; + } + return null; +} + function getNvidiaCudaBitrateMbps(options: NativeStaticLayoutExportOptions) { return Math.max(1, Math.round(options.bitrate / 1_000_000)); } @@ -2196,6 +3578,11 @@ export function buildExperimentalWindowsGpuStaticLayoutArgs( options: NativeStaticLayoutExportOptions, outputPath: string, ) { + if (options.videoCodec === "hevc") { + throw new Error( + "HEVC native static layout requires the generalized NVIDIA CUDA compositor", + ); + } const shadowPixels = Math.round(clampUnit(options.shadowIntensity ?? 0) * 64); const backgroundBlurPx = Math.max(0, options.backgroundBlurPx ?? 0); const pixelCount = options.width * options.height; @@ -2404,6 +3791,39 @@ async function prepareWindowsGpuCursorAtlas( return { atlasPath, metadataPath }; } +/** + * Resolves the native cursor asset paths that may be handed to a GPU compositor + * wrapper. The Windows GPU compositor prep writes CSV telemetry/atlas artifacts + * onto the options; the NVIDIA CUDA wrapper's `--cursor-json` contract is a JSON + * {"samples":[...]} payload (the pipeline rejects raw CSV/TSV rows), so a CSV + * path must never reach it. When overlay sidecars are present AND the cursor is + * baked into the transparent RGBA layer (cursorAtlasOwned is not true), the CUDA + * route must not draw the cursor again: stripping the assets is mandatory, never + * a silent degradation (the wrapper is never given a malformed cursor file and + * never double-renders the baked cursor). When cursorAtlasOwned is true the + * sidecar excluded cursor pixels, so the assets must pass through untouched. + */ +export function resolveNvidiaCudaCursorAssets( + options: NativeStaticLayoutExportOptions, + strip: boolean, +): Pick< + NativeStaticLayoutExportOptions, + "cursorTelemetryPath" | "cursorAtlasPath" | "cursorAtlasMetadataPath" +> { + if (!strip) { + return { + cursorTelemetryPath: options.cursorTelemetryPath ?? null, + cursorAtlasPath: options.cursorAtlasPath ?? null, + cursorAtlasMetadataPath: options.cursorAtlasMetadataPath ?? null, + }; + } + return { + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }; +} + async function prepareNvidiaCudaCursorTelemetry( options: NativeStaticLayoutExportOptions, outputPath: string, @@ -2489,16 +3909,10 @@ async function prepareNvidiaCudaCursorAtlas( return { atlasPath, metadataPath }; } -async function prepareWindowsGpuZoomTelemetry( - options: NativeStaticLayoutExportOptions, - outputPath: string, -) { - const telemetry = options.zoomTelemetry; - if (!telemetry || telemetry.length === 0) { - return null; - } - - const lines = telemetry +export function formatNativeStaticLayoutZoomTelemetryLines( + telemetry: NonNullable, +): string[] { + return telemetry .filter((sample) => { return ( Number.isFinite(sample.timeMs) && @@ -2510,13 +3924,33 @@ async function prepareWindowsGpuZoomTelemetry( .map((sample) => { const timeMs = Math.max(0, sample.timeMs); const scale = Math.max(0.01, sample.scale); + const blurStrength = Number.isFinite(sample.blurStrength) + ? Math.max(0, sample.blurStrength ?? 0) + : 0; + const blurCenterX = Number.isFinite(sample.blurCenterX) ? (sample.blurCenterX ?? 0) : 0; + const blurCenterY = Number.isFinite(sample.blurCenterY) ? (sample.blurCenterY ?? 0) : 0; return [ formatCliNumber(timeMs), formatCliNumber(scale), formatCliNumber(sample.x), formatCliNumber(sample.y), + formatCliNumber(blurStrength), + formatCliNumber(blurCenterX), + formatCliNumber(blurCenterY), ].join(","); }); +} + +async function prepareWindowsGpuZoomTelemetry( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const telemetry = options.zoomTelemetry; + if (!telemetry || telemetry.length === 0) { + return null; + } + + const lines = formatNativeStaticLayoutZoomTelemetryLines(telemetry); if (lines.length === 0) { return null; @@ -2598,13 +4032,197 @@ async function prepareWindowsGpuWebcamInput( return { inputPath: outputPath, elapsedMs: result.elapsedMs }; } +type NativeStaticLayoutSourceIdentity = { + canonicalPath: string; + device: number; + inode: number; + size: number; + mtimeMs: number; + ctimeMs: number; +}; + +/** + * A validated native source probe recorded after a successful FFmpeg metadata + * probe. The entry is only reusable when the file identity (canonical path, + * device/inode, size, mtime/ctime) AND the route requirements (requested + * output codec, encoding mode, encoder preference) match exactly. It is never + * keyed by path alone and never reused across a changed/mutated source. + */ +export interface NativeStaticLayoutSourceProbeCacheEntry { + identity: NativeStaticLayoutSourceIdentity; + requestedCodec: ExportVideoCodec; + encodingMode: NativeExportEncodingMode; + encoderPreference: ExportEncoderPreference; + metadata: NativeVideoMetadataProbe; +} + +const NATIVE_STATIC_LAYOUT_SOURCE_PROBE_CACHE_MAX = 8; +let nativeStaticLayoutSourceProbeCache = new Map(); + +/** + * Resets the bounded native source probe cache. Exposed for tests; the cache is + * session-scoped so resetting it forces the next source preparation to re-probe + * the source with FFmpeg. + */ +export function resetNativeStaticLayoutSourceProbeCache() { + nativeStaticLayoutSourceProbeCache.clear(); +} + +function buildNativeStaticLayoutSourceIdentity(stat: { + dev: number | bigint; + ino: number | bigint; + size: number | bigint; + mtimeMs: number; + ctimeMs: number; +}): NativeStaticLayoutSourceIdentity | null { + const device = Number(stat.dev); + const inode = Number(stat.ino); + // A missing/unreliable identity (e.g. zeroed device/inode) must bypass the + // cache entirely and re-probe rather than risk a false reuse. + if ( + !Number.isSafeInteger(device) || + !Number.isSafeInteger(inode) || + device === 0 || + inode === 0 + ) { + return null; + } + return { + canonicalPath: "", + device, + inode, + size: Number(stat.size), + mtimeMs: stat.mtimeMs, + ctimeMs: stat.ctimeMs, + }; +} + +function isNativeStaticLayoutSourceProbeCacheable(metadata: NativeVideoMetadataProbe) { + const codec = (metadata.codec ?? "").trim().toLowerCase(); + return codec !== "" && codec !== "unknown"; +} + +/** + * Deterministic exact-match predicate for reusing a previous successful source + * probe. Returns true only when canonical path, device/inode, size, mtime/ctime, + * requested output codec, encoding mode, and encoder preference all match. Any + * mismatch forces a re-probe (or fail closed when identity is missing). + */ +export function canReuseNativeStaticLayoutSourceProbe( + entry: NativeStaticLayoutSourceProbeCacheEntry | undefined, + current: NativeStaticLayoutSourceIdentity & { + requestedCodec: ExportVideoCodec; + encodingMode: NativeExportEncodingMode; + encoderPreference: ExportEncoderPreference; + }, +): boolean { + if (!entry) { + return false; + } + if (!isNativeStaticLayoutSourceProbeCacheable(entry.metadata)) { + return false; + } + return ( + entry.identity.canonicalPath === current.canonicalPath && + entry.identity.device === current.device && + entry.identity.inode === current.inode && + entry.identity.size === current.size && + entry.identity.mtimeMs === current.mtimeMs && + entry.identity.ctimeMs === current.ctimeMs && + entry.requestedCodec === current.requestedCodec && + entry.encodingMode === current.encodingMode && + entry.encoderPreference === current.encoderPreference + ); +} + +function rememberNativeStaticLayoutSourceProbe( + canonicalPath: string, + entry: NativeStaticLayoutSourceProbeCacheEntry, +) { + nativeStaticLayoutSourceProbeCache.set(canonicalPath, entry); + // Bound the cache; evict the oldest entry (Map preserves insertion order). + if (nativeStaticLayoutSourceProbeCache.size > NATIVE_STATIC_LAYOUT_SOURCE_PROBE_CACHE_MAX) { + const oldestKey = nativeStaticLayoutSourceProbeCache.keys().next().value; + if (oldestKey !== undefined) { + nativeStaticLayoutSourceProbeCache.delete(oldestKey); + } + } +} + +/** + * Resolves the source metadata for native static-layout source preparation, + * reusing a recent validated probe only when the file identity and the route + * requirements match exactly. Misses, mutations, changed settings, missing + * identity, uncacheable/unknown codecs, and probe failures all re-probe with + * FFmpeg (or propagate the failure); nothing is ever trusted by path alone. + */ +async function resolveNativeStaticLayoutSourceMetadata( + ffmpegPath: string, + options: Pick, + requestedCodec: ExportVideoCodec, + encoderPreference: ExportEncoderPreference, +): Promise { + const inputPath = options.inputPath; + let canonicalPath: string | null = null; + let identity: NativeStaticLayoutSourceIdentity | null = null; + try { + const stat = await fs.stat(inputPath); + const identityBase = buildNativeStaticLayoutSourceIdentity(stat); + if (identityBase) { + // Stat and realpath race minimally, but both derive from the same file; + // any mutation between them is caught on the size/mtime/ctime match. + canonicalPath = await fs.realpath(inputPath).catch(() => inputPath); + identity = { ...identityBase, canonicalPath }; + } + } catch { + // Unreadable source or missing identity: never reuse a stale entry; fall + // through to a fresh probe which will surface the real error. + identity = null; + canonicalPath = null; + } + + if (identity && canonicalPath) { + const candidate = nativeStaticLayoutSourceProbeCache.get(canonicalPath); + if ( + candidate && + canReuseNativeStaticLayoutSourceProbe(candidate, { + ...identity, + requestedCodec, + encodingMode: options.encodingMode, + encoderPreference, + }) + ) { + return candidate.metadata; + } + } + + const metadata = await probeNativeVideoMetadata(ffmpegPath, options.inputPath); + if (identity && canonicalPath && isNativeStaticLayoutSourceProbeCacheable(metadata)) { + rememberNativeStaticLayoutSourceProbe(canonicalPath, { + identity, + requestedCodec, + encodingMode: options.encodingMode, + encoderPreference, + metadata, + }); + } + return metadata; +} + async function prepareNativeStaticLayoutSourceInput( ffmpegPath: string, options: NativeStaticLayoutExportOptions, outputPath: string, session: NativeStaticLayoutExportSession, + requestedCodec: ExportVideoCodec, + encoderPreference: ExportEncoderPreference, ) { - const metadata = await probeNativeVideoMetadata(ffmpegPath, options.inputPath); + const metadata = await resolveNativeStaticLayoutSourceMetadata( + ffmpegPath, + options, + requestedCodec, + encoderPreference, + ); if (!shouldCreateNativeStaticLayoutSourceProxy(metadata, options.inputPath)) { return { inputPath: options.inputPath, @@ -2645,6 +4263,84 @@ async function prepareNativeStaticLayoutSourceInput( }; } +export function buildNativeStaticLayoutOverlayManifest( + layers: readonly (NativeStaticLayoutOverlayLayer | NativeCursorSpriteOverlayLayer)[], +) { + return { + layers: [...layers] + .sort((left, right) => left.order - right.order || left.id.localeCompare(right.id)) + .map((layer) => + isCursorSpriteOverlayLayer(layer) + ? { + // A cursor-sprite layer is a packed RGBA frame strip whose + // per-frame top-left position comes from a JSON positions + // sidecar. Base x/y are always 0; order keeps it topmost. + id: layer.id, + kind: layer.kind, + order: layer.order, + path: layer.path, + positionsPath: layer.positionsPath, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + } + : { + id: layer.id, + path: layer.path, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + // frameCount stays the logical output duration; effectiveFrameCount + // is the physical frame count the renderer wrote when identical- + // suffix dedup truncated the sidecar. Absent when every frame differs. + frameCount: layer.frameCount, + ...(layer.effectiveFrameCount !== undefined + ? { effectiveFrameCount: layer.effectiveFrameCount } + : {}), + }, + ), + }; +} + +export function getNativeStaticLayoutOverlayExpectedSidecarBytes( + layer: NativeStaticLayoutOverlayLayer, +) { + // Physical sidecar byte-size validation must use the physical frame count + // (effectiveFrameCount when present), never the logical output duration + // (frameCount), so deduped overlays are not rejected as truncated. + const physicalFrameCount = layer.effectiveFrameCount ?? layer.frameCount; + return ( + getNativeStaticLayoutOverlayFrameByteSize(layer.width, layer.height) * physicalFrameCount + ); +} + +/** + * Builds the versioned tiled/delta overlay storage descriptor from the + * renderer-prepared tiled layers. Layers are sorted by order then id so the + * native consumer blends them in deterministic z-order. The descriptor is + * session data only (never persisted) and is validated independently by + * validateNativeTiledOverlayStorageDescriptor before it reaches the wrapper. + */ +export function buildNativeStaticLayoutTiledOverlayManifest( + options: Pick< + NativeStaticLayoutExportOptions, + "width" | "height" | "frameRate" | "durationSec" + >, + layers: readonly NativeTiledOverlayLayerDescriptor[], +): NativeTiledOverlayStorageDescriptor { + return { + version: NATIVE_TILED_OVERLAY_STORAGE_VERSION, + outputWidth: options.width, + outputHeight: options.height, + frameRate: options.frameRate, + durationSec: options.durationSec, + layers: sortNativeTiledOverlayLayers(layers), + }; +} + export function buildExperimentalNvidiaCudaStaticLayoutArgs( options: NativeStaticLayoutExportOptions, outputPath: string, @@ -2670,6 +4366,8 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs( String(Math.max(1, Math.round(options.frameRate))), "--bitrate-mbps", String(getNvidiaCudaBitrateMbps(options)), + "--output-codec", + options.videoCodec ?? "h264", "--encoding-mode", options.encodingMode, "--duration-sec", @@ -2772,6 +4470,32 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs( if (options.zoomTelemetryPath) { args.push("--zoom-telemetry", options.zoomTelemetryPath); } + if (options.temporalBlur) { + // The renderer resolves temporal blur plans through + // getTemporalMotionBlurConfig, which clamps to at least + // TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT, so a plan below the minimum is + // an invariant violation. Reject it explicitly instead of silently + // dropping the effect through the sampleCount >= 3 gate below. + if (options.temporalBlur.sampleCount < TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT) { + throw new Error( + `unsupported-temporal-motion-blur: resolved temporal zoom motion blur plan uses ${options.temporalBlur.sampleCount} sample(s); the CUDA compositor minimum is ${TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT}. Refusing to silently drop the effect.`, + ); + } + args.push( + "--temporal-blur-sample-count", + String(Math.round(options.temporalBlur.sampleCount)), + "--temporal-blur-shutter-fraction", + formatCliNumber(options.temporalBlur.shutterFraction), + "--temporal-blur-weight-power", + formatCliNumber(options.temporalBlur.weightCurvePower), + ); + } + if (options.overlayManifestPath) { + args.push("--overlay-manifest", options.overlayManifestPath); + } + if (options.tiledOverlayManifestPath) { + args.push("--tiled-overlay-manifest", options.tiledOverlayManifestPath); + } if (options.timelineMapPath) { args.push("--timeline-map", options.timelineMapPath); } @@ -2810,9 +4534,67 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( const nodeCommand = resolveExperimentalNvidiaCudaNodeCommand(); const workDir = path.join(chunkDirectory, "nvidia-cuda-work"); + let effectiveOptions = options; + if (options.overlayLayers?.length) { + // GPU-preparation audit: safe GPU-side composition (native cursor atlas, + // zoom/background composition, temporal zoom blur) already runs entirely + // in the generalized NVIDIA CUDA compositor with no renderer readback of + // video pixels. Browser raster overlays (captions, annotations, webcam, + // frame visuals) intentionally remain renderer-prepared transparent RGBA + // sidecar work: the compositor would need a live DOM/canvas rasterizer to + // draw arbitrary per-frame browser content natively, which is not + // supported, and direct canvas-to-NV12 transfer is not assumed until + // runtime support is proven (AGENTS.md native raw-frame transport). The + // renderer therefore bakes those layers into a bounded RGBA sidecar that + // the CUDA compositor uploads and alpha-blends on top of the composed, + // blurred video. These layers are export-session data only, never + // persisted, and a failed/incomplete sidecar preparation returns to the + // renderer raw-frame route rather than silently dropping a layer. + const overlayManifestPath = path.join(chunkDirectory, "overlay-manifest.json"); + await fs.writeFile( + overlayManifestPath, + JSON.stringify(buildNativeStaticLayoutOverlayManifest(options.overlayLayers)), + "utf8", + ); + effectiveOptions = { + ...options, + overlayManifestPath, + }; + if (options.cursorAtlasOwned !== true) { + // Overlay sidecars already contain the renderer-baked cursor; the + // Windows GPU prep may have left CSV telemetry/atlas paths on the + // options and the CUDA wrapper JSON.parses --cursor-json (a CSV file + // crashes it). Strip the cursor assets so the wrapper is never handed + // a malformed cursor file and never double-renders the baked cursor. + // When cursorAtlasOwned is true the sidecar excluded cursor pixels, so + // the prepared JSON telemetry/atlas assets pass through untouched and + // the wrapper draws the cursor natively. + effectiveOptions = { + ...effectiveOptions, + ...resolveNvidiaCudaCursorAssets(effectiveOptions, true), + }; + } + } + if (options.tiledOverlayLayers?.length) { + // The versioned tiled/delta overlay storage descriptor is written next to + // the raw overlay manifest; the CUDA wrapper validates it independently + // and the native compositor consumes it (session data, never persisted). + const tiledOverlayManifestPath = path.join(chunkDirectory, "tiled-overlay-manifest.json"); + await fs.writeFile( + tiledOverlayManifestPath, + JSON.stringify( + buildNativeStaticLayoutTiledOverlayManifest(options, options.tiledOverlayLayers), + ), + "utf8", + ); + effectiveOptions = { + ...effectiveOptions, + tiledOverlayManifestPath, + }; + } const args = [ scriptPath, - ...buildExperimentalNvidiaCudaStaticLayoutArgs(options, outputPath, workDir), + ...buildExperimentalNvidiaCudaStaticLayoutArgs(effectiveOptions, outputPath, workDir), ]; const startedAt = getNowMs(); const startedAtIso = new Date().toISOString(); @@ -2832,6 +4614,20 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( [pathKey]: `${ffmpegDirectory}${path.delimiter}${process.env[pathKey] ?? ""}`, }; const powerGuard = startNativeStaticLayoutExportPowerGuard(); + // A capability-only prewarm child may still hold a brief NVENC probe session; + // cancel it before this real export opens its own NVENC session so the two + // never contend for the GPU. Fire-and-forget: the prewarm was never awaited. + cancelInFlightCapabilityOnlyPrewarms(); + // Expected output frames; used to frame the display-only preparation + // substates (currentFrame is always 0 during preparation). + const prepareTotalFrames = Math.max(1, Math.ceil(options.durationSec * options.frameRate)); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "wrapper-launch", + prepareTotalFrames, + getNowMs() - startedAt, + ); return await new Promise<{ elapsedMs: number; @@ -2844,15 +4640,51 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( stdio: ["ignore", "pipe", "pipe"], windowsHide: true, }); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "cuda-nvenc-init", + prepareTotalFrames, + getNowMs() - startedAt, + ); const childPriorityApplied = setNativeStaticLayoutExportProcessPriority( child.pid, "NVIDIA CUDA export wrapper", ); - console.info("[native-static-layout-export] NVIDIA CUDA runtime guard started", { - childPriorityApplied, - powerGuardStarted: powerGuard.started, - }); + // Renderer-side overlay decision visibility: what the renderer actually + // prepared for this export (rgba full-canvas layers, tiled layers, or + // cursor-sprite ROI layers). This is the ground truth for diagnosing why a + // cursor-only export may have used the baked sidecar instead of the + // cursor-sprite fast path. + const overlayKinds = (options.overlayLayers ?? []).reduce>( + (acc, layer) => { + const kind = + "kind" in layer && layer.kind === NATIVE_CURSOR_SPRITE_LAYER_KIND + ? NATIVE_CURSOR_SPRITE_LAYER_KIND + : "rgba"; + acc[kind] = (acc[kind] ?? 0) + 1; + return acc; + }, + {}, + ); + console.info( + formatLogTs(), + "[native-static-layout-export] NVIDIA CUDA runtime guard started", + { + childPriorityApplied, + powerGuardStarted: powerGuard.started, + overlayLayerKinds: overlayKinds, + tiledOverlayLayers: options.tiledOverlayLayers?.length ?? 0, + }, + ); session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below (killing an already-exited child on Windows emits an unhandled + // 'error' event when no listener is attached yet). Real failures settle + // through the dedicated error/close handlers attached below. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -2861,6 +4693,7 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( let stderr = ""; let stderrLineBuffer = ""; let lastProgressPercentage = 0; + let firstFramePrepared = false; let lastProgressForStallGuard: { currentFrame: number; percentage: number; @@ -2906,15 +4739,34 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( if (!progress) { continue; } + if (!firstFramePrepared) { + // The first helper PROGRESS line confirms the CUDA wrapper reached + // first-frame readiness (CUDA/NVENC initialized, encode producing + // frames). Emitted as a display-only preparing substate; it carries no + // FPS and does not overwrite the helper's measured encode rate. + firstFramePrepared = true; + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "first-frame", + prepareTotalFrames, + getNowMs() - startedAt, + ); + } const elapsedMs = Math.max(0, getNowMs() - startedAt); - const averageFps = - typeof progress.averageFps === "number" && - Number.isFinite(progress.averageFps) && - progress.averageFps > 0 - ? progress.averageFps - : elapsedMs > 0 && progress.currentFrame > 0 - ? (progress.currentFrame * 1000) / elapsedMs - : undefined; + const fpsFields = resolveNativeStaticLayoutFpsFields(progress, elapsedMs); + if (fpsFields.fpsSource === "estimated") { + console.warn( + formatLogTs(), + "[native-static-layout-export] Native helper has not reported measured encode FPS; using preparation-inclusive estimate", + { + backend: "nvidia-cuda-compositor", + estimatedFps: fpsFields.estimatedFps, + currentFrame: progress.currentFrame, + elapsedMs, + }, + ); + } const mappedPercentage = mapNvidiaCudaWrapperProgressPercentage(progress); lastProgressPercentage = Math.max(lastProgressPercentage, mappedPercentage); const progressForStallGuard = { @@ -2937,7 +4789,7 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( sessionId: options.sessionId, backend: "nvidia-cuda-compositor", elapsedMs, - averageFps, + ...fpsFields, }); } }); @@ -3050,6 +4902,11 @@ async function runExperimentalWindowsGpuStaticLayoutExport( stdio: ["ignore", "pipe", "pipe"], }); session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below; see the CUDA wrapper spawn for the rationale. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -3111,15 +4968,13 @@ async function runExperimentalWindowsGpuStaticLayoutExport( armStallTimeout(); } const elapsedMs = Math.max(0, getNowMs() - startedAt); + const fpsFields = resolveNativeStaticLayoutFpsFields(progress, elapsedMs); onProgress?.({ ...progress, sessionId: options.sessionId, backend: "windows-d3d11-compositor", elapsedMs, - averageFps: - elapsedMs > 0 && progress.currentFrame > 0 - ? (progress.currentFrame * 1000) / elapsedMs - : undefined, + ...fpsFields, }); } }); @@ -3190,14 +5045,167 @@ export async function exportNativeStaticLayoutVideo( ffmpegPath: string, options: NativeStaticLayoutExportOptions, onProgress?: (progress: NativeStaticLayoutExportProgress) => void, -) { +): Promise { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + if (videoCodec !== "h264" && videoCodec !== "hevc") { + throw new Error(`Unsupported native static-layout video codec: ${String(videoCodec)}`); + } + if ( + encoderPreference !== "auto" && + encoderPreference !== "hardware" && + encoderPreference !== "cpu" + ) { + throw new Error( + `Unsupported native static-layout encoder preference: ${String(encoderPreference)}`, + ); + } + options = { ...options, videoCodec, encoderPreference }; + const rawFrameFallbackReason = getNativeStaticLayoutRawFrameFallbackReason(options); + if (rawFrameFallbackReason) { + throw new Error( + `Native static-layout export requires the native rawvideo route: ${rawFrameFallbackReason}`, + ); + } + if (options.width % 2 !== 0 || options.height % 2 !== 0) { throw new Error("Native static layout export requires even output dimensions"); } if (!Number.isFinite(options.durationSec) || options.durationSec <= 0) { throw new Error("Native static layout export requires a positive duration"); } + if (options.overlayLayers?.length) { + for (const layer of sortNativeStaticLayoutOverlayLayers(options.overlayLayers)) { + const validationError = isCursorSpriteOverlayLayer(layer) + ? validateNativeCursorSpriteOverlayLayer(layer, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }) + : validateNativeStaticLayoutOverlayLayer(layer, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }); + if (validationError) { + throw new Error(`Invalid native overlay layer: ${validationError}`); + } + const stat = await fs.stat(layer.path); + const expectedBytes = getNativeStaticLayoutOverlayExpectedSidecarBytes(layer); + if (stat.size < expectedBytes) { + throw new Error( + `Native overlay layer ${layer.id} is truncated: expected ${expectedBytes} bytes, received ${stat.size}`, + ); + } + if (isCursorSpriteOverlayLayer(layer)) { + // The cursor-sprite positions sidecar must exist so the native route + // never silently drops the cursor because the per-frame positions are + // missing (full JSON contents are validated by the native reader). + const positionsStat = await fs.stat(layer.positionsPath); + if (positionsStat.size <= 0) { + throw new Error( + `Native overlay layer ${layer.id} has an empty cursor-sprite positions file`, + ); + } + } + } + } + if (options.tiledOverlayLayers?.length) { + const tiledDescriptor = buildNativeStaticLayoutTiledOverlayManifest( + options, + options.tiledOverlayLayers, + ); + const tiledValidationError = validateNativeTiledOverlayStorageDescriptor(tiledDescriptor, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }); + if (tiledValidationError) { + throw new Error(`Invalid tiled overlay descriptor: ${tiledValidationError}`); + } + for (const layer of tiledDescriptor.layers) { + const stat = await fs.stat(layer.payloadPath); + if (stat.size < layer.payloadByteLength) { + throw new Error( + `Tiled overlay layer ${layer.id} payload is truncated: expected ${layer.payloadByteLength} bytes, received ${stat.size}`, + ); + } + } + } + if ( + options.cursorAtlasOwned === true && + !( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + (options.experimentalNvidiaCudaExport === true || isExplicitNvidiaCudaExportEnabled()) + ) + ) { + // The renderer excluded cursor pixels from the overlay sidecar because + // the native atlas owns them. Only the generalized NVIDIA CUDA compositor + // can draw that atlas on top of the sidecars; the FFmpeg overlay route + // and the D3D11 helper cannot, so refusing the fallback is the only way + // to avoid silently dropping the cursor. + throw new Error( + "Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor on Windows; the FFmpeg overlay route cannot draw the cursor and the overlay sidecar excluded it.", + ); + } + if ( + options.webcamNativeOwned === true && + !( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + (options.experimentalNvidiaCudaExport === true || isExplicitNvidiaCudaExportEnabled()) + ) + ) { + // The renderer excluded webcam pixels from the overlay sidecar because + // the CUDA compositor owns the webcam natively. Only the generalized + // NVIDIA CUDA compositor can draw that webcam; the FFmpeg overlay route + // and the D3D11 helper cannot, so refusing the fallback is the only way + // to avoid silently dropping the webcam the sidecar excluded. + throw new Error( + "Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor on Windows; the FFmpeg overlay route cannot draw the webcam and the overlay sidecar excluded it.", + ); + } + if (options.webcamNativeOwned === true && !options.webcamInputPath) { + throw new Error( + "Native webcam ownership requires a webcam input path; refusing to drop the webcam the overlay sidecar excluded.", + ); + } + if (options.webcamNativeOwned === true && (options.webcamSize ?? 0) <= 0) { + throw new Error( + "Native webcam ownership requires a positive webcam size; refusing to drop the webcam the overlay sidecar excluded.", + ); + } options = await normalizeNativeStaticLayoutBackground(options); + // The FFmpeg/rawvideo fallback needs a probed encoder; the NVIDIA CUDA and + // Windows-GPU compositor routes do not use it. Resolve it lazily, only on the + // first actual FFmpeg/raw fallback branch, so a CUDA-eligible native-layout + // job never spends time on the up-to-4x15s cold encoder probe BEFORE the CUDA + // route is selected/validated. The probe result is memoized for the session + // run (and internally cached by resolveNativeVideoEncoder), so the fallback + // branches only ever pay for it once. + let resolvedNativeVideoEncoder: string | null = null; + const ensureNativeVideoEncoder = async (): Promise => { + if (resolvedNativeVideoEncoder === null) { + resolvedNativeVideoEncoder = await resolveNativeVideoEncoder( + ffmpegPath, + options.encodingMode, + videoCodec, + encoderPreference, + ); + } + return resolvedNativeVideoEncoder; + }; + // Copies the given FFmpeg-args config with the probed encoder attached. Only + // the FFmpeg/raw fallback branches call this; CUDA/GPU routes never touch it. + const withNativeVideoEncoder = async ( + config: NativeStaticLayoutExportArgsConfig, + ): Promise => { + return { ...config, videoEncoder: await ensureNativeVideoEncoder() }; + }; if ( options.webcamInputPath && !(options.experimentalWindowsGpuCompositor && process.platform === "win32") @@ -3247,22 +5255,29 @@ export async function exportNativeStaticLayoutVideo( try { nativeStaticLayoutExportSessions.set(sessionId, session); + const exportRunStartedAt = getNowMs(); await fs.mkdir(chunkDirectory, { recursive: true }); const sourceInput = await prepareNativeStaticLayoutSourceInput( ffmpegPath, options, path.join(chunkDirectory, "source-proxy.mp4"), session, + videoCodec, + encoderPreference, ); if (sourceInput.elapsedMs > 0) { metrics.staticAssetExecMs = (metrics.staticAssetExecMs ?? 0) + sourceInput.elapsedMs; } if (sourceInput.inputPath !== options.inputPath) { - console.info("[native-static-layout-export] Prepared H.264 source proxy", { - sourceCodec: sourceInput.sourceCodec, - proxyCodec: sourceInput.proxyCodec, - elapsedMs: sourceInput.elapsedMs, - }); + console.info( + formatLogTs(), + "[native-static-layout-export] Prepared H.264 source proxy", + { + sourceCodec: sourceInput.sourceCodec, + proxyCodec: sourceInput.proxyCodec, + elapsedMs: sourceInput.elapsedMs, + }, + ); options = { ...options, inputPath: sourceInput.inputPath, @@ -3284,6 +5299,7 @@ export async function exportNativeStaticLayoutVideo( const fullConfig: NativeStaticLayoutExportArgsConfig = { inputPath: options.inputPath, outputPath: videoOnlyPath, + videoCodec, width: options.width, height: options.height, frameRate: options.frameRate, @@ -3303,12 +5319,22 @@ export async function exportNativeStaticLayoutVideo( borderRadius: options.borderRadius, shadowIntensity: options.shadowIntensity, durationSec: options.durationSec, + overlayLayers: options.overlayLayers, }; const usePrecompositedLayout = shouldUsePrecompositedStaticLayout(options); let didRenderVideo = false; let didMuxAudioInline = false; - if (options.experimentalWindowsGpuCompositor && process.platform === "win32") { + if ( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + // The generalized NVIDIA CUDA compositor composites renderer-prepared + // overlay sidecars natively; the Windows D3D11 helper cannot, so it is + // skipped below when overlay layers are present. + ((options.overlayLayers?.length === 0 && !options.tiledOverlayLayers?.length) || + options.experimentalNvidiaCudaExport === true || + isExplicitNvidiaCudaExportEnabled()) + ) { try { if (session.terminating) { throw new Error("Native static layout export was cancelled"); @@ -3376,6 +5402,8 @@ export async function exportNativeStaticLayoutVideo( const nvidiaCudaSkipReason = await getExperimentalNvidiaCudaExportSkipReason(options); let shouldTryNvidiaCuda = nvidiaCudaSkipReason === null; + const requiresStrictHevcCuda = + options.videoCodec === "hevc" && options.encoderPreference === "hardware"; const validatedCudaFallbackCandidate = isValidatedNvidiaCudaFallbackCandidate(options); if ( @@ -3388,6 +5416,7 @@ export async function exportNativeStaticLayoutVideo( nvidiaCudaForceVideoOnly: true, }; console.info( + formatLogTs(), "[native-static-layout-export] NVIDIA CUDA candidate will use shared audio mux validation", { audioMode: @@ -3407,6 +5436,7 @@ export async function exportNativeStaticLayoutVideo( nvidiaCudaSkipReason !== "env-disabled" ) { console.warn( + formatLogTs(), "[native-static-layout-export] Skipping NVIDIA CUDA compositor; falling back to Windows GPU compositor", { reason: nvidiaCudaSkipReason, @@ -3416,32 +5446,109 @@ export async function exportNativeStaticLayoutVideo( }, ); } - if (shouldTryNvidiaCuda && options.cursorTelemetry?.length) { - const cursorTelemetryPath = await prepareNvidiaCudaCursorTelemetry( - options, - path.join(chunkDirectory, "cursor-telemetry.json"), + if ( + !shouldTryNvidiaCuda && + (((options.overlayLayers?.length || options.tiledOverlayLayers?.length) && + (options.zoomTelemetry?.length || options.cursorAtlasOwned === true)) || + options.temporalBlur) + ) { + throw new Error( + `CUDA composition is unavailable (${nvidiaCudaSkipReason ?? "unknown"}) while zoom motion blur${options.cursorAtlasOwned === true ? ", a native-owned cursor," : ""} and/or temporal zoom motion blur (${options.temporalBlur?.sampleCount ?? "n/a"} samples) are requested; the FFmpeg overlay route cannot preserve these effects${options.cursorAtlasOwned === true ? " and cannot draw the cursor the sidecar excluded" : ""}.`, ); - const cursorAtlas = await prepareNvidiaCudaCursorAtlas( - options, - path.join(chunkDirectory, "cursor-atlas-nvidia.png"), - path.join(chunkDirectory, "cursor-atlas-nvidia.tsv"), + } + if (options.cursorAtlasOwned === true && !options.cursorTelemetry?.length) { + throw new Error( + "Native cursor atlas ownership requires cursor telemetry; refusing to drop the cursor on a fallback route.", ); - shouldTryNvidiaCuda = Boolean(cursorTelemetryPath && cursorAtlas); - if (cursorTelemetryPath && cursorAtlas) { + } + if (shouldTryNvidiaCuda && options.cursorTelemetry?.length) { + if ( + (options.overlayLayers?.length || options.tiledOverlayLayers?.length) && + options.cursorAtlasOwned !== true + ) { + // When overlay layers are present and the cursor is baked into the + // transparent sidecar, drawing it again natively would double-render + // and the atlas is intentionally absent. The Windows-GPU prep above + // left CSV telemetry/atlas paths on the options; strip them so the + // CUDA wrapper is never handed a CSV "cursor JSON" file (it + // JSON.parses the path and crashes) and never double-renders the + // baked cursor. experimentalNvidiaCudaOptions = { ...experimentalNvidiaCudaOptions, - cursorTelemetryPath, - cursorAtlasPath: cursorAtlas.atlasPath, - cursorAtlasMetadataPath: cursorAtlas.metadataPath, + ...resolveNvidiaCudaCursorAssets(experimentalNvidiaCudaOptions, true), }; + shouldTryNvidiaCuda = true; + } else { + // No overlay layers, or the renderer excluded cursor pixels from + // the overlay sidecar (cursorAtlasOwned): the CUDA compositor + // draws the native atlas cursor on top of the composed video. + const cursorTelemetryPath = await prepareNvidiaCudaCursorTelemetry( + options, + path.join(chunkDirectory, "cursor-telemetry.json"), + ); + const cursorAtlas = await prepareNvidiaCudaCursorAtlas( + options, + path.join(chunkDirectory, "cursor-atlas-nvidia.png"), + path.join(chunkDirectory, "cursor-atlas-nvidia.tsv"), + ); + if ( + options.cursorAtlasOwned === true && + (!cursorTelemetryPath || !cursorAtlas) + ) { + throw new Error( + "Native cursor atlas ownership could not prepare cursor telemetry/atlas assets; refusing to drop the cursor on the FFmpeg overlay route.", + ); + } + shouldTryNvidiaCuda = Boolean(cursorTelemetryPath && cursorAtlas); + if (cursorTelemetryPath && cursorAtlas) { + experimentalNvidiaCudaOptions = { + ...experimentalNvidiaCudaOptions, + cursorTelemetryPath, + cursorAtlasPath: cursorAtlas.atlasPath, + cursorAtlasMetadataPath: cursorAtlas.metadataPath, + }; + } } } + const strictHevcHardFail = resolveNvidiaCudaStrictHevcHardFail( + requiresStrictHevcCuda, + shouldTryNvidiaCuda, + nvidiaCudaSkipReason, + ); + if (strictHevcHardFail) { + throw new Error(strictHevcHardFail); + } if (shouldTryNvidiaCuda) { try { const shouldMuxAudioInline = canMuxNvidiaCudaSourceAudioInline( experimentalNvidiaCudaOptions, ); + // The CUDA route is selected. Emit the two preparation substates that + // already completed before the wrapper launch (encoder capability + // probe and source validation). They are additive display-only + // progress with the NVIDIA CUDA compositor backend label from the + // very start; the wrapper-launch / cuda-nvenc-init / first-frame + // substates are emitted by the wrapper runner itself. + const prepareElapsed = () => Math.max(0, getNowMs() - exportRunStartedAt); + const prepareTotal = Math.max( + 1, + Math.ceil(options.durationSec * options.frameRate), + ); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "encoder-probe", + prepareTotal, + prepareElapsed(), + ); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "source-validation", + prepareTotal, + prepareElapsed(), + ); const cudaResult = await runExperimentalNvidiaCudaStaticLayoutExport( ffmpegPath, experimentalNvidiaCudaOptions, @@ -3456,6 +5563,7 @@ export async function exportNativeStaticLayoutVideo( durationSec: options.durationSec, targetFrames: Math.ceil(options.durationSec * options.frameRate), requiresTimelineSync: shouldMuxAudioInline, + videoCodec, }, ); if (cudaValidationIssues.length > 0) { @@ -3470,19 +5578,49 @@ export async function exportNativeStaticLayoutVideo( ); } await validateRenderedVideoOutput(); + const overlaySidecarMetrics = + resolveNvidiaCudaOverlaySidecarSummaryMetrics(options); + const tiledOverlayMetrics = + resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics(options); + if ( + (overlaySidecarMetrics || tiledOverlayMetrics) && + cudaResult.summary.nativeSummary + ) { + // Additive renderer-derived overlay sidecar metrics ride on the + // parsed native summary so the completion log and chunk metrics + // surface them without changing the helper contract. + cudaResult.summary.nativeSummary = { + ...cudaResult.summary.nativeSummary, + ...overlaySidecarMetrics, + ...tiledOverlayMetrics, + }; + } + const nativeSummaryMetrics = resolveNvidiaCudaNativeSummaryMetrics( + cudaResult.summary.nativeSummary, + ); console.info( + formatLogTs(), "[native-static-layout-export] NVIDIA CUDA compositor completed", { elapsedMs: cudaResult.elapsedMs, - fps: cudaResult.summary.fps, + // summary.fps is the configured output frame rate (stream fps, + // never a measured encode throughput). It is labeled outputFps + // so it cannot be mistaken for measured encode speed, which is + // reported separately as nativeFps from the helper summary + // (resolveNvidiaCudaNativeFps). + outputFps: cudaResult.summary.fps, targetFrames: cudaResult.summary.targetFrames, durationSec: cudaResult.summary.durationSec, - nativeEncodeMs: cudaResult.summary.timingsMs?.nativeEncode, + // timingsMs.nativeEncode is the full native helper-process wall + // time (spawn to exit: source decode, layout composition, NVENC, + // flush). It is NOT the NVENC-API encode time. The single explicit + // field is nativeEncodeWallMs; the low-level NVENC-API time is + // reported separately as nativeSummary.nvencMs (spread below via + // nativeSummaryMetrics), so no ambiguous duplicate is emitted. + nativeEncodeWallMs: cudaResult.summary.timingsMs?.nativeEncode, muxMs: cudaResult.summary.timingsMs?.mux, endToEndMs: cudaResult.summary.timingsMs?.endToEnd, - nativeFps: - cudaResult.summary.nativeSummary?.measuredFps ?? - cudaResult.summary.nativeSummary?.fps, + nativeFps: resolveNvidiaCudaNativeFps(cudaResult.summary), mappedDisplayFrames: cudaResult.summary.nativeSummary?.mappedDisplayFrames, selectedDisplayFrames: @@ -3499,6 +5637,17 @@ export async function exportNativeStaticLayoutVideo( cursorAtlas: cudaResult.summary.nativeSummary?.cursorAtlas, zoomOverlay: cudaResult.summary.nativeSummary?.zoomOverlay, zoomSamples: cudaResult.summary.nativeSummary?.zoomSamples, + overlayLayers: cudaResult.summary.nativeSummary?.overlayLayers, + ...nativeSummaryMetrics, + // The helper does not echo the configured temporal blur sample + // count in its summary yet; fall back to the renderer-resolved + // export plan so the log still exposes the requested count. + temporalBlurSampleCount: + nativeSummaryMetrics.temporalBlurSampleCount ?? + options.temporalBlur?.sampleCount, + metricInvariants: validateNvidiaCudaStageMetricInvariants( + cudaResult.summary, + ), }, ); metrics.chunkCount = 1; @@ -3519,16 +5668,67 @@ export async function exportNativeStaticLayoutVideo( if (session.terminating) { throw error; } + if (requiresStrictHevcCuda) { + throw new Error( + `HEVC Hardware NVIDIA CUDA compositor failed; refusing CPU, rawvideo, Breeze, or FFmpeg CUDA fallback (noCpuFallback:true): ${error instanceof Error ? error.message : String(error)}`, + ); + } metrics.fallbackChunkCount++; console.warn( "[native-static-layout-export] Experimental NVIDIA CUDA compositor failed or produced invalid output; falling back to Windows GPU compositor:", error, ); await removeTemporaryExportFile(videoOnlyPath); + const overlayCount = + (options.overlayLayers?.length ?? 0) + + (options.tiledOverlayLayers?.length ?? 0); + if (overlayCount > 0 && !options.overlayLayers?.length) { + // Tiled overlay layers cannot be consumed by any remaining fallback + // (Windows D3D11 or FFmpeg static layout). Fail fast so H.264 Auto / + // HEVC Auto never silently drop the sparse overlay sidecar. + throw new Error( + `CUDA composition failed with ${overlayCount} overlay sidecar(s) including ${options.tiledOverlayLayers?.length ?? 0} tiled overlay layer(s); the remaining static-layout fallback cannot consume the tiled descriptor (tiled-overlays-unsupported-in-static-layout-fallback): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + (options.overlayLayers?.length || options.tiledOverlayLayers?.length) && + (options.zoomTelemetry?.length || options.cursorAtlasOwned === true) + ) { + // Neither the D3D11 helper nor the FFmpeg overlay route can + // preserve spatial zoom blur over the transparent overlay + // sidecars, or a native-owned cursor whose pixels the sidecar + // excluded; surface the failure so the renderer falls back to raw + // frames instead of silently dropping the effect. + throw new Error( + `CUDA composition failed while zoom motion blur${options.cursorAtlasOwned === true ? ", a native-owned cursor" : ""} and ${overlayCount} overlay layer(s) are requested; the fallback route cannot preserve these effects${options.cursorAtlasOwned === true ? " and cannot draw the cursor the sidecar excluded" : ""}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (options.temporalBlur) { + // Temporal zoom motion blur is only supported by the generalized + // NVIDIA CUDA compositor; fall back would silently drop it. + throw new Error( + `CUDA composition failed while temporal zoom motion blur is requested (${options.temporalBlur.sampleCount} samples); the fallback route cannot preserve temporal blur: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (options.webcamNativeOwned) { + // The renderer excluded webcam pixels from the overlay sidecar + // because the CUDA compositor owns the webcam natively; the + // remaining D3D11/FFmpeg fallback cannot draw it, so continuing + // would silently drop the webcam. + throw new Error( + `CUDA composition failed while the CUDA compositor owns the webcam (webcamNativeOwned); the fallback route cannot draw the webcam the overlay sidecar excluded: ${error instanceof Error ? error.message : String(error)}`, + ); + } } } - if (!didRenderVideo) { + if ( + !didRenderVideo && + videoCodec !== "hevc" && + !(options.overlayLayers?.length || options.tiledOverlayLayers?.length) + ) { const gpuResult = await runExperimentalWindowsGpuStaticLayoutExport( experimentalGpuOptions, videoOnlyPath, @@ -3545,27 +5745,31 @@ export async function exportNativeStaticLayoutVideo( ); } await validateRenderedVideoOutput(); - console.info("[native-static-layout-export] Windows GPU compositor completed", { - elapsedMs: gpuResult.elapsedMs, - width: gpuResult.summary.width, - height: gpuResult.summary.height, - fps: gpuResult.summary.fps, - frames: gpuResult.summary.frames, - realtimeMultiplier: gpuResult.summary.realtimeMultiplier, - surfacePoolSize: gpuResult.summary.surfacePoolSize, - gpuDecodeSurface: gpuResult.summary.gpuDecodeSurface, - adapterIndex: gpuResult.summary.adapterIndex, - encoderBackend: gpuResult.summary.encoderBackend, - encoderTuningApplied: gpuResult.summary.encoderTuningApplied, - readMs: gpuResult.summary.readMs, - videoProcessMs: gpuResult.summary.videoProcessMs, - writeSampleMs: gpuResult.summary.writeSampleMs, - finalizeMs: gpuResult.summary.finalizeMs, - webcamOverlay: gpuResult.summary.webcamOverlay, - cursorOverlay: gpuResult.summary.cursorOverlay, - cursorAtlas: gpuResult.summary.cursorAtlas, - zoomOverlay: gpuResult.summary.zoomOverlay, - }); + console.info( + formatLogTs(), + "[native-static-layout-export] Windows GPU compositor completed", + { + elapsedMs: gpuResult.elapsedMs, + width: gpuResult.summary.width, + height: gpuResult.summary.height, + fps: gpuResult.summary.fps, + frames: gpuResult.summary.frames, + realtimeMultiplier: gpuResult.summary.realtimeMultiplier, + surfacePoolSize: gpuResult.summary.surfacePoolSize, + gpuDecodeSurface: gpuResult.summary.gpuDecodeSurface, + adapterIndex: gpuResult.summary.adapterIndex, + encoderBackend: gpuResult.summary.encoderBackend, + encoderTuningApplied: gpuResult.summary.encoderTuningApplied, + readMs: gpuResult.summary.readMs, + videoProcessMs: gpuResult.summary.videoProcessMs, + writeSampleMs: gpuResult.summary.writeSampleMs, + finalizeMs: gpuResult.summary.finalizeMs, + webcamOverlay: gpuResult.summary.webcamOverlay, + cursorOverlay: gpuResult.summary.cursorOverlay, + cursorAtlas: gpuResult.summary.cursorAtlas, + zoomOverlay: gpuResult.summary.zoomOverlay, + }, + ); const outputStat = await fs.stat(videoOnlyPath); metrics.chunkCount = 1; metrics.chunkDurationSec = options.durationSec; @@ -3585,11 +5789,29 @@ export async function exportNativeStaticLayoutVideo( if (session.terminating) { throw error; } + if (options.videoCodec === "hevc" && options.encoderPreference === "hardware") { + // Strict HEVC Hardware: the generalized NVIDIA CUDA compositor is the + // ONLY acceptable route. This outer GPU-block catch must never + // swallow the CUDA helper/noCpuFallback error (e.g. when there is no + // webcam, zoom telemetry, or native timeline) and attempt a full + // FFmpeg hevc_nvenc fallback before the renderer rejects it. Rethrow + // the original actionable error (which the inner CUDA catch already + // annotates with noCpuFallback:true and CUDA context) so no CPU, + // rawvideo, Breeze, or FFmpeg CUDA fallback path is reached. + const strictMessage = error instanceof Error ? error.message : String(error); + if (strictMessage.includes("noCpuFallback:true")) { + throw error; + } + throw new Error( + `HEVC Hardware NVIDIA CUDA compositor failed; refusing CPU, rawvideo, Breeze, or FFmpeg CUDA fallback (noCpuFallback:true): ${strictMessage}`, + ); + } if (hasNativeStaticLayoutTimeline(options)) { throw error; } metrics.fallbackChunkCount++; console.warn( + formatLogTs(), "[native-static-layout-export] Experimental Windows GPU compositor unavailable; falling back to FFmpeg static layout:", error, ); @@ -3606,6 +5828,23 @@ export async function exportNativeStaticLayoutVideo( if (!didRenderVideo && hasNativeStaticLayoutSourceCrop(options)) { throw new Error("Native crop export requires a GPU compositor backend"); } + if (!didRenderVideo && options.tiledOverlayLayers?.length) { + // The remaining static-layout paths cannot consume the versioned + // tiled/delta overlay descriptor; fail fast instead of silently dropping + // the overlay pixels on a fallback that did not render. + throw new Error( + `No GPU compositor produced output; the static-layout fallback cannot consume ${options.tiledOverlayLayers.length} tiled overlay layer(s) (tiled-overlays-unsupported-in-static-layout-fallback)`, + ); + } + if (!didRenderVideo && options.webcamNativeOwned) { + // The renderer excluded webcam pixels from the overlay sidecar because + // the CUDA compositor owns them. No fallback below (precomposited, + // FFmpeg CUDA overlay, or CPU pad) can draw that webcam, so continuing + // would silently drop it; fail fast instead. + throw new Error( + "No GPU compositor produced output while the CUDA compositor owns the webcam (webcamNativeOwned); the static-layout fallback cannot draw the webcam the overlay sidecar excluded.", + ); + } if (!didRenderVideo && usePrecompositedLayout) { const maskPath = path.join(chunkDirectory, "layout-mask.pgm"); @@ -3619,10 +5858,12 @@ export async function exportNativeStaticLayoutVideo( ), ); + const encoderConfig = await withNativeVideoEncoder(fullConfig); + const backgroundResult = await runFfmpegWithMetrics( ffmpegPath, buildNativeStaticBackgroundRenderArgs({ - ...fullConfig, + ...encoderConfig, inputPath: options.inputPath, outputPath: staticBackgroundPath, maskPath, @@ -3638,7 +5879,7 @@ export async function exportNativeStaticLayoutVideo( const fullResult = await runFfmpegWithMetrics( ffmpegPath, buildNativePrecompositedStaticLayoutArgs({ - ...fullConfig, + ...encoderConfig, staticBackgroundPath, maskPath, }), @@ -3662,9 +5903,10 @@ export async function exportNativeStaticLayoutVideo( outputBytes: outputStat.size, }); } else if (!didRenderVideo) { + const encoderConfig = await withNativeVideoEncoder(fullConfig); const primaryResult = await runFfmpegWithMetrics( ffmpegPath, - buildNativeCudaOverlayStaticLayoutArgs(fullConfig), + buildNativeCudaOverlayStaticLayoutArgs(encoderConfig), 15 * 60 * 1000, session, ); @@ -3672,6 +5914,14 @@ export async function exportNativeStaticLayoutVideo( let fullBackend: NativeStaticLayoutBackend = "cuda-overlay"; let fallbackReason: string | undefined; if (!primaryResult.success) { + const overlayCount = + (options.overlayLayers?.length ?? 0) + + (options.tiledOverlayLayers?.length ?? 0); + if (overlayCount > 0) { + throw new Error( + `CUDA overlay-layer composition failed; refusing to drop ${overlayCount} visual overlay layer(s): ${getFfmpegFailureMessage(primaryResult)}`, + ); + } fullBackend = "cuda-scale-cpu-pad"; fallbackReason = isNativeCudaOutOfMemory(primaryResult.stderr) ? "cuda-oom" @@ -3679,7 +5929,7 @@ export async function exportNativeStaticLayoutVideo( metrics.fallbackChunkCount++; fullResult = await runFfmpegWithMetrics( ffmpegPath, - buildNativeCudaScaleCpuPadStaticLayoutArgs(fullConfig), + buildNativeCudaScaleCpuPadStaticLayoutArgs(encoderConfig), 15 * 60 * 1000, session, ); @@ -3714,6 +5964,7 @@ export async function exportNativeStaticLayoutVideo( const baseConfig: NativeStaticLayoutExportArgsConfig = { inputPath: options.inputPath, outputPath, + videoCodec, width: options.width, height: options.height, frameRate: options.frameRate, @@ -3724,6 +5975,7 @@ export async function exportNativeStaticLayoutVideo( offsetX: options.offsetX, offsetY: options.offsetY, backgroundColor: options.backgroundColor, + videoEncoder: await ensureNativeVideoEncoder(), startSec: chunk.startSec, durationSec: chunk.durationSec, }; @@ -3798,6 +6050,10 @@ export async function exportNativeStaticLayoutVideo( return { outputPath: videoOnlyPath, metrics, + videoCodec, + encoderPreference, + encoderName: "nvidia-cuda-compositor", + route: "nvidia-cuda-compositor", }; } const audioMuxProgressStart = 97.25; @@ -3827,9 +6083,26 @@ export async function exportNativeStaticLayoutVideo( ); Object.assign(metrics, finalized.metrics); outputPathToKeep = finalized.outputPath; + const route = metrics.chunks[0]?.backend; + if (!route) { + throw new Error("Native static-layout export did not report a route"); + } + // CUDA and D3D11 GPU-compositor routes do not use the FFmpeg encoder (it + // is resolved lazily, only on FFmpeg/raw fallback), so report the actual + // backend as the encoder name for those routes rather than a probed name. + const encoderName = + route === "nvidia-cuda-compositor" + ? "nvidia-cuda-compositor" + : route === "windows-d3d11-compositor" + ? "windows-d3d11-compositor" + : (resolvedNativeVideoEncoder ?? "cuda-static-composite"); return { outputPath: finalized.outputPath, metrics, + videoCodec, + encoderPreference, + encoderName, + route, }; } catch (error) { await removeTemporaryExportFile(videoOnlyPath); @@ -3899,8 +6172,8 @@ export async function probeNativeVideoEncoder( const args = buildNativeVideoExportArgs( encoderName, { - width: 64, - height: 64, + width: NATIVE_ENCODER_PROBE_DIMENSION, + height: NATIVE_ENCODER_PROBE_DIMENSION, frameRate: 1, bitrate: 1_500_000, encodingMode, @@ -3926,11 +6199,20 @@ export async function probeNativeVideoEncoder( stderrOutput += chunk.toString(); }); + // Swallow child errors (e.g. EPIPE on stdin, or a SIGKILL on an already + // exited probe process). Without a listener these surface as uncaught + // "The process not found" noise on Windows. The close handler below + // settles the probe result. + process.on("error", () => { + /* probe failure settles via close; nothing to do here */ + }); + process.on("close", (code) => { clearTimeout(timeout); void removeTemporaryExportFile(outputPath); if (code !== 0 && stderrOutput.trim().length > 0) { console.warn( + formatLogTs(), `[native-export] Encoder probe failed for ${encoderName}:`, stderrOutput.trim(), ); @@ -3938,38 +6220,66 @@ export async function probeNativeVideoEncoder( resolve(code === 0); }); - process.stdin.end(Buffer.alloc(getNativeVideoInputByteSize(64, 64), 0)); + process.stdin.end( + Buffer.alloc( + getNativeVideoInputByteSize( + NATIVE_ENCODER_PROBE_DIMENSION, + NATIVE_ENCODER_PROBE_DIMENSION, + ), + 0, + ), + ); }); } export async function resolveNativeVideoEncoder( ffmpegPath: string, encodingMode: NativeExportEncodingMode, + codec: "h264" | "hevc" = "h264", + preference: "auto" | "hardware" | "cpu" = "auto", ) { if ( cachedNativeVideoEncoder?.ffmpegPath === ffmpegPath && - cachedNativeVideoEncoder?.encodingMode === encodingMode + cachedNativeVideoEncoder?.encodingMode === encodingMode && + cachedNativeVideoEncoder?.codec === codec && + cachedNativeVideoEncoder?.preference === preference ) { return cachedNativeVideoEncoder.encoderName; } const availableEncoders = await getAvailableNativeVideoEncoders(ffmpegPath); - const candidates = [ - ...new Set([...getPreferredNativeVideoEncoders(process.platform), "libx264"]), - ]; + const candidates = getNativeEncoderCandidates(codec, preference, process.platform); + const usableCandidates = candidates.filter((encoderName) => availableEncoders.has(encoderName)); - for (const encoderName of candidates) { - if (!availableEncoders.has(encoderName)) { - continue; - } + if (usableCandidates.length === 0) { + throw new Error( + `No usable FFmpeg ${codec.toUpperCase()} encoder was available for native export (preference: ${preference})`, + ); + } + for (const encoderName of usableCandidates) { if (await probeNativeVideoEncoder(ffmpegPath, encoderName, encodingMode)) { - setCachedNativeVideoEncoder({ ffmpegPath, encodingMode, encoderName }); + setCachedNativeVideoEncoder({ + ffmpegPath, + encodingMode, + codec, + preference, + encoderName, + }); return encoderName; } } - throw new Error("No usable FFmpeg encoder was available for native export"); + if (preference === "hardware") { + throw new Error( + `No usable hardware FFmpeg ${codec.toUpperCase()} encoder was available for native export. ` + + `Tried: ${usableCandidates.join(", ")}. Install a supported hardware encoder or choose CPU/auto.`, + ); + } + + throw new Error( + `No usable FFmpeg ${codec.toUpperCase()} encoder was available for native export (preference: ${preference})`, + ); } export function canCopyAudioCodecIntoMp4(codec?: string | null) { @@ -4127,7 +6437,7 @@ export async function muxNativeVideoExportAudio( const ffmpegExecStartedAt = getNowMs(); await runFfmpegAudioMux(ffmpegPath, args, 15 * 60 * 1000, options, onProgress, session); metrics.ffmpegExecMs = getNowMs() - ffmpegExecStartedAt; - console.info("[native-video-export] Audio mux completed", { + console.info(formatLogTs(), "[native-video-export] Audio mux completed", { ffmpegExecMs: metrics.ffmpegExecMs, audioMode: options.audioMode, tempVideoBytes: metrics.tempVideoBytes, diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts index 36bcaf064..3f9bfd4cb 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts @@ -46,7 +46,7 @@ describe("planNativeStaticLayoutRoutes", () => { d3d11: d3d11Probe, source, }), - ).toEqual({ + ).toMatchObject({ selectedRoute: "nvidia-cuda-compositor", decisions: [ { @@ -124,7 +124,7 @@ describe("planNativeStaticLayoutRoutes", () => { d3d11, source, }), - ).toEqual({ + ).toMatchObject({ selectedRoute: "ffmpeg-static-layout", decisions: [ { @@ -149,6 +149,130 @@ describe("planNativeStaticLayoutRoutes", () => { }); }); + it("routes HEVC only through NVIDIA CUDA", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.videoCodec).toBe("hevc"); + expect(plan.encoderPreference).toBe("auto"); + expect(plan.decisions).toEqual( + expect.arrayContaining([ + { + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + { + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + ]), + ); + }); + + it("keeps the strict no-CPU-fallback flag on HEVC Hardware post-route failures", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "hardware", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + // Route is selected (CUDA available), but if it fails after planning the + // plan must still forbid CPU/rawvideo/Breeze fallback. + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.noCpuFallback).toBe(true); + expect(plan.fallbackRoute).toBe("hard-fail"); + expect(plan.fallbackReason).toBe("hevc-hardware-route-failed"); + }); + + it("keeps the HEVC Auto selected plan non-strict for post-route failures", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.noCpuFallback).toBe(false); + expect(plan.fallbackRoute).toBeNull(); + expect(plan.fallbackReason).toBeNull(); + }); + + it("hard-fails when HEVC CUDA is unavailable and Hardware is strict", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "hardware", + cuda: { ...cudaProbe, skipReason: "nvidia-gpu-unavailable" }, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("hard-fail"); + expect(plan.noCpuFallback).toBe(true); + expect(plan.fallbackReason).toBe("hevc-hardware-route-unavailable:nvidia-gpu-unavailable"); + expect(plan.decisions).toEqual( + expect.arrayContaining([ + { + route: "nvidia-cuda-compositor", + status: "rejected", + reasons: ["nvidia-gpu-unavailable"], + }, + { + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + { + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + ]), + ); + }); + + it("keeps HEVC Auto rawvideo fallback non-strict", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: { ...cudaProbe, skipReason: "nvidia-gpu-unavailable" }, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.noCpuFallback).toBe(false); + expect(plan.fallbackReason).toBe("hevc-cuda-unavailable:nvidia-gpu-unavailable"); + }); + + it("keeps CPU preference out of every GPU route", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "h264", + encoderPreference: "cpu", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.fallbackReason).toBe("encoder-preference-cpu-requires-native-rawvideo"); + expect(plan.decisions.every((decision) => decision.status === "rejected")).toBe(true); + }); + it("preserves proxy source metadata for route diagnostics", () => { const proxiedSource = { inputCodec: "vp9", diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts index 3ee6c4e73..25c0d57d0 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts @@ -1,9 +1,14 @@ -import type { NativeVideoExportAudioMode } from "../nativeVideoExport"; +import type { + ExportEncoderPreference, + ExportVideoCodec, + NativeVideoExportAudioMode, +} from "../nativeVideoExport"; export type NativeStaticLayoutRoute = | "nvidia-cuda-compositor" | "windows-d3d11-compositor" | "ffmpeg-static-layout"; +export type NativeStaticLayoutFallbackRoute = "native-rawvideo" | "hard-fail"; export interface NativeStaticLayoutRouteDecision { route: NativeStaticLayoutRoute; @@ -44,21 +49,158 @@ export interface NativeStaticLayoutRouteSource { } export interface NativeStaticLayoutRoutePlan { - selectedRoute: NativeStaticLayoutRoute; + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + selectedRoute: NativeStaticLayoutRoute | null; + fallbackRoute: NativeStaticLayoutFallbackRoute | null; + fallbackReason: string | null; + /** + * Strict HEVC Hardware policy: when true the export MUST hard-fail with an + * actionable error instead of falling back to renderer raw frames, Breeze, + * or CPU. No consumer may convert this plan into a rawvideo fallback. + */ + noCpuFallback: boolean; decisions: NativeStaticLayoutRouteDecision[]; cuda: NvidiaCudaExportCapabilityProbe; d3d11: WindowsD3D11ExportCapabilityProbe; source: NativeStaticLayoutRouteSource; } +function createRawVideoFallbackPlan(options: { + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + cuda: NvidiaCudaExportCapabilityProbe; + d3d11: WindowsD3D11ExportCapabilityProbe; + source: NativeStaticLayoutRouteSource; + reason: string; + cudaReason: string; + noCpuFallback?: boolean; +}) { + const { videoCodec, encoderPreference, cuda, d3d11, source } = options; + const noCpuFallback = options.noCpuFallback === true; + return { + videoCodec, + encoderPreference, + selectedRoute: null, + fallbackRoute: noCpuFallback ? "hard-fail" : "native-rawvideo", + fallbackReason: options.reason, + noCpuFallback, + decisions: [ + { + route: "nvidia-cuda-compositor" as const, + status: "rejected" as const, + reasons: [options.cudaReason], + }, + { + route: "windows-d3d11-compositor" as const, + status: "rejected" as const, + reasons: [ + videoCodec === "hevc" ? "hevc-requires-nvidia-cuda-compositor" : options.reason, + ], + }, + { + route: "ffmpeg-static-layout" as const, + status: "rejected" as const, + reasons: [ + videoCodec === "hevc" ? "hevc-requires-nvidia-cuda-compositor" : options.reason, + ], + }, + ], + cuda, + d3d11, + source, + } satisfies NativeStaticLayoutRoutePlan; +} + export function planNativeStaticLayoutRoutes(options: { + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; cuda: NvidiaCudaExportCapabilityProbe; d3d11: WindowsD3D11ExportCapabilityProbe; source: NativeStaticLayoutRouteSource; }): NativeStaticLayoutRoutePlan { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; const { cuda, d3d11, source } = options; const decisions: NativeStaticLayoutRouteDecision[] = []; + if (encoderPreference === "cpu") { + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: "encoder-preference-cpu-requires-native-rawvideo", + cudaReason: "encoder-preference-cpu-never-enters-gpu-compositor", + }); + } + + if (videoCodec === "hevc") { + if (!cuda.skipReason) { + decisions.push({ + route: "nvidia-cuda-compositor", + status: "selected", + reasons: ["cuda-wrapper-and-nvidia-gpu-available-for-hevc"], + }); + decisions.push({ + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }); + decisions.push({ + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }); + // Strict HEVC Hardware policy survives route selection: if the selected + // CUDA compositor fails after planning, the export must still hard-fail + // instead of falling back to renderer raw frames, Breeze, or CPU. HEVC + // Auto keeps the non-strict contract (rawvideo fallback is allowed). + const strictHevcHardware = encoderPreference === "hardware"; + return { + videoCodec, + encoderPreference, + selectedRoute: "nvidia-cuda-compositor", + fallbackRoute: strictHevcHardware ? "hard-fail" : null, + fallbackReason: strictHevcHardware ? "hevc-hardware-route-failed" : null, + noCpuFallback: strictHevcHardware, + decisions, + cuda, + d3d11, + source, + }; + } + + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: + encoderPreference === "hardware" + ? `hevc-hardware-route-unavailable:${cuda.skipReason}` + : `hevc-cuda-unavailable:${cuda.skipReason}`, + cudaReason: cuda.skipReason, + // Strict HEVC Hardware policy: the NVIDIA CUDA compositor is the ONLY + // acceptable route. Never fall back to renderer rawvideo, Breeze, or CPU. + noCpuFallback: encoderPreference === "hardware", + }); + } + + if (encoderPreference === "hardware") { + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: "encoder-preference-hardware-requires-native-rawvideo", + cudaReason: "explicit-hardware-preference-requires-native-rawvideo", + }); + } + if (!cuda.skipReason) { decisions.push({ route: "nvidia-cuda-compositor", @@ -78,7 +220,12 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["native-gpu-runtime-fallback"], }); return { + videoCodec, + encoderPreference, selectedRoute: "nvidia-cuda-compositor", + fallbackRoute: null, + fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, @@ -104,7 +251,12 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["windows-d3d11-runtime-fallback"], }); return { + videoCodec, + encoderPreference, selectedRoute: "windows-d3d11-compositor", + fallbackRoute: null, + fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, @@ -123,7 +275,12 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["native-gpu-routes-unavailable"], }); return { + videoCodec, + encoderPreference, selectedRoute: "ffmpeg-static-layout", + fallbackRoute: null, + fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index 5992cadf3..cc910b1a7 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { captureCanvasFrameForNativeExport } from "../../src/lib/exporter/nativeFrameCapture"; import { ATEMPO_FILTER_EPSILON } from "./ffmpeg/filters"; import { buildEditedTrackSourceAudioFilter, @@ -8,11 +9,85 @@ import { buildNativePrecompositedStaticLayoutArgs, buildNativeStaticBackgroundRenderArgs, buildNativeStaticLayoutChunks, + buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, createNativeSquircleMaskPgmBuffer, + getCpuEncoderForCodec, + getNativeEncoderCandidates, isNativeCudaOutOfMemory, } from "./nativeVideoExport"; +const childProcessMocks = vi.hoisted(() => ({ + encoderNames: "", + failingEncoders: new Set(), + execFile: vi.fn( + ( + _file: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, result?: { stdout: string; stderr: string }) => void, + ) => { + if (Array.isArray(args) && args.includes("-encoders")) { + cb(null, { stdout: childProcessMocks.encoderNames, stderr: "" }); + } else { + cb(null, { stdout: "", stderr: "" }); + } + }, + ), + spawn: vi.fn((_file: string, args: string[]) => { + let encoder = ""; + const codecIndex = args.indexOf("-c:v"); + if (codecIndex >= 0) { + encoder = args[codecIndex + 1]; + } + const exitCode = childProcessMocks.failingEncoders.has(encoder) ? 1 : 0; + return { + stdin: { + end: vi.fn(() => undefined), + destroy: vi.fn(), + destroyed: false, + writableEnded: false, + }, + stderr: { on: vi.fn() }, + on: vi.fn((event: string, cb: (code: number) => void) => { + if (event === "close") { + setTimeout(() => cb(exitCode), 0); + } + }), + kill: vi.fn(), + }; + }), +})); + +vi.mock("electron", () => ({ + app: { + getAppPath: vi.fn(() => process.cwd()), + getGPUInfo: vi.fn(async () => ({ gpuDevice: [] })), + getPath: vi.fn(() => process.env.TEMP ?? process.cwd()), + isPackaged: false, + }, + powerSaveBlocker: { + start: vi.fn(() => 1), + isStarted: vi.fn(() => false), + stop: vi.fn(), + }, +})); + +vi.mock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: vi.fn(() => "ffmpeg"), + getFfprobeBinaryPath: vi.fn(() => "ffprobe"), +})); + +vi.mock("node:child_process", () => ({ + execFile: childProcessMocks.execFile, + spawn: childProcessMocks.spawn, +})); + +import { resolveNativeVideoEncoder } from "./export/native-video"; +// Use the real in-memory state module so we can seed/read the native encoder +// cache through its real setter for the cache-identity tests below. +import { setCachedNativeVideoEncoder } from "./state"; + describe("buildTrimmedSourceAudioFilter", () => { it("concatenates trimmed source segments into a single output label", () => { expect( @@ -160,6 +235,62 @@ describe("native static layout command builders", () => { expect(args).toEqual(expect.arrayContaining(["-ss", "120.000", "-t", "60.000"])); }); + it("selects HEVC NVENC without changing the CUDA filtergraph", () => { + const args = buildNativeCudaOverlayStaticLayoutArgs({ + ...baseConfig, + videoCodec: "hevc", + }); + + expect(args).toContain("hevc_nvenc"); + expect(args).not.toContain("h264_nvenc"); + expect(args).toEqual( + expect.arrayContaining([expect.stringContaining("overlay_cuda=192:108")]), + ); + }); + + it("adds sorted transparent RGBA overlay inputs to the CUDA filtergraph", () => { + const args = buildNativeCudaOverlayStaticLayoutArgs({ + ...baseConfig, + overlayLayers: [ + { + id: "caption", + order: 2, + path: "caption.rgba", + x: 0, + y: 800, + width: 1920, + height: 280, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "cursor", + order: 1, + path: "cursor.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filter = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual( + expect.arrayContaining(["-f", "rawvideo", "cursor.rgba", "caption.rgba"]), + ); + expect(filter).toContain("[1:v]format=rgba[overlay_0]"); + expect(filter).toContain("overlay=0:0"); + expect(filter).toContain("[2:v]format=rgba[overlay_1]"); + expect(filter).toContain("overlay=0:800"); + }); + it("builds the stable CUDA scale plus CPU pad fallback command", () => { const args = buildNativeCudaScaleCpuPadStaticLayoutArgs(baseConfig); @@ -248,6 +379,102 @@ describe("native static layout command builders", () => { expect(pixels[4 * 8 + 4]).toBe(255); }); + it("preserves overlay layers in the precomposited CPU composition branch", () => { + const args = buildNativePrecompositedStaticLayoutArgs({ + ...baseConfig, + staticBackgroundPath: "background.png", + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual(expect.arrayContaining(["-f", "rawvideo", "effects.rgba"])); + expect(filterComplex).toContain("[2:v]format=rgba[overlay_0]"); + expect(filterComplex).toContain("[layout][overlay_0]overlay=x=0:y=0:format=auto"); + }); + + it("sorts shuffled overlay layers by order then id in the precomposited branch", () => { + const args = buildNativePrecompositedStaticLayoutArgs({ + ...baseConfig, + staticBackgroundPath: "background.png", + overlayLayers: [ + { + id: "caption", + order: 3, + path: "caption.rgba", + x: 0, + y: 800, + width: 1920, + height: 280, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "cursor", + order: 1, + path: "cursor.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "annotation", + order: 2, + path: "annotation.rgba", + x: 0, + y: 540, + width: 1920, + height: 540, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual( + expect.arrayContaining([ + "-f", + "rawvideo", + "cursor.rgba", + "annotation.rgba", + "caption.rgba", + ]), + ); + expect(filterComplex).toContain("[2:v]format=rgba[overlay_0]"); + expect(filterComplex).toContain("[layout][overlay_0]overlay=x=0:y=0:format=auto"); + expect(filterComplex).toContain("[3:v]format=rgba[overlay_1]"); + expect(filterComplex).toContain( + "[layout_overlay_0][overlay_1]overlay=x=0:y=540:format=auto", + ); + expect(filterComplex).toContain("[4:v]format=rgba[overlay_2]"); + expect(filterComplex).toContain( + "[layout_overlay_1][overlay_2]overlay=x=0:y=800:format=auto", + ); + }); + it("splits long exports into bounded chunks", () => { expect(buildNativeStaticLayoutChunks(367.5, 120)).toEqual([ { index: 0, startSec: 0, durationSec: 120 }, @@ -286,3 +513,234 @@ describe("native static layout command builders", () => { expect(isNativeCudaOutOfMemory("FFmpeg exited with code 1")).toBe(false); }); }); + +describe("getNativeEncoderCandidates", () => { + it("orders H.265 hardware candidates highest for Auto on Windows", () => { + expect(getNativeEncoderCandidates("hevc", "auto", "win32")).toEqual([ + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + "libx265", + ]); + }); + + it("returns only hardware candidates for the hardware preference", () => { + expect(getNativeEncoderCandidates("hevc", "hardware", "win32")).toEqual([ + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + ]); + expect(getNativeEncoderCandidates("hevc", "hardware", "darwin")).toEqual([ + "hevc_videotoolbox", + ]); + expect(getNativeEncoderCandidates("h264", "hardware", "linux")).toEqual([ + "h264_nvenc", + "h264_qsv", + ]); + }); + + it("returns only the CPU encoder for the cpu preference", () => { + expect(getNativeEncoderCandidates("hevc", "cpu", "linux")).toEqual(["libx265"]); + expect(getNativeEncoderCandidates("h264", "cpu", "win32")).toEqual(["libx264"]); + }); + + it("orders H.264 Linux hardware before the CPU fallback for Auto", () => { + expect(getNativeEncoderCandidates("h264", "auto", "linux")).toEqual([ + "h264_nvenc", + "h264_qsv", + "libx264", + ]); + }); + + it("maps each codec to its CPU encoder", () => { + expect(getCpuEncoderForCodec("h264")).toBe("libx264"); + expect(getCpuEncoderForCodec("hevc")).toBe("libx265"); + }); +}); + +describe("native raw-frame orientation", () => { + it("keeps synthetic top and bottom rows top-down without an FFmpeg flip", async () => { + class SyntheticVideoFrame { + async copyTo(destination: Uint8Array): Promise { + destination.set([255, 0, 0, 255, 0, 0, 255, 255]); + } + + close(): void {} + } + + vi.stubGlobal("VideoFrame", SyntheticVideoFrame); + const frame = await captureCanvasFrameForNativeExport( + { width: 1, height: 2 } as HTMLCanvasElement, + 0, + true, + ); + expect([...frame]).toEqual([255, 0, 0, 255, 0, 0, 255, 255]); + + const args = buildNativeVideoExportArgs( + "libx264", + { + width: 1, + height: 2, + frameRate: 30, + bitrate: 1_500_000, + encodingMode: "fast", + inputMode: "rawvideo", + }, + "out.mp4", + ); + expect(args).toEqual(expect.arrayContaining(["-f", "rawvideo", "-pix_fmt", "rgba"])); + expect(args).not.toContain("-vf"); + expect(args).not.toContain("vflip"); + }); +}); + +describe("buildNativeVideoExportArgs codec-aware", () => { + const base = { + width: 1920, + height: 1080, + frameRate: 60, + bitrate: 8_000_000, + encodingMode: "quality" as const, + }; + + it("libx265 includes bitrate, GOP, pixel format, and MP4 flags", () => { + const args = buildNativeVideoExportArgs( + "libx265", + { ...base, videoCodec: "hevc", encoderPreference: "cpu" }, + "out.mp4", + ); + expect(args[args.indexOf("-c:v") + 1]).toBe("libx265"); + expect(args[args.indexOf("-preset") + 1]).toBe("slow"); + expect(args[args.indexOf("-g") + 1]).toBe("300"); + expect(args).toContain("-b:v"); + expect(args[args.lastIndexOf("-pix_fmt") + 1]).toBe("yuv420p"); + expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart"); + expect(args).toContain("out.mp4"); + }); + + it("hevc_nvenc includes bitrate, GOP, pixel format, and MP4 flags", () => { + const args = buildNativeVideoExportArgs( + "hevc_nvenc", + { ...base, videoCodec: "hevc", encoderPreference: "hardware" }, + "out.mp4", + ); + expect(args[args.indexOf("-c:v") + 1]).toBe("hevc_nvenc"); + expect(args).toContain("-preset"); + expect(args[args.indexOf("-g") + 1]).toBe("300"); + expect(args).toContain("-b:v"); + expect(args[args.lastIndexOf("-pix_fmt") + 1]).toBe("yuv420p"); + expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart"); + expect(args).toContain("out.mp4"); + }); + + it("libx264 CPU keeps its existing preset tuning", () => { + const args = buildNativeVideoExportArgs( + "libx264", + { ...base, videoCodec: "h264", encoderPreference: "cpu" }, + "out.mp4", + ); + expect(args[args.indexOf("-preset") + 1]).toBe("slow"); + // quality mode => -preset slow on libx264 (no -tune for quality). + expect(args).toContain("-preset"); + }); +}); + +describe("resolveNativeVideoEncoder", () => { + const encoderListing = (...names: string[]) => + names.map((name) => ` V....D ${name} ffmpeg-${name} encoder\r\n`).join(""); + + beforeEach(() => { + childProcessMocks.execFile.mockClear(); + childProcessMocks.spawn.mockClear(); + childProcessMocks.failingEncoders = new Set(); + setCachedNativeVideoEncoder(null); + childProcessMocks.encoderNames = encoderListing("libx264", "libx265"); + }); + + it("resolves H.264 CPU to libx264", async () => { + childProcessMocks.encoderNames = encoderListing("libx264"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "cpu")).resolves.toBe( + "libx264", + ); + const probeArgs = childProcessMocks.spawn.mock.calls[0]?.[1] as string[]; + expect(probeArgs).toContain("libx264"); + }); + + it("resolves HEVC CPU to libx265", async () => { + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "cpu")).resolves.toBe( + "libx265", + ); + const probeArgs = childProcessMocks.spawn.mock.calls[0]?.[1] as string[]; + expect(probeArgs).toContain("libx265"); + }); + + it("auto falls back from a failing hardware encoder to CPU", async () => { + const hardware = getNativeEncoderCandidates("hevc", "hardware", process.platform); + childProcessMocks.encoderNames = encoderListing(...hardware, "libx265"); + childProcessMocks.failingEncoders = new Set(hardware); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "auto")).resolves.toBe( + "libx265", + ); + }); + + it("hardware-only does not silently fall back to CPU", async () => { + const hardware = getNativeEncoderCandidates("hevc", "hardware", process.platform); + childProcessMocks.encoderNames = encoderListing(...hardware, "libx265"); + childProcessMocks.failingEncoders = new Set(hardware); + await expect( + resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "hardware"), + ).rejects.toThrow(/hardware/i); + const probedEncoders = ( + childProcessMocks.spawn.mock.calls as Array<[string, string[]]> + ).map(([, args]) => args[args.indexOf("-c:v") + 1]); + expect(probedEncoders).not.toContain("libx265"); + }); + + it("errors when no available hardware encoder can be probed", async () => { + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect( + resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "hardware"), + ).rejects.toThrow(/hardware/i); + expect(childProcessMocks.spawn).not.toHaveBeenCalled(); + }); + + it("includes codec and preference in the cache identity", async () => { + // Seed the cache with an H.264/auto entry via the real state setter. + setCachedNativeVideoEncoder({ + ffmpegPath: "ffmpeg", + encodingMode: "balanced", + codec: "h264", + preference: "auto", + encoderName: "libx264", + }); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "auto")).resolves.toBe( + "libx264", + ); + expect(childProcessMocks.execFile).not.toHaveBeenCalled(); + expect(childProcessMocks.spawn).not.toHaveBeenCalled(); + + // A different codec must not reuse the cached H.264/auto entry. + setCachedNativeVideoEncoder(null); + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "auto")).resolves.toBe( + "libx265", + ); + + // A different preference must not collide with the cached H.264/auto entry. + setCachedNativeVideoEncoder({ + ffmpegPath: "ffmpeg", + encodingMode: "balanced", + codec: "h264", + preference: "auto", + encoderName: "libx264", + }); + childProcessMocks.encoderNames = encoderListing("libx264"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "cpu")).resolves.toBe( + "libx264", + ); + expect(childProcessMocks.execFile).toHaveBeenCalled(); + }); +}); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 07aa64a91..0077a6a80 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -1,8 +1,12 @@ +import type { NativeStaticLayoutOverlayLayer } from "../../src/lib/exporter/nativeStaticLayoutOverlays"; import { getShadowFilterPadding, VIDEO_SHADOW_LAYER_PROFILES, } from "../../src/lib/exporter/shadowProfile"; +import type { ExportEncoderPreference, ExportVideoCodec } from "../../src/lib/exporter/types"; import { getSquirclePathPoints } from "../../src/lib/geometry/squircle"; +export type { ExportEncoderPreference, ExportVideoCodec }; + import { ATEMPO_FILTER_EPSILON, buildAtempoFilters } from "./ffmpeg/filters"; const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; @@ -23,6 +27,8 @@ export interface NativeVideoExportStartOptions { bitrate: number; encodingMode: NativeExportEncodingMode; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; } export interface NativeVideoExportAudioSegment { @@ -67,6 +73,7 @@ export type NativeStaticLayoutBackend = export interface NativeStaticLayoutExportArgsConfig { inputPath: string; outputPath: string; + videoCodec?: ExportVideoCodec; width: number; height: number; frameRate: number; @@ -89,6 +96,12 @@ export interface NativeStaticLayoutExportArgsConfig { shadowIntensity?: number; startSec?: number; durationSec?: number; + overlayLayers?: NativeStaticLayoutOverlayLayer[]; + videoEncoder?: string; +} + +function getNativeStaticLayoutVideoEncoder(codec: ExportVideoCodec = "h264") { + return codec === "hevc" ? "hevc_nvenc" : "h264_nvenc"; } export interface NativeStaticLayoutChunk { @@ -114,19 +127,60 @@ export function parseAvailableFfmpegEncoders(stdout: string): Set { return encoders; } -export function getPreferredNativeVideoEncoders(platform: NodeJS.Platform): string[] { +function getHardwareEncoderCandidates( + codec: ExportVideoCodec, + platform: NodeJS.Platform, +): string[] { + if (codec === "hevc") { + switch (platform) { + case "darwin": + return ["hevc_videotoolbox"]; + case "win32": + return ["hevc_nvenc", "hevc_qsv", "hevc_amf", "hevc_mf"]; + case "linux": + return ["hevc_nvenc", "hevc_qsv"]; + default: + return []; + } + } + switch (platform) { case "darwin": - return ["h264_videotoolbox", "libx264"]; + return ["h264_videotoolbox"]; case "win32": - return ["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"]; + return ["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf"]; case "linux": - return ["h264_nvenc", "h264_qsv", "libx264"]; + return ["h264_nvenc", "h264_qsv"]; default: - return ["libx264"]; + return []; } } +export function getCpuEncoderForCodec(codec: ExportVideoCodec): string { + return codec === "hevc" ? "libx265" : "libx264"; +} + +export function getNativeEncoderCandidates( + codec: ExportVideoCodec, + preference: ExportEncoderPreference, + platform: NodeJS.Platform, +): string[] { + const hardware = getHardwareEncoderCandidates(codec, platform); + const cpu = [getCpuEncoderForCodec(codec)]; + + if (preference === "cpu") { + return cpu; + } + if (preference === "hardware") { + return hardware; + } + return [...hardware, ...cpu]; +} + +export function getPreferredNativeVideoEncoders(platform: NodeJS.Platform): string[] { + return getNativeEncoderCandidates("h264", "auto", platform); +} + function getLibx264ModeArgs(encodingMode: NativeExportEncodingMode): string[] { switch (encodingMode) { case "fast": @@ -139,6 +193,18 @@ function getLibx264ModeArgs(encodingMode: NativeExportEncodingMode): string[] { } } +function getLibx265ModeArgs(encodingMode: NativeExportEncodingMode): string[] { + switch (encodingMode) { + case "fast": + return ["-preset", "ultrafast"]; + case "quality": + return ["-preset", "slow"]; + case "balanced": + default: + return ["-preset", "medium"]; + } +} + function getBitrateArgs(bitrate: number): string[] { const effectiveBitrate = Math.max(1_500_000, Math.round(bitrate)); const maxRate = Math.max(effectiveBitrate, Math.round(effectiveBitrate * 1.2)); @@ -295,8 +361,6 @@ export function buildNativeVideoExportArgs( String(options.frameRate), "-i", "pipe:0", - "-vf", - "vflip", "-an", "-c:v", encoder, @@ -307,6 +371,10 @@ export function buildNativeVideoExportArgs( if (encoder === "libx264") { args.push(...getLibx264ModeArgs(options.encodingMode)); + } else if (encoder === "libx265") { + args.push(...getLibx265ModeArgs(options.encodingMode)); + } else if (encoder === "hevc_nvenc") { + args.push(...getNvencStaticLayoutModeArgs(options.encodingMode)); } args.push("-pix_fmt", "yuv420p", "-movflags", "+faststart", outputPath); @@ -319,23 +387,63 @@ export function buildNativeCudaOverlayStaticLayoutArgs( const backgroundColor = formatFfmpegColor(config.backgroundColor); const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); const args = ["-y", "-hide_banner", "-loglevel", "error"]; + const overlayLayers = [...(config.overlayLayers ?? [])].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); + args.push("-hwaccel", "cuda", "-hwaccel_output_format", "cuda", "-i", config.inputPath); + for (const layer of overlayLayers) { + args.push( + "-f", + "rawvideo", + "-pix_fmt", + "rgba", + "-s:v", + `${layer.width}x${layer.height}`, + "-framerate", + String(layer.frameRate), + "-i", + layer.path, + ); + } + + let filterComplex = + `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];` + + `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];` + + `[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`; + if (overlayLayers.length > 0) { + const filterParts = [ + `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=rgba[bg]`, + `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fg]`, + `[bg][fg]overlay=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat[layout]`, + ]; + let currentLabel = "layout"; + for (const [index, layer] of overlayLayers.entries()) { + const inputIndex = index + 1; + const overlayLabel = `overlay_${index}`; + const nextLabel = index === overlayLayers.length - 1 ? "out" : `layout_${index}`; + filterParts.push( + `[${inputIndex}:v]format=rgba[${overlayLabel}]`, + `[${currentLabel}][${overlayLabel}]overlay=${layer.x}:${layer.y}:shortest=0:repeatlast=1:eof_action=repeat[${nextLabel}]`, + ); + currentLabel = nextLabel; + } + filterParts.push( + `[${currentLabel}]trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`, + ); + filterComplex = filterParts.join(";"); + } + args.push( - "-hwaccel", - "cuda", - "-hwaccel_output_format", - "cuda", - "-i", - config.inputPath, "-filter_complex", - `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`, + filterComplex, "-map", "[out]", "-an", "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-movflags", @@ -348,6 +456,9 @@ export function buildNativeCudaOverlayStaticLayoutArgs( export function buildNativeCudaScaleCpuPadStaticLayoutArgs( config: NativeStaticLayoutExportArgsConfig, ): string[] { + if (config.overlayLayers?.length) { + throw new Error("CUDA scale/pad fallback cannot preserve native overlay layers"); + } const backgroundColor = formatFfmpegColor(config.backgroundColor); const args = ["-y", "-hide_banner", "-loglevel", "error"]; pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); @@ -366,7 +477,7 @@ export function buildNativeCudaScaleCpuPadStaticLayoutArgs( "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-pix_fmt", @@ -482,6 +593,9 @@ export function buildNativePrecompositedStaticLayoutArgs( const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); const useMask = Boolean(config.maskPath && (config.borderRadius ?? 0) > 0.5); + const overlayLayers = [...(config.overlayLayers ?? [])].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); const args = ["-y", "-hide_banner", "-loglevel", "error"]; pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); args.push( @@ -513,11 +627,43 @@ export function buildNativePrecompositedStaticLayoutArgs( config.maskPath, ); } + for (const layer of overlayLayers) { + args.push( + "-f", + "rawvideo", + "-pix_fmt", + "rgba", + "-s:v", + `${layer.width}x${layer.height}`, + "-framerate", + String(layer.frameRate), + "-i", + layer.path, + ); + } const foregroundFilter = `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fgbase]`; const maskFilter = useMask ? ";[2:v]format=gray[mask];[fgbase][mask]alphamerge[fg]" : ""; const foregroundLabel = useMask ? "fg" : "fgbase"; - const filterComplex = `${foregroundFilter}${maskFilter};[1:v]format=rgba[bg];[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto,trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`; + const filterParts = [ + `${foregroundFilter}${maskFilter}`, + `[1:v]format=rgba[bg]`, + `[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto[layout]`, + ]; + let currentLabel = "layout"; + const firstOverlayInputIndex = useMask ? 3 : 2; + for (const [index, layer] of overlayLayers.entries()) { + const nextLabel = `layout_overlay_${index}`; + filterParts.push( + `[${firstOverlayInputIndex + index}:v]format=rgba[overlay_${index}]`, + `[${currentLabel}][overlay_${index}]overlay=x=${layer.x}:y=${layer.y}:format=auto[${nextLabel}]`, + ); + currentLabel = nextLabel; + } + filterParts.push( + `[${currentLabel}]trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`, + ); + const filterComplex = filterParts.join(";"); args.push( "-filter_complex", @@ -528,7 +674,7 @@ export function buildNativePrecompositedStaticLayoutArgs( "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-pix_fmt", diff --git a/electron/ipc/project/manager.test.ts b/electron/ipc/project/manager.test.ts index 508730924..25f86cf95 100644 --- a/electron/ipc/project/manager.test.ts +++ b/electron/ipc/project/manager.test.ts @@ -95,14 +95,14 @@ describe("local media path policy", () => { const { isAllowedMediaPath } = await import("../../mediaServer"); // Unapproved external paths are rejected before they ever reach the media server. - expect(isAllowedMediaPath(videoPath)).toBe(false); + await expect(isAllowedMediaPath(videoPath)).resolves.toBe(false); await expect(resolveApprovedLocalMediaPath(videoPath)).resolves.toBeNull(); // Once the user opts in (via dialog/export/etc.) the path is approved. await rememberApprovedLocalReadPath(videoPath); await expect(resolveApprovedLocalMediaPath(videoPath)).resolves.toBe(resolvedVideoPath); - expect(isAllowedMediaPath(videoPath)).toBe(true); + await expect(isAllowedMediaPath(videoPath)).resolves.toBe(true); }); it("rejects existing non-media files when resolving local media URLs", async () => { @@ -115,7 +115,109 @@ describe("local media path policy", () => { const { isAllowedMediaPath } = await import("../../mediaServer"); await expect(resolveApprovedLocalMediaPath(textPath)).resolves.toBeNull(); - expect(isAllowedMediaPath(textPath)).toBe(false); + await expect(isAllowedMediaPath(textPath)).resolves.toBe(false); + }); + + it("allows m4a audio assets inside the recordings directory", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + + const { resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath); + }); + + it("allows m4a assets inside a configured custom recordings directory", async () => { + const customRecordingsPath = path.join(tempRoot, "Custom Recordings"); + const audioPath = path.join(customRecordingsPath, "recording-2026-08-03.mic.m4a"); + await fs.mkdir(customRecordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + await fs.writeFile( + path.join(userDataPath, "recordings-settings.json"), + JSON.stringify({ recordingsDir: customRecordingsPath }), + "utf-8", + ); + + const { resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath); + }); + + it("matches policy roots case-insensitively on Windows", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + + const { resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + if (process.platform === "win32") { + const caseVariantRoot = recordingsPath.replace(/^([a-zA-Z]):/, (drive) => + drive.toLowerCase() === drive ? drive.toUpperCase() : drive.toLowerCase(), + ); + const caseVariantPath = audioPath.replace(recordingsPath, caseVariantRoot); + if (caseVariantPath !== audioPath) { + await expect(resolveApprovedLocalMediaPath(caseVariantPath)).resolves.toBe( + resolvedAudioPath, + ); + } + } + }); + + it("folds Windows case and extended-length prefixes in policy comparisons", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.system.wav"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const { isPathInsideDirectory, resolveLocalMediaUrlPath } = await import("./manager"); + const { foldPathComparisonKey } = await import("../state"); + + if (process.platform !== "win32") { + expect(isPathInsideDirectory(audioPath, recordingsPath)).toBe(true); + return; + } + + const caseVariantPath = audioPath.replace(/^([a-zA-Z]):/, (drive) => + drive.toLowerCase() === drive ? drive.toUpperCase() : drive.toLowerCase(), + ); + const extendedPath = `\\\\?\\${audioPath}`; + + // All three forms fold to one comparison key. + expect(foldPathComparisonKey(caseVariantPath)).toBe(foldPathComparisonKey(audioPath)); + expect(foldPathComparisonKey(extendedPath)).toBe(foldPathComparisonKey(audioPath)); + + // Containment accepts the extended-prefix form inside the plain root. + expect(isPathInsideDirectory(extendedPath, recordingsPath)).toBe(true); + expect(isPathInsideDirectory(caseVariantPath, recordingsPath)).toBe(true); + + // A pending grant made through the extended-prefix form authorizes the + // plain real path once the file appears (serve-time re-validation). + const resolution = await resolveLocalMediaUrlPath(extendedPath); + expect(resolution.status).toBe("pending"); + await fs.writeFile(audioPath, "test-audio"); + const { isAllowedMediaPath } = await import("../../mediaServer"); + await expect(isAllowedMediaPath(await fs.realpath(audioPath))).resolves.toBe(true); + }); + + it("normalizes file:/// URLs in media path approval", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + + const { resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + const fileUrl = + process.platform === "win32" + ? `file:///${audioPath.replace(/\\/g, "/")}` + : `file://${audioPath}`; + + await expect(resolveApprovedLocalMediaPath(fileUrl)).resolves.toBe(resolvedAudioPath); }); it("rejects symlinks under allowed prefixes that point outside the allowlist", async () => { @@ -143,6 +245,145 @@ describe("local media path policy", () => { await expect(resolveApprovedLocalMediaPath(symlinkInsideUserData)).resolves.toBeNull(); }); + describe("resolveLocalMediaUrlPath pending sidecar candidates", () => { + it("grants a pending URL for a missing supported sidecar under the recordings root", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const m4aPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + + await expect(resolveLocalMediaUrlPath(m4aPath)).resolves.toEqual({ + status: "pending", + path: m4aPath, + }); + }); + + it("grants a pending URL for a missing webm sidecar under a custom recordings root", async () => { + const customRecordingsPath = path.join(tempRoot, "Custom Recordings"); + const webmPath = path.join(customRecordingsPath, "recording-2026-08-03.mic.webm"); + await fs.mkdir(customRecordingsPath, { recursive: true }); + await fs.writeFile( + path.join(userDataPath, "recordings-settings.json"), + JSON.stringify({ recordingsDir: customRecordingsPath }), + "utf-8", + ); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + + await expect(resolveLocalMediaUrlPath(webmPath)).resolves.toEqual({ + status: "pending", + path: webmPath, + }); + }); + + it("rejects a missing candidate outside the allowed roots", async () => { + const downloadsPath = path.join(tempRoot, "Downloads"); + const missingPath = path.join(downloadsPath, "recording.system.m4a"); + await fs.mkdir(downloadsPath, { recursive: true }); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + + await expect(resolveLocalMediaUrlPath(missingPath)).resolves.toEqual({ + status: "rejected", + reason: "outside-allowed-roots", + }); + }); + + it("rejects a missing candidate with an unsupported extension under a root", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const notesPath = path.join(recordingsPath, "recording-2026-08-03.notes.txt"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + + await expect(resolveLocalMediaUrlPath(notesPath)).resolves.toEqual({ + status: "rejected", + reason: "unsupported-type", + }); + }); + + it("approves an existing media file through the URL resolver", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.system.wav"); + await fs.mkdir(recordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + await expect(resolveLocalMediaUrlPath(audioPath)).resolves.toEqual({ + status: "approved", + path: resolvedAudioPath, + }); + }); + + it("approves an existing file:/// media URL through the URL resolver", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const audioPath = path.join(recordingsPath, "recording-2026-08-03.mic.wav"); + await fs.mkdir(recordingsPath, { recursive: true }); + await fs.writeFile(audioPath, "test-audio"); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + const fileUrl = + process.platform === "win32" + ? `file:///${audioPath.replace(/\\/g, "/")}` + : `file://${audioPath}`; + + await expect(resolveLocalMediaUrlPath(fileUrl)).resolves.toEqual({ + status: "approved", + path: resolvedAudioPath, + }); + }); + + it("rejects a symlink under a root that points outside as a pending/approved URL", async () => { + const outsideTarget = path.join(tempRoot, "outside-secret.mp4"); + const symlinkInsideUserData = path.join(userDataPath, "shortcut-to-secret.mp4"); + await fs.writeFile(outsideTarget, "secret-bytes"); + + try { + await fs.symlink(outsideTarget, symlinkInsideUserData); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EPERM") { + return; + } + throw error; + } + + const { resolveLocalMediaUrlPath } = await import("./manager"); + + await expect(resolveLocalMediaUrlPath(symlinkInsideUserData)).resolves.toEqual({ + status: "rejected", + reason: "symlink-escape", + }); + }); + + it("serves a pending in-root sidecar through mediaServer once the file appears", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const m4aPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const { resolveLocalMediaUrlPath } = await import("./manager"); + const { isAllowedMediaPath } = await import("../../mediaServer"); + + const resolution = await resolveLocalMediaUrlPath(m4aPath); + expect(resolution.status).toBe("pending"); + + // Simulate the mux rename completing after the pending grant: the + // media-server serve-time realpath check must now accept the file. + await fs.writeFile(m4aPath, "test-audio"); + const realPath = await fs.realpath(m4aPath); + await expect(isAllowedMediaPath(realPath)).resolves.toBe(true); + + // A symlink escape still resolves outside the roots and must stay blocked. + const outsideTarget = path.join(tempRoot, "outside-secret.m4a"); + await fs.writeFile(outsideTarget, "secret-bytes"); + const escaped = await fs.realpath(outsideTarget); + await expect(isAllowedMediaPath(escaped)).resolves.toBe(false); + }); + }); + it("preserves an existing project thumbnail when no replacement is provided", async () => { const projectPath = path.join(tempRoot, "Projects", "demo.recordly"); const thumbnailDataUrl = `data:image/png;base64,${Buffer.from("png-thumbnail").toString("base64")}`; diff --git a/electron/ipc/project/manager.ts b/electron/ipc/project/manager.ts index 7828fa8b6..a0ced70f5 100644 --- a/electron/ipc/project/manager.ts +++ b/electron/ipc/project/manager.ts @@ -16,6 +16,8 @@ import { import { approvedLocalReadPaths, currentProjectPath, + customRecordingsDir, + foldPathComparisonKey, setCurrentProjectPath, setCurrentRecordingSession, setCurrentVideoPath, @@ -43,20 +45,36 @@ export function getAssetRootPath() { export function isPathInsideDirectory(candidatePath: string, directoryPath: string) { const normalizedCandidatePath = normalizePath(candidatePath); const normalizedDirectoryPath = normalizePath(directoryPath); + const foldedCandidatePath = foldPathComparisonKey(normalizedCandidatePath); + const foldedDirectoryPath = foldPathComparisonKey(normalizedDirectoryPath); return ( - normalizedCandidatePath === normalizedDirectoryPath || - normalizedCandidatePath.startsWith(`${normalizedDirectoryPath}${path.sep}`) + foldedCandidatePath === foldedDirectoryPath || + foldedCandidatePath.startsWith(`${foldedDirectoryPath}${path.sep}`) ); } -export function isAllowedLocalReadPath(candidatePath: string) { - const allowedPrefixes = [ - RECORDINGS_DIR, +// Approved roots are derived from the live app paths plus the configured +// recordings directory (state keeps the value loaded from the recordings +// settings file; the async media path re-reads it through getRecordingsDir). +// Hard-coding a literal recordings path here would silently reject users who +// configured a custom recordings directory outside the default userData path. +export function getAllowedLocalReadRootsSync() { + return [ + customRecordingsDir ?? RECORDINGS_DIR, USER_DATA_PATH, getAssetRootPath(), app.getPath("temp"), ]; +} + +export async function getAllowedLocalReadRoots() { + return [await getRecordingsDir(), USER_DATA_PATH, getAssetRootPath(), app.getPath("temp")]; +} + +export function isAllowedLocalReadPath(candidatePath: string) { + const allowedPrefixes = getAllowedLocalReadRootsSync(); const normalizedCandidatePath = normalizePath(candidatePath); + const foldedCandidatePath = foldPathComparisonKey(normalizedCandidatePath); // Canonicalize so a symlink placed under an allowed prefix can't smuggle in a // target that lives outside it. realpathSync throws when the path doesn't @@ -79,7 +97,7 @@ export function isAllowedLocalReadPath(candidatePath: string) { // for read-local-file and the local media URL handler. const lexicalAllowed = allowedPrefixes.some((prefix) => isPathInsideDirectory(normalizedCandidatePath, prefix)) || - approvedLocalReadPaths.has(normalizedCandidatePath); + approvedLocalReadPaths.has(foldedCandidatePath); if (!lexicalAllowed) { return false; } @@ -90,7 +108,7 @@ export function isAllowedLocalReadPath(candidatePath: string) { return ( allowedPrefixes.some((prefix) => isPathInsideDirectory(canonicalCandidatePath, prefix)) || - approvedLocalReadPaths.has(canonicalCandidatePath) + approvedLocalReadPaths.has(foldPathComparisonKey(canonicalCandidatePath)) ); } @@ -100,7 +118,38 @@ export function isAllowedLocalReadPath(candidatePath: string) { // become fetchable inside the app. export async function isAllowedLocalMediaPath(candidatePath: string) { const normalizedCandidatePath = normalizePath(candidatePath); - return isAllowedLocalReadPath(normalizedCandidatePath); + return isAllowedLocalReadPathWithRoots( + normalizedCandidatePath, + await getAllowedLocalReadRoots(), + ); +} + +async function isAllowedLocalReadPathWithRoots(candidatePath: string, allowedPrefixes: string[]) { + const normalizedCandidatePath = normalizePath(candidatePath); + const foldedCandidatePath = foldPathComparisonKey(normalizedCandidatePath); + + let canonicalCandidatePath = normalizedCandidatePath; + try { + canonicalCandidatePath = normalizePath(realpathSync(normalizedCandidatePath)); + } catch { + // File may not exist yet; keep the lexical path. + } + + const lexicalAllowed = + allowedPrefixes.some((prefix) => isPathInsideDirectory(normalizedCandidatePath, prefix)) || + approvedLocalReadPaths.has(foldedCandidatePath); + if (!lexicalAllowed) { + return false; + } + + if (canonicalCandidatePath === normalizedCandidatePath) { + return true; + } + + return ( + allowedPrefixes.some((prefix) => isPathInsideDirectory(canonicalCandidatePath, prefix)) || + approvedLocalReadPaths.has(foldPathComparisonKey(canonicalCandidatePath)) + ); } async function collectApprovedLocalReadPaths(filePath?: string | null): Promise { @@ -109,13 +158,13 @@ async function collectApprovedLocalReadPaths(filePath?: string | null): Promise< return []; } - const approvedPaths = [normalizePath(normalizedPath)]; + const approvedPaths = [foldPathComparisonKey(normalizePath(normalizedPath))]; try { - const realPath = await fs.realpath(approvedPaths[0]); - const normalizedRealPath = normalizePath(realPath); - if (!approvedPaths.includes(normalizedRealPath)) { - approvedPaths.push(normalizedRealPath); + const realPath = await fs.realpath(normalizePath(normalizedPath)); + const foldedRealPath = foldPathComparisonKey(normalizePath(realPath)); + if (!approvedPaths.includes(foldedRealPath)) { + approvedPaths.push(foldedRealPath); } } catch { // Ignore missing files; the eventual read will surface the real error. @@ -136,9 +185,84 @@ export async function rememberApprovedLocalReadPath(filePath?: string | null) { } } +export type LocalMediaUrlResolution = + | { status: "approved"; path: string } + | { status: "pending"; path: string } + | { + status: "rejected"; + reason: + | "missing" + | "unsupported-type" + | "outside-allowed-roots" + | "not-file" + | "symlink-escape"; + }; + +// Resolve a candidate for the local media server URL. Existing files go +// through the strict realpath + allowlist path. Files that do not exist yet +// (speculative audio sidecar candidates requested before the mux rename +// completes on Windows) may still receive a PENDING url when they are a +// supported media type on a lexical path inside an allowed recordings root; +// the media server re-validates realpath and symlink containment at serve time +// when the file actually appears, so no canonical security is weakened. +export async function resolveLocalMediaUrlPath( + candidatePath: string, +): Promise { + const normalizedCandidatePath = normalizeVideoSourcePath(candidatePath) ?? candidatePath; + const resolvedCandidatePath = normalizePath(normalizedCandidatePath); + + const realPath = await fs.realpath(resolvedCandidatePath).catch((error: unknown) => { + return (error as NodeJS.ErrnoException)?.code === "ENOENT" ? null : undefined; + }); + + if (realPath === undefined) { + return { status: "rejected", reason: "missing" }; + } + + if (realPath === null) { + // The file does not exist yet. Grant a pending URL only for a supported + // media type under an allowed root; never for arbitrary missing paths. + if (!isSupportedLocalMediaPath(resolvedCandidatePath)) { + return { status: "rejected", reason: "unsupported-type" }; + } + const allowedRoots = await getAllowedLocalReadRoots(); + if (!(await isAllowedLocalReadPathWithRoots(resolvedCandidatePath, allowedRoots))) { + return { status: "rejected", reason: "outside-allowed-roots" }; + } + await rememberApprovedLocalReadPath(normalizedCandidatePath); + return { status: "pending", path: resolvedCandidatePath }; + } + + const stat = await fs.stat(realPath).catch(() => null); + if (!stat?.isFile()) { + return { status: "rejected", reason: "not-file" }; + } + if (!isSupportedLocalMediaPath(realPath)) { + return { status: "rejected", reason: "unsupported-type" }; + } + const allowedRoots = await getAllowedLocalReadRoots(); + const lexicalAllowed = await isAllowedLocalReadPathWithRoots( + resolvedCandidatePath, + allowedRoots, + ); + if (!(await isAllowedLocalMediaPath(realPath))) { + // The real path is not accepted by the roots or the approved set; if the + // lexical path looked allowed, this is a symlink/reparse-point escape. + return { + status: "rejected", + reason: lexicalAllowed ? "symlink-escape" : "outside-allowed-roots", + }; + } + await rememberApprovedLocalReadPath(normalizedCandidatePath); + return { status: "approved", path: realPath }; +} + export async function resolveApprovedLocalMediaPath(candidatePath: string): Promise { - const normalizedCandidatePath = normalizePath(candidatePath); - const realPath = await fs.realpath(normalizedCandidatePath).catch(() => null); + // Accept file:/// URLs and bare paths; persisted project sources can carry + // either form and both must resolve through the local media server. + const normalizedCandidatePath = normalizeVideoSourcePath(candidatePath) ?? candidatePath; + const resolvedCandidatePath = normalizePath(normalizedCandidatePath); + const realPath = await fs.realpath(resolvedCandidatePath).catch(() => null); if (!realPath) { return null; @@ -153,7 +277,7 @@ export async function resolveApprovedLocalMediaPath(candidatePath: string): Prom return null; } - await rememberApprovedLocalReadPath(candidatePath); + await rememberApprovedLocalReadPath(normalizedCandidatePath); return realPath; } diff --git a/electron/ipc/register/export.test.ts b/electron/ipc/register/export.test.ts index 33941eb18..0a4b3802f 100644 --- a/electron/ipc/register/export.test.ts +++ b/electron/ipc/register/export.test.ts @@ -9,14 +9,11 @@ vi.mock("electron", () => ({ getPath: () => process.env.TEMP ?? process.cwd(), isPackaged: false, }, - BrowserWindow: { - fromWebContents: () => null, - }, - dialog: { - showSaveDialog: vi.fn(), - }, + BrowserWindow: { fromWebContents: () => null }, + dialog: { showSaveDialog: vi.fn() }, ipcMain: { handle: vi.fn(), + on: vi.fn(), }, powerSaveBlocker: { isStarted: () => true, @@ -26,10 +23,14 @@ vi.mock("electron", () => ({ })); vi.mock("../ffmpeg/binary", () => ({ - getFfmpegBinaryPath: () => "ffmpeg", + getFfmpegBinaryPath: () => path.join(process.cwd(), "recordly-missing-ffmpeg-binary"), })); -import { moveExportedTempFile } from "./export"; +import { ipcMain } from "electron"; + +import * as nativeVideo from "../export/native-video"; +import { type NativeVideoExportSession, nativeVideoExportSessions } from "../export/native-video"; +import { moveExportedTempFile, registerExportHandlers } from "./export"; const tempDirs: string[] = []; @@ -55,9 +56,7 @@ describe("moveExportedTempFile", () => { await moveExportedTempFile(tempPath, destinationPath); - await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe( - "recordly-export", - ); + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("recordly-export"); await expect(fs.access(tempPath)).rejects.toThrow(); }); @@ -86,3 +85,230 @@ describe("moveExportedTempFile", () => { await expect(fs.access(tempPath)).rejects.toThrow(); }); }); + +describe("registerExportHandlers native-video-export-start observability", () => { + function captureStartHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-start"); + expect(entry).toBeDefined(); + return entry?.[1] as (event: unknown, options: Record) => Promise; + } + + it("logs the incoming request settings before encoder resolution and on failure", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await handler( + { sender: {} }, + { + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 0, + encodingMode: "quality", + inputMode: "rawvideo", + videoCodec: "hevc", + encoderPreference: "hardware", + }, + ); + + // Missing ffmpeg binary forces the native encoder resolution to fail, so the + // handler still emits its pre-resolution request log before the failure log. + expect(result).toMatchObject({ success: false }); + + const logLines = logSpy.mock.calls.map((args) => String(args[1])); + const startRequest = logLines.find((line) => line.includes("Start request")); + expect(startRequest).toBeDefined(); + expect(startRequest).toMatch(/session=recordly-export-/); + expect(startRequest).toMatch(/codec=hevc/); + expect(startRequest).toMatch(/preference=hardware/); + expect(startRequest).toMatch(/input=rawvideo/); + expect(startRequest).toMatch(/mode=quality/); + expect(startRequest).toMatch(/1920x1080/); + expect(startRequest).toMatch(/fps=30/); + + const errorLines = errorSpy.mock.calls.map((args) => String(args[1])); + const failure = errorLines.find((line) => line.includes("Failed to start")); + expect(failure).toBeDefined(); + expect(failure).toMatch(/session=recordly-export-/); + expect(failure).toMatch(/codec=hevc/); + expect(failure).toMatch(/preference=hardware/); + expect(failure).toMatch(/1920x1080/); + }); + + it("logs effective defaults when codec/preference/input are omitted", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await handler( + { sender: {} }, + { + width: 640, + height: 360, + frameRate: 24, + bitrate: 0, + encodingMode: "fast", + }, + ); + + const logLines = logSpy.mock.calls.map((args) => String(args[1])); + const startRequest = logLines.find((line) => line.includes("Start request")); + expect(startRequest).toBeDefined(); + expect(startRequest).toMatch(/codec=h264/); + expect(startRequest).toMatch(/preference=auto/); + expect(startRequest).toMatch(/input=rawvideo/); + expect(startRequest).toMatch(/640x360/); + }); +}); + +describe("registerExportHandlers prewarm teardown hard-fail", () => { + function captureStartHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-start"); + expect(entry).toBeDefined(); + return entry?.[1] as (event: unknown, options: Record) => Promise; + } + + it("hard-fails HEVC Hardware exports with noCpuFallback when the prewarm teardown fails", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(nativeVideo, "resolveNativeVideoEncoder").mockResolvedValue("hevc_nvenc"); + const cancelPrewarmSpy = vi + .spyOn(nativeVideo, "cancelInFlightCapabilityOnlyPrewarms") + .mockImplementation(() => { + throw new Error("capability-only prewarm child failed to terminate"); + }); + + const result = await handler( + { sender: {} }, + { + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 0, + encodingMode: "quality", + inputMode: "rawvideo", + videoCodec: "hevc", + encoderPreference: "hardware", + }, + ); + + expect(result).toMatchObject({ success: false }); + const error = String((result as { error?: unknown }).error); + expect(error).toContain("noCpuFallback:true"); + expect(error).toContain("capability-only prewarm child failed to terminate"); + expect(cancelPrewarmSpy).toHaveBeenCalledTimes(1); + // The handler returns the strict hard-fail before the FFmpeg process spawn + // can ever run, so the export stops instead of continuing. + expect(error).toContain("before opening its encoder session"); + }); +}); + +describe("registerExportHandlers native-video-export-finish finalization state", () => { + function captureFinishHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-finish"); + expect(entry).toBeDefined(); + return entry?.[1] as ( + event: unknown, + sessionId: string, + options?: unknown, + ) => Promise; + } + + function captureWriteFrameHandler() { + const registrations = vi.mocked(ipcMain.on).mock.calls; + const entry = registrations.find( + ([channel]) => channel === "native-video-export-write-frame-async", + ); + expect(entry).toBeDefined(); + return entry?.[1] as unknown as ( + event: { + sender: { send: (...args: unknown[]) => void; isDestroyed: () => boolean }; + }, + payload: { sessionId: string; requestId: number; frameData: Uint8Array }, + ) => void; + } + + it("rejects frame writes after finish has started", async () => { + registerExportHandlers(); + const finishHandler = captureFinishHandler(); + const writeFrameHandler = captureWriteFrameHandler(); + + let resolveWriteSequence!: () => void; + const writeSequence = new Promise((resolve) => { + resolveWriteSequence = resolve; + }); + + const sender = { send: vi.fn(), isDestroyed: () => false }; + const stdin = { + destroyed: false, + writableEnded: false, + writable: true, + writableLength: 0, + end: vi.fn(), + write: vi.fn(), + destroy: vi.fn(), + on: vi.fn(), + once: vi.fn(), + off: vi.fn(), + }; + const session = { + ffmpegProcess: { + stdin, + stderr: { on: vi.fn() }, + on: vi.fn(), + once: vi.fn(), + kill: vi.fn(), + }, + outputPath: path.join(os.tmpdir(), "recordly-finish-test.mp4"), + inputByteSize: 1920 * 1080 * 4, + inputMode: "rawvideo", + maxQueuedWriteBytes: 32 * 1024 * 1024, + stderrOutput: "", + encoderName: "hevc_nvenc", + processError: null, + stdinError: null, + terminating: false, + writeSequence, + completionPromise: Promise.resolve(), + sender: null, + pendingWriteRequestIds: new Set(), + framePort: null, + framePortReady: false, + nextFrameSequence: 0, + pendingFrameRequests: new Map(), + highestAcceptedFrameRequestId: -1, + } as unknown as NativeVideoExportSession; + + nativeVideoExportSessions.set("finish-test-session", session); + + const finishPromise = finishHandler(undefined, "finish-test-session"); + + writeFrameHandler( + { sender }, + { + sessionId: "finish-test-session", + requestId: 7, + frameData: new Uint8Array(1920 * 1080 * 4), + }, + ); + + expect(sender.send).toHaveBeenCalledWith("native-video-export-write-frame-result", { + sessionId: "finish-test-session", + requestId: 7, + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + + resolveWriteSequence(); + const result = await finishPromise; + expect(result).toMatchObject({ success: true }); + }); +}); diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index c4410a271..57dc4558e 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -5,12 +5,6 @@ import path from "node:path"; import type { Readable, Writable } from "node:stream"; import type { SaveDialogOptions } from "electron"; import { app, BrowserWindow, dialog, ipcMain } from "electron"; -import { - parseCaptionSidecarPayload, - type CaptionSidecarPayload, - withCaptionSidecarMessage, - writeCaptionSidecarsBestEffort, -} from "./exportCaptionSidecars"; import { closeExportStream, isOwnedExportPath, @@ -20,9 +14,13 @@ import { writeToExportStream, } from "../export/exportStream"; import { + attachNativeVideoExportFramePort, + cancelInFlightCapabilityOnlyPrewarms, + closeNativeVideoExportFramePort, enqueueNativeVideoExportFrameWrite, enqueueNativeVideoExportFrameWrites, exportNativeStaticLayoutVideo, + flushNativeVideoExportFramePortPendingRequests, flushNativeVideoExportPendingWriteRequests, getNativeExportCapabilities, getNativeVideoExportMaxQueuedWriteBytes, @@ -38,19 +36,29 @@ import { probeNativeVideoMetadata, removeTemporaryExportFile, resolveNativeVideoEncoder, + sendNativeVideoExportFramePortError, sendNativeVideoExportWriteFrameResult, settleNativeVideoExportWriteFrameRequest, } from "../export/native-video"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { formatLogTs } from "../log"; import { buildNativeH264StreamExportArgs, buildNativeVideoExportArgs, + type ExportEncoderPreference, + type ExportVideoCodec, getNativeVideoInputByteSize, type NativeExportEncodingMode, type NativeVideoExportFinishOptions, } from "../nativeVideoExport"; import { isAllowedLocalReadPath, resolveApprovedLocalMediaPath } from "../project/manager"; import { approveUserPath } from "../utils"; +import { + type CaptionSidecarPayload, + parseCaptionSidecarPayload, + withCaptionSidecarMessage, + writeCaptionSidecarsBestEffort, +} from "./exportCaptionSidecars"; function getPartialExportDestinationPath(destinationPath: string) { const parsed = path.parse(destinationPath); @@ -60,6 +68,42 @@ function getPartialExportDestinationPath(destinationPath: string) { const MAX_IN_MEMORY_EXPORT_BYTES = 0x7fffffff; +/** + * Native video export sessions that entered the finalization ("finish") phase. + * Marked before the finish handler awaits the write sequence so no further + * frame-write requests are accepted while already-accepted writes drain; the + * frame port is closed after the sequence settles. A WeakSet keeps the session + * contract in native-video.ts untouched and lets entries be collected once the + * session is removed from nativeVideoExportSessions. + */ +const finishingNativeVideoExportSessions = new WeakSet(); + +/** + * Structured, timestamped route/settings summary logged at the `native-video- + * export-start` IPC boundary. Captures only high-level request settings (codec, + * encoder preference, input/mode, dimensions, frame rate) so operators can see + * the exact route requested before encoder resolution. Never includes media + * bytes, source paths, or runtime encoder names. + */ +function formatNativeExportRequestSettings(settings: { + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + inputMode: "rawvideo" | "h264-stream"; + encodingMode: NativeExportEncodingMode; + width: number; + height: number; + frameRate: number; +}) { + return ( + `codec=${settings.videoCodec} ` + + `preference=${settings.encoderPreference} ` + + `input=${settings.inputMode} ` + + `mode=${settings.encodingMode} ` + + `${settings.width}x${settings.height} ` + + `fps=${settings.frameRate}` + ); +} + function getInMemoryExportTooLargeMessage(byteLength: number) { if (byteLength <= MAX_IN_MEMORY_EXPORT_BYTES) { return null; @@ -75,12 +119,7 @@ export async function moveExportedTempFile(tempPath: string, destinationPath: st return; } catch (error) { const code = (error as NodeJS.ErrnoException).code; - if ( - code !== "EXDEV" && - code !== "EPERM" && - code !== "ENOTEMPTY" && - code !== "EEXIST" - ) { + if (code !== "EXDEV" && code !== "EPERM" && code !== "ENOTEMPTY" && code !== "EEXIST") { throw error; } // Cross-device or Windows permission quirks — fall back to copy + unlink so @@ -113,9 +152,7 @@ export async function moveExportedTempFile(tempPath: string, destinationPath: st await fs.rename(partialDestinationPath, destinationPath); } catch (replaceError) { if (movedExistingDestination) { - await fs - .rename(backupDestinationPath, destinationPath) - .catch(() => undefined); + await fs.rename(backupDestinationPath, destinationPath).catch(() => undefined); } throw replaceError; } @@ -194,6 +231,35 @@ async function sanitizeNativeStaticLayoutExportOptions( inputPath: await resolveAllowedReadableFilePath(options.inputPath, "Native input"), }; const mutableOptions = sanitized as unknown as Record; + if (sanitized.overlayLayers) { + for (const layer of sanitized.overlayLayers) { + if (typeof layer.path !== "string" || layer.path.trim().length === 0) { + throw new Error(`Native overlay layer ${layer.id} requires a file path`); + } + layer.path = await resolveAllowedReadableFilePath( + layer.path, + `Native overlay ${layer.id}`, + { + mediaOnly: false, + }, + ); + } + } + + if (sanitized.tiledOverlayLayers) { + for (const layer of sanitized.tiledOverlayLayers) { + if (typeof layer.payloadPath !== "string" || layer.payloadPath.trim().length === 0) { + throw new Error(`Native tiled overlay layer ${layer.id} requires a payload path`); + } + layer.payloadPath = await resolveAllowedReadableFilePath( + layer.payloadPath, + `Native tiled overlay payload ${layer.id}`, + { + mediaOnly: false, + }, + ); + } + } for (const [field, label] of [ ["backgroundImagePath", "Native background image"], @@ -264,22 +330,57 @@ export function registerExportHandlers() { bitrate: number; encodingMode: NativeExportEncodingMode; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; }, ) => { + const inputMode = options.inputMode ?? "rawvideo"; + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + let sessionId = ""; + // Build the request summary once, before the try, so the same string is + // reused for the start and failure logs (formatNativeExportRequestSettings + // is a pure string formatter over already-normalized locals; it has no + // side effects that must stay inside the try). + const requestSettings = formatNativeExportRequestSettings({ + videoCodec, + encoderPreference, + inputMode, + encodingMode: options.encodingMode, + width: options.width, + height: options.height, + frameRate: options.frameRate, + }); try { if (options.width % 2 !== 0 || options.height % 2 !== 0) { throw new Error("Native export requires even output dimensions"); } const ffmpegPath = getFfmpegBinaryPath(); - const inputMode = options.inputMode ?? "rawvideo"; - const sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const outputPath = path.join(app.getPath("temp"), `${sessionId}.mp4`); + // Recommend the real route once an encoder has been resolved; the raw + // request line below is emitted before encoder resolution so a mismatch + // between the requested prewarm (e.g. hevc/hardware) and the encoder the + // export actually started with is immediately visible in one place. + console.log( + formatLogTs(), + `[native-export] Start request session=${sessionId} ${requestSettings}`, + ); + let encoderName: string; let ffmpegArgs: string[]; - if (inputMode === "h264-stream") { + // Preserve the existing zero-copy H.264 stream-copy path for the default + // H.264 + Auto selection. Explicit hardware/CPU H.264, and any HEVC + // selection, go through the native raw-frame route. + const useH264StreamCopy = + inputMode === "h264-stream" && + videoCodec === "h264" && + encoderPreference === "auto"; + + if (useH264StreamCopy) { // Pre-encoded H.264 Annex B from browser VideoEncoder — just stream-copy into MP4 encoderName = "h264-stream-copy"; ffmpegArgs = buildNativeH264StreamExportArgs({ @@ -287,10 +388,40 @@ export function registerExportHandlers() { outputPath, }); } else { - encoderName = await resolveNativeVideoEncoder(ffmpegPath, options.encodingMode); + encoderName = await resolveNativeVideoEncoder( + ffmpegPath, + options.encodingMode, + videoCodec, + encoderPreference, + ); ffmpegArgs = buildNativeVideoExportArgs(encoderName, options, outputPath); } + // A capability-only prewarm child may still hold a brief NVENC probe + // session; cancel it before this real export opens its real encoder + // session so they never contend for the GPU/hardware encoder. Await + // the teardown so no in-flight capability probe is still running when + // the FFmpeg process spawns below. + try { + await cancelInFlightCapabilityOnlyPrewarms(); + } catch (error) { + // Strict HEVC Hardware forbids every fallback: a prewarm that cannot + // be torn down must stop the export rather than let a stale NVENC + // probe contend with the real encoder session. + if (videoCodec === "hevc" && encoderPreference === "hardware") { + throw new Error( + `HEVC Hardware export requires tearing down the in-flight capability-only prewarm before opening its encoder session; prewarm teardown failed (noCpuFallback:true): ${error instanceof Error ? error.message : String(error)}`, + ); + } + // Non-strict routes keep the prewarm best-effort: a stale capability + // probe must never block a compatible export. + console.warn( + formatLogTs(), + `[native-export] Capability-only prewarm teardown failed session=${sessionId || "unknown"} ${requestSettings}:`, + error, + ); + } + const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, { stdio: ["pipe", "ignore", "pipe"], }) as ChildProcessByStdio; @@ -317,6 +448,11 @@ export function registerExportHandlers() { writeSequence: Promise.resolve(), sender: event.sender, pendingWriteRequestIds: new Set(), + framePort: null, + framePortReady: false, + nextFrameSequence: 0, + pendingFrameRequests: new Map(), + highestAcceptedFrameRequestId: -1, completionPromise: new Promise((resolve, reject) => { ffmpegProcess.once("error", (error) => { const processError = @@ -340,6 +476,11 @@ export function registerExportHandlers() { } session.stdinError = stdinError; + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + getNativeVideoExportSessionError(session, stdinError.message), + ); }); ffmpegProcess.once("close", (code, signal) => { if (session.terminating) { @@ -363,7 +504,17 @@ export function registerExportHandlers() { }); }), }; - void session.completionPromise.catch(() => undefined); + void session.completionPromise.catch((error: unknown) => { + const processError = error instanceof Error ? error : new Error(String(error)); + if (!session.processError) { + session.processError = processError; + } + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + getNativeVideoExportSessionError(session, processError.message), + ); + }); ffmpegProcess.stderr.on("data", (chunk: Buffer) => { session.stderrOutput += chunk.toString(); @@ -372,7 +523,8 @@ export function registerExportHandlers() { nativeVideoExportSessions.set(sessionId, session); console.log( - `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? "hardware" : "software"} session ${sessionId} with ${encoderName}`, + formatLogTs(), + `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? "hardware" : "software"} session=${sessionId} encoder=${encoderName} route=${useH264StreamCopy ? "h264-stream-copy" : "native-raw"} ${requestSettings}`, ); return { @@ -382,7 +534,8 @@ export function registerExportHandlers() { }; } catch (error) { console.error( - "[native-export] Failed to start native video export session:", + formatLogTs(), + `[native-export] Failed to start native video export session session=${sessionId || "unknown"} ${requestSettings}:`, error, ); return { @@ -412,7 +565,7 @@ export function registerExportHandlers() { metadata, }; } catch (error) { - console.warn("[probe-native-video-metadata] Failed:", error); + console.warn(formatLogTs(), "[probe-native-video-metadata] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), @@ -427,7 +580,7 @@ export function registerExportHandlers() { capabilities: await getNativeExportCapabilities(), }; } catch (error) { - console.warn("[native-export-capabilities] Failed:", error); + console.warn(formatLogTs(), "[native-export-capabilities] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), @@ -460,18 +613,19 @@ export function registerExportHandlers() { return { success: true, tempPath: result.outputPath, + videoCodec: result.videoCodec, + encoderPreference: result.encoderPreference, + route: result.route, encoderName: primaryBackend === "nvidia-cuda-compositor" ? "nvidia-cuda-compositor" : primaryBackend === "windows-d3d11-compositor" ? "windows-d3d11-compositor" - : result.metrics.chunkCount > 1 - ? "chunked-h264-nvenc" - : "static-layout-h264-nvenc", + : result.encoderName, metrics: result.metrics, }; } catch (error) { - console.warn("[native-static-layout-export] Failed:", error); + console.warn(formatLogTs(), "[native-static-layout-export] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), @@ -496,6 +650,36 @@ export function registerExportHandlers() { return { success: true }; }); + ipcMain.on("native-video-export-frame-channel", (event, payload: { sessionId?: string }) => { + const port = event.ports[0]; + const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : ""; + if (!port) { + return; + } + + const session = nativeVideoExportSessions.get(sessionId); + if (!session) { + sendNativeVideoExportFramePortError(port, sessionId, "Invalid native export session", { + fallbackAvailable: true, + }); + port.close(); + return; + } + + if (finishingNativeVideoExportSessions.has(session)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session is finishing; no more frames are accepted", + { fallbackAvailable: false }, + ); + port.close(); + return; + } + + attachNativeVideoExportFramePort(sessionId, session, port, event.sender); + }); + ipcMain.on( "native-video-export-write-frames-async", ( @@ -539,6 +723,14 @@ export function registerExportHandlers() { return; } + if (finishingNativeVideoExportSessions.has(session)) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + return; + } + if ( session.inputMode !== "h264-stream" && frameDataList.some((frameData) => frameData.byteLength !== session.inputByteSize) @@ -607,6 +799,14 @@ export function registerExportHandlers() { return; } + if (finishingNativeVideoExportSessions.has(session)) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + return; + } + if ( session.inputMode !== "h264-stream" && frameData.byteLength !== session.inputByteSize @@ -645,6 +845,10 @@ export function registerExportHandlers() { return { success: false, error: "Invalid native export session" }; } + // Enter the finalization phase before settling the write sequence so + // subsequent frame-write requests are rejected while already-accepted + // writes drain; the frame port is closed after the sequence settles. + finishingNativeVideoExportSessions.add(session); try { await session.writeSequence; if ( @@ -659,6 +863,11 @@ export function registerExportHandlers() { session.outputPath, options ?? {}, ); + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export session finished", + ); nativeVideoExportSessions.delete(sessionId); // Register the finalized path so only app-produced paths can flow back // through finalize-exported-video / discard-exported-temp. @@ -680,6 +889,7 @@ export function registerExportHandlers() { metrics: finalized.metrics, }; } catch (error) { + closeNativeVideoExportFramePort(sessionId, session, String(error)); flushNativeVideoExportPendingWriteRequests(sessionId, session, String(error)); nativeVideoExportSessions.delete(sessionId); await removeTemporaryExportFile(session.outputPath); @@ -805,6 +1015,11 @@ export function registerExportHandlers() { session.terminating = true; nativeVideoExportSessions.delete(sessionId); + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export session was cancelled", + ); flushNativeVideoExportPendingWriteRequests( sessionId, session, diff --git a/electron/ipc/register/project.ts b/electron/ipc/register/project.ts index f1fa43e26..1e0d604f1 100644 --- a/electron/ipc/register/project.ts +++ b/electron/ipc/register/project.ts @@ -22,7 +22,7 @@ import { rememberRecentProject, replaceApprovedSessionLocalReadPaths, rememberApprovedLocalReadPath, - resolveApprovedLocalMediaPath, + resolveLocalMediaUrlPath, saveProjectThumbnail, saveRecentProjectPaths, } from "../project/manager"; @@ -687,13 +687,35 @@ export function registerProjectHandlers() { if (!baseUrl || !filePath) { return { success: false as const }; } - const resolved = await resolveApprovedLocalMediaPath(filePath); - if (!resolved) { - const normalized = path.resolve(filePath); - console.warn(`[get-local-media-url] Blocked disallowed path: ${normalized}`); - return { success: false as const }; + // Normalize file:/// URLs (and bare paths) before the path policy check so + // persisted project sources round-trip through the local media server. + const normalizedFilePath = normalizeVideoSourcePath(filePath) ?? filePath; + const resolution = await resolveLocalMediaUrlPath(normalizedFilePath); + if (resolution.status === 'approved') { + return { success: true as const, url: buildMediaUrl(baseUrl, resolution.path) }; + } + if (resolution.status === 'pending') { + // Expected speculative audio sidecar candidate (e.g. .m4a/.webm variants + // requested before the Windows mux rename completes). The media server + // re-validates realpath + symlink containment when the file appears, so + // this is not a security grant for an arbitrary missing path. + console.info( + `[get-local-media-url] Pending approved media path (may appear after mux): ${resolution.path}`, + ); + return { + success: true as const, + url: buildMediaUrl(baseUrl, resolution.path), + pending: true as const, + }; } - return { success: true as const, url: buildMediaUrl(baseUrl, resolved) }; + // True policy violation / unsupported candidate: log distinctly from the + // expected pending sidecar case above. + console.warn( + `[get-local-media-url] Rejected media path (${resolution.reason}): ${path.resolve( + normalizedFilePath, + )}`, + ); + return { success: false as const }; }); } diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..d5d851b26 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -21,6 +21,19 @@ export let currentRecordingSession: RecordingSessionData | null = null; // ── Security: approved read paths ───────────────────────────────────────────── export const approvedLocalReadPaths = new Set(); +// Windows paths are case-insensitive and may surface as extended-length +// (`\\?\`) paths from realpath. Fold both forms so policy comparisons do not +// reject a path just because its drive letter or directory casing differs from +// the approved root. Non-Windows paths are compared verbatim. +export function foldPathComparisonKey(filePath: string) { + if (process.platform !== "win32") { + return filePath; + } + + const withoutExtendedPrefix = filePath.replace(/^\\\\\?\\/, "").replace(/^\\.\\/, ""); + return withoutExtendedPrefix.toLowerCase(); +} + // ── Native macOS capture ────────────────────────────────────────────────────── export let nativeScreenRecordingActive = false; export let nativeCaptureProcess: ChildProcessWithoutNullStreams | null = null; @@ -96,6 +109,8 @@ export let cachedNativeMacWindowSourcesAtMs = 0; export let cachedNativeVideoEncoder: { ffmpegPath: string; encodingMode: string; + codec: string; + preference: string; encoderName: string; } | null = null; @@ -283,7 +298,13 @@ export function setCachedNativeMacWindowSourcesAtMs(v: number) { } export function setCachedNativeVideoEncoder( - v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null, + v: { + ffmpegPath: string; + encodingMode: string; + codec: string; + preference: string; + encoderName: string; + } | null, ) { cachedNativeVideoEncoder = v; } diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 3f2efb065..c060d07d7 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -8,6 +8,7 @@ import { AUTO_RECORDING_PREFIX, RECORDINGS_SETTINGS_FILE } from "./constants"; import { approvedLocalReadPaths, customRecordingsDir, + foldPathComparisonKey, recordingsDirLoaded, setCustomRecordingsDir, setRecordingsDirLoaded, @@ -124,9 +125,8 @@ export function getMacPrivacySettingsUrl(pane: "screen" | "accessibility" | "mic export function approveUserPath(filePath: string | null | undefined): void { if (!filePath) return; try { - approvedLocalReadPaths.add(path.resolve(filePath)); + approvedLocalReadPaths.add(foldPathComparisonKey(path.resolve(filePath))); } catch { // Ignore invalid paths; later reads will surface the underlying error. } } - diff --git a/electron/mediaServer.test.ts b/electron/mediaServer.test.ts index cf6cd01ba..406046149 100644 --- a/electron/mediaServer.test.ts +++ b/electron/mediaServer.test.ts @@ -56,18 +56,168 @@ describe("media server path policy", () => { const { isAllowedMediaPath } = await import("./mediaServer"); const { rememberApprovedLocalReadPath } = await import("./ipc/project/manager"); - expect(isAllowedMediaPath(videoPath)).toBe(false); + await expect(isAllowedMediaPath(videoPath)).resolves.toBe(false); await rememberApprovedLocalReadPath(videoPath); - expect(isAllowedMediaPath(videoPath)).toBe(true); + await expect(isAllowedMediaPath(videoPath)).resolves.toBe(true); }); it("rejects missing media files outside the allowed directories", async () => { const missingPath = path.join(tempRoot, "Downloads", "missing.mp4"); const { isAllowedMediaPath } = await import("./mediaServer"); - expect(isAllowedMediaPath(missingPath)).toBe(false); + await expect(isAllowedMediaPath(missingPath)).resolves.toBe(false); + }); +}); + +describe("pending media URL serve-time authorization (real HTTP)", () => { + let tempRoot: string; + let appDataPath: string; + let userDataPath: string; + let tempPath: string; + let appPath: string; + + beforeEach(async () => { + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-media-server-http-")); + appDataPath = path.join(tempRoot, "AppData"); + userDataPath = path.join(tempRoot, "UserData"); + tempPath = path.join(tempRoot, "Temp"); + appPath = path.join(tempRoot, "App"); + + await Promise.all( + [appDataPath, userDataPath, tempPath, appPath].map((dirPath) => + fs.mkdir(dirPath, { recursive: true }), + ), + ); + + vi.resetModules(); + vi.doMock("electron", () => ({ + app: { + isPackaged: false, + getAppPath: () => appPath, + getPath: (name: string) => { + if (name === "appData") return appDataPath; + if (name === "userData") return userDataPath; + if (name === "temp") return tempPath; + return tempRoot; + }, + setPath: () => undefined, + }, + })); + }); + + afterEach(async () => { + vi.resetModules(); + vi.doUnmock("electron"); + if (tempRoot) { + await fs.rm(tempRoot, { recursive: true, force: true }); + } + }); + + async function mintPendingUrl(candidatePath: string) { + const { resolveLocalMediaUrlPath } = await import("./ipc/project/manager"); + const { ensureMediaServer, buildMediaUrl } = await import("./mediaServer"); + + const resolution = await resolveLocalMediaUrlPath(candidatePath); + expect(resolution.status).toBe("pending"); + const baseUrl = await ensureMediaServer(); + return buildMediaUrl(baseUrl, resolution.path); + } + + it("serves a pending in-root sidecar whose file appears during the request", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const m4aPath = path.join(recordingsPath, "recording-2026-08-03.system.m4a"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const pendingUrl = await mintPendingUrl(m4aPath); + + // The renderer fetch races ahead of the mux rename: the file is still + // missing when the request arrives and appears while it is in flight. + const fetchPromise = fetch(pendingUrl); + await new Promise((resolve) => setTimeout(resolve, 150)); + await fs.writeFile(m4aPath, "test-audio"); + + const response = await fetchPromise; + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("audio/mp4"); + expect(await response.text()).toBe("test-audio"); + }); + + it("serves a pending sidecar under a custom recordings root once it appears", async () => { + const customRecordingsPath = path.join(tempRoot, "Custom Recordings"); + const webmPath = path.join(customRecordingsPath, "recording-2026-08-03.mic.webm"); + await fs.mkdir(customRecordingsPath, { recursive: true }); + await fs.writeFile( + path.join(userDataPath, "recordings-settings.json"), + JSON.stringify({ recordingsDir: customRecordingsPath }), + "utf-8", + ); + + const pendingUrl = await mintPendingUrl(webmPath); + + const fetchPromise = fetch(pendingUrl); + await new Promise((resolve) => setTimeout(resolve, 150)); + await fs.writeFile(webmPath, "test-audio"); + + const response = await fetchPromise; + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("video/webm"); + }); + + it("returns 404 for a pending-approved in-root path that never appears", async () => { + const recordingsPath = path.join(userDataPath, "recordings"); + const wavPath = path.join(recordingsPath, "recording-2026-08-03.system.wav"); + await fs.mkdir(recordingsPath, { recursive: true }); + + const pendingUrl = await mintPendingUrl(wavPath); + const { setPendingMediaAppearTimeoutMsForTests } = await import("./mediaServer"); + setPendingMediaAppearTimeoutMsForTests(150); + + const response = await fetch(pendingUrl); + expect(response.status).toBe(404); + }); + + it("rejects an outside path over HTTP even when the file exists", async () => { + const downloadsPath = path.join(tempRoot, "Downloads"); + const videoPath = path.join(downloadsPath, "personal-video.mp4"); + await fs.mkdir(downloadsPath, { recursive: true }); + await fs.writeFile(videoPath, "outside-bytes"); + + const { ensureMediaServer, buildMediaUrl } = await import("./mediaServer"); + const baseUrl = await ensureMediaServer(); + const response = await fetch(buildMediaUrl(baseUrl, videoPath)); + + expect(response.status).toBe(403); + }); + + it("rejects a pending in-root symlink whose target appears outside the roots", async () => { + const outsideTarget = path.join(tempRoot, "outside-secret.m4a"); + const symlinkInsideUserData = path.join( + userDataPath, + "recordings", + "recording-2026-08-03.system.m4a", + ); + await fs.mkdir(path.dirname(symlinkInsideUserData), { recursive: true }); + + try { + // The link exists but its target does not yet, so the URL mint sees a + // missing path and grants a pending URL for the lexical in-root path. + await fs.symlink(outsideTarget, symlinkInsideUserData); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EPERM") { + return; + } + throw error; + } + + const pendingUrl = await mintPendingUrl(symlinkInsideUserData); + + // The target appears after the pending grant; the serve-time realpath now + // lands outside every allowed root, so the escape must stay blocked. + await fs.writeFile(outsideTarget, "secret-bytes"); + const response = await fetch(pendingUrl); + expect(response.status).toBe(403); }); }); diff --git a/electron/mediaServer.ts b/electron/mediaServer.ts index c979a84e8..dc421b767 100644 --- a/electron/mediaServer.ts +++ b/electron/mediaServer.ts @@ -2,12 +2,31 @@ import { createReadStream } from "node:fs"; import fs from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import path from "node:path"; -import { approvedLocalReadPaths } from "./ipc/state"; +import { approvedLocalReadPaths, foldPathComparisonKey } from "./ipc/state"; import { getMediaContentType } from "./mediaTypes"; let mediaServerBaseUrl: string | null = null; let mediaServerStartPromise: Promise | null = null; +// A pending URL is minted for a supported in-root sidecar candidate before the +// Windows mux rename completes, so the media server may receive a request for a +// file that does not exist yet. When the lexical path was granted a pending URL +// (inside an allowed root AND present in the approved set), wait a short bounded +// time for the file to appear before falling back to a rejection. The wait only +// applies to already-approved in-root paths; after the file appears the standard +// realpath + containment authorization still runs, so symlink escapes and +// outside roots are rejected exactly as before. +export const PENDING_MEDIA_APPEAR_POLL_INTERVAL_MS = 100; + +export const PENDING_MEDIA_APPEAR_TIMEOUT_MS = 3000; + +let pendingMediaAppearTimeoutMs = PENDING_MEDIA_APPEAR_TIMEOUT_MS; + +// Test-only override so regression tests do not stall for the full window. +export function setPendingMediaAppearTimeoutMsForTests(timeoutMs: number) { + pendingMediaAppearTimeoutMs = timeoutMs; +} + export function resolveHttpByteRange( rangeHeader: string, fileSize: number, @@ -58,8 +77,45 @@ async function resolveRealPath(filePath: string): Promise { } } -export function isAllowedMediaPath(realPath: string): boolean { - return approvedLocalReadPaths.has(realPath); +async function isPendingApprovedMediaPath(lexicalPath: string): Promise { + const { getAllowedLocalReadRootsSync, isPathInsideDirectory } = await import( + "./ipc/project/manager" + ); + const rootContained = getAllowedLocalReadRootsSync().some((root) => + isPathInsideDirectory(lexicalPath, root), + ); + return rootContained && approvedLocalReadPaths.has(foldPathComparisonKey(lexicalPath)); +} + +async function waitForPendingMediaFile(lexicalPath: string): Promise { + const deadline = Date.now() + pendingMediaAppearTimeoutMs; + while (Date.now() < deadline) { + const resolvedPath = await resolveRealPath(lexicalPath); + if (resolvedPath) { + return resolvedPath; + } + await new Promise((resolve) => setTimeout(resolve, PENDING_MEDIA_APPEAR_POLL_INTERVAL_MS)); + } + return null; +} + +export async function isAllowedMediaPath(realPath: string): Promise { + const resolvedRealPath = path.resolve(realPath); + // Accept any real file contained inside an allowed root (this covers pending + // sidecar URLs granted for a lexical in-root path that only appeared after + // the mux rename) OR an explicitly approved path. realpath has already been + // resolved by the caller, so a symlink/reparse-point escape from inside a + // root lands outside the roots here and is rejected. The manager module is + // imported lazily so importing this module does not pull the app-paths chain + // (pure helpers like resolveHttpByteRange stay importable in tests without an + // Electron mock). + const { getAllowedLocalReadRootsSync, isPathInsideDirectory } = await import( + "./ipc/project/manager" + ); + const rootContained = getAllowedLocalReadRootsSync().some((root) => + isPathInsideDirectory(resolvedRealPath, root), + ); + return rootContained || approvedLocalReadPaths.has(foldPathComparisonKey(resolvedRealPath)); } async function handleMediaRequest( @@ -82,8 +138,26 @@ async function handleMediaRequest( return; } - const resolvedPath = await resolveRealPath(rawPath); - if (!resolvedPath || !isAllowedMediaPath(resolvedPath)) { + const lexicalPath = path.resolve(rawPath); + let resolvedPath = await resolveRealPath(lexicalPath); + let pendingApproved = false; + if (!resolvedPath) { + pendingApproved = await isPendingApprovedMediaPath(lexicalPath); + if (pendingApproved) { + // The file was still inside the mux rename window when the request + // arrived. Wait briefly for it to appear; the authorization below + // re-runs against the real path once it does. + resolvedPath = await waitForPendingMediaFile(lexicalPath); + } + } + if (!resolvedPath || !(await isAllowedMediaPath(resolvedPath))) { + if (pendingApproved && !resolvedPath) { + // Pending-approved path that never appeared: not found (kept distinct + // from the forbidden response used for unapproved paths). + response.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }); + response.end("Not Found"); + return; + } console.warn(`[media-server] Blocked access to unapproved path: ${rawPath}`); response.writeHead(403, { "Content-Type": "text/plain; charset=utf-8" }); response.end("Forbidden"); diff --git a/electron/mediaTypes.ts b/electron/mediaTypes.ts index 5c4307b63..7c3e09648 100644 --- a/electron/mediaTypes.ts +++ b/electron/mediaTypes.ts @@ -9,6 +9,7 @@ export const MEDIA_CONTENT_TYPES: Record = { ".wav": "audio/wav", ".mp3": "audio/mpeg", ".ogg": "audio/ogg", + ".m4a": "audio/mp4", ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", diff --git a/electron/native/bin/win32-x64/cursor-monitor.exe b/electron/native/bin/win32-x64/cursor-monitor.exe index ef71282d9..db889d7f9 100644 Binary files a/electron/native/bin/win32-x64/cursor-monitor.exe and b/electron/native/bin/win32-x64/cursor-monitor.exe differ diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.json index c5080b665..af8a653b6 100644 --- a/electron/native/bin/win32-x64/helpers-manifest.json +++ b/electron/native/bin/win32-x64/helpers-manifest.json @@ -5,31 +5,31 @@ "helpers": { "wgc-capture": { "binaryName": "wgc-capture.exe", - "binarySha256": "298b41f371c3881046061048b466e12ed70dd93fa761bade2fd57d1ccddf3cb9", + "binarySha256": "6e79b3eebd3f63e48a366047ae4ec97ac0a5bb85e1f24b4629715f92220f774f", "sourceDir": "electron/native/wgc-capture", - "sourceFingerprint": "6ee457080c27dc939ff4b61965f86b6d73995e40200440a1dc44865f9708d39f", - "updatedAt": "2026-05-24T19:49:15.077Z" + "sourceFingerprint": "c6dac250c9d16f7aa881998353441b4ae3fb59731b802e3b8491a136e79726b0", + "updatedAt": "2026-08-03T14:06:52.964Z" }, "cursor-monitor": { "binaryName": "cursor-monitor.exe", - "binarySha256": "6ae6d91103b6e891a851e8ea5791e1c1f9aaab700134c18bc4c46cfffd7fdd12", + "binarySha256": "0e06a3c5847b3cb39faceedd3b0634d56b8de285fb3804bed24b10a6a99d3128", "sourceDir": "electron/native/cursor-monitor", - "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", - "updatedAt": "2026-05-07T15:22:18.173Z" + "sourceFingerprint": "45bb72e4039e061a354af87317d7c42ec3d2118369b3f4379046ff497b701e72", + "updatedAt": "2026-08-03T14:06:57.157Z" }, "recordly-gpu-export": { "binaryName": "recordly-gpu-export.exe", - "binarySha256": "4cb3a293fd36f718af55906820d9b3fd78babc855888c2e248f0b918ec1aff3c", + "binarySha256": "220b32ece91edc68722709983aa366feb473d8304cdd040f632aa5416e5d9019", "sourceDir": "electron/native/gpu-export-probe", - "sourceFingerprint": "743b386a5f1bbcc99cec5465c3de228d2b045061dead31dfcbf25cf6a1e61de5", - "updatedAt": "2026-05-07T20:13:48.585Z" + "sourceFingerprint": "37a5842eba63cdeccfddd02cc278a98207248fb0aa6b56153fb85ed152c908c1", + "updatedAt": "2026-08-03T14:06:54.934Z" }, "recordly-nvidia-cuda-compositor": { "binaryName": "recordly-nvidia-cuda-compositor.exe", - "binarySha256": "250a3f8cac7c6ea38a873434d23d4b2be7d6555e42cc0b405aa26f774169159c", + "binarySha256": "a3caa267f02f108884fe3a6ee0986d1a46c40bfe41d6c3b2baad1de3afc30b4d", "sourceDir": "electron/native/nvidia-cuda-compositor", - "sourceFingerprint": "de1219228ce326e96d1f4815a3763b10d5f235cc1286bc6542c99707a85d5947", - "updatedAt": "2026-05-27T11:29:32.957Z" + "sourceFingerprint": "5b56fc43acae2e5aae9e1245738e74162336a914edcda0227271d46b105c75b4", + "updatedAt": "2026-08-04T14:24:46.417Z" } } } diff --git a/electron/native/bin/win32-x64/recordly-gpu-export.exe b/electron/native/bin/win32-x64/recordly-gpu-export.exe index 3d0f57f58..eb5d8a54d 100644 Binary files a/electron/native/bin/win32-x64/recordly-gpu-export.exe and b/electron/native/bin/win32-x64/recordly-gpu-export.exe differ diff --git a/electron/native/bin/win32-x64/recordly-nvidia-cuda-compositor.exe b/electron/native/bin/win32-x64/recordly-nvidia-cuda-compositor.exe index c23c14f90..e9a7568e3 100644 Binary files a/electron/native/bin/win32-x64/recordly-nvidia-cuda-compositor.exe and b/electron/native/bin/win32-x64/recordly-nvidia-cuda-compositor.exe differ diff --git a/electron/native/bin/win32-x64/wgc-capture.exe b/electron/native/bin/win32-x64/wgc-capture.exe index 8180edd00..690c49577 100644 Binary files a/electron/native/bin/win32-x64/wgc-capture.exe and b/electron/native/bin/win32-x64/wgc-capture.exe differ diff --git a/electron/native/nvidia-cuda-compositor/CMakeLists.txt b/electron/native/nvidia-cuda-compositor/CMakeLists.txt index e79d9630c..fa2cda64f 100644 --- a/electron/native/nvidia-cuda-compositor/CMakeLists.txt +++ b/electron/native/nvidia-cuda-compositor/CMakeLists.txt @@ -1,5 +1,14 @@ cmake_minimum_required(VERSION 3.24) +# Target the common NVENC-capable architectures with native SASS plus PTX +# fallback. The legacy default (compute_75 only) cannot run on Blackwell +# (sm_120) GPUs. Override with -DCMAKE_CUDA_ARCHITECTURES for exotic targets. +# Must be set before project() so CUDA language init does not pin it to the +# toolkit default. +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES "75;86;89;90;120") +endif() + project(recordly_nvidia_cuda_compositor LANGUAGES CXX CUDA) set(CMAKE_CXX_STANDARD 17) @@ -7,6 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CUDA_STANDARD 17) set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(RECORDLY_NVENC_HEADERS_DIR + "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/nv-codec-headers/include/ffnvcodec" + CACHE PATH + "Path to the nv-codec-headers include dir (nvEncodeAPI.h 12+/13+ for Blackwell)" +) set(RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/video-sdk-samples" CACHE PATH @@ -27,8 +41,12 @@ add_executable(recordly-nvidia-cuda-compositor ) target_include_directories(recordly-nvidia-cuda-compositor PRIVATE + "${RECORDLY_NVENC_HEADERS_DIR}" "${NVIDIA_SAMPLES_DIR}" "${NVCODEC_DIR}" + "${NVCODEC_DIR}/NvEncoder" + "${NVCODEC_DIR}/NvDecoder" + "${NVCODEC_DIR}/../Utils" ) target_compile_definitions(recordly-nvidia-cuda-compositor PRIVATE diff --git a/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs b/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs new file mode 100644 index 000000000..fa83b86e5 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs @@ -0,0 +1,191 @@ +// Cursor telemetry contract shared by the NVIDIA CUDA compositor wrapper and +// its callers. +// +// The canonical `--cursor-json` payload is a JSON object with a `samples` +// array; each sample is +// { timeMs, cx, cy, cursorType?, cursorTypeIndex?, interactionType?, +// bounceScale?, visible? } +// The exact generated telemetry row format (TSV inside the pipeline, CSV from +// the Windows GPU compositor telemetry prep) is +// timeMscxcycursorTypeIndexbounceScalevisible +// (6 whitespace- or comma-separated fields). Parsing must NEVER hand raw rows +// to JSON.parse; a mis-wired caller that points --cursor-json at a CSV/TSV +// samples file must be accepted or rejected with an actionable error, not a +// SyntaxError. + +import { writeFileSync } from "node:fs"; + +export const CURSOR_SAMPLE_TYPES = [ + "arrow", + "text", + "pointer", + "crosshair", + "open-hand", + "closed-hand", + "resize-ew", + "resize-ns", + "not-allowed", +]; +export const cursorTypeIndexes = new Map(CURSOR_SAMPLE_TYPES.map((type, index) => [type, index])); +export const CURSOR_SAMPLE_MAX_TYPE_INDEX = CURSOR_SAMPLE_TYPES.length - 1; + +const CLICK_TYPES = new Set(["click", "double-click", "right-click", "middle-click"]); +const DEFAULT_BOUNCE_DURATION_MS = 180; + +function isCursorClickType(interactionType) { + return CLICK_TYPES.has(interactionType); +} + +export function cursorBounceScale(interactionType, ageMs, durationMs = DEFAULT_BOUNCE_DURATION_MS) { + if (!isCursorClickType(interactionType)) { + return 1; + } + if (ageMs < 0 || ageMs > durationMs) { + return 1; + } + const progress = 1 - ageMs / durationMs; + return Math.max(0.72, 1 - Math.sin(progress * Math.PI) * 0.08); +} + +export function latestClickSample(samples, sampleIndex) { + for (let index = sampleIndex; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample && isCursorClickType(sample.interactionType)) { + return sample; + } + } + return null; +} + +export function isValidCursorSample(sample) { + return ( + sample !== null && + typeof sample === "object" && + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.cx) && + Number.isFinite(sample.cy) + ); +} + +function normalizeCursorTypeIndex(value) { + if (typeof value === "string" && cursorTypeIndexes.has(value)) { + return cursorTypeIndexes.get(value); + } + if (Number.isFinite(value)) { + return Math.max(0, Math.min(CURSOR_SAMPLE_MAX_TYPE_INDEX, Math.round(value))); + } + return 0; +} + +// Formats samples into the exact TSV sidecar consumed by the native compositor +// (`--cursor-samples`): timeMs, cx, cy, cursorTypeIndex, bounceScale, visible, +// tab-separated, one row per line, in input order. Preserves renderer-resolved +// click bounce (bounceScale) and click type when present. +export function formatCursorSamplesTsv(samples) { + const rows = []; + for (let index = 0; index < samples.length; index += 1) { + const sample = samples[index]; + if (!isValidCursorSample(sample)) { + continue; + } + const clickSample = latestClickSample(samples, index); + const bounceScale = Number.isFinite(sample.bounceScale) + ? sample.bounceScale + : clickSample + ? cursorBounceScale(clickSample.interactionType, sample.timeMs - clickSample.timeMs) + : 1; + rows.push( + [ + sample.timeMs, + sample.cx, + sample.cy, + normalizeCursorTypeIndex(sample.cursorTypeIndex), + Number(bounceScale.toFixed(4)), + sample.visible === false ? 0 : 1, + ].join("\t"), + ); + } + return rows.join("\n"); +} + +export function writeCursorSamplesFile(samples, outputPath) { + const lines = formatCursorSamplesTsv(samples); + writeFileSync(outputPath, lines ? `${lines}\n` : ""); + return samples.length; +} + +function clampUnit(value) { + return Math.min(1, Math.max(0, value)); +} + +function parseCursorTelemetryRows(text, sourcePath) { + const samples = []; + const lines = text.split(/\r?\n/); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex += 1) { + const line = lines[lineIndex].trim(); + if (!line) { + continue; + } + let fields = line.split("\t"); + if (fields.length === 1) { + fields = line.split(","); + } + if (fields.length < 3 || fields.length > 6) { + throw new Error( + `${sourcePath} line ${lineIndex + 1} is not a cursor telemetry row; expected 6 TSV/CSV fields (timeMs,cx,cy,cursorTypeIndex,bounceScale,visible), received ${fields.length}: ${line}`, + ); + } + const numbers = fields.map(Number); + if (!numbers.slice(0, 3).every(Number.isFinite)) { + throw new Error( + `${sourcePath} line ${lineIndex + 1} has invalid cursor position fields: ${line}`, + ); + } + samples.push({ + timeMs: Math.max(0, numbers[0]), + cx: clampUnit(numbers[1]), + cy: clampUnit(numbers[2]), + cursorTypeIndex: normalizeCursorTypeIndex(numbers[3]), + bounceScale: Number.isFinite(numbers[4]) ? Math.max(0.1, Math.min(2, numbers[4])) : 1, + visible: Number.isFinite(numbers[5]) ? numbers[5] !== 0 : true, + }); + } + if (samples.length === 0) { + throw new Error( + `${sourcePath} contains no parseable cursor telemetry samples; expected a JSON {"samples":[...]} payload or TSV/CSV rows (timeMs,cx,cy,cursorTypeIndex,bounceScale,visible)`, + ); + } + return samples; +} + +// Parses a --cursor-json file into cursor samples. Accepts the canonical JSON +// payload ({"samples":[...]} or a bare array) and the exact generated TSV/CSV +// row format so a mis-wired CSV/TSV telemetry file never reaches JSON.parse. +export function parseCursorTelemetrySamples(text, sourcePath = "cursor telemetry") { + if (typeof text !== "string") { + throw new Error(`${sourcePath} must contain text content`); + } + const trimmed = text.trim(); + if (!trimmed) { + return []; + } + + let payload = null; + let jsonError = null; + try { + payload = JSON.parse(trimmed); + } catch (error) { + jsonError = error; + } + if (jsonError === null) { + const samples = Array.isArray(payload) ? payload : payload?.samples; + if (Array.isArray(samples)) { + return samples.filter((sample) => isValidCursorSample(sample)); + } + throw new Error( + `${sourcePath} is valid JSON but does not contain a samples array; expected {"samples":[...]}`, + ); + } + + return parseCursorTelemetryRows(trimmed, sourcePath); +} diff --git a/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs b/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs new file mode 100644 index 000000000..4ee8e747b --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { + formatCursorSamplesTsv, + parseCursorTelemetrySamples, + writeCursorSamplesFile, +} from "./cursorTelemetry.mjs"; + +// Regression guard for the CUDA compositor contract bug where a raw CSV/TSV +// cursor telemetry file was handed to JSON.parse (run-mp4-pipeline.mjs +// "JSON.parse receives 0,0.523828,0.731944,0,1,1"). The parser must accept the +// exact generated telemetry row format without ever JSON.parsing raw rows. + +const CSV_TELEMETRY = [ + "0,0.523828,0.731944,0,1,1", + "1000,0.62,0.7,2,0.9784,1", + "2000,0.3,0.4,1,0.8675,0", +].join("\n"); + +const JSON_TELEMETRY = JSON.stringify({ + samples: [ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { timeMs: 1000, cx: 0.62, cy: 0.7, cursorTypeIndex: 2, bounceScale: 0.9784, visible: true }, + { timeMs: 2000, cx: 0.3, cy: 0.4, cursorTypeIndex: 1, bounceScale: 0.8675, visible: false }, + ], +}); + +describe("parseCursorTelemetrySamples", () => { + it("parses the exact generated CSV row format without JSON.parse", () => { + expect(() => JSON.parse(CSV_TELEMETRY)).toThrow(); + + const samples = parseCursorTelemetrySamples(CSV_TELEMETRY, "cursor-telemetry.csv"); + expect(samples).toEqual([ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { + timeMs: 1000, + cx: 0.62, + cy: 0.7, + cursorTypeIndex: 2, + bounceScale: 0.9784, + visible: true, + }, + { + timeMs: 2000, + cx: 0.3, + cy: 0.4, + cursorTypeIndex: 1, + bounceScale: 0.8675, + visible: false, + }, + ]); + }); + + it("parses the exact generated TSV row format (pipeline cursor sidecar)", () => { + const tsv = ["0\t0.523828\t0.731944\t0\t1\t1", "1000\t0.62\t0.7\t2\t0.9784\t1"].join("\n"); + const samples = parseCursorTelemetrySamples(tsv, "cursor.tsv"); + expect(samples).toEqual([ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { + timeMs: 1000, + cx: 0.62, + cy: 0.7, + cursorTypeIndex: 2, + bounceScale: 0.9784, + visible: true, + }, + ]); + }); + + it("parses the canonical JSON payload with a samples array", () => { + const samples = parseCursorTelemetrySamples(JSON_TELEMETRY, "cursor-telemetry.json"); + expect(samples).toHaveLength(3); + expect(samples[1]).toMatchObject({ timeMs: 1000, cx: 0.62, cursorTypeIndex: 2 }); + expect(samples[2].visible).toBe(false); + }); + + it("accepts a bare JSON array of samples", () => { + const samples = parseCursorTelemetrySamples( + JSON.stringify(JSON.parse(JSON_TELEMETRY).samples), + ); + expect(samples).toHaveLength(3); + }); + + it("clamps row values to the telemetry contract bounds", () => { + const samples = parseCursorTelemetrySamples("0,-0.5,1.7,99,5,0", "cursor.csv"); + expect(samples[0]).toEqual({ + timeMs: 0, + cx: 0, + cy: 1, + cursorTypeIndex: 8, + bounceScale: 2, + visible: false, + }); + }); + + it("accepts partial 3-field rows with defaults like the native loader", () => { + const samples = parseCursorTelemetrySamples("0\t0.5\t0.5", "cursor.tsv"); + expect(samples[0]).toEqual({ + timeMs: 0, + cx: 0.5, + cy: 0.5, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }); + }); + + it("round-trips the generated row format without value drift", () => { + const parsed = parseCursorTelemetrySamples(CSV_TELEMETRY, "cursor-telemetry.csv"); + const tsv = formatCursorSamplesTsv(parsed); + const reparsed = parseCursorTelemetrySamples(tsv, "roundtrip.tsv"); + expect(reparsed).toEqual(parsed); + }); + + it("rejects non-telemetry text with an actionable error instead of JSON.parse noise", () => { + expect(() => parseCursorTelemetrySamples("hello world", "bad.txt")).toThrow( + /bad\.txt line 1 is not a cursor telemetry row/, + ); + expect(parseCursorTelemetrySamples("", "empty.csv")).toEqual([]); + }); + + it("rejects valid JSON that is not a samples payload", () => { + expect(() => parseCursorTelemetrySamples('{"layers":[]}', "overlay.json")).toThrow( + /does not contain a samples array/, + ); + }); +}); + +describe("writeCursorSamplesFile", () => { + it("writes the TSV sidecar consumed by the native compositor", async () => { + const os = await import("node:os"); + const path = await import("node:path"); + const fs = await import("node:fs"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "recordly-cursor-telemetry-")); + const outputPath = path.join(dir, "cursor.tsv"); + try { + const samples = parseCursorTelemetrySamples(JSON_TELEMETRY, "cursor-telemetry.json"); + const count = writeCursorSamplesFile(samples, outputPath); + expect(count).toBe(3); + const written = fs.readFileSync(outputPath, "utf8"); + expect(written).toBe( + "0\t0.523828\t0.731944\t0\t1\t1\n1000\t0.62\t0.7\t2\t0.9784\t1\n2000\t0.3\t0.4\t1\t0.8675\t0\n", + ); + expect(parseCursorTelemetrySamples(written, outputPath)).toEqual(samples); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs new file mode 100644 index 000000000..fe6cd7f59 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs @@ -0,0 +1,260 @@ +// Renderer-prepared transparent RGBA overlay sidecar manifest reader for the +// NVIDIA CUDA compositor wrapper (run-mp4-pipeline.mjs). +// +// Manifest layers carry a logical frameCount plus an optional +// effectiveFrameCount. When renderer-side deduplication truncated an identical +// suffix, the sidecar physically stores only effectiveFrameCount frames +// (1 <= effectiveFrameCount <= frameCount) and the final physical frame repeats +// for output indices [effectiveFrameCount, frameCount). Byte validation and the +// native --overlay descriptor must therefore use the physical frame count while +// the returned metadata/summary keeps the logical count. Manifests without +// effectiveFrameCount are fully dynamic layers and behave exactly as before. +// +// Two layer kinds are accepted: +// - "rgba" (default when `kind` is absent): a fixed-position raw RGBA overlay +// sidecar with a single `path`. Behavior is unchanged. Every layer carries +// its manifest `kind` and `order` (a safe-integer manifest order, otherwise +// a deterministic default), so classification and z-order survive JS-side +// filtering and the native descriptor keeps the renderer's global z-order. +// - "cursor-sprite": a tightly packed raw RGBA frame strip at `path` (one +// width*height*4 frame per output frame) whose per-frame top-left {x,y} +// position comes from a JSON `positionsPath` sidecar (exactly frameCount +// integer entries, top-down output pixels). Base x/y are 0 and ignored; +// positions are clamped to the output canvas and malformed/missing/truncated +// input fails closed so the cursor is never silently omitted. +// Unknown layer kinds are rejected rather than dropped. + +import { existsSync, readFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +function fail(message) { + throw new Error(message); +} + +// Clamps a top-left position so a partially off-canvas sprite keeps its visible +// part on screen. Sprite dimensions must already be validated against the +// canvas before clamping. +function clampPosition(position, spriteWidth, spriteHeight, outputWidth, outputHeight) { + const maxX = Math.max(0, outputWidth - spriteWidth); + const maxY = Math.max(0, outputHeight - spriteHeight); + return { + x: Math.max(0, Math.min(maxX, Number(position.x))), + y: Math.max(0, Math.min(maxY, Number(position.y))), + }; +} + +function readCursorSpritePositions( + positionsPath, + frameCount, + outputSize, + layerId, + spriteWidth, + spriteHeight, +) { + const resolvedPositionsPath = resolve(positionsPath); + if (!existsSync(resolvedPositionsPath)) { + fail(`Cursor-sprite layer ${layerId} positions do not exist: ${resolvedPositionsPath}`); + } + + let parsed; + try { + parsed = JSON.parse(readFileSync(resolvedPositionsPath, "utf8")); + } catch (error) { + fail(`Invalid cursor-sprite positions ${resolvedPositionsPath}: ${error.message}`); + } + const positions = Array.isArray(parsed) ? parsed : parsed?.positions; + if (!Array.isArray(positions)) { + fail( + `Cursor-sprite layer ${layerId} positions must be a JSON array of {x,y} objects: ` + + resolvedPositionsPath, + ); + } + if (positions.length !== frameCount) { + fail( + `Cursor-sprite layer ${layerId} positions must contain exactly one {x,y} per output ` + + `frame: expected ${frameCount}, received ${positions.length}`, + ); + } + + const { outputWidth, outputHeight } = outputSize; + return positions.map((position, index) => { + const x = Number(position?.x); + const y = Number(position?.y); + if (!Number.isSafeInteger(x) || !Number.isSafeInteger(y)) { + fail(`Cursor-sprite layer ${layerId} has a malformed position at frame ${index}`); + } + // Clamp the visible part of a partially off-canvas cursor rather than + // silently dropping it. + return clampPosition({ x, y }, spriteWidth, spriteHeight, outputWidth, outputHeight); + }); +} + +export function readOverlayManifest(manifestPath, outputSize) { + if (!manifestPath) { + return []; + } + const { outputWidth, outputHeight } = outputSize; + const resolvedPath = resolve(manifestPath); + if (!existsSync(resolvedPath)) { + fail(`Overlay manifest does not exist: ${resolvedPath}`); + } + + let manifest; + try { + manifest = JSON.parse(readFileSync(resolvedPath, "utf8")); + } catch (error) { + fail(`Invalid overlay manifest ${resolvedPath}: ${error.message}`); + } + if (!Array.isArray(manifest.layers)) { + fail(`Overlay manifest requires a layers array: ${resolvedPath}`); + } + + const layers = []; + for (const layer of manifest.layers) { + const id = typeof layer?.id === "string" ? layer.id : ""; + const kind = layer?.kind ?? "rgba"; + const layerPath = typeof layer?.path === "string" ? layer.path : ""; + const x = Number(layer?.x); + const y = Number(layer?.y); + const width = Number(layer?.width); + const height = Number(layer?.height); + const frameCount = Number(layer?.frameCount); + const effectiveFrameCount = + layer?.effectiveFrameCount === undefined || layer?.effectiveFrameCount === null + ? null + : Number(layer.effectiveFrameCount); + if (!id || !layerPath) { + fail(`Overlay manifest layer requires an id and path: ${resolvedPath}`); + } + if ( + ![x, y, width, height, frameCount].every(Number.isSafeInteger) || + width <= 0 || + height <= 0 || + frameCount <= 0 || + x < 0 || + y < 0 + ) { + fail(`Invalid overlay manifest layer ${id}: ${resolvedPath}`); + } + if (kind === "cursor-sprite") { + if (width > outputWidth || height > outputHeight) { + fail(`Cursor-sprite layer ${id} exceeds the output canvas: ${resolvedPath}`); + } + if (effectiveFrameCount !== null) { + fail( + `Cursor-sprite layer ${id} does not support effectiveFrameCount: ${resolvedPath}`, + ); + } + // Base x/y are always 0 for a cursor-sprite; positions carry the + // per-frame top-left. + const positionsPath = + typeof layer?.positionsPath === "string" ? layer.positionsPath : ""; + if (!positionsPath) { + fail(`Cursor-sprite layer ${id} requires a positionsPath: ${resolvedPath}`); + } + const positions = readCursorSpritePositions( + positionsPath, + frameCount, + outputSize, + id, + width, + height, + ); + const resolvedLayerPath = resolve(layerPath); + if (!existsSync(resolvedLayerPath)) { + fail(`Cursor-sprite layer ${id} does not exist: ${resolvedLayerPath}`); + } + const expectedBytes = width * height * 4 * frameCount; + const stat = statSync(resolvedLayerPath); + if (stat.size < expectedBytes) { + fail( + `Cursor-sprite layer ${id} is truncated: expected at least ${expectedBytes} ` + + `bytes, received ${stat.size}`, + ); + } + // Cursor sprite layers blend above the fixed-position overlays; a + // manifest order takes precedence, otherwise default to a high value so + // the cursor stays sharp/topmost. + const order = Number.isSafeInteger(Number(layer?.order)) ? Number(layer.order) : 10000; + layers.push({ + id, + kind, + order, + path: resolvedLayerPath, + positionsPath: resolve(positionsPath), + x: 0, + y: 0, + width, + height, + frameCount, + positions, + }); + continue; + } + if (kind !== "rgba") { + fail(`Overlay manifest layer ${id} has an unexpected kind "${kind}": ${resolvedPath}`); + } + if (effectiveFrameCount !== null) { + // Mirror the renderer contract (validateNativeStaticLayoutOverlayLayer): + // the physical sidecar count must be a positive integer no greater than + // the logical count. Malformed values fail here with the same generic + // invalid-layer message instead of a confusing truncation error. + if ( + !Number.isSafeInteger(effectiveFrameCount) || + effectiveFrameCount < 1 || + effectiveFrameCount > frameCount + ) { + fail(`Invalid overlay manifest layer ${id}: ${resolvedPath}`); + } + } + if (x + width > outputWidth || y + height > outputHeight) { + fail(`Overlay layer ${id} exceeds the output canvas: ${resolvedPath}`); + } + const resolvedLayerPath = resolve(layerPath); + if (!existsSync(resolvedLayerPath)) { + fail(`Overlay layer ${id} does not exist: ${resolvedLayerPath}`); + } + const physicalFrameCount = effectiveFrameCount ?? frameCount; + const expectedBytes = width * height * 4 * physicalFrameCount; + const stat = statSync(resolvedLayerPath); + if (stat.size < expectedBytes) { + fail( + `Overlay layer ${id} is truncated: expected at least ${expectedBytes} bytes, received ${stat.size}`, + ); + } + // A manifest order takes precedence; otherwise default to the layer's + // position in the sorted manifest so relative z-order survives even when + // the producer omits the field (mirrors the native --overlay insertion + // default). Mixed rgba/cursor-sprite manifests keep ascending z-order and + // the cursor-sprite default (10000) stays above the fixed-position layers. + const order = Number.isSafeInteger(Number(layer?.order)) + ? Number(layer.order) + : layers.length; + layers.push({ + id, + kind, + order, + path: resolvedLayerPath, + x, + y, + width, + height, + frameCount, + ...(effectiveFrameCount !== null ? { effectiveFrameCount } : {}), + }); + } + return layers; +} + +// Sorts overlay layers by ascending manifest z-order (order, then id) so the +// consumer's kind filters and the native descriptor always see the renderer's +// global z-order regardless of the manifest's physical order. Mirrors the sort +// used by the renderer-side native arg builders (order asc, then id +// localeCompare). Cursor-sprite layers keep their high default order (10000) +// when the producer omits the field, so they stay above fixed rgba layers even +// when the manifest lists them first. +export function sortOverlayLayersByOrder(layers) { + return [...layers].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); +} diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs new file mode 100644 index 000000000..7b36434ac --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs @@ -0,0 +1,680 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { readOverlayManifest, sortOverlayLayersByOrder } from "./overlayManifest.mjs"; + +// Contract for renderer-prepared transparent RGBA overlay sidecars: +// - frameCount is the logical frame count (durationSec * frameRate). +// - effectiveFrameCount (optional) is the physical frame count when renderer +// deduplication truncated an identical suffix; the sidecar then stores only +// effectiveFrameCount frames and the final physical frame repeats for output +// indices [effectiveFrameCount, frameCount). +// - Byte validation must use the physical count; metadata must keep the logical +// count; the native --overlay descriptor must receive the physical count so +// OverlayFrameSource clamps/repeats the final frame. + +const OUTPUT_SIZE = { outputWidth: 1920, outputHeight: 1080 }; +const FRAME_BYTES = 4 * 4 * 4; // 4x4 RGBA + +function makeTempDir(prefix = "recordly-overlay-manifest-") { + return mkdtempSync(join(tmpdir(), prefix)); +} + +function writeManifest(dir, layers, name = "overlay-manifest.json") { + const manifestPath = join(dir, name); + writeFileSync(manifestPath, JSON.stringify({ layers })); + return manifestPath; +} + +function writeSidecar(dir, byteLength, name = "overlay.rgba") { + const sidecarPath = join(dir, name); + writeFileSync(sidecarPath, Buffer.alloc(byteLength, 0x7f)); + return sidecarPath; +} + +function layer(overrides = {}) { + return { + id: "overlay-a", + path: "", + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + ...overrides, + }; +} + +describe("readOverlayManifest", () => { + it("returns an empty array when no manifest path is provided", () => { + expect(readOverlayManifest("", OUTPUT_SIZE)).toEqual([]); + expect(readOverlayManifest(null, OUTPUT_SIZE)).toEqual([]); + }); + + it("rejects a missing manifest file", () => { + expect(() => readOverlayManifest("/missing/overlay.json", OUTPUT_SIZE)).toThrow( + "Overlay manifest does not exist:", + ); + }); + + it("rejects invalid JSON and a missing layers array", () => { + const dir = makeTempDir(); + try { + const badJson = join(dir, "bad.json"); + writeFileSync(badJson, "{not json"); + expect(() => readOverlayManifest(badJson, OUTPUT_SIZE)).toThrow( + /Invalid overlay manifest/, + ); + + const noLayers = join(dir, "no-layers.json"); + writeFileSync(noLayers, JSON.stringify({ frames: 10 })); + expect(() => readOverlayManifest(noLayers, OUTPUT_SIZE)).toThrow( + "Overlay manifest requires a layers array:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts a manifest without effectiveFrameCount and validates bytes against the logical count", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar })]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toEqual([ + { + id: "overlay-a", + kind: "rgba", + order: 0, + path: sidecar, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts an effectiveFrameCount sidecar with only physical frames and preserves the logical count", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 2); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toEqual([ + { + id: "overlay-a", + kind: "rgba", + order: 0, + path: sidecar, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + effectiveFrameCount: 2, + }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts effectiveFrameCount equal to frameCount (no dedup truncation)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 10 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].frameCount).toBe(10); + expect(layers[0].effectiveFrameCount).toBe(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("treats a null effectiveFrameCount as absent (backward compatible)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, effectiveFrameCount: null }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0]).not.toHaveProperty("effectiveFrameCount"); + expect(layers[0].frameCount).toBe(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects effectiveFrameCount values outside 1..frameCount with the invalid-layer message", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + for (const effectiveFrameCount of [0, -1, 11, 1.5, "not-a-number"]) { + const manifestPath = writeManifest( + dir, + [layer({ path: sidecar, frameCount: 10, effectiveFrameCount })], + `invalid-${String(effectiveFrameCount)}.json`, + ); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("validates physical bytes against effectiveFrameCount, not the logical count", () => { + const dir = makeTempDir(); + try { + // Logical count implies 10 frames (10 * FRAME_BYTES) but the sidecar + // physically stores 2; with the physical-count rule this is valid. + const sidecar = writeSidecar(dir, FRAME_BYTES * 2); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + expect(readOverlayManifest(manifestPath, OUTPUT_SIZE)).toHaveLength(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fails a sidecar truncated below the physical count (with effectiveFrameCount)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 2 - 1); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Overlay layer overlay-a is truncated: expected at least ${ + FRAME_BYTES * 2 + } bytes, received ${FRAME_BYTES * 2 - 1}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps the backward-compatible truncation message for manifests without effectiveFrameCount", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 9); + const manifestPath = writeManifest(dir, [layer({ path: sidecar })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Overlay layer overlay-a is truncated: expected at least ${ + FRAME_BYTES * 10 + } bytes, received ${FRAME_BYTES * 9}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("enforces output-canvas bounds for every layer", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar, x: 1918, width: 4 })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Overlay layer overlay-a exceeds the output canvas:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects layers missing an id/path or with invalid geometry", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const missingId = writeManifest(dir, [layer({ path: sidecar, id: "" })]); + expect(() => readOverlayManifest(missingId, OUTPUT_SIZE)).toThrow( + "Overlay manifest layer requires an id and path:", + ); + + const missingPath = writeManifest(dir, [layer({ path: "" })]); + expect(() => readOverlayManifest(missingPath, OUTPUT_SIZE)).toThrow( + "Overlay manifest layer requires an id and path:", + ); + + const badGeometry = writeManifest(dir, [layer({ path: sidecar, width: 0 })]); + expect(() => readOverlayManifest(badGeometry, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + + const negativeX = writeManifest(dir, [layer({ path: sidecar, x: -1 })]); + expect(() => readOverlayManifest(negativeX, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a missing sidecar file", () => { + const dir = makeTempDir(); + try { + const manifestPath = writeManifest(dir, [layer({ path: join(dir, "nope.rgba") })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Overlay layer overlay-a does not exist:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("handles mixed layers (deduped and fully dynamic) in one manifest", () => { + const dir = makeTempDir(); + try { + const fullSidecar = writeSidecar(dir, FRAME_BYTES * 10, "full.rgba"); + const dedupedSidecar = writeSidecar(dir, FRAME_BYTES * 3, "deduped.rgba"); + const manifestPath = writeManifest(dir, [ + layer({ id: "a", path: fullSidecar, frameCount: 10 }), + layer({ id: "b", path: dedupedSidecar, frameCount: 10, effectiveFrameCount: 3 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "a", frameCount: 10 }); + expect(layers[0]).not.toHaveProperty("effectiveFrameCount"); + expect(layers[1]).toMatchObject({ id: "b", frameCount: 10, effectiveFrameCount: 3 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +// Cursor-sprite overlay kind. A cursor-sprite layer is a tightly packed raw +// RGBA frame strip (one width*height*4 frame per output frame) whose per-frame +// top-left {x,y} comes from a JSON positions sidecar (exactly frameCount +// entries, top-down output pixels). Base x/y are always 0; positions are +// clamped to keep the visible part of a partially off-canvas cursor on screen, +// and malformed/missing/truncated input fails closed (never silently omits). + +function cursorSpriteLayer(overrides = {}) { + return { + id: "cursor-sprite", + kind: "cursor-sprite", + path: "", + positionsPath: "", + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + ...overrides, + }; +} + +function writePositions(dir, positions, name = "cursor.positions.json") { + const positionsPath = join(dir, name); + writeFileSync(positionsPath, JSON.stringify(positions)); + return positionsPath; +} + +describe("cursor-sprite overlay layers", () => { + it("accepts a cursor-sprite layer with a per-frame positions sidecar", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, (_, index) => ({ x: index, y: 10 - index })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(1); + expect(layers[0]).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + path: sprite, + positionsPath, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + }); + expect(layers[0].positions).toEqual( + Array.from({ length: 10 }, (_, index) => ({ x: index, y: 10 - index })), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts a positions file wrapped in a { positions } object", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions(dir, { + positions: Array.from({ length: 10 }, (_, index) => ({ x: index, y: 0 })), + }); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].positions).toHaveLength(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("clamps a partially off-canvas cursor position instead of dropping it", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, (_, index) => ({ + x: index === 0 ? -5 : 1919, + y: index === 0 ? 5 : 1080, + })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].positions[0]).toEqual({ x: 0, y: 5 }); + expect(layers[0].positions[9]).toEqual({ x: 1919 - 3, y: 1080 - 4 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a missing or malformed positions sidecar", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const missing = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath: join(dir, "nope.json") }), + ]); + expect(() => readOverlayManifest(missing, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite positions do not exist:", + ); + + const badJsonPath = join(dir, "bad.json"); + writeFileSync(badJsonPath, "{not json"); + const badJson = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath: badJsonPath }), + ]); + expect(() => readOverlayManifest(badJson, OUTPUT_SIZE)).toThrow( + "Invalid cursor-sprite positions", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a positions count that does not match the frame count", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 9 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite positions must contain exactly one {x,y} per output frame: expected 10, received 9", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a malformed (non-integer or negative) position", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positions = Array.from({ length: 10 }, () => ({ x: 0, y: 0 })); + positions[3] = { x: 0.5, y: 0 }; + const positionsPath = writePositions(dir, positions); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite has a malformed position at frame 3", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a truncated cursor-sprite frame strip", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 9, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Cursor-sprite layer cursor-sprite is truncated: expected at least ${ + FRAME_BYTES * 10 + } bytes, received ${FRAME_BYTES * 9}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a cursor-sprite layer whose sprite exceeds the output canvas", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath, width: 1921 }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite exceeds the output canvas:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects unexpected layer kinds rather than dropping them", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar, kind: "unknown" })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + 'Overlay manifest layer overlay-a has an unexpected kind "unknown":', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects effectiveFrameCount on a cursor-sprite layer", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ + path: sprite, + positionsPath, + effectiveFrameCount: 3, + }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite does not support effectiveFrameCount:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("mixes rgba and cursor-sprite layers in one manifest", () => { + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + layer({ id: "a", path: rgbaSidecar }), + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "a", kind: "rgba" }); + expect(layers[1]).toMatchObject({ id: "cursor", kind: "cursor-sprite" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +// Layer classification and z-order regression: the reader must attach the +// manifest `kind`/`order` fields to every returned layer so the wrapper's +// kind filters never drop a layer and the native descriptor keeps the +// renderer-side global z-order (rgba layers included). + +describe("layer classification and z-order", () => { + it("attaches the manifest kind and order to every layer so classification and z-order survive", () => { + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + layer({ id: "bottom", path: rgbaSidecar, order: 5 }), + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath, order: 7 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "bottom", kind: "rgba", order: 5 }); + expect(layers[1]).toMatchObject({ id: "cursor", kind: "cursor-sprite", order: 7 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("defaults rgba layers to a deterministic order when the manifest omits it", () => { + const dir = makeTempDir(); + try { + const firstSidecar = writeSidecar(dir, FRAME_BYTES * 10, "first.rgba"); + const secondSidecar = writeSidecar(dir, FRAME_BYTES * 10, "second.rgba"); + const manifestPath = writeManifest(dir, [ + layer({ id: "first", path: firstSidecar }), + layer({ id: "second", path: secondSidecar }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "first", kind: "rgba", order: 0 }); + expect(layers[1]).toMatchObject({ id: "second", kind: "rgba", order: 1 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("sorts a mixed manifest by (order, id) so the cursor-sprite stays above the rgba layers", () => { + // Regression for the CodeRabbit round-3 finding: readOverlayManifest + // preserves manifest order, so the consumer must sort by (order, id) + // before its kind filters. A deliberately non-sorted mixed manifest + // must still produce an ordering with the cursor-sprite layer above + // the lower-order rgba layer (and the id tie-break must be stable for + // equal orders). + const dir = makeTempDir(); + try { + const bottomSidecar = writeSidecar(dir, FRAME_BYTES * 10, "bottom.rgba"); + const topSidecar = writeSidecar(dir, FRAME_BYTES * 10, "top.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + // Non-sorted on purpose: cursor-sprite first, then the rgba + // layers in reverse order with equal orders to exercise the id + // tie-break. + cursorSpriteLayer({ id: "cursor-sprite", path: sprite, positionsPath, order: 3 }), + layer({ id: "z-rgba-top", path: topSidecar, order: 2 }), + layer({ id: "a-rgba-bottom", path: bottomSidecar, order: 2 }), + ]); + const layers = sortOverlayLayersByOrder(readOverlayManifest(manifestPath, OUTPUT_SIZE)); + expect(layers.map(({ id, kind, order }) => ({ id, kind, order }))).toEqual([ + { id: "a-rgba-bottom", kind: "rgba", order: 2 }, + { id: "z-rgba-top", kind: "rgba", order: 2 }, + { id: "cursor-sprite", kind: "cursor-sprite", order: 3 }, + ]); + // The consumer filters the sorted list into kind groups, so the + // cursor-sprite layer (order 3) ends up above every rgba layer + // (order 2) regardless of the manifest's physical order. + const rgba = layers.filter((layer) => layer.kind === "rgba"); + const cursor = layers.filter((layer) => layer.kind === "cursor-sprite"); + expect(rgba.map((layer) => layer.order)).toEqual([2, 2]); + expect(cursor.map((layer) => layer.order)).toEqual([3]); + expect(cursor[0].order).toBeGreaterThan(Math.max(...rgba.map((layer) => layer.order))); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps a default-order cursor-sprite above rgba layers when the manifest lists the cursor first", () => { + // With omitted orders the reader defaults rgba layers to their manifest + // position (shifted by the cursor-sprite layer listed before them) and + // cursor-sprite layers to 10000; sorting must keep the cursor on top + // even though the manifest lists it before the rgba layers. + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath }), + layer({ id: "first", path: rgbaSidecar }), + layer({ id: "second", path: rgbaSidecar }), + ]); + const layers = sortOverlayLayersByOrder(readOverlayManifest(manifestPath, OUTPUT_SIZE)); + expect(layers.map(({ id, kind, order }) => ({ id, kind, order }))).toEqual([ + { id: "first", kind: "rgba", order: 1 }, + { id: "second", kind: "rgba", order: 2 }, + { id: "cursor", kind: "cursor-sprite", order: 10000 }, + ]); + const rgba = layers.filter((layer) => layer.kind === "rgba"); + const cursor = layers.filter((layer) => layer.kind === "cursor-sprite"); + expect(cursor[0].order).toBeGreaterThan(Math.max(...rgba.map((layer) => layer.order))); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs index 43324b051..c103da29d 100644 --- a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs +++ b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs @@ -48,18 +48,14 @@ function resolveToolCommand(envNames, moduleName, fallbackName) { const ffmpegCommand = resolveToolCommand(["RECORDLY_FFMPEG_EXE"], "ffmpeg-static", "ffmpeg"); const ffprobeCommand = resolveToolCommand(["RECORDLY_FFPROBE_EXE"], "ffprobe-static", "ffprobe"); -const cursorTypes = [ - "arrow", - "text", - "pointer", - "crosshair", - "open-hand", - "closed-hand", - "resize-ew", - "resize-ns", - "not-allowed", -]; -const cursorTypeIndexes = new Map(cursorTypes.map((type, index) => [type, index])); +import { parseCursorTelemetrySamples, writeCursorSamplesFile } from "./cursorTelemetry.mjs"; +import { readOverlayManifest, sortOverlayLayersByOrder } from "./overlayManifest.mjs"; +import { shouldProbeSourcePts } from "./sourcePtsPlan.mjs"; +import { + readTiledOverlayManifest, + resolveTiledOverlayLayerMetrics, + resolveTiledOverlayRawFallbackReason, +} from "./tiledOverlayManifest.mjs"; function fail(message) { throw new Error(message); @@ -284,6 +280,7 @@ function emitPreparationProgress(totalFrames, percentage, stage) { const progressStage = stage ?? "preparing"; const finalizing = progressStage === "finalizing"; const payload = { + outputCodec, currentFrame: finalizing ? Math.max(1, Math.floor(totalFrames)) : 0, totalFrames: Math.max(1, Math.floor(totalFrames)), percentage: Number(Math.min(99, Math.max(0, percentage)).toFixed(2)), @@ -451,69 +448,6 @@ function summarizeGpuSamples(samples) { return summary; } -function cursorBounceScale(interactionType, ageMs, durationMs = 180) { - if (!["click", "double-click", "right-click", "middle-click"].includes(interactionType)) { - return 1; - } - if (ageMs < 0 || ageMs > durationMs) { - return 1; - } - const progress = 1 - ageMs / durationMs; - return Math.max(0.72, 1 - Math.sin(progress * Math.PI) * 0.08); -} - -function latestClickSample(samples, sampleIndex) { - for (let index = sampleIndex; index >= 0; index -= 1) { - const sample = samples[index]; - if ( - ["click", "double-click", "right-click", "middle-click"].includes( - sample?.interactionType, - ) - ) { - return sample; - } - } - return null; -} - -function writeCursorSamples(cursorPayload, outputPath) { - const samples = Array.isArray(cursorPayload.samples) ? cursorPayload.samples : []; - const cursorLines = samples - .map((sample, index) => { - if ( - !Number.isFinite(sample?.timeMs) || - !Number.isFinite(sample?.cx) || - !Number.isFinite(sample?.cy) - ) { - return null; - } - const clickSample = latestClickSample(samples, index); - const bounceScale = Number.isFinite(sample.bounceScale) - ? sample.bounceScale - : clickSample - ? cursorBounceScale( - clickSample.interactionType, - sample.timeMs - clickSample.timeMs, - ) - : 1; - return [ - sample.timeMs, - sample.cx, - sample.cy, - cursorTypeIndexes.get(sample.cursorType) ?? - (Number.isFinite(sample.cursorTypeIndex) - ? Math.max(0, Math.min(8, Math.round(sample.cursorTypeIndex))) - : 0), - Number(bounceScale.toFixed(4)), - sample.visible === false ? 0 : 1, - ].join("\t"); - }) - .filter(Boolean) - .join("\n"); - writeFileSync(outputPath, cursorLines ? `${cursorLines}\n` : ""); - return samples.length; -} - function renderTahoeCursorAtlas(workDir) { const rgbaPath = join(workDir, "tahoe-cursor-atlas.rgba"); const metadataPath = join(workDir, "tahoe-cursor-atlas.tsv"); @@ -698,7 +632,11 @@ function getVideoInfo(inputPath) { fail(`No video stream found in ${inputPath}`); } if (stream.codec_name !== "h264") { - fail(`The NVIDIA CUDA compositor currently expects H.264 input, got ${stream.codec_name}`); + fail( + `The NVIDIA CUDA compositor only supports H.264 input video; got ${ + stream.codec_name ?? "unknown" + }`, + ); } const durationSec = Number(stream.duration); if (!Number.isFinite(durationSec) || durationSec <= 0) { @@ -930,6 +868,11 @@ const inputPath = resolve(getArg("--input")); const outputPath = resolve( getArg("--output", join(scriptDir, "recordly-nvdec-nvenc-mp4-output.mp4")), ); +const outputCodec = getArg("--output-codec", "h264"); +if (!["h264", "hevc"].includes(outputCodec)) { + throw new Error(`Unsupported --output-codec: ${outputCodec}; expected h264 or hevc`); +} +const elementaryStreamFormat = outputCodec; const requestedOutputWidth = Math.round(getNumberArg("--width", 0)); const requestedOutputHeight = Math.round(getNumberArg("--height", 0)); const fps = Math.round(getNumberArg("--fps", 30)); @@ -982,6 +925,11 @@ const cursorAtlasPng = getArg("--cursor-atlas-png", ""); const cursorAtlasMetadata = getArg("--cursor-atlas-metadata", ""); const zoomTelemetry = getArg("--zoom-telemetry", ""); const timelineMap = getArg("--timeline-map", ""); +const overlayManifest = getArg("--overlay-manifest", ""); +const tiledOverlayManifest = getArg("--tiled-overlay-manifest", ""); +const temporalBlurSampleCount = getNumberArg("--temporal-blur-sample-count", 0); +const temporalBlurShutterFraction = getNumberArg("--temporal-blur-shutter-fraction", 0); +const temporalBlurWeightPower = getNumberArg("--temporal-blur-weight-power", 1); if (!existsSync(inputPath)) { fail(`Input does not exist: ${inputPath}`); @@ -1021,6 +969,17 @@ function resolveNativeProbePath() { } const nativeProbe = resolveNativeProbePath(); +// The --help capability probe output is stable for a given helper build; +// compute it once and reuse it for every feature check in this export instead +// of re-invoking the native binary once per overlay/temporal feature. +let nativeHelpCache = null; +function readNativeHelp() { + if (nativeHelpCache === null) { + nativeHelpCache = run(nativeProbe, ["--help"]).stdout; + } + return nativeHelpCache; +} + const baseName = basename(inputPath).replace(/\.[^.]+$/, ""); const webcamBaseName = webcamInput ? basename(webcamInput).replace(/\.[^.]+$/, "") @@ -1028,7 +987,7 @@ const webcamBaseName = webcamInput const annexBPath = join(workDir, `${baseName}.annexb.h264`); const webcamAnnexBPath = join(workDir, `${webcamBaseName}.annexb.h264`); const cursorSamplesPath = join(workDir, `${baseName}.cursor.tsv`); -const encodedPath = join(workDir, `${baseName}.mapped-callback.h264`); +const encodedPath = join(workDir, `${baseName}.mapped-callback.${outputCodec}`); const shouldBakeStaticShadow = Boolean(backgroundImage) && contentWidth > 0 && @@ -1235,11 +1194,26 @@ const demuxPromise = endPercentage: 2, }, ); -const sourcePtsPromise = writeFramePtsSidecarAsync(inputPath, sourceDurationSec, sourcePtsPath); +// Source frame PTS is only required for timeline-map exports and for inline +// audio mux validation (the wrapper checks the native summary reports a +// timestamp-aligned mode before trusting the muxed audio). For plain video-only +// exports the per-packet ffprobe scan is pure overhead (it dominates the wall +// time of short 4K exports), so skip it unless it is actually consumed. +const needsSourcePts = shouldProbeSourcePts({ + hasTimelineSegments: timelineSegments.length > 0, + videoOnly, + forceSourcePts: process.env.RECORDLY_NVIDIA_CUDA_FORCE_SOURCE_PTS, +}); +const sourcePtsPromise = needsSourcePts + ? writeFramePtsSidecarAsync(inputPath, sourceDurationSec, sourcePtsPath) + : Promise.resolve(zeroElapsed()); if (cursorJson) { - const cursorPayload = JSON.parse(readFileSync(resolve(cursorJson), "utf8")); - writeCursorSamples(cursorPayload, cursorSamplesPath); + const cursorPayload = parseCursorTelemetrySamples( + readFileSync(resolve(cursorJson), "utf8"), + resolve(cursorJson), + ); + writeCursorSamplesFile(cursorPayload, cursorSamplesPath); } const cursorAtlas = cursorJson && cursorHeight > 0 && cursorAtlasPng && cursorAtlasMetadata @@ -1274,6 +1248,8 @@ const encodeArgs = [ annexBPath, "--output", encodedPath, + "--output-codec", + outputCodec, "--fps", String(fps), "--input-frames", @@ -1402,6 +1378,149 @@ if (contentWidth > 0 && contentHeight > 0) { if (zoomTelemetry) { encodeArgs.push("--zoom-samples", resolve(zoomTelemetry)); } +if (temporalBlurSampleCount > 0) { + // The native compositor must advertise --temporal-blur-sample-count in its + // --help usage before the wrapper forwards the resolved temporal zoom + // motion blur plan (mirror of the tiled-overlay probe below). Until then + // temporal blur cannot be composited and the export fails fast instead of + // silently dropping the effect. + const nativeHelp = readNativeHelp(); + if (!nativeHelp.includes("--temporal-blur-sample-count")) { + fail( + "unsupported-temporal-motion-blur: the native NVIDIA CUDA compositor does not support temporal zoom motion blur yet; " + + "main.cu must consume --temporal-blur-sample-count (this build) or the renderer must " + + "keep the effect on a CUDA-capable helper.", + ); + } +} +if (temporalBlurSampleCount >= 3) { + encodeArgs.push( + "--temporal-blur-sample-count", + String(temporalBlurSampleCount), + "--temporal-blur-shutter-fraction", + String(temporalBlurShutterFraction), + "--temporal-blur-weight-power", + String(temporalBlurWeightPower), + ); +} else if (temporalBlurSampleCount > 0) { + // The TS-side invariant rejects resolved plans below the minimum (3), but a + // direct wrapper invocation could still request 1-2 samples. The native + // compositor accepts 3..61 samples only, so fail fast with the established + // unsupported-result contract instead of a warning and a silently dropped + // effect (mirroring the unsupported-temporal-motion-blur fail above). + fail( + `unsupported-temporal-motion-blur: temporal zoom motion blur requested with ${temporalBlurSampleCount} sample(s), below the minimum of 3; ` + + "main.cu only consumes --temporal-blur-sample-count values in the supported 3..61 range.", + ); +} +// The manifest may mix fixed-position rgba layers and cursor-sprite layers. +// rgba layers keep the proven per-layer --overlay descriptor; cursor-sprite +// layers are forwarded to the native cursor-sprite compositor route that owns +// the packed frame strip + per-frame positions validation. Layers are sorted +// by ascending (order, id) before the kind filters so mixed manifests keep the +// renderer's global z-order regardless of manifest order and cursor-sprite +// layers stay above the fixed rgba layers. +const overlayLayers = sortOverlayLayersByOrder( + readOverlayManifest(overlayManifest, { + outputWidth, + outputHeight, + }), +); +const rgbaOverlayLayers = overlayLayers.filter((layer) => layer.kind === "rgba"); +const cursorSpriteLayers = overlayLayers.filter((layer) => layer.kind === "cursor-sprite"); +if (rgbaOverlayLayers.length) { + for (const layer of rgbaOverlayLayers) { + encodeArgs.push( + "--overlay", + layer.path, + String(layer.x), + String(layer.y), + String(layer.width), + String(layer.height), + // The native OverlayFrameSource clamps/repeats the final physical frame + // for output indices beyond the physical count, so the descriptor must + // carry the physical sidecar count (effectiveFrameCount when renderer + // dedup truncated an identical suffix, otherwise the logical count). + String(layer.effectiveFrameCount ?? layer.frameCount), + // Optional 7th argument is the renderer-side global z-order; the native + // compositor merges raw/tiled/cursor-sprite layers by this ascending + // value so a manifest order survives classification and filtering. + String(layer.order), + ); + } +} +if (cursorSpriteLayers.length) { + const nativeHelp = readNativeHelp(); + if (!nativeHelp.includes("--cursor-sprite")) { + fail( + "The native NVIDIA CUDA compositor does not support cursor-sprite overlays yet; " + + "main.cu must consume --cursor-sprite (this build) or the renderer must keep " + + "the baked cursor overlay sidecar fallback.", + ); + } + for (const layer of cursorSpriteLayers) { + // positions are validated/clamped on the JS side above; the native + // compositor re-validates the positions file and hard-fails (noCpuFallback) + // so the cursor is never silently omitted on a strict native route. + encodeArgs.push( + "--cursor-sprite", + layer.id, + String(layer.order), + layer.path, + resolve(layer.positionsPath), + String(layer.width), + String(layer.height), + String(layer.frameCount), + ); + } +} +// Tiled/delta sparse overlay stream: the versioned descriptor was validated by +// readTiledOverlayManifest (independently of the TS side). The native CUDA +// compositor consumes the descriptor itself; it must advertise +// --tiled-overlay-manifest in its --help usage before the wrapper forwards it. +// Until then a tiled stream cannot be composited and the export fails fast +// instead of silently dropping overlay pixels. +const tiledOverlayLayers = readTiledOverlayManifest(tiledOverlayManifest, { + outputWidth, + outputHeight, + frameRate: fps, + durationSec, +}); +const tiledOverlayMetrics = tiledOverlayLayers.map((layer) => { + const layerMetrics = resolveTiledOverlayLayerMetrics(layer); + return { + layer: { + id: layer.id, + order: layer.order, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + frameRate: layer.frameRate, + durationSec: layer.durationSec, + tileSize: layer.tileSize, + pixelFormat: layer.pixelFormat, + payloadPath: layer.payloadPath, + payloadByteLength: layer.payloadByteLength, + staticTileCount: layer.staticTiles.length, + frameDeltaCount: layer.frameDeltas.length, + }, + metrics: layerMetrics, + rawFallbackReason: resolveTiledOverlayRawFallbackReason(layer, layerMetrics), + }; +}); +if (tiledOverlayLayers.length) { + const nativeHelp = readNativeHelp(); + if (!nativeHelp.includes("--tiled-overlay-manifest")) { + fail( + "The native NVIDIA CUDA compositor does not support tiled overlay manifests yet; " + + "main.cu must consume --tiled-overlay-manifest (follow-up) or the renderer must " + + "keep the raw RGBA overlay sidecar fallback.", + ); + } + encodeArgs.push("--tiled-overlay-manifest", resolve(tiledOverlayManifest)); +} const encode = reuseIntermediates && existsSync(encodedPath) ? { elapsedMs: 0, stdout: "", gpuSummary: null } @@ -1411,7 +1530,44 @@ const encode = sampleGpuDuringEncode ? gpuSampleIntervalMs : 0, ); const nativeSummary = encode.stdout ? parseProbeSummary(encode.stdout) : null; +if (nativeSummary && tiledOverlayLayers.length) { + // Additive renderer-derived tiled throughput bookkeeping rides on the native + // summary so the main-process normalization surfaces it unchanged. Values are + // aggregated across layers; rawFallbackReason is the first conservative + // eligibility decision that forced the raw full-frame fallback. These are + // diagnostic only and never claim zero-copy. + nativeSummary.tiledOverlayLayers = tiledOverlayLayers.length; + nativeSummary.changedTileCount = tiledOverlayMetrics.reduce( + (total, entry) => total + entry.metrics.changedTileCount, + 0, + ); + nativeSummary.uploadedTileBytes = tiledOverlayMetrics.reduce( + (total, entry) => total + entry.metrics.uploadedTileBytes, + 0, + ); + nativeSummary.cachedTileCount = tiledOverlayMetrics.reduce( + (total, entry) => total + entry.metrics.cachedTileCount, + 0, + ); + const firstFallbackReason = tiledOverlayMetrics.find( + (entry) => entry.rawFallbackReason !== null, + )?.rawFallbackReason; + if (firstFallbackReason) { + nativeSummary.rawFallbackReason = firstFallbackReason; + } +} +if (nativeSummary?.outputCodec && nativeSummary.outputCodec !== outputCodec) { + fail(`Native output codec mismatch: expected ${outputCodec}, got ${nativeSummary.outputCodec}`); +} +const elementaryStreamInputArgs = [ + "-f", + elementaryStreamFormat, + "-framerate", + String(fps), + "-i", + encodedPath, +]; const mux = skipMux ? { elapsedMs: 0 } : videoOnly @@ -1423,10 +1579,7 @@ const mux = skipMux "-loglevel", "error", "-stats", - "-framerate", - String(fps), - "-i", - encodedPath, + ...elementaryStreamInputArgs, "-map", "0:v:0", "-c:v", @@ -1449,10 +1602,7 @@ const mux = skipMux "-loglevel", "error", "-stats", - "-framerate", - String(fps), - "-i", - encodedPath, + ...elementaryStreamInputArgs, "-i", inputPath, "-map", @@ -1487,6 +1637,13 @@ const outputInfo = skipMux const outputStreams = outputInfo.streams ?? []; const outputVideo = outputStreams.find((stream) => stream.codec_type === "video") ?? null; const outputAudio = outputStreams.find((stream) => stream.codec_type === "audio") ?? null; +if (!skipMux && outputVideo?.codec_name !== outputCodec) { + fail( + `Muxed output codec mismatch: expected ${outputCodec}, got ${ + outputVideo?.codec_name ?? "none" + }`, + ); +} console.log( JSON.stringify( @@ -1497,6 +1654,8 @@ console.log( requestedOutputPath: outputPath, encodedPath, fps, + outputCodec, + elementaryStreamFormat, bitrateMbps, encodingMode, streamSync, @@ -1566,6 +1725,45 @@ console.log( inputPath: resolve(zoomTelemetry), } : null, + overlay: + overlayLayers.length || tiledOverlayLayers.length + ? { + layers: rgbaOverlayLayers.map((layer) => ({ + id: layer.id, + path: layer.path, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + ...(layer.effectiveFrameCount !== undefined + ? { + effectiveFrameCount: + layer.effectiveFrameCount, + } + : {}), + physicalFrameCount: + layer.effectiveFrameCount ?? layer.frameCount, + })), + cursorSprite: cursorSpriteLayers.length + ? { + layers: cursorSpriteLayers.map((layer) => ({ + id: layer.id, + order: layer.order, + path: layer.path, + positionsPath: layer.positionsPath, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + positionsCount: layer.positions.length, + })), + } + : null, + tiled: tiledOverlayLayers.length + ? { layers: tiledOverlayMetrics } + : null, + } + : null, } : null, gpuSampleIntervalMs: sampleGpuDuringEncode ? gpuSampleIntervalMs : null, diff --git a/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs new file mode 100644 index 000000000..5b10ffe4f --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs @@ -0,0 +1,14 @@ +// Source PTS sidecar decision for the CUDA export pipeline. +// +// The per-packet ffprobe scan that writes the frame PTS sidecar dominates the +// wall time of short 4K exports (~1.7s per 6s clip). It is only consumed when +// a timeline map is present (mapped-callback frame selection needs source PTS) +// or when the wrapper will inline-mux audio (the wrapper validates the native +// summary reports a timestamp-aligned mode before trusting the muxed audio). +// Plain video-only exports with no timeline skip the probe entirely; the native +// compositor's decoder-policy frame selection produces the same output frames. + +export function shouldProbeSourcePts(options) { + const { hasTimelineSegments, videoOnly, forceSourcePts } = options; + return hasTimelineSegments === true || videoOnly !== true || forceSourcePts === "1"; +} diff --git a/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs new file mode 100644 index 000000000..229381ec6 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { shouldProbeSourcePts } from "./sourcePtsPlan.mjs"; + +describe("shouldProbeSourcePts", () => { + it("probes when a timeline map is present (mapped-callback needs source PTS)", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: true, + videoOnly: true, + forceSourcePts: undefined, + }), + ).toBe(true); + }); + + it("probes when the wrapper will inline-mux audio (non-video-only)", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: false, + forceSourcePts: undefined, + }), + ).toBe(true); + }); + + it("skips the probe for plain video-only exports without a timeline", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: true, + forceSourcePts: undefined, + }), + ).toBe(false); + }); + + it("honors the force override for diagnostics", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: true, + forceSourcePts: "1", + }), + ).toBe(true); + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/src/main.cu b/electron/native/nvidia-cuda-compositor/src/main.cu index 28c72123d..2e3488e60 100644 --- a/electron/native/nvidia-cuda-compositor/src/main.cu +++ b/electron/native/nvidia-cuda-compositor/src/main.cu @@ -2,19 +2,26 @@ #include #include +#include #include #include +#include #include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include +#include +#include #include #include "NvDecoder/NvDecoder.h" @@ -33,9 +40,114 @@ struct TimelineSegment { double speed = 1.0; }; +// Renderer-prepared transparent RGBA overlay sidecar layer. The sidecar is a +// raw top-down RGBA stream with frameCount frames of width*height*4 bytes at +// the export frame rate. Layers are composited in the order they appear in +// options.overlayLayers (z-order) after the video layout and zoom blur, which +// matches the renderer contract (overlays are drawn above the blurred video). +// The `order` field is the renderer-side global z-order; mixed raw and tiled +// overlays are merged and blended by this ascending value. +struct OverlayLayerDescriptor { + std::string id; + int order = 0; + std::string path; + int x = 0; + int y = 0; + int width = 0; + int height = 0; + int frameCount = 0; // logical frame count (renderer-side) + int effectiveFrameCount = 0; // physical sidecar frames (clamped for reads) + double frameRate = 0.0; + double durationSec = 0.0; +}; + +// --------------------------------------------------------------------------- +// Renderer-prepared tiled/delta transparent RGBA overlay stream (storage +// version 1). Mirrors the TS contract in nativeStaticLayoutOverlays.ts: fixed +// 128x128 lossless raw RGBA tiles, staticTiles define the full initial layer +// state emitted once, frameDeltas carry per-frame changed tile payloads +// (ascending, unique frame indices in [0, frameCount)). Every payload region +// is written exactly once into the bounded payload stream +// (payloadPath/payloadByteLength), so sparse 4K overlays never duplicate +// unchanged pixels. The helper validates the descriptor independently before +// encoding (it never trusts the renderer blob) and rejects malformed or +// truncated input with an actionable JSON failure; there is no silent raw +// fallback inside the helper. +// --------------------------------------------------------------------------- +struct TiledOverlayTileRecord { + int tileIndex = 0; + int64_t byteOffset = 0; + int64_t byteLength = 0; +}; + +struct TiledOverlayFrameDelta { + int frameIndex = 0; + std::vector changedTiles; +}; + +struct TiledOverlayLayerDescriptor { + std::string id; + int order = 0; + int x = 0; + int y = 0; + int width = 0; + int height = 0; + double frameRate = 0.0; + double durationSec = 0.0; + int frameCount = 0; + int tileSize = 0; + std::string pixelFormat; + std::string payloadPath; + int64_t payloadByteLength = 0; + std::vector staticTiles; + std::vector frameDeltas; + // Derived at load time (never part of the wire descriptor). + int tileColumns = 0; + int tileRows = 0; + int tileCount = 0; + int64_t tileByteSize = 0; + int64_t maxDeltaBytes = 0; + std::string rawFallbackReason; +}; + +constexpr int kTiledOverlayStorageVersion = 1; +constexpr int kTiledOverlayTileSize = 128; +constexpr int64_t kTiledOverlayTileByteSize = 128LL * 128LL * 4LL; + +// Unified z-order entry for the native composition loop. The renderer emits +// both raw RGBA sidecar layers and tiled/delta layers with a single global +// `order` value; the compositor must blend them in that exact order rather +// than grouping all raw layers below all tiled layers. +struct CompositeLayer { + enum class Kind { Raw, Tiled }; + Kind kind = Kind::Raw; + int sourceIndex = 0; // index into the matching source (raw or tiled) + int order = 0; + int x = 0; + int y = 0; + int width = 0; + int height = 0; +}; + +// Conservative tiled-vs-raw density/size heuristics mirrored from the TS side +// (NATIVE_TILED_OVERLAY_MIN_TILE_COUNT / _MAX_CHANGED_TILE_FRACTION / +// _MAX_PAYLOAD_BYTES_FRACTION). Layers that trip a heuristic are still valid +// tiled streams the helper composites losslessly; the reason is only reported +// as an observable diagnostic so a raw full-frame fallback (which the renderer +// may keep for dense layers) is never indistinguishable from a tiled export. +constexpr int kTiledOverlayMinTileCount = 4; +constexpr double kTiledOverlayMaxChangedTileFraction = 0.5; +constexpr double kTiledOverlayMaxPayloadBytesFraction = 0.7; + +enum class OutputCodec { + H264, + HEVC, +}; + struct Options { std::string inputPath; std::string outputPath = "recordly-nvidia-cuda-compositor.h264"; + OutputCodec outputCodec = OutputCodec::H264; std::string sourcePtsPath; std::string timelineMapPath; std::vector timelineSegments; @@ -87,15 +199,53 @@ struct Options { int cursorAtlasWidth = 0; int cursorAtlasHeight = 0; std::string zoomSamplesPath; + // Renderer-resolved temporal zoom motion blur plan (see temporalMotionBlur.ts): + // the compositor derives per-frame sample offsets/weights from these three + // values plus the output frame duration. 0 sample count disables temporal + // blur so the existing spatial blur telemetry path is used. + int temporalBlurSampleCount = 0; + double temporalBlurShutterFraction = 0.0; + double temporalBlurWeightPower = 1.0; + std::vector overlayLayers; + std::string overlayManifestPath; + std::vector compositeLayers; + std::string tiledOverlayManifestPath; + // Validated tiled/delta overlay stream (version 1 descriptor). Loaded in + // parseOptions before encoding so malformed/truncated/unsupported + // descriptors fail with an actionable JSON failure before any decode work. + std::vector tiledOverlayLayers; }; constexpr int kMaxCursorAtlasEntries = 16; constexpr int kWebcamPrefetchOutputFrames = 900; +// Bounded overlay frame ring: slots are keyed by the clamped frame index so +// single-frame layers and tail-repeated frames are read from disk once and +// served from the device slot for every following output frame. Two slots of +// head room give a two-frame read-ahead without unbounded memory; the depth is +// always prefetchSlots - 2 so the ring never overwrites the slot the current +// blend is reading. +constexpr int kOverlayPrefetchSlots = 4; +constexpr int kOverlayPrefetchDepth = kOverlayPrefetchSlots - 2; [[noreturn]] void fail(const std::string& message) { throw std::runtime_error(message); } +const char* outputCodecName(OutputCodec codec) { + return codec == OutputCodec::HEVC ? "hevc" : "h264"; +} + +OutputCodec parseOutputCodec(const char* value) { + const std::string codec = value; + if (codec == "h264") { + return OutputCodec::H264; + } + if (codec == "hevc") { + return OutputCodec::HEVC; + } + fail("Unsupported --output-codec: " + codec + "; expected h264 or hevc"); +} + void checkCuda(cudaError_t status, const char* expression) { if (status != cudaSuccess) { std::ostringstream stream; @@ -147,7 +297,912 @@ double parseFiniteDouble(const char* value, const char* name) { stream << "Invalid " << name << ": " << value; fail(stream.str()); } - return parsed; + return parsed; +} + +// --------------------------------------------------------------------------- +// Minimal standards-compliant JSON parser for the tiled overlay descriptor. +// The descriptor is session data from the renderer (never persisted); the +// parser rejects malformed input with an actionable message that includes the +// offending byte position. Depth and element caps keep corrupted or +// adversarial input from exhausting memory. +// --------------------------------------------------------------------------- +struct JsonValue { + enum class Type { + Null, + Bool, + Number, + String, + Array, + Object, + }; + Type type = Type::Null; + bool boolean = false; + double number = 0.0; + std::string string; + std::vector array; + std::vector> object; +}; + +class JsonParser { +public: + explicit JsonParser(const std::string& text) : text_(text) {} + + JsonValue parse() { + skipWhitespace(); + JsonValue root = parseValue(0); + skipWhitespace(); + if (position_ < text_.size()) { + failAt("Unexpected trailing characters"); + } + return root; + } + +private: + static constexpr int kMaxDepth = 64; + static constexpr size_t kMaxElements = 1u << 20u; + + [[noreturn]] void failAt(const std::string& message) const { + std::ostringstream stream; + stream << message << " at byte " << position_; + throw std::runtime_error(stream.str()); + } + + static bool isDigit(char c) { + return c >= '0' && c <= '9'; + } + + static bool isHexDigit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + static int hexValue(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; + } + + void skipWhitespace() { + while (position_ < text_.size()) { + const char c = text_[position_]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + ++position_; + } else { + break; + } + } + } + + bool consume(char expected) { + if (position_ < text_.size() && text_[position_] == expected) { + ++position_; + return true; + } + return false; + } + + JsonValue parseValue(int depth) { + if (depth > kMaxDepth) { + failAt("JSON nesting too deep"); + } + if (position_ >= text_.size()) { + failAt("Unexpected end of JSON input"); + } + const char c = text_[position_]; + if (c == '{') { + return parseObject(depth); + } + if (c == '[') { + return parseArray(depth); + } + if (c == '"') { + return parseString(); + } + if (c == 't') { + return parseKeyword("true", JsonValue::Type::Bool, true); + } + if (c == 'f') { + return parseKeyword("false", JsonValue::Type::Bool, false); + } + if (c == 'n') { + return parseKeyword("null", JsonValue::Type::Null, false); + } + if (c == '-' || isDigit(c)) { + return parseNumber(); + } + failAt("Unexpected token"); + } + + JsonValue parseKeyword(const char* keyword, JsonValue::Type type, bool boolean) { + const size_t length = std::strlen(keyword); + if (text_.compare(position_, length, keyword) != 0) { + failAt("Invalid JSON token"); + } + position_ += length; + JsonValue result; + result.type = type; + result.boolean = boolean; + return result; + } + + JsonValue parseObject(int depth) { + consume('{'); + JsonValue result; + result.type = JsonValue::Type::Object; + skipWhitespace(); + if (consume('}')) { + return result; + } + while (true) { + if (result.object.size() >= kMaxElements) { + failAt("JSON object too large"); + } + skipWhitespace(); + if (position_ >= text_.size() || text_[position_] != '"') { + failAt("Expected a string object key"); + } + JsonValue keyValue = parseString(); + skipWhitespace(); + if (!consume(':')) { + failAt("Expected ':' after object key"); + } + skipWhitespace(); + result.object.emplace_back(std::move(keyValue.string), parseValue(depth + 1)); + skipWhitespace(); + if (consume('}')) { + break; + } + if (!consume(',')) { + failAt("Expected ',' or '}' in object"); + } + skipWhitespace(); + } + return result; + } + + JsonValue parseArray(int depth) { + consume('['); + JsonValue result; + result.type = JsonValue::Type::Array; + skipWhitespace(); + if (consume(']')) { + return result; + } + while (true) { + if (result.array.size() >= kMaxElements) { + failAt("JSON array too large"); + } + skipWhitespace(); + result.array.push_back(parseValue(depth + 1)); + skipWhitespace(); + if (consume(']')) { + break; + } + if (!consume(',')) { + failAt("Expected ',' or ']' in array"); + } + skipWhitespace(); + } + return result; + } + + // Appends the UTF-8 encoding of codepoint to out. + static void appendUtf8(std::string& out, unsigned int codepoint) { + if (codepoint < 0x80) { + out.push_back(static_cast(codepoint)); + } else if (codepoint < 0x800) { + out.push_back(static_cast(0xC0 | (codepoint >> 6))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } else if (codepoint < 0x10000) { + out.push_back(static_cast(0xE0 | (codepoint >> 12))); + out.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } else { + out.push_back(static_cast(0xF0 | (codepoint >> 18))); + out.push_back(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + unsigned int parseHexQuad() { + if (position_ + 4 > text_.size()) { + failAt("Incomplete unicode escape"); + } + unsigned int value = 0; + for (int index = 0; index < 4; ++index) { + const char c = text_[position_++]; + if (!isHexDigit(c)) { + failAt("Invalid unicode escape"); + } + value = (value << 4) | static_cast(hexValue(c)); + } + return value; + } + + JsonValue parseString() { + consume('"'); + std::string value; + while (true) { + if (position_ >= text_.size()) { + failAt("Unterminated string"); + } + const unsigned char c = static_cast(text_[position_]); + if (c == '"') { + ++position_; + break; + } + if (c == '\\') { + ++position_; + if (position_ >= text_.size()) { + failAt("Unterminated escape sequence"); + } + const char escape = text_[position_++]; + switch (escape) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + case 'u': { + const unsigned int first = parseHexQuad(); + unsigned int codepoint = first; + if (first >= 0xD800 && first <= 0xDBFF) { + // High surrogate: expect a low surrogate escape. + if (position_ + 1 < text_.size() && text_[position_] == '\\' && + text_[position_ + 1] == 'u') { + position_ += 2; + const unsigned int second = parseHexQuad(); + if (second >= 0xDC00 && second <= 0xDFFF) { + codepoint = + 0x10000 + ((first - 0xD800) << 10) + (second - 0xDC00); + } else { + codepoint = 0xFFFD; + } + } else { + codepoint = 0xFFFD; + } + } else if (first >= 0xDC00 && first <= 0xDFFF) { + // Lone low surrogate. + codepoint = 0xFFFD; + } + appendUtf8(value, codepoint); + break; + } + default: failAt("Invalid escape sequence"); + } + continue; + } + if (c < 0x20) { + failAt("Unescaped control character in string"); + } + value.push_back(static_cast(c)); + ++position_; + } + JsonValue result; + result.type = JsonValue::Type::String; + result.string = std::move(value); + return result; + } + + JsonValue parseNumber() { + const size_t start = position_; + if (consume('-') && position_ >= text_.size()) { + failAt("Invalid JSON number"); + } + if (text_[position_] == '0') { + ++position_; + } else if (isDigit(text_[position_])) { + while (position_ < text_.size() && isDigit(text_[position_])) { + ++position_; + } + } else { + failAt("Invalid JSON number"); + } + if (position_ < text_.size() && text_[position_] == '.') { + ++position_; + if (position_ >= text_.size() || !isDigit(text_[position_])) { + failAt("Invalid JSON number fraction"); + } + while (position_ < text_.size() && isDigit(text_[position_])) { + ++position_; + } + } + if (position_ < text_.size() && (text_[position_] == 'e' || text_[position_] == 'E')) { + ++position_; + if (position_ < text_.size() && (text_[position_] == '+' || text_[position_] == '-')) { + ++position_; + } + if (position_ >= text_.size() || !isDigit(text_[position_])) { + failAt("Invalid JSON number exponent"); + } + while (position_ < text_.size() && isDigit(text_[position_])) { + ++position_; + } + } + const std::string token = text_.substr(start, position_ - start); + char* end = nullptr; + const double number = std::strtod(token.c_str(), &end); + if (!end || *end != '\0' || !std::isfinite(number)) { + failAt("Invalid JSON number"); + } + JsonValue result; + result.type = JsonValue::Type::Number; + result.number = number; + return result; + } + + const std::string& text_; + size_t position_ = 0; +}; + +const JsonValue* jsonObjectFind(const JsonValue& value, const std::string& key) { + if (value.type != JsonValue::Type::Object) { + return nullptr; + } + for (const auto& entry : value.object) { + if (entry.first == key) { + return &entry.second; + } + } + return nullptr; +} + +bool jsonHasString(const JsonValue& value, const std::string& key, std::string* out) { + const JsonValue* field = jsonObjectFind(value, key); + if (!field || field->type != JsonValue::Type::String) { + return false; + } + if (out) { + *out = field->string; + } + return true; +} + +bool jsonNumberField(const JsonValue& value, const std::string& key, double* out) { + const JsonValue* field = jsonObjectFind(value, key); + if (!field || field->type != JsonValue::Type::Number) { + return false; + } + *out = field->number; + return true; +} + +bool jsonIsSafeInteger(double value) { + return std::isfinite(value) && std::floor(value) == value && + std::fabs(value) <= 9007199254740991.0; // 2^53 - 1, mirrors Number.isSafeInteger. +} + +bool jsonSafeIntField( + const JsonValue& value, + const std::string& key, + int64_t* out, + int64_t minimum, + int64_t maximum) { + double number = 0.0; + if (!jsonNumberField(value, key, &number) || !jsonIsSafeInteger(number)) { + return false; + } + if (number < static_cast(minimum) || number > static_cast(maximum)) { + return false; + } + *out = static_cast(number); + return true; +} + +int64_t tiledOverlayTileCountForSize(int width, int height) { + const int columns = std::max(1, (width + kTiledOverlayTileSize - 1) / kTiledOverlayTileSize); + const int rows = std::max(1, (height + kTiledOverlayTileSize - 1) / kTiledOverlayTileSize); + return static_cast(columns) * static_cast(rows); +} + +// Conservative tiled-vs-raw eligibility heuristic mirrored from the TS side +// (resolveNativeTiledOverlayRawFallbackReason). Returns "" when eligible. +std::string resolveTiledOverlayRawFallbackReason(const TiledOverlayLayerDescriptor& layer) { + if (layer.tileCount < kTiledOverlayMinTileCount) { + return "small-layer"; + } + for (const auto& delta : layer.frameDeltas) { + if (static_cast(delta.changedTiles.size()) > + static_cast(layer.tileCount) * kTiledOverlayMaxChangedTileFraction) { + return "dense-frame-delta"; + } + } + int64_t changedCount = 0; + for (const auto& delta : layer.frameDeltas) { + changedCount += static_cast(delta.changedTiles.size()); + } + const int64_t uploadedTileBytes = (layer.tileCount + changedCount) * layer.tileByteSize; + const int64_t rawPhysicalBytes = + static_cast(layer.width) * static_cast(layer.height) * 4LL * + static_cast(layer.frameCount); + if (rawPhysicalBytes > 0 && + static_cast(uploadedTileBytes) / + static_cast(rawPhysicalBytes) >= + kTiledOverlayMaxPayloadBytesFraction) { + return "payload-bytes-exceed-raw"; + } + return ""; +} + +// Validates one tile record against the layer contract and fills out the +// record. Mirrors validateTiledOverlayTileRecord on the TS side: tileIndex in +// [0, tileCount), byteLength must be exactly tileSize^2*4, and the range must +// stay inside the bounded payload stream. +bool parseTiledOverlayTileRecord( + const JsonValue& record, + const std::string& layerId, + int64_t tileCount, + int64_t payloadByteLength, + TiledOverlayTileRecord* out) { + if (record.type != JsonValue::Type::Object) { + return false; + } + int64_t tileIndex = 0; + int64_t byteOffset = 0; + int64_t byteLength = 0; + if (!jsonSafeIntField(record, "tileIndex", &tileIndex, 0, 1LL << 40) || + !jsonSafeIntField(record, "byteOffset", &byteOffset, 0, 1LL << 50) || + !jsonSafeIntField(record, "byteLength", &byteLength, 0, 1LL << 50)) { + return false; + } + if (tileIndex >= tileCount || byteLength != kTiledOverlayTileByteSize || + byteOffset + byteLength > payloadByteLength) { + return false; + } + out->tileIndex = static_cast(tileIndex); + out->byteOffset = byteOffset; + out->byteLength = byteLength; + return true; +} + +// Loads and validates the version-1 tiled overlay storage descriptor from +// manifestPath. Mirrors readTiledOverlayManifest/validateNativeTiledOverlay +// (Storage|Layer)Descriptor on the TS side so the CUDA helper never trusts an +// opaque blob: every layer, tile record, and payload range is checked and +// malformed/truncated/unsupported descriptors fail with an actionable message +// (surfaced as a JSON failure by main). outputWidth/outputHeight/fps/duration +// are the helper's resolved values (0 = not yet known); the manifest's own +// top-level fields are always authoritative for layer-bounds validation. +std::vector loadTiledOverlayManifest( + const std::string& manifestPath, + int outputWidth, + int outputHeight, + int fps, + double durationSec) { + if (manifestPath.empty()) { + return {}; + } + + std::ifstream manifestFile(manifestPath, std::ios::binary); + if (!manifestFile) { + fail("Tiled overlay manifest does not exist: " + manifestPath); + } + std::ostringstream buffer; + buffer << manifestFile.rdbuf(); + if (manifestFile.bad()) { + fail("Failed to read tiled overlay manifest: " + manifestPath); + } + const std::string text = buffer.str(); + + JsonValue root; + try { + root = JsonParser(text).parse(); + } catch (const std::exception& error) { + fail("Invalid tiled overlay manifest " + manifestPath + ": " + error.what()); + } + + int64_t version = 0; + if (!jsonSafeIntField(root, "version", &version, 0, 1 << 30)) { + fail("Tiled overlay manifest requires a version: " + manifestPath); + } + if (version != kTiledOverlayStorageVersion) { + fail( + "Unsupported tiled overlay storage version " + std::to_string(version) + ": " + + manifestPath); + } + + int64_t manifestWidth = 0; + int64_t manifestHeight = 0; + double manifestFrameRate = 0.0; + double manifestDurationSec = 0.0; + if (!jsonSafeIntField(root, "outputWidth", &manifestWidth, 1, 1 << 20) || + !jsonSafeIntField(root, "outputHeight", &manifestHeight, 1, 1 << 20)) { + fail("Tiled overlay manifest requires positive output dimensions: " + manifestPath); + } + if (outputWidth > 0 && (manifestWidth != outputWidth || manifestHeight != outputHeight)) { + fail("Tiled overlay storage dimensions do not match the output: " + manifestPath); + } + if (!jsonNumberField(root, "frameRate", &manifestFrameRate) || + !std::isfinite(manifestFrameRate) || manifestFrameRate <= 0.0) { + fail("Tiled overlay manifest requires a positive frame rate: " + manifestPath); + } + if (fps > 0 && std::fabs(manifestFrameRate - static_cast(fps)) > 0.01) { + fail("Tiled overlay storage frame rate does not match the output: " + manifestPath); + } + if (!jsonNumberField(root, "durationSec", &manifestDurationSec) || + !std::isfinite(manifestDurationSec) || manifestDurationSec <= 0.0) { + fail("Tiled overlay manifest requires a positive duration: " + manifestPath); + } + if (durationSec > 0.0 && + std::fabs(manifestDurationSec - durationSec) > 1.0 / manifestFrameRate) { + fail("Tiled overlay storage duration does not match the output: " + manifestPath); + } + + const JsonValue* rawLayers = jsonObjectFind(root, "layers"); + if (!rawLayers || rawLayers->type != JsonValue::Type::Array) { + fail("Tiled overlay manifest requires a layers array: " + manifestPath); + } + + std::vector layers; + layers.reserve(rawLayers->array.size()); + int previousOrder = -1; + std::string previousId; + for (const JsonValue& rawLayer : rawLayers->array) { + TiledOverlayLayerDescriptor layer; + if (!jsonHasString(rawLayer, "id", &layer.id) || layer.id.empty() || + !jsonHasString(rawLayer, "payloadPath", &layer.payloadPath) || + layer.payloadPath.empty()) { + fail("Tiled overlay layer requires an id and payload path: " + manifestPath); + } + int64_t order = 0; + int64_t x = 0; + int64_t y = 0; + int64_t width = 0; + int64_t height = 0; + int64_t frameCount = 0; + int64_t tileSize = 0; + int64_t payloadByteLength = 0; + if (!jsonSafeIntField(rawLayer, "order", &order, 0, 1 << 30) || + !jsonSafeIntField(rawLayer, "x", &x, 0, 1 << 20) || + !jsonSafeIntField(rawLayer, "y", &y, 0, 1 << 20) || + !jsonSafeIntField(rawLayer, "width", &width, 1, 1 << 20) || + !jsonSafeIntField(rawLayer, "height", &height, 1, 1 << 20) || + !jsonSafeIntField(rawLayer, "frameCount", &frameCount, 1, 1 << 30)) { + fail("Invalid tiled overlay layer " + layer.id + ": " + manifestPath); + } + if (x + width > manifestWidth || y + height > manifestHeight) { + fail("Tiled overlay layer " + layer.id + " exceeds the output canvas: " + manifestPath); + } + if (!jsonNumberField(rawLayer, "frameRate", &layer.frameRate) || + !std::isfinite(layer.frameRate) || layer.frameRate <= 0.0 || + std::fabs(layer.frameRate - manifestFrameRate) > 0.01) { + fail("Tiled overlay layer " + layer.id + " has an incompatible frame rate: " + manifestPath); + } + if (!jsonNumberField(rawLayer, "durationSec", &layer.durationSec) || + !std::isfinite(layer.durationSec) || layer.durationSec <= 0.0 || + std::fabs(layer.durationSec - manifestDurationSec) > 1.0 / manifestFrameRate) { + fail("Tiled overlay layer " + layer.id + " has an incompatible duration: " + manifestPath); + } + const int64_t expectedFrameCount = + static_cast(std::ceil(layer.durationSec * layer.frameRate)); + if (frameCount < expectedFrameCount) { + fail("Tiled overlay layer " + layer.id + " does not contain enough frames: " + manifestPath); + } + if (!jsonSafeIntField(rawLayer, "tileSize", &tileSize, 1, 1 << 20) || + tileSize != kTiledOverlayTileSize) { + fail( + "Tiled overlay layer " + layer.id + " must use " + + std::to_string(kTiledOverlayTileSize) + "px tiles: " + manifestPath); + } + std::string pixelFormat; + if (!jsonHasString(rawLayer, "pixelFormat", &pixelFormat) || pixelFormat != "rgba") { + fail("Tiled overlay layer " + layer.id + " must use RGBA tiles: " + manifestPath); + } + if (!jsonSafeIntField(rawLayer, "payloadByteLength", &payloadByteLength, 0, 1LL << 50)) { + fail("Tiled overlay layer " + layer.id + " has an invalid payload byte length: " + manifestPath); + } + const JsonValue* staticTiles = jsonObjectFind(rawLayer, "staticTiles"); + const JsonValue* frameDeltas = jsonObjectFind(rawLayer, "frameDeltas"); + if (!staticTiles || staticTiles->type != JsonValue::Type::Array || + !frameDeltas || frameDeltas->type != JsonValue::Type::Array) { + fail("Tiled overlay layer " + layer.id + " requires static tiles and frame deltas: " + manifestPath); + } + if (order < previousOrder || (order == previousOrder && layer.id <= previousId)) { + fail("Tiled overlay layers must be sorted by order then id: " + manifestPath); + } + previousOrder = static_cast(order); + previousId = layer.id; + + layer.order = static_cast(order); + layer.x = static_cast(x); + layer.y = static_cast(y); + layer.width = static_cast(width); + layer.height = static_cast(height); + layer.frameCount = static_cast(frameCount); + layer.tileSize = static_cast(tileSize); + layer.payloadByteLength = payloadByteLength; + layer.tileColumns = std::max(1, (layer.width + kTiledOverlayTileSize - 1) / kTiledOverlayTileSize); + layer.tileRows = std::max(1, (layer.height + kTiledOverlayTileSize - 1) / kTiledOverlayTileSize); + layer.tileCount = static_cast(tiledOverlayTileCountForSize(layer.width, layer.height)); + layer.tileByteSize = static_cast(layer.tileSize) * layer.tileSize * 4LL; + layer.staticTiles.reserve(staticTiles->array.size()); + + std::set seenStaticTiles; + std::set> payloadRanges; + for (const JsonValue& record : staticTiles->array) { + TiledOverlayTileRecord parsed; + if (!parseTiledOverlayTileRecord( + record, + layer.id, + layer.tileCount, + layer.payloadByteLength, + &parsed)) { + fail("Tiled overlay layer " + layer.id + " has an invalid static tile record: " + manifestPath); + } + if (!seenStaticTiles.insert(parsed.tileIndex).second) { + fail("Tiled overlay layer " + layer.id + " emits duplicate static tile: " + manifestPath); + } + if (!payloadRanges.insert({parsed.byteOffset, parsed.byteLength}).second) { + fail("Tiled overlay layer " + layer.id + " duplicates tile payload bytes: " + manifestPath); + } + layer.staticTiles.push_back(parsed); + } + if (seenStaticTiles.size() != static_cast(layer.tileCount)) { + fail("Tiled overlay layer " + layer.id + " does not fully define the static tile base: " + manifestPath); + } + + layer.frameDeltas.reserve(frameDeltas->array.size()); + int64_t maxDeltaBytes = layer.tileByteSize; + int previousFrameIndex = -1; + for (const JsonValue& rawDelta : frameDeltas->array) { + TiledOverlayFrameDelta delta; + int64_t deltaFrameIndex = 0; + if (!jsonSafeIntField(rawDelta, "frameIndex", &deltaFrameIndex, 0, 1 << 30) || + deltaFrameIndex >= frameCount) { + fail("Tiled overlay layer " + layer.id + " has an invalid delta frame index: " + manifestPath); + } + if (deltaFrameIndex <= previousFrameIndex) { + fail("Tiled overlay layer " + layer.id + " has unsorted or duplicate delta frame indices: " + manifestPath); + } + previousFrameIndex = static_cast(deltaFrameIndex); + delta.frameIndex = static_cast(deltaFrameIndex); + const JsonValue* changedTiles = jsonObjectFind(rawDelta, "changedTiles"); + if (!changedTiles || changedTiles->type != JsonValue::Type::Array) { + fail("Tiled overlay layer " + layer.id + " requires a changedTiles array: " + manifestPath); + } + std::set seenDeltaTiles; + delta.changedTiles.reserve(changedTiles->array.size()); + for (const JsonValue& record : changedTiles->array) { + TiledOverlayTileRecord parsed; + if (!parseTiledOverlayTileRecord( + record, + layer.id, + layer.tileCount, + layer.payloadByteLength, + &parsed)) { + fail("Tiled overlay layer " + layer.id + " has an invalid changed tile record: " + manifestPath); + } + if (!seenDeltaTiles.insert(parsed.tileIndex).second) { + fail("Tiled overlay layer " + layer.id + " repeats a tile within a frame delta: " + manifestPath); + } + if (!payloadRanges.insert({parsed.byteOffset, parsed.byteLength}).second) { + fail("Tiled overlay layer " + layer.id + " duplicates tile payload bytes: " + manifestPath); + } + delta.changedTiles.push_back(parsed); + } + maxDeltaBytes = std::max( + maxDeltaBytes, + static_cast(delta.changedTiles.size()) * layer.tileByteSize); + layer.frameDeltas.push_back(std::move(delta)); + } + if (1 + static_cast(layer.frameDeltas.size()) > layer.frameCount) { + fail("Tiled overlay layer " + layer.id + " has more state versions than frames: " + manifestPath); + } + layer.maxDeltaBytes = maxDeltaBytes; + layer.rawFallbackReason = resolveTiledOverlayRawFallbackReason(layer); + + const std::string resolvedPayloadPath = layer.payloadPath; + std::ifstream payload(resolvedPayloadPath, std::ios::binary); + if (!payload) { + fail("Tiled overlay layer " + layer.id + " does not exist: " + resolvedPayloadPath); + } + payload.seekg(0, std::ios::end); + const std::streampos end = payload.tellg(); + if (end < 0 || static_cast(end) < layer.payloadByteLength) { + fail( + "Tiled overlay layer " + layer.id + " payload is truncated: expected at least " + + std::to_string(layer.payloadByteLength) + " bytes, received " + + std::to_string(end < 0 ? 0 : static_cast(end))); + } + layers.push_back(std::move(layer)); + } + return layers; +} + +// Loads and validates the version-1 raw RGBA overlay sidecar manifest. Mirrors +// overlayManifest.mjs / validateNativeStaticLayoutOverlayLayer: layers carry a +// global `order`, bounds, frameRate, duration, frameCount and optional +// effectiveFrameCount. The helper never trusts the blob and rejects malformed +// or truncated manifests with an actionable JSON failure. +std::vector loadOverlayManifest( + const std::string& manifestPath, + int outputWidth, + int outputHeight, + int fps, + double durationSec) { + if (manifestPath.empty()) { + return {}; + } + + std::ifstream manifestFile(manifestPath, std::ios::binary); + if (!manifestFile) { + fail("Overlay manifest does not exist: " + manifestPath); + } + std::ostringstream buffer; + buffer << manifestFile.rdbuf(); + if (manifestFile.bad()) { + fail("Failed to read overlay manifest: " + manifestPath); + } + const std::string rawText = buffer.str(); + + JsonValue root; + try { + root = JsonParser(rawText).parse(); + } catch (const std::exception& error) { + fail("Invalid overlay manifest " + manifestPath + ": " + error.what()); + } + + int64_t version = 0; + if (!jsonSafeIntField(root, "version", &version, 0, 1 << 30)) { + fail("Overlay manifest requires a version: " + manifestPath); + } + if (version != 1) { + fail("Unsupported overlay manifest version " + std::to_string(version) + ": " + manifestPath); + } + + int64_t manifestWidth = 0; + int64_t manifestHeight = 0; + double manifestFrameRate = 0.0; + double manifestDurationSec = 0.0; + if (!jsonSafeIntField(root, "outputWidth", &manifestWidth, 1, 1 << 20) || + !jsonSafeIntField(root, "outputHeight", &manifestHeight, 1, 1 << 20)) { + fail("Overlay manifest requires positive output dimensions: " + manifestPath); + } + if (outputWidth > 0 && (manifestWidth != outputWidth || manifestHeight != outputHeight)) { + fail("Overlay manifest dimensions do not match the output: " + manifestPath); + } + if (!jsonNumberField(root, "frameRate", &manifestFrameRate) || + !std::isfinite(manifestFrameRate) || manifestFrameRate <= 0.0) { + fail("Overlay manifest requires a positive frame rate: " + manifestPath); + } + if (fps > 0 && std::fabs(manifestFrameRate - static_cast(fps)) > 0.01) { + fail("Overlay manifest frame rate does not match the output: " + manifestPath); + } + if (!jsonNumberField(root, "durationSec", &manifestDurationSec) || + !std::isfinite(manifestDurationSec) || manifestDurationSec <= 0.0) { + fail("Overlay manifest requires a positive duration: " + manifestPath); + } + if (durationSec > 0.0 && + std::fabs(manifestDurationSec - durationSec) > 1.0 / manifestFrameRate) { + fail("Overlay manifest duration does not match the output: " + manifestPath); + } + + const JsonValue* rawLayers = jsonObjectFind(root, "layers"); + if (!rawLayers || rawLayers->type != JsonValue::Type::Array) { + fail("Overlay manifest requires a layers array: " + manifestPath); + } + + std::vector layers; + layers.reserve(rawLayers->array.size()); + int previousOrder = -1; + std::string previousId; + for (const JsonValue& rawLayer : rawLayers->array) { + OverlayLayerDescriptor layer; + if (!jsonHasString(rawLayer, "id", &layer.id) || layer.id.empty() || + !jsonHasString(rawLayer, "path", &layer.path) || layer.path.empty()) { + fail("Overlay layer requires an id and path: " + manifestPath); + } + int64_t order = 0; + int64_t x = 0; + int64_t y = 0; + int64_t width = 0; + int64_t height = 0; + int64_t frameCount = 0; + if (!jsonSafeIntField(rawLayer, "order", &order, 0, 1 << 30) || + !jsonSafeIntField(rawLayer, "x", &x, 0, 1 << 20) || + !jsonSafeIntField(rawLayer, "y", &y, 0, 1 << 20) || + !jsonSafeIntField(rawLayer, "width", &width, 1, 1 << 20) || + !jsonSafeIntField(rawLayer, "height", &height, 1, 1 << 20) || + !jsonSafeIntField(rawLayer, "frameCount", &frameCount, 1, 1 << 30)) { + fail("Invalid overlay layer " + layer.id + ": " + manifestPath); + } + if (x + width > manifestWidth || y + height > manifestHeight) { + fail("Overlay layer " + layer.id + " exceeds the output canvas: " + manifestPath); + } + if (!jsonNumberField(rawLayer, "frameRate", &layer.frameRate) || + !std::isfinite(layer.frameRate) || layer.frameRate <= 0.0 || + std::fabs(layer.frameRate - manifestFrameRate) > 0.01) { + fail("Overlay layer " + layer.id + " has an incompatible frame rate: " + manifestPath); + } + if (!jsonNumberField(rawLayer, "durationSec", &layer.durationSec) || + !std::isfinite(layer.durationSec) || layer.durationSec <= 0.0 || + std::fabs(layer.durationSec - manifestDurationSec) > 1.0 / manifestFrameRate) { + fail("Overlay layer " + layer.id + " has an incompatible duration: " + manifestPath); + } + const int64_t expectedFrameCount = + static_cast(std::ceil(layer.durationSec * layer.frameRate)); + if (frameCount < expectedFrameCount) { + fail("Overlay layer " + layer.id + " does not contain enough frames: " + manifestPath); + } + int64_t effectiveFrameCount = 0; + const JsonValue* effectiveField = jsonObjectFind(rawLayer, "effectiveFrameCount"); + if (effectiveField && effectiveField->type != JsonValue::Type::Null) { + if (!jsonSafeIntField(rawLayer, "effectiveFrameCount", &effectiveFrameCount, 1, frameCount)) { + fail("Overlay layer " + layer.id + " has an invalid effective frame count: " + manifestPath); + } + } else { + effectiveFrameCount = frameCount; + } + if (order < previousOrder || (order == previousOrder && layer.id <= previousId)) { + fail("Overlay layers must be sorted by order then id: " + manifestPath); + } + previousOrder = static_cast(order); + previousId = layer.id; + + layer.order = static_cast(order); + layer.x = static_cast(x); + layer.y = static_cast(y); + layer.width = static_cast(width); + layer.height = static_cast(height); + layer.frameCount = static_cast(frameCount); + layer.effectiveFrameCount = static_cast(effectiveFrameCount); + layers.push_back(std::move(layer)); + } + return layers; +} + +// Cross-checks the loaded tiled layers against the resolved output canvas. The +// descriptor's own outputWidth/outputHeight were validated at load; without +// explicit --width/--height the canvas is only known after decode, so this is +// called (like validateOverlayBounds) before the first encoded frame. +void validateTiledOverlayBounds(const Options& options, int outputWidth, int outputHeight) { + for (const auto& layer : options.tiledOverlayLayers) { + if (layer.x < 0 || layer.y < 0 || + layer.x + layer.width > outputWidth || + layer.y + layer.height > outputHeight) { + fail("Tiled overlay layer exceeds the output canvas: " + layer.id); + } + } +} + +// Descriptor-derived tiled throughput bookkeeping (same formulas as the TS +// resolveNativeTiledOverlayMetrics). Used for the failure summary, where the +// runtime-measured counters may not exist yet; diagnostics only. +struct TiledOverlayDerivedMetrics { + int64_t changedTileCount = 0; + int64_t uploadedTileBytes = 0; + int64_t cachedTileCount = 0; + std::string rawFallbackReason; +}; + +TiledOverlayDerivedMetrics computeTiledOverlayDerivedMetrics( + const std::vector& layers) { + TiledOverlayDerivedMetrics metrics; + for (const auto& layer : layers) { + int64_t changedCount = 0; + for (const auto& delta : layer.frameDeltas) { + changedCount += static_cast(delta.changedTiles.size()); + } + const int64_t uploadedCount = + static_cast(layer.staticTiles.size()) + changedCount; + metrics.changedTileCount += changedCount; + metrics.uploadedTileBytes += uploadedCount * layer.tileByteSize; + metrics.cachedTileCount += std::max( + 0, + static_cast(layer.tileCount) * layer.frameCount - uploadedCount); + if (metrics.rawFallbackReason.empty() && !layer.rawFallbackReason.empty()) { + metrics.rawFallbackReason = layer.rawFallbackReason; + } + } + return metrics; } Options parseOptions(int argc, char** argv) { @@ -167,6 +1222,8 @@ Options parseOptions(int argc, char** argv) { options.inputPath = requireValue("--input"); } else if (arg == "--output") { options.outputPath = requireValue("--output"); + } else if (arg == "--output-codec") { + options.outputCodec = parseOutputCodec(requireValue("--output-codec")); } else if (arg == "--source-pts") { options.sourcePtsPath = requireValue("--source-pts"); } else if (arg == "--width") { @@ -278,9 +1335,44 @@ Options parseOptions(int argc, char** argv) { parsePositiveInt(requireValue("--cursor-atlas-height"), "--cursor-atlas-height"); } else if (arg == "--zoom-samples") { options.zoomSamplesPath = requireValue("--zoom-samples"); + } else if (arg == "--temporal-blur-sample-count") { + options.temporalBlurSampleCount = + parsePositiveInt(requireValue("--temporal-blur-sample-count"), "--temporal-blur-sample-count"); + } else if (arg == "--temporal-blur-shutter-fraction") { + options.temporalBlurShutterFraction = parseFiniteDouble( + requireValue("--temporal-blur-shutter-fraction"), + "--temporal-blur-shutter-fraction"); + } else if (arg == "--temporal-blur-weight-power") { + options.temporalBlurWeightPower = parseFiniteDouble( + requireValue("--temporal-blur-weight-power"), + "--temporal-blur-weight-power"); + } else if (arg == "--overlay") { + OverlayLayerDescriptor layer; + layer.path = requireValue("--overlay"); + layer.x = parseNonNegativeInt(requireValue("--overlay"), "--overlay x"); + layer.y = parseNonNegativeInt(requireValue("--overlay"), "--overlay y"); + layer.width = parsePositiveInt(requireValue("--overlay"), "--overlay width"); + layer.height = parsePositiveInt(requireValue("--overlay"), "--overlay height"); + layer.effectiveFrameCount = + parsePositiveInt(requireValue("--overlay"), "--overlay frameCount"); + layer.frameCount = layer.effectiveFrameCount; + // Optional 7th argument is the renderer-side global z-order. + // Default to the current index so legacy callers still blend raw + // layers in the order they were supplied. + if (index + 1 < argc && argv[index + 1][0] != '-') { + layer.order = parseNonNegativeInt(argv[++index], "--overlay order"); + } else { + layer.order = static_cast(options.overlayLayers.size()); + } + options.overlayLayers.push_back(layer); + } else if (arg == "--overlay-manifest") { + options.overlayManifestPath = requireValue("--overlay-manifest"); + } else if (arg == "--tiled-overlay-manifest") { + options.tiledOverlayManifestPath = requireValue("--tiled-overlay-manifest"); } else if (arg == "--help") { std::cout << "Usage: recordly-nvidia-cuda-compositor --input input.annexb.h264 " - "[--output out.h264] [--source-pts source-pts.csv] [--width N --height N] [--fps 30] " + "[--output out.h264] [--output-codec h264|hevc] " + "[--source-pts source-pts.csv] [--width N --height N] [--fps 30] " "[--max-frames N] [--bitrate-mbps N] [--encoding-mode fast|balanced|quality] " "[--post-select] [--callback-encode] [--stream-sync] [--prewarm-ms N] [--chunk-mb N] " "[--content-x N --content-y N --content-width N --content-height N --radius N] " @@ -291,7 +1383,12 @@ Options parseOptions(int argc, char** argv) { "[--cursor-samples cursor.tsv --cursor-height N] " "[--cursor-atlas-rgba cursor.rgba --cursor-atlas-metadata cursor.tsv " "--cursor-atlas-width N --cursor-atlas-height N] " - "[--zoom-samples zoom.csv]\n"; + "[--zoom-samples zoom.csv] " + "[--temporal-blur-sample-count N --temporal-blur-shutter-fraction F " + "--temporal-blur-weight-power P] " + "[--overlay overlay.rgba x y width height frameCount [order]]... " + "[--overlay-manifest overlay-manifest.json] " + "[--tiled-overlay-manifest tiled-overlay-manifest.json]\n"; std::exit(0); } else { std::ostringstream stream; @@ -308,9 +1405,103 @@ Options parseOptions(int argc, char** argv) { if (options.width > 0 && (options.width % 2 != 0 || options.height % 2 != 0)) { fail("--width and --height must be even numbers for NV12 encoding"); } + for (const auto& layer : options.overlayLayers) { + if (layer.width <= 0 || layer.height <= 0 || layer.frameCount <= 0) { + fail("Invalid --overlay layer dimensions: " + layer.path); + } + } + if (options.temporalBlurSampleCount > 0) { + if (options.temporalBlurSampleCount < 3 || options.temporalBlurSampleCount > 61) { + fail("Invalid --temporal-blur-sample-count: " + + std::to_string(options.temporalBlurSampleCount)); + } + if (!std::isfinite(options.temporalBlurShutterFraction) || + options.temporalBlurShutterFraction < 0.18 || + options.temporalBlurShutterFraction > 3.0) { + fail("Invalid --temporal-blur-shutter-fraction"); + } + } + if (!options.tiledOverlayManifestPath.empty()) { + // Load + validate the tiled overlay descriptor before encoding starts: + // malformed/truncated/unsupported descriptors fail fast here instead of + // after decode work, and there is no silent raw fallback inside the + // helper. Duration cross-check uses the requested output timeline when + // targetFrames is known (the wrapper always passes it); layer bounds + // are validated against the descriptor's own output dimensions and + // re-cross-checked against the resolved canvas at sink creation. + const double expectedDurationSec = options.targetFrames > 0 + ? static_cast(options.targetFrames) / static_cast(options.fps) + : 0.0; + options.tiledOverlayLayers = loadTiledOverlayManifest( + options.tiledOverlayManifestPath, + options.width, + options.height, + options.fps, + expectedDurationSec); + } + if (!options.overlayManifestPath.empty()) { + const double expectedDurationSec = options.targetFrames > 0 + ? static_cast(options.targetFrames) / static_cast(options.fps) + : 0.0; + options.overlayLayers = loadOverlayManifest( + options.overlayManifestPath, + options.width, + options.height, + options.fps, + expectedDurationSec); + } + + // Build one global z-order list from raw and tiled overlay layers. The + // renderer already sorted each manifest by (order, id), so a stable sort + // on order produces the exact cross-kind ordering the renderer expects. + options.compositeLayers.reserve( + options.overlayLayers.size() + options.tiledOverlayLayers.size()); + for (size_t i = 0; i < options.overlayLayers.size(); ++i) { + const auto& layer = options.overlayLayers[i]; + CompositeLayer entry; + entry.kind = CompositeLayer::Kind::Raw; + entry.sourceIndex = static_cast(i); + entry.order = layer.order; + entry.x = layer.x; + entry.y = layer.y; + entry.width = layer.width; + entry.height = layer.height; + options.compositeLayers.push_back(entry); + } + for (size_t i = 0; i < options.tiledOverlayLayers.size(); ++i) { + const auto& layer = options.tiledOverlayLayers[i]; + CompositeLayer entry; + entry.kind = CompositeLayer::Kind::Tiled; + entry.sourceIndex = static_cast(i); + entry.order = layer.order; + entry.x = layer.x; + entry.y = layer.y; + entry.width = layer.width; + entry.height = layer.height; + options.compositeLayers.push_back(entry); + } + std::stable_sort( + options.compositeLayers.begin(), + options.compositeLayers.end(), + [](const CompositeLayer& a, const CompositeLayer& b) { return a.order < b.order; }); + return options; } +// The overlay canvas bounds depend on the output dimensions. With explicit +// --width/--height they are known at parse time; without them the canvas is +// resolved from the decoded source, so the caller validates with the resolved +// dimensions before the first frame is encoded. +void validateOverlayBounds(const Options& options, int outputWidth, int outputHeight) { + for (const auto& layer : options.overlayLayers) { + if (layer.x < 0 || layer.y < 0 || + layer.x + layer.width > outputWidth || + layer.y + layer.height > outputHeight) { + fail("Overlay layer exceeds the output canvas: " + layer.id + ": " + layer.path); + } + } +} + bool shouldEncodeFrame(int sourceFrameIndex, int encodedFrames, const Options& options) { if (options.inputFrames <= 0 || options.targetFrames <= 0) { return true; @@ -578,6 +1769,10 @@ struct ProgressCounters { double decodeWallMs = 0.0; double encodeMs = 0.0; double compositeMs = 0.0; + double compositeGpuMs = 0.0; + double zoomBlurGpuMs = 0.0; + double overlayBlendGpuMs = 0.0; + double overlayUploadMs = 0.0; double nvencMs = 0.0; double packetWriteMs = 0.0; double webcamDecodeMs = 0.0; @@ -585,11 +1780,34 @@ struct ProgressCounters { int roiCompositeFrames = 0; int monolithicCompositeFrames = 0; int copyCompositeFrames = 0; + int zoomBlurFrames = 0; + int overlayBlendFrames = 0; + int temporalBlurFrames = 0; + int64_t temporalBlurSamplesTotal = 0; + int temporalBlurBgPrecomposedFrames = 0; + int temporalBlurStationaryFrames = 0; + int temporalBgCacheBuilds = 0; + int64_t temporalBgCacheHits = 0; + int64_t overlayStaticRegionBlends = 0; + int64_t overlayFileLoads = 0; + int64_t overlayCacheHits = 0; + int64_t overlayPinnedHits = 0; + int64_t overlayReadWaits = 0; + int64_t overlayPendingReadsPeak = 0; + double overlayHostReadMs = 0.0; + double overlayH2DEnqueueMs = 0.0; + // Tiled/delta overlay stream (additive, measured). cumulative over the + // whole run; the interval fields below are deltas between reports. + int tiledOverlayLayers = 0; + int64_t changedTileCount = 0; + int64_t uploadedTileBytes = 0; + int64_t cachedTileCount = 0; }; struct ProgressReportState { std::chrono::steady_clock::time_point startedAt; std::chrono::steady_clock::time_point lastReportAt; + const char* outputCodec = "h264"; int lastReportedFrame = 0; ProgressCounters lastCounters; }; @@ -773,6 +1991,14 @@ struct ZoomSample { double scale = 1.0; double x = 0.0; double y = 0.0; + // Renderer-equivalent radial zoom-blur parameters for the step that ends at + // this sample. blurStrength is the ZoomBlurFilter strength (0 = no blur); + // the center is in output pixels. The JS side computes these from the same + // camera-step analysis the interactive renderer uses, so the native + // compositor reproduces the spatial zoom blur without re-deriving it. + double blurStrength = 0.0; + double blurCenterX = 0.0; + double blurCenterY = 0.0; }; struct ZoomTrack { @@ -813,48 +2039,932 @@ struct ZoomTrack { left.scale + (right.scale - left.scale) * t, left.x + (right.x - left.x) * t, left.y + (right.y - left.y) * t, + left.blurStrength + (right.blurStrength - left.blurStrength) * t, + left.blurCenterX + (right.blurCenterX - left.blurCenterX) * t, + left.blurCenterY + (right.blurCenterY - left.blurCenterY) * t, }; } }; +struct TemporalBlurSample { + double offsetUs = 0.0; + double weight = 0.0; +}; + +// Mirrors buildTemporalSamplePlanUs from src/lib/exporter/temporalMotionBlur.ts: +// symmetric shutter window centered on the frame, cos-tapered weights normalized +// to sum to 1. The weight floor (0.22) and taper are part of the renderer's +// contract; the compositor must reproduce them so native output matches the +// configured high-level temporal sample plan. +std::vector buildTemporalSamplePlan( + int sampleCount, + double shutterFraction, + double weightCurvePower, + double frameDurationUs) { + const int safeSampleCount = std::max(1, sampleCount); + if (safeSampleCount <= 1) { + return {{0.0, 1.0}}; + } + + const double shutterWindowUs = + std::max(1.0, frameDurationUs) * std::max(0.0, std::min(3.0, shutterFraction)); + const double startOffsetUs = -shutterWindowUs / 2.0; + const double stepUs = shutterWindowUs / static_cast(safeSampleCount - 1); + std::vector offsetsUs; + offsetsUs.reserve(safeSampleCount); + for (int index = 0; index < safeSampleCount; ++index) { + offsetsUs.push_back(startOffsetUs + stepUs * static_cast(index)); + } + + constexpr double kWeightFloor = 0.22; + const double centerIndex = static_cast(safeSampleCount - 1) / 2.0; + std::vector rawWeights; + rawWeights.reserve(safeSampleCount); + double totalWeight = 0.0; + for (int index = 0; index < safeSampleCount; ++index) { + const double normalizedDistance = + std::abs(static_cast(index) - centerIndex) / std::max(1.0, centerIndex); + const double taperedWeight = std::cos(normalizedDistance * (3.14159265358979323846 / 2.0)); + const double rawWeight = + kWeightFloor + + (1.0 - kWeightFloor) * + std::pow(std::max(0.0, taperedWeight), weightCurvePower); + rawWeights.push_back(rawWeight); + totalWeight += rawWeight; + } + + std::vector samples; + samples.reserve(safeSampleCount); + for (int index = 0; index < safeSampleCount; ++index) { + samples.push_back({ + offsetsUs[index], + totalWeight > 0.0 ? rawWeights[index] / totalWeight : 1.0 / safeSampleCount, + }); + } + return samples; +} + std::unique_ptr loadZoomTrack(const Options& options) { if (options.zoomSamplesPath.empty()) { return nullptr; } - std::ifstream input(options.zoomSamplesPath); - if (!input) { - fail("Failed to open zoom samples: " + options.zoomSamplesPath); + std::ifstream input(options.zoomSamplesPath); + if (!input) { + fail("Failed to open zoom samples: " + options.zoomSamplesPath); + } + + auto track = std::make_unique(); + std::string line; + while (std::getline(input, line)) { + if (line.empty()) { + continue; + } + std::replace(line.begin(), line.end(), ',', ' '); + std::istringstream row(line); + ZoomSample sample; + if (!(row >> sample.timeMs >> sample.scale >> sample.x >> sample.y)) { + continue; + } + if (!std::isfinite(sample.timeMs) || !std::isfinite(sample.scale) || + !std::isfinite(sample.x) || !std::isfinite(sample.y)) { + continue; + } + // Optional renderer-computed zoom-blur fields (columns 5-7). Older + // telemetry files with only timeMs/scale/x/y keep blurStrength = 0. + if (!(row >> sample.blurStrength)) { + sample.blurStrength = 0.0; + } else if (!std::isfinite(sample.blurStrength)) { + sample.blurStrength = 0.0; + } + if (!(row >> sample.blurCenterX)) { + sample.blurCenterX = 0.0; + } else if (!std::isfinite(sample.blurCenterX)) { + sample.blurCenterX = 0.0; + } + if (!(row >> sample.blurCenterY)) { + sample.blurCenterY = 0.0; + } else if (!std::isfinite(sample.blurCenterY)) { + sample.blurCenterY = 0.0; + } + sample.timeMs = std::max(0.0, sample.timeMs); + sample.scale = std::max(0.01, sample.scale); + sample.blurStrength = std::max(0.0, sample.blurStrength); + track->samples.push_back(sample); + } + if (track->samples.empty()) { + fail("No zoom samples were loaded: " + options.zoomSamplesPath); + } + std::sort(track->samples.begin(), track->samples.end(), [](const auto& left, const auto& right) { + return left.timeMs < right.timeMs; + }); + return track; +} + +// Blend launch rectangle for a renderer-prepared RGBA overlay layer. Dynamic +// (multi-frame) layers always use the full layer rect; physical single-frame +// layers get a one-time alpha bound (see computeStaticAlphaBounds) so the blend +// kernel only visits pixels that can write, with the full rect as the fallback. +struct OverlayBlendRegion { + int x = 0; + int y = 0; + int width = 0; + int height = 0; + bool bounded = false; +}; + +OverlayBlendRegion fullOverlayBlendRegion(const OverlayLayerDescriptor& descriptor) { + OverlayBlendRegion region; + region.x = 0; + region.y = 0; + region.width = descriptor.width; + region.height = descriptor.height; + region.bounded = false; + return region; +} + +// Scans the first frame of a physical single-frame overlay layer for the +// bounding box of pixels with nonzero alpha, expanded by one pixel so every 2x2 +// chroma block that touches an alpha pixel is inside the launch region. The +// bound is computed once per layer; pixels outside it have alpha == 0 for the +// whole layer, so the blend kernel writes nothing there and the bounded launch +// is bit-identical to the full-frame blend. A fully transparent layer gets an +// empty region (the blend launch is skipped entirely, which is also exact). +void computeStaticAlphaBounds( + const unsigned char* rgba, + int width, + int height, + OverlayBlendRegion& region) { + int minX = width; + int minY = height; + int maxX = -1; + int maxY = -1; + for (int y = 0; y < height; ++y) { + const unsigned char* row = + rgba + static_cast(y) * static_cast(width) * 4; + for (int x = 0; x < width; ++x) { + if (row[x * 4 + 3] > 0) { + minX = std::min(minX, x); + minY = std::min(minY, y); + maxX = std::max(maxX, x); + maxY = std::max(maxY, y); + } + } + } + if (maxX < minX) { + region = {0, 0, 0, 0, true}; + return; + } + region.x = std::max(0, minX - 1); + region.y = std::max(0, minY - 1); + region.width = std::min(width, maxX + 2) - region.x; + region.height = std::min(height, maxY + 2) - region.y; + region.bounded = true; +} + +// Streaming source for renderer-prepared transparent RGBA overlay sidecars. +// Frames are raw top-down RGBA and are consumed sequentially by the output +// frame index. Each dynamic layer owns a bounded background reader thread that +// reads sidecar frames from disk into persistent pinned ring buffers while the +// encode thread keeps running, so the encode loop never blocks on file I/O. +// The encode thread only enqueues H2D copies (pinned -> device) on the +// compositor stream, ordered ahead of the blend kernels on the same stream, so +// the single per-frame cudaStreamSynchronize stays sufficient. The 4-slot ring +// semantics are unchanged: frame ordering, tail-repeat clamping, the read-once +// static cache, and bounded memory are all preserved. +class OverlayFrameSource { +public: + explicit OverlayFrameSource(const std::vector& layers) { + layers_.reserve(layers.size()); + for (const auto& descriptor : layers) { + std::unique_ptr layer = loadLayer(descriptor); + if (layer->staticLayer) { + // The constructor read of the static layer's single frame is a + // disk read with an upload (synchronous), so it counts as a + // file load like the streaming path counts its reads. + ++fileLoads_; + } + layers_.push_back(std::move(layer)); + } + // Start one reader thread per dynamic layer only after every layer is + // fully built (vector is stable and static frames are staged), so a + // reader can never observe a partially initialized layer. + for (auto& layer : layers_) { + if (!layer->staticLayer) { + startReader(*layer); + } + } + } + + ~OverlayFrameSource() { + // Stop and join every reader before freeing the pinned buffers the + // readers write into. Reader threads never touch CUDA, so joining is + // safe while the primary context is current. + for (auto& layer : layers_) { + stopReader(*layer); + } + for (auto& layer : layers_) { + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + if (layer->deviceFrames[slot]) { + cudaFree(layer->deviceFrames[slot]); + } + if (layer->pinnedFrames[slot]) { + cudaFreeHost(layer->pinnedFrames[slot]); + } + } + } + } + + bool empty() const { + return layers_.empty(); + } + + size_t layerCount() const { + return layers_.size(); + } + + // Total streaming-path overlay time (background host reads + H2D + // enqueues). Static constructor staging is intentionally excluded, matching + // the pre-background-reader semantics of uploadMs. + double uploadMs() const { + return hostReadMs() + h2dEnqueueMs(); + } + + // Wall time the background reader threads spent reading sidecar bytes from + // disk (not the H2D transfer time). + double hostReadMs() const { + return static_cast(hostReadUs_.load()) / 1000.0; + } + + // Wall time the encode thread spent enqueuing H2D copies (cudaMemcpyAsync + // API calls) on the compositor stream. + double h2dEnqueueMs() const { + return h2dEnqueueMs_; + } + + // Number of overlay frames read from disk (one per unique requested frame; + // static single-frame layers count their one constructor read). + int64_t fileLoads() const { + return fileLoads_.load(); + } + + // Number of times a requested overlay frame was already device-resident in + // its ring slot, so neither a file read nor an H2D copy was needed. Static + // single-frame layers, tail-repeated frames, and read-ahead frames all + // count here. + int64_t cacheHits() const { + return cacheHits_; + } + + // Number of times a requested overlay frame was already in pinned memory + // (the background reader had finished the file read) and only the H2D + // enqueue was needed. + int64_t pinnedHits() const { + return pinnedHits_; + } + + // Number of times the encode thread had to wait for the background reader + // to finish a file read before it could enqueue the H2D copy. + int64_t readWaits() const { + return readWaits_; + } + + // Peak depth of the bounded background-reader queue across all layers. + int64_t pendingReadsPeak() const { + return pendingReadsPeak_.load(); + } + + const OverlayLayerDescriptor& descriptor(size_t index) const { + return layers_[index]->descriptor; + } + + // Blend launch rectangle for the layer. Dynamic (multi-frame) layers return + // the full layer rect; physical single-frame layers return the one-time + // alpha bound (or an empty rect for a fully transparent layer). + OverlayBlendRegion blendRegion(size_t index) const { + return layers_[index]->blendRegion; + } + + // Prepares the overlay frame for the given output frame index for every + // layer. Call this before launching the blend kernels: it waits (host-side) + // only when the background reader has not finished the requested frame, + // then enqueues the H2D copy on the compositor stream so blends stay + // ordered. Slots are keyed by the clamped frame index inside a small + // bounded ring, so a single-frame layer or a tail-repeated last frame is + // read from disk once and served from its device slot for every following + // output frame. This never syncs the compositor stream; the encode loop + // keeps its single per-frame cudaStreamSynchronize. + void beginFrame(int outputFrameIndex, cudaStream_t copyStream) { + if (layers_.empty()) { + return; + } + + for (size_t index = 0; index < layers_.size(); ++index) { + auto& layer = *layers_[index]; + const int frameIndex = clampedFrameIndex(layer, outputFrameIndex); + const int slot = slotFor(frameIndex); + if (layer.staticLayer) { + // Static layers are fully staged in the constructor; slot 0 is + // always device-resident for frame 0. + ++cacheHits_; + continue; + } + waitForFrame(layer, frameIndex, slot, copyStream); + } + } + + const unsigned char* frameDevicePtr(size_t layerIndex, int outputFrameIndex) const { + const auto& layer = *layers_[layerIndex]; + return layer.deviceFrames[slotFor(clampedFrameIndex(layer, outputFrameIndex))]; + } + + // Must be called after beginFrame + the blend kernels are queued (after the + // per-frame stream sync). Dispatches bounded background reads for the next + // overlay frames so the following output frames do not stall on file I/O; + // when a read-ahead frame's pinned data is already available it also + // enqueues the H2D copy immediately so the transfer overlaps NVENC. + // Read-ahead depth is bounded by the ring size minus one and never targets + // the slot the current blend is reading, so the pipeline stays ordered with + // bounded memory. + void prefetchNextFrame(int outputFrameIndex, cudaStream_t copyStream) { + if (layers_.empty()) { + return; + } + + for (size_t index = 0; index < layers_.size(); ++index) { + auto& layer = *layers_[index]; + if (layer.staticLayer) { + continue; + } + const int currentFrameIndex = clampedFrameIndex(layer, outputFrameIndex); + const int currentSlot = slotFor(currentFrameIndex); + for (int depth = 1; depth <= kOverlayPrefetchDepth; ++depth) { + const int frameIndex = clampedFrameIndex(layer, outputFrameIndex + depth); + if (frameIndex == currentFrameIndex || slotFor(frameIndex) == currentSlot) { + continue; + } + requestRead(layer, frameIndex, slotFor(frameIndex), copyStream); + } + } + } + +private: + enum class SlotState { + Empty, + Reading, + PinnedReady, + DeviceReady, + }; + + struct LoadedLayer { + OverlayLayerDescriptor descriptor; + size_t frameBytes = 0; + std::ifstream input; + unsigned char* deviceFrames[kOverlayPrefetchSlots] = {}; + unsigned char* pinnedFrames[kOverlayPrefetchSlots] = {}; + int loadedSlots[kOverlayPrefetchSlots] = {}; + SlotState slotStates[kOverlayPrefetchSlots] = {}; + OverlayBlendRegion blendRegion; + bool staticLayer = false; + // Bounded background reader state (dynamic layers only). The pending + // queue never holds more than one entry per ring slot, so it is bounded + // by kOverlayPrefetchSlots; the reader thread is the only accessor of + // input and pinnedFrames outside the constructor. + std::mutex mutex; + std::condition_variable cv; + std::deque> pendingReads; + bool stop = false; + bool readerStarted = false; + std::thread readerThread; + std::string readError; + }; + + static int clampedFrameIndex(const LoadedLayer& layer, int outputFrameIndex) { + return std::min(outputFrameIndex, std::max(0, layer.descriptor.effectiveFrameCount - 1)); + } + + static int slotFor(int frameIndex) { + return frameIndex % kOverlayPrefetchSlots; + } + + static std::unique_ptr loadLayer(const OverlayLayerDescriptor& descriptor) { + std::unique_ptr layer = std::make_unique(); + layer->descriptor = descriptor; + layer->frameBytes = static_cast(descriptor.width) * + static_cast(descriptor.height) * 4; + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + layer->loadedSlots[slot] = -1; + layer->slotStates[slot] = SlotState::Empty; + } + + layer->input.open(descriptor.path, std::ios::binary); + if (!layer->input) { + fail("Failed to open overlay layer: " + descriptor.path); + } + layer->input.seekg(0, std::ios::end); + const std::streampos end = layer->input.tellg(); + layer->input.seekg(0, std::ios::beg); + if (end < 0 || + static_cast(end) < layer->frameBytes * static_cast(descriptor.effectiveFrameCount)) { + fail("Overlay layer is truncated: " + descriptor.id + ": " + descriptor.path); + } + + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + checkCuda(cudaMalloc(&layer->deviceFrames[slot], layer->frameBytes), "cudaMalloc overlay frame"); + checkCuda(cudaMallocHost(&layer->pinnedFrames[slot], layer->frameBytes), "cudaMallocHost overlay frame"); + } + + // Physical single-frame layers are invariant for the whole export: read + // the single frame once, compute the alpha bounds once, and stage the + // device copy now so beginFrame serves it from slot 0 without a second + // file read. The ring is keyed by the clamped frame index, which is + // always 0 for an effectiveFrameCount == 1 layer, so slot 0 stays valid forever. + layer->staticLayer = descriptor.effectiveFrameCount == 1; + layer->blendRegion = fullOverlayBlendRegion(descriptor); + if (layer->staticLayer) { + layer->input.seekg(0, std::ios::beg); + layer->input.read( + reinterpret_cast(layer->pinnedFrames[0]), + static_cast(layer->frameBytes)); + if (static_cast(layer->input.gcount()) != layer->frameBytes) { + fail("Failed to read overlay frame 0: " + descriptor.path); + } + computeStaticAlphaBounds( + layer->pinnedFrames[0], + descriptor.width, + descriptor.height, + layer->blendRegion); + // Static staging (read + upload) is intentionally not timed: it is + // a one-time constructor cost and the streaming-path timing metrics + // (hostReadMs/h2dEnqueueMs) exclude it, matching the historical + // uploadMs semantics. + checkCuda( + cudaMemcpy( + layer->deviceFrames[0], + layer->pinnedFrames[0], + layer->frameBytes, + cudaMemcpyHostToDevice), + "cudaMemcpy overlay static frame 0"); + layer->loadedSlots[0] = 0; + layer->slotStates[0] = SlotState::DeviceReady; + } + return layer; + } + + void startReader(LoadedLayer& layer) { + std::unique_lock lock(layer.mutex); + layer.readerStarted = true; + layer.readerThread = std::thread(&OverlayFrameSource::readerLoop, this, &layer); + } + + void stopReader(LoadedLayer& layer) { + { + std::unique_lock lock(layer.mutex); + layer.stop = true; + } + layer.cv.notify_all(); + if (layer.readerStarted && layer.readerThread.joinable()) { + layer.readerThread.join(); + } + } + + // Background reader main loop: pops the oldest queued (slot, frameIndex) + // read, performs the file read into the persistent pinned buffer, and + // publishes the PinnedReady state. The queue is bounded (one entry per ring + // slot) and the loop never touches CUDA, so cancellation is a simple stop + // flag + join; a read failure is captured and re-thrown on the encode + // thread at the next beginFrame. + void readerLoop(LoadedLayer* layer) { + while (true) { + std::pair request; + { + std::unique_lock lock(layer->mutex); + layer->cv.wait(lock, [&] { + return layer->stop || !layer->pendingReads.empty(); + }); + if (layer->stop) { + return; + } + request = layer->pendingReads.front(); + layer->pendingReads.pop_front(); + } + readFrameIntoPinned(*layer, request.first, request.second); + } + } + + void readFrameIntoPinned(LoadedLayer& layer, int slot, int frameIndex) { + const auto readStart = std::chrono::steady_clock::now(); + try { + layer.input.seekg( + static_cast(layer.frameBytes * static_cast(frameIndex)), + std::ios::beg); + layer.input.read( + reinterpret_cast(layer.pinnedFrames[slot]), + static_cast(layer.frameBytes)); + if (static_cast(layer.input.gcount()) != layer.frameBytes) { + throw std::runtime_error( + "Failed to read overlay frame " + std::to_string(frameIndex) + ": " + + layer.descriptor.path); + } + } catch (const std::exception& error) { + std::unique_lock lock(layer.mutex); + layer.readError = error.what(); + layer.stop = true; + layer.cv.notify_all(); + return; + } + hostReadUs_ += static_cast(elapsedMs(readStart, std::chrono::steady_clock::now()) * 1000.0); + ++fileLoads_; + { + std::unique_lock lock(layer.mutex); + layer.loadedSlots[slot] = frameIndex; + layer.slotStates[slot] = SlotState::PinnedReady; + layer.cv.notify_all(); + } + } + + // Queues a background read for (slot, frameIndex) unless one is already in + // flight/queued for that slot. A newer request supersedes a stale queued + // target for the same slot (the older frame's blend already consumed its + // device data, so overwriting the pinned buffer is safe). The queue is + // bounded to one entry per ring slot; if it is full the request is dropped + // and the caller's wait loop retries once the reader drains an entry. + // Must be called with layer.mutex held. + void requestReadLocked(LoadedLayer& layer, int frameIndex, int slot) { + if (layer.slotStates[slot] == SlotState::Reading && + layer.loadedSlots[slot] == frameIndex) { + return; + } + for (auto& entry : layer.pendingReads) { + if (entry.first == slot) { + if (entry.second != frameIndex) { + entry.second = frameIndex; + } + layer.cv.notify_one(); + return; + } + } + if (layer.pendingReads.size() >= static_cast(kOverlayPrefetchSlots)) { + return; + } + layer.pendingReads.push_back({slot, frameIndex}); + pendingReadsPeak_.store( + std::max(pendingReadsPeak_.load(), static_cast(layer.pendingReads.size()))); + layer.slotStates[slot] = SlotState::Reading; + layer.loadedSlots[slot] = frameIndex; + layer.cv.notify_one(); + } + + // Non-blocking read-ahead request (prefetch path): queues the background + // read and, when the pinned data is already available, enqueues the H2D + // copy immediately so it overlaps NVENC instead of the next beginFrame. + void requestRead(LoadedLayer& layer, int frameIndex, int slot, cudaStream_t copyStream) { + std::unique_lock lock(layer.mutex); + if (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex) { + ++cacheHits_; + return; + } + if (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) { + ++pinnedHits_; + lock.unlock(); + enqueueH2D(layer, slot, copyStream); + lock.lock(); + layer.slotStates[slot] = SlotState::DeviceReady; + return; + } + requestReadLocked(layer, frameIndex, slot); + } + + // Ensures the requested overlay frame's pinned data is available and its + // H2D copy is enqueued on the compositor stream. Waits on the background + // reader are host-side (condition variable) and never sync the stream; the + // encode loop keeps its single per-frame cudaStreamSynchronize. + void waitForFrame(LoadedLayer& layer, int frameIndex, int slot, cudaStream_t copyStream) { + std::unique_lock lock(layer.mutex); + while (true) { + if (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex) { + ++cacheHits_; + return; + } + if (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) { + ++pinnedHits_; + lock.unlock(); + enqueueH2D(layer, slot, copyStream); + lock.lock(); + layer.slotStates[slot] = SlotState::DeviceReady; + return; + } + if (!layer.readError.empty()) { + fail(layer.readError); + } + if (layer.stop) { + fail("Overlay reader stopped before frame " + std::to_string(frameIndex)); + } + requestReadLocked(layer, frameIndex, slot); + ++readWaits_; + layer.cv.wait(lock, [&] { + return layer.stop || !layer.readError.empty() || + (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) || + (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex); + }); + } + } + + // Enqueues the H2D copy for a PinnedReady slot on the compositor stream. + // Main thread only; the transfer is ordered ahead of the blend kernels on + // the same stream. + void enqueueH2D(LoadedLayer& layer, int slot, cudaStream_t copyStream) { + const auto enqueueStart = std::chrono::steady_clock::now(); + checkCuda( + cudaMemcpyAsync( + layer.deviceFrames[slot], + layer.pinnedFrames[slot], + layer.frameBytes, + cudaMemcpyHostToDevice, + copyStream), + "cudaMemcpyAsync overlay frame"); + h2dEnqueueMs_ += elapsedMs(enqueueStart, std::chrono::steady_clock::now()); + } + + std::vector> layers_; + std::atomic fileLoads_{0}; + std::atomic hostReadUs_{0}; + std::atomic pendingReadsPeak_{0}; + double h2dEnqueueMs_ = 0.0; + int64_t cacheHits_ = 0; + int64_t pinnedHits_ = 0; + int64_t readWaits_ = 0; +}; + +// Device tile cache for the renderer-prepared tiled/delta RGBA overlay stream. +// Each layer owns one contiguous device canvas (tileCount x 128x128x4 bytes, +// i.e. exactly one full layer frame) plus a bounded pinned staging buffer sized +// to the largest frame delta. The static tile base is read and uploaded once +// synchronously before encoding (like the raw static-layer staging, and +// excluded from the streaming-path timing metrics). At each logical frame only +// the changed tile payloads are read from the payload stream (bounded byte +// ranges into the pinned staging) and enqueued as ordered H2D copies on the +// compositor stream, ahead of the blend kernels on the same stream, so the +// single per-frame cudaStreamSynchronize stays sufficient and memory stays +// bounded (device canvas == one layer frame, staging == largest delta). The +// cached tile state is blended in z-order by the tiled blend kernel. +class TiledOverlayFrameSource { +public: + explicit TiledOverlayFrameSource(const std::vector& layers) { + layers_.reserve(layers.size()); + for (const auto& descriptor : layers) { + layers_.push_back(loadLayer(descriptor)); + // Static base bytes are uploaded once (at load); they count toward + // uploadedTileBytes so the measured invariant matches the renderer + // bookkeeping (uploaded = static base + all changed tiles). + uploadedTileBytes_ += + static_cast(descriptor.tileCount) * descriptor.tileByteSize; + if (rawFallbackReason_.empty() && !descriptor.rawFallbackReason.empty()) { + rawFallbackReason_ = descriptor.rawFallbackReason; + } + } + } + + ~TiledOverlayFrameSource() { + for (auto& layer : layers_) { + if (layer->tileCanvas) { + cudaFree(layer->tileCanvas); + layer->tileCanvas = nullptr; + } + if (layer->pinnedStaging) { + cudaFreeHost(layer->pinnedStaging); + layer->pinnedStaging = nullptr; + } + } + } + + bool empty() const { + return layers_.empty(); + } + + size_t layerCount() const { + return layers_.size(); + } + + const TiledOverlayLayerDescriptor& descriptor(size_t index) const { + return layers_[index]->descriptor; + } + + // Contiguous per-layer device tile canvas: tile t occupies + // [t * tileByteSize, (t + 1) * tileByteSize) in row-major tile order. + const unsigned char* tileCanvasDevicePtr(size_t index) const { + return layers_[index]->tileCanvas; + } + + // Applies every frame delta with frameIndex <= the clamped logical frame of + // the output index: bounded payload reads into the pinned staging buffer, + // then one ordered H2D copy per changed tile on the compositor stream. A + // delta with empty changedTiles (or no delta at this frame) reuses the + // current cache with no payload read. This never syncs the compositor + // stream; the encode loop keeps its single per-frame cudaStreamSynchronize. + void beginFrame(int outputFrameIndex, cudaStream_t copyStream) { + for (auto& layer : layers_) { + const int frameIndex = + std::min(outputFrameIndex, layer->descriptor.frameCount - 1); + if (frameIndex < 0) { + continue; + } + int changedThisFrame = 0; + while (layer->nextDeltaIndex < layer->descriptor.frameDeltas.size()) { + const TiledOverlayFrameDelta& delta = + layer->descriptor.frameDeltas[layer->nextDeltaIndex]; + if (delta.frameIndex > frameIndex) { + break; + } + changedThisFrame += static_cast(delta.changedTiles.size()); + uploadDelta(*layer, delta, copyStream); + ++layer->nextDeltaIndex; + } + changedTileCount_ += changedThisFrame; + // Frame 0 is fully defined by the static base (uploaded once at + // load), so it contributes no cache hits; every later frame serves + // every tile that did not change this frame from the device cache. + if (frameIndex > 0) { + cachedTileCount_ += + static_cast(layer->descriptor.tileCount) - changedThisFrame; + } + } + } + + // Tile payloads uploaded from frame deltas (excludes the static base; the + // renderer derives the same value from the descriptor). int64, no overflow: + // bounded by the validated payload stream length. + int64_t changedTileCount() const { + return changedTileCount_; + } + + // Tile payload bytes uploaded once (static base + all changed tiles). + int64_t uploadedTileBytes() const { + return uploadedTileBytes_; + } + + // Tile-state lookups served from previously uploaded payloads across the + // output timeline (diagnostic only; never claims zero-copy). + int64_t cachedTileCount() const { + return cachedTileCount_; + } + + // Wall time the encode thread spent reading changed tile payloads from + // disk (not H2D transfer time; static base reads are excluded, matching the + // raw streaming-path hostReadMs semantics). + double hostReadMs() const { + return static_cast(hostReadUs_.load()) / 1000.0; + } + + // Wall time the encode thread spent enqueuing per-tile H2D copies on the + // compositor stream. + double h2dEnqueueMs() const { + return h2dEnqueueMs_; + } + + int64_t cacheHits() const { + return cachedTileCount_; + } + + // First conservative tiled-vs-raw eligibility decision ("" when every layer + // is eligible). Diagnostic only; the helper still composites every layer as + // a lossless tiled stream (there is no silent raw fallback in the helper). + const std::string& rawFallbackReason() const { + return rawFallbackReason_; + } + +private: + struct LoadedLayer { + TiledOverlayLayerDescriptor descriptor; + std::ifstream input; + unsigned char* tileCanvas = nullptr; + unsigned char* pinnedStaging = nullptr; + size_t nextDeltaIndex = 0; + }; + + static std::unique_ptr loadLayer( + const TiledOverlayLayerDescriptor& descriptor) { + std::unique_ptr layer = std::make_unique(); + layer->descriptor = descriptor; + const int64_t tileByteSize = descriptor.tileByteSize; + const int64_t canvasBytes = static_cast(descriptor.tileCount) * tileByteSize; + if (canvasBytes <= 0 || descriptor.maxDeltaBytes < tileByteSize) { + fail("Invalid tiled overlay layer: " + descriptor.id); + } + + layer->input.open(descriptor.payloadPath, std::ios::binary); + if (!layer->input) { + fail("Failed to open tiled overlay payload: " + descriptor.payloadPath); + } + layer->input.seekg(0, std::ios::end); + const std::streampos end = layer->input.tellg(); + layer->input.seekg(0, std::ios::beg); + if (end < 0 || static_cast(end) < descriptor.payloadByteLength) { + fail("Tiled overlay payload is truncated: " + descriptor.payloadPath); + } + + checkCuda( + cudaMalloc(&layer->tileCanvas, static_cast(canvasBytes)), + "cudaMalloc tiled overlay canvas"); + checkCuda( + cudaMallocHost( + &layer->pinnedStaging, + static_cast(descriptor.maxDeltaBytes)), + "cudaMallocHost tiled overlay staging"); + + // Static base: read every tile's initial payload once and upload it + // synchronously before encoding. One-time cost, not timed (matching the + // raw static-layer staging semantics); the tiles stay device-resident. + for (const auto& record : descriptor.staticTiles) { + layer->input.seekg(static_cast(record.byteOffset), std::ios::beg); + layer->input.read( + reinterpret_cast(layer->pinnedStaging), + static_cast(kTiledOverlayTileByteSize)); + if (static_cast(layer->input.gcount()) != kTiledOverlayTileByteSize) { + fail("Failed to read static tile of tiled overlay layer: " + descriptor.id); + } + checkCuda( + cudaMemcpy( + layer->tileCanvas + + static_cast(record.tileIndex) * + static_cast(kTiledOverlayTileByteSize), + layer->pinnedStaging, + static_cast(kTiledOverlayTileByteSize), + cudaMemcpyHostToDevice), + "cudaMemcpy tiled overlay static tile"); + } + return layer; } - auto track = std::make_unique(); - std::string line; - while (std::getline(input, line)) { - if (line.empty()) { - continue; + // Reads the delta's changed tile payloads into the pinned staging buffer + // and enqueues one H2D copy per tile on the compositor stream. The staging + // buffer is bounded to the largest delta in the descriptor (validated at + // load), and each H2D copy reads a distinct staging region, so reusing the + // buffer across frames is safe: the enqueued copies complete by the single + // per-frame cudaStreamSynchronize before the buffer is rewritten. + void uploadDelta( + LoadedLayer& layer, + const TiledOverlayFrameDelta& delta, + cudaStream_t copyStream) { + if (delta.changedTiles.empty()) { + return; } - std::replace(line.begin(), line.end(), ',', ' '); - std::istringstream row(line); - ZoomSample sample; - if (!(row >> sample.timeMs >> sample.scale >> sample.x >> sample.y)) { - continue; + const size_t deltaBytes = static_cast(delta.changedTiles.size()) * + static_cast(kTiledOverlayTileByteSize); + if (deltaBytes > static_cast(layer.descriptor.maxDeltaBytes)) { + fail("Tiled overlay delta exceeds the bounded staging buffer: " + layer.descriptor.id); } - if (!std::isfinite(sample.timeMs) || !std::isfinite(sample.scale) || - !std::isfinite(sample.x) || !std::isfinite(sample.y)) { - continue; + + const auto readStart = std::chrono::steady_clock::now(); + size_t stagingOffset = 0; + for (const auto& record : delta.changedTiles) { + layer.input.seekg(static_cast(record.byteOffset), std::ios::beg); + layer.input.read( + reinterpret_cast(layer.pinnedStaging + stagingOffset), + static_cast(kTiledOverlayTileByteSize)); + if (static_cast(layer.input.gcount()) != kTiledOverlayTileByteSize) { + fail( + "Failed to read changed tile payload of tiled overlay layer: " + + layer.descriptor.id); + } + stagingOffset += static_cast(kTiledOverlayTileByteSize); } - sample.timeMs = std::max(0.0, sample.timeMs); - sample.scale = std::max(0.01, sample.scale); - track->samples.push_back(sample); - } - if (track->samples.empty()) { - fail("No zoom samples were loaded: " + options.zoomSamplesPath); + hostReadUs_ += static_cast( + elapsedMs(readStart, std::chrono::steady_clock::now()) * 1000.0); + + const auto enqueueStart = std::chrono::steady_clock::now(); + stagingOffset = 0; + for (const auto& record : delta.changedTiles) { + checkCuda( + cudaMemcpyAsync( + layer.tileCanvas + + static_cast(record.tileIndex) * + static_cast(kTiledOverlayTileByteSize), + layer.pinnedStaging + stagingOffset, + static_cast(kTiledOverlayTileByteSize), + cudaMemcpyHostToDevice, + copyStream), + "cudaMemcpyAsync tiled overlay changed tile"); + stagingOffset += static_cast(kTiledOverlayTileByteSize); + } + h2dEnqueueMs_ += elapsedMs(enqueueStart, std::chrono::steady_clock::now()); + uploadedTileBytes_ += static_cast(deltaBytes); } - std::sort(track->samples.begin(), track->samples.end(), [](const auto& left, const auto& right) { - return left.timeMs < right.timeMs; - }); - return track; -} + + std::vector> layers_; + std::atomic hostReadUs_{0}; + int64_t changedTileCount_ = 0; + int64_t uploadedTileBytes_ = 0; + int64_t cachedTileCount_ = 0; + double h2dEnqueueMs_ = 0.0; + std::string rawFallbackReason_; +}; struct CursorAtlasEntry { int x = 0; @@ -1484,6 +3594,84 @@ __device__ int sampleCursorAtlasShadowAlpha( return min(255, weightedAlpha / 100); } +__device__ __forceinline__ unsigned char temporalAccumulateByte( + unsigned char current, + unsigned char value, + unsigned int weightFixed, + int accumulateMode) { + if (accumulateMode == 0) { + // Legacy direct write (non-temporal composites). + return value; + } + const int weighted = (static_cast(weightFixed) * static_cast(value) + 128) >> 8; + if (accumulateMode == 1) { + // First temporal sample: replace (the target is not pre-zeroed, so this + // must not read stale buffer contents). Matches the previous + // zero-fill + (weight * value + 128) >> 8 accumulate exactly. + return static_cast(min(255, weighted)); + } + // Subsequent temporal samples: saturating accumulate into the target. + return static_cast(min(255, static_cast(current) + weighted)); +} + +// Accumulates the temporal sample weights applied to the invariant background +// into one full-frame pass. Every pixel outside the transformed content +// bounding box maps outside the content rect for every temporal sample, so its +// per-sample composite value is always the background; the saturating weighted +// sum of the background is therefore identical for all samples and can be +// computed once per output frame. The term-for-term math reproduces the +// replace-then-saturate-accumulate chain of compositeStaticNv12Kernel exactly +// (same (weight * value + 128) >> 8 per sample, same saturation), including +// per-sample rounding, so pixels served by this pass are bit-identical to the +// previous per-sample full-frame composites. +__global__ void accumulateBackgroundNv12Kernel( + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + unsigned char backgroundY, + unsigned char backgroundU, + unsigned char backgroundV, + const unsigned char* background, + const unsigned int* sampleWeights, + int sampleCount) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= dstWidth || y >= dstHeight || sampleCount <= 0) { + return; + } + + const unsigned int bgY = background ? background[y * dstWidth + x] : backgroundY; + unsigned int yAcc = (sampleWeights[0] * bgY + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int term = (sampleWeights[index] * bgY + 128u) >> 8; + yAcc = min(255u, yAcc + term); + } + dst[y * dstPitch + x] = static_cast(yAcc); + + if ((x % 2) == 0 && (y % 2) == 0) { + unsigned int bgU = backgroundU; + unsigned int bgV = backgroundV; + if (background) { + const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; + bgU = bgUv[0]; + bgV = bgUv[1]; + } + unsigned int uAcc = (sampleWeights[0] * bgU + 128u) >> 8; + unsigned int vAcc = (sampleWeights[0] * bgV + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int uTerm = (sampleWeights[index] * bgU + 128u) >> 8; + const unsigned int vTerm = (sampleWeights[index] * bgV + 128u) >> 8; + uAcc = min(255u, uAcc + uTerm); + vAcc = min(255u, vAcc + vTerm); + } + unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + dstUv[0] = static_cast(uAcc); + dstUv[1] = static_cast(vAcc); + } +} + __global__ void compositeStaticNv12Kernel( const unsigned char* src, int srcPitch, @@ -1495,6 +3683,10 @@ __global__ void compositeStaticNv12Kernel( int dstChromaOffset, int dstWidth, int dstHeight, + int regionX, + int regionY, + int regionWidth, + int regionHeight, int contentX, int contentY, int contentWidth, @@ -1533,10 +3725,17 @@ __global__ void compositeStaticNv12Kernel( bool zoomEnabled, float zoomScale, float zoomX, - float zoomY) { - const int x = blockIdx.x * blockDim.x + threadIdx.x; - const int y = blockIdx.y * blockDim.y + threadIdx.y; - if (x >= dstWidth || y >= dstHeight) { + float zoomY, + unsigned int temporalWeightFixed, + int temporalAccumulateMode) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight) { + return; + } + const int x = regionX + localX; + const int y = regionY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { return; } @@ -1638,10 +3837,250 @@ __global__ void compositeStaticNv12Kernel( outY = 16; } } - dst[y * dstPitch + x] = outY; + dst[y * dstPitch + x] = temporalAccumulateByte( + dst[y * dstPitch + x], + outY, + temporalWeightFixed, + temporalAccumulateMode); + + if ((x % 2) == 0 && (y % 2) == 0) { + unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + unsigned char outU = backgroundU; + unsigned char outV = backgroundV; + const float uvLayoutXf = + zoomActive ? (static_cast(x + 1) - zoomX) / safeZoomScale : static_cast(x + 1); + const float uvLayoutYf = + zoomActive ? (static_cast(y + 1) - zoomY) / safeZoomScale : static_cast(y + 1); + const int uvLayoutX = static_cast(floorf(uvLayoutXf)); + const int uvLayoutY = static_cast(floorf(uvLayoutYf)); + const bool uvInside = isInsideRoundedRect( + uvLayoutX, + uvLayoutY, + contentX, + contentY, + contentWidth, + contentHeight, + radius); + if (uvInside) { + const float localX = fminf(static_cast(contentWidth - 1), fmaxf(0.0f, uvLayoutXf - contentX)); + const float localY = fminf(static_cast(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY)); + const int suvX = + min(srcWidth - 2, (cropX + static_cast((localX * cropWidth) / contentWidth)) & ~1); + const int suvY = + min((srcHeight / 2) - 1, (cropY + static_cast(localY * cropHeight / contentHeight)) / 2); + const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX; + outU = srcUv[0]; + outV = srcUv[1]; + } else { + if (background) { + const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; + outU = bgUv[0]; + outV = bgUv[1]; + } else { + outU = backgroundU; + outV = backgroundV; + } + } + if (webcam && + isInsideRoundedRect( + x + 1, + y + 1, + webcamX, + webcamY, + webcamSize, + webcamSize, + webcamRadius)) { + const int localX = max(0, min(webcamSize - 1, x + 1 - webcamX)); + const int localY = max(0, min(webcamSize - 1, y + 1 - webcamY)); + const int sampleX = min(webcamFrameWidth - 1, (localX * webcamFrameWidth) / webcamSize); + const int sampleY = min(webcamFrameHeight - 1, (localY * webcamFrameHeight) / webcamSize); + const int mirroredX = webcamMirror ? webcamFrameWidth - 1 - sampleX : sampleX; + const int webcamUvX = min(webcamFrameWidth - 2, mirroredX & ~1); + const int webcamUvY = min((webcamFrameHeight / 2) - 1, sampleY / 2); + const unsigned char* webcamUv = + webcam + webcamFrameWidth * webcamFrameHeight + webcamUvY * webcamFrameWidth + webcamUvX; + outU = webcamUv[0]; + outV = webcamUv[1]; + } + unsigned char cursorUvY = 0; + unsigned char cursorUvU = 128; + unsigned char cursorUvV = 128; + int cursorUvAlpha = 0; + const int cursorUvShadowAlpha = cursorVisible && cursorAtlasRgba + ? sampleCursorAtlasShadowAlpha( + cursorAtlasRgba, + cursorAtlasWidth, + cursorAtlasHeight, + cursorAtlasEntryX, + cursorAtlasEntryY, + cursorAtlasEntryWidth, + cursorAtlasEntryHeight, + cursorX, + cursorY, + cursorWidth, + cursorHeight, + x + 1, + y + 1) + : 0; + if (cursorUvShadowAlpha > 0) { + outU = blendByte(outU, 128, cursorUvShadowAlpha); + outV = blendByte(outV, 128, cursorUvShadowAlpha); + } + const bool cursorAtlasUvHit = + cursorVisible && + sampleCursorAtlasNv12( + cursorAtlasRgba, + cursorAtlasWidth, + cursorAtlasHeight, + cursorAtlasEntryX, + cursorAtlasEntryY, + cursorAtlasEntryWidth, + cursorAtlasEntryHeight, + cursorX, + cursorY, + cursorWidth, + cursorHeight, + x + 1, + y + 1, + &cursorUvY, + &cursorUvU, + &cursorUvV, + &cursorUvAlpha); + if (cursorAtlasUvHit) { + outU = blendByte(outU, cursorUvU, cursorUvAlpha); + outV = blendByte(outV, cursorUvV, cursorUvAlpha); + } else { + const int cursorUvMask = + cursorVisible && !cursorAtlasRgba + ? cursorMaskAt(x + 1, y + 1, cursorX, cursorY, cursorWidth, cursorHeight) + : 0; + if (cursorUvMask > 0) { + outU = 128; + outV = 128; + } + } + dstUv[0] = temporalAccumulateByte( + dstUv[0], + outU, + temporalWeightFixed, + temporalAccumulateMode); + dstUv[1] = temporalAccumulateByte( + dstUv[1], + outV, + temporalWeightFixed, + temporalAccumulateMode); + } +} + +// Fused constant-transform temporal composition: evaluates the source + layout +// composite value exactly once per pixel (the same content/bg/shadow selection +// compositeStaticNv12Kernel makes for the temporal path, where webcam/cursor +// are applied afterward) and then applies the existing fixed-point weights in +// order: sample 0 replaces with (w0 * v + 128) >> 8 and later samples +// saturate-accumulate (w * v + 128) >> 8. This is only launched when the +// stationary shutter-window check proved every sample resolves to the same +// camera transform (bit-identical scale/x/y), so the per-sample composite value +// is identical for every sample and the term-for-term math reproduces the +// per-sample replace-then-accumulate chain exactly, including per-sample +// rounding and progressive saturation. +__global__ void compositeStaticStationaryNv12Kernel( + const unsigned char* src, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + int regionX, + int regionY, + int regionWidth, + int regionHeight, + int contentX, + int contentY, + int contentWidth, + int contentHeight, + int sourceCropX, + int sourceCropY, + int sourceCropWidth, + int sourceCropHeight, + int radius, + unsigned char backgroundY, + unsigned char backgroundU, + unsigned char backgroundV, + const unsigned char* background, + int shadowOffsetY, + int shadowIntensityPct, + bool zoomEnabled, + float zoomScale, + float zoomX, + float zoomY, + const unsigned int* sampleWeights, + int sampleCount) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight || sampleCount <= 0) { + return; + } + const int x = regionX + localX; + const int y = regionY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + + const bool zoomActive = zoomEnabled && zoomScale > 0.01f; + const float safeZoomScale = fmaxf(zoomScale, 0.01f); + const float layoutXf = + zoomActive ? (static_cast(x) - zoomX) / safeZoomScale : static_cast(x); + const float layoutYf = + zoomActive ? (static_cast(y) - zoomY) / safeZoomScale : static_cast(y); + const int layoutX = static_cast(floorf(layoutXf)); + const int layoutY = static_cast(floorf(layoutYf)); + + const int cropX = max(0, min(sourceCropX, srcWidth - 1)); + const int cropY = max(0, min(sourceCropY, srcHeight - 1)); + const int cropWidth = max(1, min(sourceCropWidth > 0 ? sourceCropWidth : srcWidth, srcWidth - cropX)); + const int cropHeight = max(1, min(sourceCropHeight > 0 ? sourceCropHeight : srcHeight, srcHeight - cropY)); + const bool inside = + isInsideRoundedRect(layoutX, layoutY, contentX, contentY, contentWidth, contentHeight, radius); + unsigned char outY = background ? background[y * dstWidth + x] : backgroundY; + if (inside) { + const float localX = + fminf(static_cast(contentWidth - 1), fmaxf(0.0f, layoutXf - contentX)); + const float localY = + fminf(static_cast(contentHeight - 1), fmaxf(0.0f, layoutYf - contentY)); + const int sx = min(srcWidth - 1, cropX + static_cast((localX * cropWidth) / contentWidth)); + const int sy = min(srcHeight - 1, cropY + static_cast((localY * cropHeight) / contentHeight)); + outY = src[sy * srcPitch + sx]; + } else { + const bool shadowInside = + shadowIntensityPct > 0 && + isInsideRoundedRect( + layoutX, + layoutY, + contentX, + contentY + shadowOffsetY, + contentWidth, + contentHeight, + radius + 8); + if (shadowInside) { + const int darkenPct = min(75, max(0, shadowIntensityPct / 2)); + outY = static_cast((static_cast(outY) * (100 - darkenPct)) / 100); + } + } + unsigned int yAcc = (sampleWeights[0] * outY + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int term = (sampleWeights[index] * outY + 128u) >> 8; + yAcc = min(255u, yAcc + term); + } + dst[y * dstPitch + x] = static_cast(yAcc); if ((x % 2) == 0 && (y % 2) == 0) { unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + unsigned char outU = backgroundU; + unsigned char outV = backgroundV; const float uvLayoutXf = zoomActive ? (static_cast(x + 1) - zoomX) / safeZoomScale : static_cast(x + 1); const float uvLayoutYf = @@ -1657,103 +4096,32 @@ __global__ void compositeStaticNv12Kernel( contentHeight, radius); if (uvInside) { - const float localX = fminf(static_cast(contentWidth - 1), fmaxf(0.0f, uvLayoutXf - contentX)); - const float localY = fminf(static_cast(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY)); + const float localX = + fminf(static_cast(contentWidth - 1), fmaxf(0.0f, uvLayoutXf - contentX)); + const float localY = + fminf(static_cast(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY)); const int suvX = min(srcWidth - 2, (cropX + static_cast((localX * cropWidth) / contentWidth)) & ~1); const int suvY = min((srcHeight / 2) - 1, (cropY + static_cast(localY * cropHeight / contentHeight)) / 2); const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX; - dstUv[0] = srcUv[0]; - dstUv[1] = srcUv[1]; - } else { - if (background) { - const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; - dstUv[0] = bgUv[0]; - dstUv[1] = bgUv[1]; - } else { - dstUv[0] = backgroundU; - dstUv[1] = backgroundV; - } - } - if (webcam && - isInsideRoundedRect( - x + 1, - y + 1, - webcamX, - webcamY, - webcamSize, - webcamSize, - webcamRadius)) { - const int localX = max(0, min(webcamSize - 1, x + 1 - webcamX)); - const int localY = max(0, min(webcamSize - 1, y + 1 - webcamY)); - const int sampleX = min(webcamFrameWidth - 1, (localX * webcamFrameWidth) / webcamSize); - const int sampleY = min(webcamFrameHeight - 1, (localY * webcamFrameHeight) / webcamSize); - const int mirroredX = webcamMirror ? webcamFrameWidth - 1 - sampleX : sampleX; - const int webcamUvX = min(webcamFrameWidth - 2, mirroredX & ~1); - const int webcamUvY = min((webcamFrameHeight / 2) - 1, sampleY / 2); - const unsigned char* webcamUv = - webcam + webcamFrameWidth * webcamFrameHeight + webcamUvY * webcamFrameWidth + webcamUvX; - dstUv[0] = webcamUv[0]; - dstUv[1] = webcamUv[1]; - } - unsigned char cursorUvY = 0; - unsigned char cursorUvU = 128; - unsigned char cursorUvV = 128; - int cursorUvAlpha = 0; - const int cursorUvShadowAlpha = cursorVisible && cursorAtlasRgba - ? sampleCursorAtlasShadowAlpha( - cursorAtlasRgba, - cursorAtlasWidth, - cursorAtlasHeight, - cursorAtlasEntryX, - cursorAtlasEntryY, - cursorAtlasEntryWidth, - cursorAtlasEntryHeight, - cursorX, - cursorY, - cursorWidth, - cursorHeight, - x + 1, - y + 1) - : 0; - if (cursorUvShadowAlpha > 0) { - dstUv[0] = blendByte(dstUv[0], 128, cursorUvShadowAlpha); - dstUv[1] = blendByte(dstUv[1], 128, cursorUvShadowAlpha); + outU = srcUv[0]; + outV = srcUv[1]; + } else if (background) { + const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; + outU = bgUv[0]; + outV = bgUv[1]; } - const bool cursorAtlasUvHit = - cursorVisible && - sampleCursorAtlasNv12( - cursorAtlasRgba, - cursorAtlasWidth, - cursorAtlasHeight, - cursorAtlasEntryX, - cursorAtlasEntryY, - cursorAtlasEntryWidth, - cursorAtlasEntryHeight, - cursorX, - cursorY, - cursorWidth, - cursorHeight, - x + 1, - y + 1, - &cursorUvY, - &cursorUvU, - &cursorUvV, - &cursorUvAlpha); - if (cursorAtlasUvHit) { - dstUv[0] = blendByte(dstUv[0], cursorUvU, cursorUvAlpha); - dstUv[1] = blendByte(dstUv[1], cursorUvV, cursorUvAlpha); - } else { - const int cursorUvMask = - cursorVisible && !cursorAtlasRgba - ? cursorMaskAt(x + 1, y + 1, cursorX, cursorY, cursorWidth, cursorHeight) - : 0; - if (cursorUvMask > 0) { - dstUv[0] = 128; - dstUv[1] = 128; - } + unsigned int uAcc = (sampleWeights[0] * outU + 128u) >> 8; + unsigned int vAcc = (sampleWeights[0] * outV + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int uTerm = (sampleWeights[index] * outU + 128u) >> 8; + const unsigned int vTerm = (sampleWeights[index] * outV + 128u) >> 8; + uAcc = min(255u, uAcc + uTerm); + vAcc = min(255u, vAcc + vTerm); } + dstUv[0] = static_cast(uAcc); + dstUv[1] = static_cast(vAcc); } } @@ -1963,6 +4331,338 @@ __global__ void overlayCursorNv12Kernel( } } +__device__ float zoomBlurHash01(int x, int y) { + unsigned int value = static_cast(x) * 747796405u + + static_cast(y) * 2891336453u + 0x9e3779b9u; + value = value * 1664525u + 1013904223u; + value ^= value >> 13; + return static_cast(value & 0x00ffffffu) / 16777216.0f; +} + +// Spatial radial zoom blur equivalent to the renderer's ZoomBlurFilter applied +// to the transformed content. For each pixel the ray toward the blur center is +// sampled with a tent weight profile (4*(p-p^2)) over a fixed sample count, +// matching the pixi-filters zoom-blur shader with innerRadius=0/radius=-1 that +// the interactive renderer configures. Blur is restricted to the content region +// so webcam/cursor/background stay sharp, like the renderer's camera container. +// NV12 chroma is blurred at half resolution with the same radial ray. +__global__ void zoomBlurNv12Kernel( + const unsigned char* src, + int srcPitch, + int srcChromaOffset, + int dstWidth, + int dstHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int regionLeft, + int regionTop, + int regionRight, + int regionBottom, + float centerX, + float centerY, + float strength) { + constexpr int kZoomBlurSamples = 13; + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x < regionLeft || x >= regionRight || y < regionTop || y >= regionBottom || + x >= dstWidth || y >= dstHeight) { + return; + } + + const float dirX = centerX - static_cast(x); + const float dirY = centerY - static_cast(y); + const float offset = zoomBlurHash01(x, y); + float total = 0.0f; + float acc = 0.0f; + for (int t = 0; t < kZoomBlurSamples; ++t) { + const float percent = (static_cast(t) + offset) / static_cast(kZoomBlurSamples); + const float weight = 4.0f * (percent - percent * percent); + const int sx = static_cast(static_cast(x) + dirX * strength * percent); + const int sy = static_cast(static_cast(y) + dirY * strength * percent); + const int clampedX = min(regionRight - 1, max(regionLeft, sx)); + const int clampedY = min(regionBottom - 1, max(regionTop, sy)); + acc += weight * static_cast(src[clampedY * srcPitch + clampedX]); + total += weight; + } + dst[y * dstPitch + x] = static_cast(acc / total + 0.5f); + + if ((x % 2) == 0 && (y % 2) == 0) { + const int ux = x / 2; + const int uy = y / 2; + const int uLeft = regionLeft / 2; + const int uTop = regionTop / 2; + const int uRight = min(dstWidth / 2, (regionRight + 1) / 2); + const int uBottom = min(dstHeight / 2, (regionBottom + 1) / 2); + if (ux >= uLeft && ux < uRight && uy >= uTop && uy < uBottom) { + const float uCenterX = centerX * 0.5f; + const float uCenterY = centerY * 0.5f; + const float uDirX = uCenterX - static_cast(ux); + const float uDirY = uCenterY - static_cast(uy); + const float uOffset = zoomBlurHash01(ux, uy); + float uTotal = 0.0f; + float uAcc = 0.0f; + float vAcc = 0.0f; + for (int t = 0; t < kZoomBlurSamples; ++t) { + const float percent = (static_cast(t) + uOffset) / static_cast(kZoomBlurSamples); + const float weight = 4.0f * (percent - percent * percent); + const int sux = min(uRight - 1, max(uLeft, static_cast(static_cast(ux) + uDirX * strength * percent))); + const int suy = min(uBottom - 1, max(uTop, static_cast(static_cast(uy) + uDirY * strength * percent))); + const unsigned char* uv = src + srcChromaOffset + suy * srcPitch + sux * 2; + uAcc += weight * static_cast(uv[0]); + vAcc += weight * static_cast(uv[1]); + uTotal += weight; + } + unsigned char* dstUv = dst + dstChromaOffset + uy * dstPitch + ux * 2; + dstUv[0] = static_cast(uAcc / uTotal + 0.5f); + dstUv[1] = static_cast(vAcc / uTotal + 0.5f); + } + } +} + +__device__ void rgbaToNv12Yuv(int r, int g, int b, unsigned char& y, unsigned char& u, unsigned char& v) { + y = clampByteDevice(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16); + u = clampByteDevice(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128); + v = clampByteDevice(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128); +} + +// Blends a transparent top-down RGBA overlay layer over the composed NV12 +// frame. Luma is blended per pixel; chroma is averaged over the 2x2 block using +// only the pixels covered by the layer, then blended with the same average +// alpha. This reproduces the renderer contract: the overlay sidecar is drawn +// above the zoom-blurred video layout. The launch rectangle may be the full +// layer (dynamic layers) or a one-time alpha bound (static single-frame +// layers); threads are indexed by region-local coordinates and mapped back to +// layer-local coordinates, so a bounded launch visits exactly the pixels the +// full-frame launch could write. +__global__ void blendOverlayRgbaNv12Kernel( + const unsigned char* overlay, + int overlayWidth, + int overlayHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + int layerX, + int layerY, + int layerWidth, + int layerHeight, + int regionX, + int regionY, + int regionWidth, + int regionHeight) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight) { + return; + } + + const int layerLocalX = regionX + localX; + const int layerLocalY = regionY + localY; + if (layerLocalX < 0 || layerLocalY < 0 || + layerLocalX >= layerWidth || layerLocalY >= layerHeight) { + return; + } + + const int x = layerX + layerLocalX; + const int y = layerY + layerLocalY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + + const int pixelOffset = (layerLocalY * layerWidth + layerLocalX) * 4; + const int alpha = overlay[pixelOffset + 3]; + if (alpha > 0) { + const int r = overlay[pixelOffset]; + const int g = overlay[pixelOffset + 1]; + const int b = overlay[pixelOffset + 2]; + unsigned char overlayY = 0; + unsigned char overlayU = 0; + unsigned char overlayV = 0; + rgbaToNv12Yuv(r, g, b, overlayY, overlayU, overlayV); + unsigned char* yPtr = dst + y * dstPitch + x; + *yPtr = blendByte(*yPtr, overlayY, alpha); + } + + if ((x % 2) == 0 && (y % 2) == 0 && x + 1 < dstWidth && y + 1 < dstHeight) { + int alphaSum = 0; + int uSum = 0; + int vSum = 0; + int samples = 0; + for (int dy = 0; dy < 2; ++dy) { + for (int dx = 0; dx < 2; ++dx) { + const int sampleX = x + dx; + const int sampleY = y + dy; + const int layerLocalX = sampleX - layerX; + const int layerLocalY = sampleY - layerY; + if (layerLocalX < 0 || layerLocalY < 0 || + layerLocalX >= layerWidth || layerLocalY >= layerHeight) { + continue; + } + const int sampleOffset = (layerLocalY * layerWidth + layerLocalX) * 4; + const int sampleAlpha = overlay[sampleOffset + 3]; + if (sampleAlpha <= 0) { + continue; + } + const int r = overlay[sampleOffset]; + const int g = overlay[sampleOffset + 1]; + const int b = overlay[sampleOffset + 2]; + unsigned char sampleYValue = 0; + unsigned char sampleU = 0; + unsigned char sampleV = 0; + rgbaToNv12Yuv(r, g, b, sampleYValue, sampleU, sampleV); + alphaSum += sampleAlpha; + uSum += static_cast(sampleU) * sampleAlpha; + vSum += static_cast(sampleV) * sampleAlpha; + ++samples; + } + } + if (samples > 0) { + const int avgAlpha = alphaSum / samples; + const int avgU = uSum / alphaSum; + const int avgV = vSum / alphaSum; + unsigned char* uvPtr = dst + dstChromaOffset + (y / 2) * dstPitch + x; + uvPtr[0] = blendByte( + uvPtr[0], + static_cast(clampByteDevice(avgU)), + avgAlpha); + uvPtr[1] = blendByte( + uvPtr[1], + static_cast(clampByteDevice(avgV)), + avgAlpha); + } + } +} + +// Layer-local pixel offset into the contiguous per-tile device canvas: tile +// (tileX, tileY) in row-major tile order, pixel (withinX, withinY) inside the +// tile. Each output pixel maps to exactly one tile, so tile edges are exact and +// never sampled twice (no seams); edge tiles of partial layers are masked by +// the caller's layer-bounds checks exactly like the raw full-frame blend. +__device__ size_t tiledOverlayPixelOffset( + int localX, + int localY, + int tileSize, + int tileColumns, + int tileByteSize) { + const int tileX = localX / tileSize; + const int tileY = localY / tileSize; + const int withinX = localX - tileX * tileSize; + const int withinY = localY - tileY * tileSize; + return static_cast(tileY * tileColumns + tileX) * + static_cast(tileByteSize) + + static_cast((withinY * tileSize + withinX) * 4); +} + +// Blends one tiled/delta overlay layer (device tile canvas) over the composed +// NV12 frame in z-order. Pixel-for-pixel identical to the raw RGBA blend: each +// layer-local coordinate samples the same RGBA byte the raw sidecar would hold +// (the tiles tile the layer exactly), the luma blend and the per-2x2-block +// alpha-weighted chroma blend reuse the same math (blendByte, rgbaToNv12Yuv, +// clampByteDevice), and the layer-bounds checks mask partial edge tiles. +__global__ void blendTiledOverlayRgbaNv12Kernel( + const unsigned char* tiles, + int tileSize, + int tileColumns, + int tileByteSize, + int layerWidth, + int layerHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + int layerX, + int layerY) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= layerWidth || localY >= layerHeight) { + return; + } + + const int x = layerX + localX; + const int y = layerY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + + const size_t pixelOffset = + tiledOverlayPixelOffset(localX, localY, tileSize, tileColumns, tileByteSize); + const int alpha = tiles[pixelOffset + 3]; + if (alpha > 0) { + const int r = tiles[pixelOffset]; + const int g = tiles[pixelOffset + 1]; + const int b = tiles[pixelOffset + 2]; + unsigned char overlayY = 0; + unsigned char overlayU = 0; + unsigned char overlayV = 0; + rgbaToNv12Yuv(r, g, b, overlayY, overlayU, overlayV); + unsigned char* yPtr = dst + y * dstPitch + x; + *yPtr = blendByte(*yPtr, overlayY, alpha); + } + + if ((x % 2) == 0 && (y % 2) == 0 && x + 1 < dstWidth && y + 1 < dstHeight) { + int alphaSum = 0; + int uSum = 0; + int vSum = 0; + int samples = 0; + for (int dy = 0; dy < 2; ++dy) { + for (int dx = 0; dx < 2; ++dx) { + const int sampleX = x + dx; + const int sampleY = y + dy; + const int layerLocalX = sampleX - layerX; + const int layerLocalY = sampleY - layerY; + if (layerLocalX < 0 || layerLocalY < 0 || + layerLocalX >= layerWidth || layerLocalY >= layerHeight) { + continue; + } + const size_t sampleOffset = tiledOverlayPixelOffset( + layerLocalX, + layerLocalY, + tileSize, + tileColumns, + tileByteSize); + const int sampleAlpha = tiles[sampleOffset + 3]; + if (sampleAlpha <= 0) { + continue; + } + const int r = tiles[sampleOffset]; + const int g = tiles[sampleOffset + 1]; + const int b = tiles[sampleOffset + 2]; + unsigned char sampleYValue = 0; + unsigned char sampleU = 0; + unsigned char sampleV = 0; + rgbaToNv12Yuv(r, g, b, sampleYValue, sampleU, sampleV); + alphaSum += sampleAlpha; + uSum += static_cast(sampleU) * sampleAlpha; + vSum += static_cast(sampleV) * sampleAlpha; + ++samples; + } + } + if (samples > 0) { + const int avgAlpha = alphaSum / samples; + const int avgU = uSum / alphaSum; + const int avgV = vSum / alphaSum; + unsigned char* uvPtr = dst + dstChromaOffset + (y / 2) * dstPitch + x; + uvPtr[0] = blendByte( + uvPtr[0], + static_cast(clampByteDevice(avgU)), + avgAlpha); + uvPtr[1] = blendByte( + uvPtr[1], + static_cast(clampByteDevice(avgV)), + avgAlpha); + } + } +} + +// Accumulates one weighted temporal sample into the composed frame using the +// renderer's cos-tapered shutter plan: dst = clamp(dst + (weightFixed*src)>>8). +// Weights are normalized to sum to 1, so the accumulation is a weighted average; +// NV12 chroma is accumulated per 2x2 block the same way the other blend kernels +// handle it. The target must start at zero (luma 0 / chroma 0) before the first +// sample. __global__ void prewarmKernel(unsigned int* state, unsigned int seed) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; unsigned int value = seed ^ (index * 747796405u + 2891336453u); @@ -1994,8 +4694,23 @@ void prewarmCuda(int durationMs) { checkCuda(cudaFree(state), "cudaFree prewarm"); } +// Map the high-level encoding mode to the current NVENC preset family. The +// legacy HP/HQ preset GUIDs cannot initialize on Blackwell-era drivers; the +// P1/P4/P6 presets must be paired with a valid tuningInfo (see the nvEncodeAPI +// note: "Presets P1-P7 are only supported with valid +// NV_ENC_INITIALIZE_PARAMS::tuningInfo"). GUID getNvencPresetGuid(const std::string& encodingMode) { - return encodingMode == "fast" ? NV_ENC_PRESET_HP_GUID : NV_ENC_PRESET_HQ_GUID; + if (encodingMode == "fast") { + return NV_ENC_PRESET_P1_GUID; + } + if (encodingMode == "quality") { + return NV_ENC_PRESET_P6_GUID; + } + return NV_ENC_PRESET_P4_GUID; +} + +NV_ENC_TUNING_INFO getNvencTuningInfo() { + return NV_ENC_TUNING_INFO_HIGH_QUALITY; } uint32_t getNvencMaxBitrate(uint32_t bitrate, const std::string& encodingMode) { @@ -2011,6 +4726,165 @@ uint32_t getNvencBufferSize(uint32_t bitrate, const std::string& encodingMode) { std::min(0xffffffffu, static_cast(bitrate) * multiplier)); } +// NVENC capability/version diagnostics captured before encoder creation. The +// compositor never claims codec or rate-control support the device does not +// list; the probe result feeds a minimal-first NV_ENC_CONFIG so optional fields +// (custom VBV, AQ) are only enabled when the hardware reports them. +struct NvencCapabilityProbe { + bool apiLoaded = false; + bool sessionOpened = false; + uint32_t driverMaxApiVersion = 0; + uint32_t sdkApiVersion = NVENCAPI_VERSION; + bool h264Supported = false; + bool hevcSupported = false; + int supportedRateControlModes = 0; + bool customVbvBufferSizeSupported = false; + bool asyncEncodeSupported = false; + bool temporalAqSupported = false; + int widthMax = 0; + int heightMax = 0; + int mbPerSecMax = 0; + std::string deviceName; + int cudaDriverVersion = 0; + int cudaComputeMajor = 0; + int cudaComputeMinor = 0; + std::string error; +}; + +// Which optional NVENC fields were actually applied after the capability probe. +// Reported so diagnostics never claim a feature (AQ, custom VBV) the hardware +// did not accept. +struct NvencConfigUsed { + bool customVbv = false; + bool aq = false; + std::string rcMode = "vbr"; +}; + +#if defined(_WIN32) +NvencCapabilityProbe probeNvencCapabilities(CUcontext context, GUID requestedCodecGuid) { + NvencCapabilityProbe probe; + probe.apiLoaded = false; + probe.sessionOpened = false; + + HMODULE module = LoadLibraryW(L"nvEncodeAPI64.dll"); + if (!module) { + probe.error = "nvEncodeAPI64.dll could not be loaded"; + return probe; + } + + typedef NVENCSTATUS(NVENCAPI* NvEncodeAPIGetMaxSupportedVersion_Type)(uint32_t*); + typedef NVENCSTATUS(NVENCAPI* NvEncodeAPICreateInstance_Type)(NV_ENCODE_API_FUNCTION_LIST*); + auto getMaxSupportedVersion = reinterpret_cast( + GetProcAddress(module, "NvEncodeAPIGetMaxSupportedVersion")); + auto createInstance = reinterpret_cast( + GetProcAddress(module, "NvEncodeAPICreateInstance")); + if (!getMaxSupportedVersion || !createInstance) { + probe.error = "NVENC API entry points not found"; + FreeLibrary(module); + return probe; + } + + NVENCSTATUS status = getMaxSupportedVersion(&probe.driverMaxApiVersion); + if (status != NV_ENC_SUCCESS) { + probe.error = "NvEncodeAPIGetMaxSupportedVersion failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + probe.apiLoaded = true; + + NV_ENCODE_API_FUNCTION_LIST functionList = {NV_ENCODE_API_FUNCTION_LIST_VER}; + status = createInstance(&functionList); + if (status != NV_ENC_SUCCESS) { + probe.error = "NvEncodeAPICreateInstance failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + + // Open a real NVENC session so the capability reads reflect the actual + // device. nvEncGetEncodeCaps requires a valid encoder handle; a null handle + // makes every caps query fail, which would silently degrade the encoder + // config to CBR without custom VBV or AQ. The session is opened against the + // same CUDA primary context the runtime allocations and the export encoder + // use and is destroyed before the real encoder session is created. + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams = {NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER}; + openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA; + openParams.device = context; + openParams.apiVersion = NVENCAPI_VERSION; + void* encoder = nullptr; + status = functionList.nvEncOpenEncodeSessionEx(&openParams, &encoder); + if (status != NV_ENC_SUCCESS) { + probe.error = "nvEncOpenEncodeSessionEx failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + probe.sessionOpened = true; + + auto queryCaps = [&](GUID codecGuid, NV_ENC_CAPS cap, int* value) -> bool { + NV_ENC_CAPS_PARAM capsParam = {NV_ENC_CAPS_PARAM_VER}; + capsParam.capsToQuery = cap; + return functionList.nvEncGetEncodeCaps(encoder, codecGuid, &capsParam, value) == + NV_ENC_SUCCESS; + }; + // Codec support is reported per codec: each query uses its own GUID so a + // device that only lists H.264 never reports HEVC support and vice versa. + int h264Value = 0; + int hevcValue = 0; + const bool h264CapsStatus = queryCaps(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES, &h264Value); + const bool hevcCapsStatus = queryCaps(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES, &hevcValue); + probe.h264Supported = h264CapsStatus; + probe.hevcSupported = hevcCapsStatus; + // Rate control / VBV / AQ / dimension caps are consumed by the encoder + // config for the requested output codec, so query them against that codec + // rather than always H.264. + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES, &probe.supportedRateControlModes); + int customVbv = 0; + int asyncEncode = 0; + int temporalAq = 0; + int widthMax = 0; + int heightMax = 0; + int mbPerSecMax = 0; + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE, &customVbv); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT, &asyncEncode); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ, &temporalAq); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_WIDTH_MAX, &widthMax); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_HEIGHT_MAX, &heightMax); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_MB_PER_SEC_MAX, &mbPerSecMax); + probe.customVbvBufferSizeSupported = customVbv != 0; + probe.asyncEncodeSupported = asyncEncode != 0; + probe.temporalAqSupported = temporalAq != 0; + probe.widthMax = widthMax; + probe.heightMax = heightMax; + probe.mbPerSecMax = mbPerSecMax; + + char deviceName[256] = {}; + const CUresult deviceNameResult = cuDeviceGetName(deviceName, sizeof(deviceName), 0); + if (deviceNameResult == CUDA_SUCCESS) { + probe.deviceName = deviceName; + } + int computeMajor = 0; + int computeMinor = 0; + const CUresult computeResult = + cuDeviceComputeCapability(&computeMajor, &computeMinor, 0); + if (computeResult == CUDA_SUCCESS) { + probe.cudaComputeMajor = computeMajor; + probe.cudaComputeMinor = computeMinor; + } + cuDriverGetVersion(&probe.cudaDriverVersion); + + if (functionList.nvEncDestroyEncoder) { + functionList.nvEncDestroyEncoder(encoder); + } + FreeLibrary(module); + return probe; +} +#else +NvencCapabilityProbe probeNvencCapabilities(CUcontext, GUID) { + NvencCapabilityProbe probe; + probe.error = "NVENC probe is only implemented on Windows"; + return probe; +} +#endif + class NvencSink { public: NvencSink( @@ -2020,52 +4894,125 @@ public: int fps, uint32_t bitrate, const std::string& outputPath, - bool streamSync, Options layoutOptions, const WebcamFrameCache* webcamCache, const CursorTrack* cursorTrack, - const ZoomTrack* zoomTrack) + const ZoomTrack* zoomTrack, + OverlayFrameSource* overlaySource, + TiledOverlayFrameSource* tiledOverlaySource, + const NvencCapabilityProbe& capabilityProbe) : encoder_(context, width, height, NV_ENC_BUFFER_FORMAT_NV12), width_(width), height_(height), fps_(fps), - streamSync_(streamSync), layoutOptions_(layoutOptions), webcamCache_(webcamCache), cursorTrack_(cursorTrack), - zoomTrack_(zoomTrack) { + zoomTrack_(zoomTrack), + overlaySource_(overlaySource), + tiledOverlaySource_(tiledOverlaySource), + compositeLayers_(layoutOptions.compositeLayers), + hasOverlayLayers_(!compositeLayers_.empty()) { loadBackgroundFrame(); loadWebcamFrame(); loadCursorAtlas(); - if (streamSync_) { - checkCuda(cudaStreamCreateWithFlags(©Stream_, cudaStreamNonBlocking), "cudaStreamCreateWithFlags"); + temporalBlurSampleCount_ = layoutOptions_.temporalBlurSampleCount; + temporalBlurShutterFraction_ = layoutOptions_.temporalBlurShutterFraction; + temporalBlurWeightPower_ = layoutOptions_.temporalBlurWeightPower; + // The temporal sample plan depends only on the sample count, shutter + // fraction, weight curve power, and frame duration; cache it once instead + // of rebuilding the cos-tapered weights for every output frame. + if (temporalBlurSampleCount_ >= 3) { + temporalSamplePlan_ = buildTemporalSamplePlan( + temporalBlurSampleCount_, + temporalBlurShutterFraction_, + temporalBlurWeightPower_, + 1000000.0 / static_cast(fps_)); } + // Always use a non-blocking compositor stream: it keeps composite/zoom + // blur/overlay kernels ordered without implicitly serializing against the + // legacy default stream, and a single cudaStreamSynchronize before + // NVENC's synchronous input copy is the only per-frame sync needed. + checkCuda(cudaStreamCreateWithFlags(©Stream_, cudaStreamNonBlocking), "cudaStreamCreateWithFlags"); + checkCuda(cudaEventCreate(&compositeStartEvent_), "cudaEventCreate compositeStart"); + checkCuda(cudaEventCreate(&compositeEndEvent_), "cudaEventCreate compositeEnd"); + checkCuda(cudaEventCreate(&blurStartEvent_), "cudaEventCreate blurStart"); + checkCuda(cudaEventCreate(&blurEndEvent_), "cudaEventCreate blurEnd"); + checkCuda(cudaEventCreate(&overlayStartEvent_), "cudaEventCreate overlayStart"); + checkCuda(cudaEventCreate(&overlayEndEvent_), "cudaEventCreate overlayEnd"); + + // Query NVENC capability/version diagnostics before building the config. + // Optional fields (custom VBV, AQ) are only enabled when the device + // reports them, which avoids NV_ENC_ERR_INVALID_CALL (8) style failures on + // hardware/driver combinations that do not support the requested fields. + capabilityProbe_ = capabilityProbe; + const bool vbrSupported = + (capabilityProbe_.supportedRateControlModes & (1 << NV_ENC_PARAMS_RC_VBR)) != 0; + const bool customVbvSupported = capabilityProbe_.customVbvBufferSizeSupported; + const bool aqSupported = capabilityProbe_.temporalAqSupported; NV_ENC_INITIALIZE_PARAMS initializeParams = {NV_ENC_INITIALIZE_PARAMS_VER}; NV_ENC_CONFIG encodeConfig = {NV_ENC_CONFIG_VER}; initializeParams.encodeConfig = &encodeConfig; - encoder_.CreateDefaultEncoderParams( - &initializeParams, - NV_ENC_CODEC_H264_GUID, - getNvencPresetGuid(layoutOptions_.encodingMode)); - + const GUID codecGuid = layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_CODEC_HEVC_GUID + : NV_ENC_CODEC_H264_GUID; + // Build the encoder config explicitly instead of relying on + // nvEncGetEncodePresetConfig: on current SDK/driver combos the preset + // query can return an empty NV_ENC_CONFIG (rc=CONSTQP, no bitrate, + // chromaFormatIDC=0), which makes nvEncInitializeEncoder fail with + // NV_ENC_ERR_INVALID_PARAM (error 8) even for a valid NV12 export. The + // explicit minimal config below is valid on every supported NVENC device. + initializeParams.encodeGUID = codecGuid; + initializeParams.presetGUID = getNvencPresetGuid(layoutOptions_.encodingMode); + initializeParams.tuningInfo = getNvencTuningInfo(); + initializeParams.encodeWidth = static_cast(width); + initializeParams.encodeHeight = static_cast(height); + initializeParams.darWidth = static_cast(width); + initializeParams.darHeight = static_cast(height); + initializeParams.maxEncodeWidth = static_cast(width); + initializeParams.maxEncodeHeight = static_cast(height); + initializeParams.enablePTD = 1; initializeParams.frameRateNum = static_cast(fps); initializeParams.frameRateDen = 1; + // Async NVENC is the default on every supported device and is required for + // the compositor's stream-ordered pipeline; the capability probe reports it + // where available, but the sync fallback is never selected on failure. initializeParams.enableEncodeAsync = 1; - encodeConfig.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID; + encodeConfig.profileGUID = layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_HEVC_PROFILE_MAIN_GUID + : NV_ENC_H264_PROFILE_HIGH_GUID; encodeConfig.gopLength = static_cast(fps * 2); encodeConfig.frameIntervalP = 1; - encodeConfig.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR; + // Minimal-first rate control: VBR when the device lists it, otherwise CBR. + encodeConfig.rcParams.rateControlMode = + vbrSupported ? NV_ENC_PARAMS_RC_VBR : NV_ENC_PARAMS_RC_CBR; encodeConfig.rcParams.averageBitRate = bitrate; - encodeConfig.rcParams.maxBitRate = getNvencMaxBitrate(bitrate, layoutOptions_.encodingMode); - encodeConfig.rcParams.vbvBufferSize = getNvencBufferSize(bitrate, layoutOptions_.encodingMode); - encodeConfig.rcParams.vbvInitialDelay = bitrate; - if (layoutOptions_.encodingMode != "fast") { + nvencConfigUsed_.customVbv = customVbvSupported; + if (customVbvSupported) { + encodeConfig.rcParams.maxBitRate = + getNvencMaxBitrate(bitrate, layoutOptions_.encodingMode); + encodeConfig.rcParams.vbvBufferSize = + getNvencBufferSize(bitrate, layoutOptions_.encodingMode); + encodeConfig.rcParams.vbvInitialDelay = bitrate; + } + nvencConfigUsed_.aq = aqSupported && layoutOptions_.encodingMode != "fast"; + if (nvencConfigUsed_.aq) { encodeConfig.rcParams.enableAQ = 1; - encodeConfig.rcParams.aqStrength = layoutOptions_.encodingMode == "quality" ? 10 : 8; + encodeConfig.rcParams.aqStrength = + layoutOptions_.encodingMode == "quality" ? 10 : 8; + } + if (layoutOptions_.outputCodec == OutputCodec::HEVC) { + encodeConfig.encodeCodecConfig.hevcConfig.idrPeriod = encodeConfig.gopLength; + encodeConfig.encodeCodecConfig.hevcConfig.chromaFormatIDC = 1; + } else { + encodeConfig.encodeCodecConfig.h264Config.idrPeriod = encodeConfig.gopLength; + encodeConfig.encodeCodecConfig.h264Config.chromaFormatIDC = 1; } - encodeConfig.encodeCodecConfig.h264Config.idrPeriod = encodeConfig.gopLength; encoder_.CreateEncoder(&initializeParams); + nvencConfigUsed_.rcMode = + encodeConfig.rcParams.rateControlMode == NV_ENC_PARAMS_RC_VBR ? "vbr" : "cbr"; + refreshCapabilityProbeFromEncoder(); output_.open(outputPath, std::ios::binary); if (!output_) { @@ -2141,12 +5088,71 @@ public: (std::abs(zoomSample.scale - 1.0) > 0.001 || std::abs(zoomSample.x) > 0.5 || std::abs(zoomSample.y) > 0.5); + const float safeZoomScale = std::max(0.01f, static_cast(zoomSample.scale)); + int blurRegionLeft = layoutOptions_.contentX; + int blurRegionTop = layoutOptions_.contentY; + int blurRegionRight = layoutOptions_.contentX + layoutOptions_.contentWidth; + int blurRegionBottom = layoutOptions_.contentY + layoutOptions_.contentHeight; + if (zoomChangesLayout) { + blurRegionLeft = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentX * safeZoomScale + zoomSample.x))); + blurRegionTop = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentY * safeZoomScale + zoomSample.y))); + blurRegionRight = std::min( + width_, + static_cast(std::ceil( + (layoutOptions_.contentX + layoutOptions_.contentWidth) * safeZoomScale + + zoomSample.x))); + blurRegionBottom = std::min( + height_, + static_cast(std::ceil( + (layoutOptions_.contentY + layoutOptions_.contentHeight) * safeZoomScale + + zoomSample.y))); + } + blurRegionRight = std::max(blurRegionLeft + 2, blurRegionRight); + blurRegionBottom = std::max(blurRegionTop + 2, blurRegionBottom); const bool useFastRoiComposite = canUseFastRoiComposite(zoomChangesLayout); const bool useLayeredStaticRoiComposite = !useFastRoiComposite && canUseLayeredStaticRoiComposite(zoomChangesLayout); + const bool useTemporalBlur = temporalBlurActive(); const auto compositeStart = std::chrono::steady_clock::now(); - if (useFastRoiComposite) { + checkCuda(cudaEventRecord(compositeStartEvent_, copyStream_), "cudaEventRecord compositeStart"); + if (useTemporalBlur) { + // Temporal zoom motion blur: re-composite the same decoded content at + // the renderer's symmetric shutter sample offsets (cos-tapered weights) + // with the camera transform interpolated from the zoom telemetry, then + // accumulate the weighted samples. This reproduces the configured + // high-level temporal sample plan natively instead of substituting the + // spatial blur. Webcam/cursor are applied once afterward (sharp) and + // the RGBA sidecar is blended last, so overlays stay crisp. + compositeTemporalBlurSamples( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + outputFrameTimeMs); + applySharpOverlays( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + webcamFrame, + cursorPosition, + cursorX, + cursorY, + cursorWidth, + cursorHeight, + useCursorAtlas, + cursorEntry, + block); + ++roiCompositeFrames_; + } else if (useFastRoiComposite) { copyNv12Kernel<<>>( srcFrame, srcPitch, @@ -2197,7 +5203,9 @@ public: } } - if (cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0) { + const bool drawCursor = cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0 && + !hasOverlayLayers_; + if (drawCursor) { const int cursorPadding = useCursorAtlas ? 4 : 2; const int regionX = std::max(0, cursorX - cursorPadding); const int regionY = std::max(0, cursorY - cursorPadding); @@ -2426,7 +5434,9 @@ public: } } - if (cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0) { + const bool drawCursor = cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0 && + !hasOverlayLayers_; + if (drawCursor) { const int cursorPadding = useCursorAtlas ? 4 : 2; const int regionX = std::max(0, cursorX - cursorPadding); const int regionY = std::max(0, cursorY - cursorPadding); @@ -2476,6 +5486,10 @@ public: static_cast(inputFrame->chromaOffsets[0]), width_, height_, + 0, + 0, + width_, + height_, layoutOptions_.contentX, layoutOptions_.contentY, layoutOptions_.contentWidth, @@ -2514,7 +5528,9 @@ public: zoomEnabled, static_cast(zoomSample.scale), static_cast(zoomSample.x), - static_cast(zoomSample.y)); + static_cast(zoomSample.y), + 0, + 0); checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel"); ++monolithicCompositeFrames_; } else { @@ -2532,10 +5548,121 @@ public: checkCuda(cudaGetLastError(), "copyNv12Kernel"); ++copyCompositeFrames_; } - if (streamSync_) { - checkCuda(cudaStreamSynchronize(copyStream_), "cudaStreamSynchronize copy"); - } else { - checkCuda(cudaDeviceSynchronize(), "cudaDeviceSynchronize"); + checkCuda(cudaEventRecord(compositeEndEvent_, copyStream_), "cudaEventRecord compositeEnd"); + if (!useTemporalBlur && zoomTrack_ && zoomSample.blurStrength > 0.001) { + checkCuda(cudaEventRecord(blurStartEvent_, copyStream_), "cudaEventRecord blurStart"); + applyZoomBlurFrame( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + blurRegionLeft, + blurRegionTop, + blurRegionRight, + blurRegionBottom, + static_cast(zoomSample.blurCenterX), + static_cast(zoomSample.blurCenterY), + static_cast(zoomSample.blurStrength)); + checkCuda(cudaEventRecord(blurEndEvent_, copyStream_), "cudaEventRecord blurEnd"); + zoomBlurRecorded_ = true; + } + if (hasOverlayLayers_) { + checkCuda(cudaEventRecord(overlayStartEvent_, copyStream_), "cudaEventRecord overlayStart"); + if (overlaySource_ && !overlaySource_->empty()) { + overlaySource_->beginFrame(outputFrameIndex, copyStream_); + } + if (tiledOverlaySource_ && !tiledOverlaySource_->empty()) { + tiledOverlaySource_->beginFrame(outputFrameIndex, copyStream_); + } + for (const auto& entry : compositeLayers_) { + if (entry.kind == CompositeLayer::Kind::Raw) { + const size_t layerIndex = static_cast(entry.sourceIndex); + const auto& layer = overlaySource_->descriptor(layerIndex); + const OverlayBlendRegion overlayRegion = overlaySource_->blendRegion(layerIndex); + if (overlayRegion.width <= 0 || overlayRegion.height <= 0) { + // Fully transparent static layer: the bounded region is empty + // and the full-frame blend would write nothing either. + continue; + } + const dim3 overlayGrid( + (overlayRegion.width + block.x - 1) / block.x, + (overlayRegion.height + block.y - 1) / block.y); + blendOverlayRgbaNv12Kernel<<>>( + overlaySource_->frameDevicePtr(layerIndex, outputFrameIndex), + layer.width, + layer.height, + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + width_, + height_, + layer.x, + layer.y, + layer.width, + layer.height, + overlayRegion.x, + overlayRegion.y, + overlayRegion.width, + overlayRegion.height); + checkCuda(cudaGetLastError(), "blendOverlayRgbaNv12Kernel"); + if (overlayRegion.bounded) { + ++overlayStaticRegionBlends_; + } + } else { + const size_t layerIndex = static_cast(entry.sourceIndex); + const auto& layer = tiledOverlaySource_->descriptor(layerIndex); + const dim3 tiledGrid( + (layer.width + block.x - 1) / block.x, + (layer.height + block.y - 1) / block.y); + blendTiledOverlayRgbaNv12Kernel<<>>( + tiledOverlaySource_->tileCanvasDevicePtr(layerIndex), + layer.tileSize, + layer.tileColumns, + static_cast(layer.tileByteSize), + layer.width, + layer.height, + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + width_, + height_, + layer.x, + layer.y); + checkCuda(cudaGetLastError(), "blendTiledOverlayRgbaNv12Kernel"); + } + } + if (tiledOverlaySource_ && !tiledOverlaySource_->empty()) { + ++tiledOverlayBlendFrames_; + } + ++overlayBlendFrames_; + checkCuda(cudaEventRecord(overlayEndEvent_, copyStream_), "cudaEventRecord overlayEnd"); + overlayRecorded_ = true; + } + // Single per-frame synchronization on the compositor stream. Composite, + // zoom blur, and overlay blends are all queued on the copy stream, and + // NVENC's synchronous input copy needs them complete, so one + // cudaStreamSynchronize is both necessary and sufficient; the previous + // double sync (and any global device sync) added a full round trip per + // frame without improving correctness. + checkCuda(cudaStreamSynchronize(copyStream_), "cudaStreamSynchronize frame"); + // The overlay read-ahead prefetch is dispatched after the required frame + // sync. File reads run on the bounded background reader threads, so the + // encode thread does not block on disk I/O; ready pinned frames get + // their H2D copy enqueued here so the transfer overlaps NVENC. The + // uploads stay ordered before the next frame's beginFrame/blend because + // they are queued on the same non-blocking compositor stream, and the + // bounded ring never targets the slot the current blend read, so the + // next blend starts only after its frame is resident. + if (overlaySource_ && !overlaySource_->empty()) { + overlaySource_->prefetchNextFrame(outputFrameIndex, copyStream_); + } + accumulateStageGpuTime(compositeStartEvent_, compositeEndEvent_, compositeGpuMs_); + if (zoomBlurRecorded_) { + accumulateStageGpuTime(blurStartEvent_, blurEndEvent_, zoomBlurGpuMs_); + zoomBlurRecorded_ = false; + } + if (overlayRecorded_) { + accumulateStageGpuTime(overlayStartEvent_, overlayEndEvent_, overlayBlendGpuMs_); + overlayRecorded_ = false; } const auto compositeEnd = std::chrono::steady_clock::now(); compositeMs_ += elapsedMs(compositeStart, compositeEnd); @@ -2562,6 +5689,20 @@ public: checkCuda(cudaStreamDestroy(copyStream_), "cudaStreamDestroy"); copyStream_ = nullptr; } + cudaEvent_t stageEvents[] = { + compositeStartEvent_, + compositeEndEvent_, + blurStartEvent_, + blurEndEvent_, + overlayStartEvent_, + overlayEndEvent_, + }; + for (cudaEvent_t& event : stageEvents) { + if (event) { + checkCuda(cudaEventDestroy(event), "cudaEventDestroy stage"); + event = nullptr; + } + } if (backgroundDevice_) { checkCuda(cudaFree(backgroundDevice_), "cudaFree backgroundDevice"); backgroundDevice_ = nullptr; @@ -2574,6 +5715,19 @@ public: checkCuda(cudaFree(cursorAtlasDevice_), "cudaFree cursorAtlasDevice"); cursorAtlasDevice_ = nullptr; } + if (zoomBlurScratch_) { + checkCuda(cudaFree(zoomBlurScratch_), "cudaFree zoomBlurScratch"); + zoomBlurScratch_ = nullptr; + } + if (temporalWeightsDevice_) { + checkCuda(cudaFree(temporalWeightsDevice_), "cudaFree temporalWeightsDevice"); + temporalWeightsDevice_ = nullptr; + temporalWeightsDeviceCount_ = 0; + } + if (temporalBgCacheDevice_) { + checkCuda(cudaFree(temporalBgCacheDevice_), "cudaFree temporalBgCacheDevice"); + temporalBgCacheDevice_ = nullptr; + } } uint64_t outputBytes() const { @@ -2604,7 +5758,819 @@ public: return copyCompositeFrames_; } + int zoomBlurFrames() const { + return zoomBlurFrames_; + } + + int overlayBlendFrames() const { + return overlayBlendFrames_; + } + + int temporalBlurFrames() const { + return temporalBlurFrames_; + } + + int temporalBlurBgPrecomposedFrames() const { + return temporalBlurBgPrecomposedFrames_; + } + + int temporalBlurStationaryFrames() const { + return temporalBlurStationaryFrames_; + } + + int temporalBgCacheBuilds() const { + return temporalBgCacheBuilds_; + } + + int64_t temporalBgCacheHits() const { + return temporalBgCacheHits_; + } + + int64_t overlayStaticRegionBlends() const { + return overlayStaticRegionBlends_; + } + + int64_t overlayFileLoads() const { + return overlaySource_ ? overlaySource_->fileLoads() : 0; + } + + int64_t overlayCacheHits() const { + return (overlaySource_ ? overlaySource_->cacheHits() : 0) + + (tiledOverlaySource_ ? tiledOverlaySource_->cacheHits() : 0); + } + + int64_t overlayPinnedHits() const { + return overlaySource_ ? overlaySource_->pinnedHits() : 0; + } + + int64_t overlayReadWaits() const { + return overlaySource_ ? overlaySource_->readWaits() : 0; + } + + int64_t overlayPendingReadsPeak() const { + return overlaySource_ ? overlaySource_->pendingReadsPeak() : 0; + } + + int tiledOverlayLayerCount() const { + return tiledOverlaySource_ ? static_cast(tiledOverlaySource_->layerCount()) : 0; + } + + int tiledOverlayBlendFrames() const { + return tiledOverlayBlendFrames_; + } + + // Tile payloads uploaded from frame deltas (measured; excludes the static + // base, matching the renderer-derived changedTileCount). + int64_t tiledChangedTileCount() const { + return tiledOverlaySource_ ? tiledOverlaySource_->changedTileCount() : 0; + } + + // Tile payload bytes uploaded once (static base + changed tiles; measured). + int64_t tiledUploadedTileBytes() const { + return tiledOverlaySource_ ? tiledOverlaySource_->uploadedTileBytes() : 0; + } + + // Tile-state lookups served from previously uploaded payloads (measured; + // diagnostic only, never a zero-copy claim). + int64_t tiledCachedTileCount() const { + return tiledOverlaySource_ ? tiledOverlaySource_->cachedTileCount() : 0; + } + + // First conservative tiled-vs-raw eligibility decision, "" when every tiled + // layer is eligible. Observable diagnostic; the helper never silently + // falls back to a raw/CPU path for a requested tiled stream. + const std::string& tiledRawFallbackReason() const { + static const std::string kEmpty; + return tiledOverlaySource_ ? tiledOverlaySource_->rawFallbackReason() : kEmpty; + } + + int64_t temporalBlurSamplesTotal() const { + return temporalBlurSamplesTotal_; + } + + const NvencCapabilityProbe& capabilityProbe() const { + return capabilityProbe_; + } + + const NvencConfigUsed& nvencConfigUsed() const { + return nvencConfigUsed_; + } + + double compositeGpuMs() const { + return compositeGpuMs_; + } + + double zoomBlurGpuMs() const { + return zoomBlurGpuMs_; + } + + double overlayBlendGpuMs() const { + return overlayBlendGpuMs_; + } + + double overlayUploadMs() const { + return (overlaySource_ ? overlaySource_->uploadMs() : 0.0) + + (tiledOverlaySource_ ? tiledOverlaySource_->hostReadMs() + + tiledOverlaySource_->h2dEnqueueMs() + : 0.0); + } + + double overlayHostReadMs() const { + return (overlaySource_ ? overlaySource_->hostReadMs() : 0.0) + + (tiledOverlaySource_ ? tiledOverlaySource_->hostReadMs() : 0.0); + } + + double overlayH2DEnqueueMs() const { + return (overlaySource_ ? overlaySource_->h2dEnqueueMs() : 0.0) + + (tiledOverlaySource_ ? tiledOverlaySource_->h2dEnqueueMs() : 0.0); + } + private: + void refreshCapabilityProbeFromEncoder() { + // The probe session can fail caps queries on some driver/GPU combos + // (NV_ENC_ERR_ENCODER_NOT_INITIALIZED on the probe session even though + // the real encoder session works). Re-query the caps through the live + // encoder session so diagnostics report what the device truly supports + // instead of conservative fallbacks. Codec support is re-queried per + // codec (H.264 caps for h264Supported, HEVC caps for hevcSupported); + // the rate-control/VBV/AQ/dimension caps are re-queried for the codec + // the live encoder was created with. + const GUID codecGuid = codecGuidForEncoder(); + auto queryCap = [&](GUID queryCodecGuid, NV_ENC_CAPS cap) -> int { + return encoder_.GetCapabilityValue(queryCodecGuid, cap); + }; + capabilityProbe_.h264Supported = + queryCap(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES) >= 0; + capabilityProbe_.hevcSupported = + queryCap(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES) >= 0; + const int rcModes = queryCap(codecGuid, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES); + if (rcModes >= 0) { + capabilityProbe_.supportedRateControlModes = rcModes; + } + capabilityProbe_.customVbvBufferSizeSupported = + queryCap(codecGuid, NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE) > 0; + capabilityProbe_.asyncEncodeSupported = + queryCap(codecGuid, NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT) > 0; + capabilityProbe_.temporalAqSupported = + queryCap(codecGuid, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ) > 0; + capabilityProbe_.widthMax = queryCap(codecGuid, NV_ENC_CAPS_WIDTH_MAX); + capabilityProbe_.heightMax = queryCap(codecGuid, NV_ENC_CAPS_HEIGHT_MAX); + capabilityProbe_.mbPerSecMax = queryCap(codecGuid, NV_ENC_CAPS_MB_PER_SEC_MAX); + } + + GUID codecGuidForEncoder() const { + return layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_CODEC_HEVC_GUID + : NV_ENC_CODEC_H264_GUID; + } + + void accumulateStageGpuTime( + cudaEvent_t startEvent, + cudaEvent_t endEvent, + double& accumulator) { + if (!startEvent || !endEvent) { + return; + } + float elapsed = 0.0f; + checkCuda( + cudaEventElapsedTime(&elapsed, startEvent, endEvent), + "cudaEventElapsedTime stage"); + accumulator += elapsed; + } + + bool temporalBlurActive() const { + return temporalBlurSampleCount_ > 0 && + zoomTrack_ != nullptr && + hasStaticLayout(layoutOptions_); + } + + void compositeTemporalBlurSamples( + unsigned char* target, + int targetPitch, + int targetChromaOffset, + const unsigned char* srcFrame, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, + double outputFrameTimeMs) { + const std::vector& samples = temporalSamplePlan_; + + const dim3 block(16, 16); + const dim3 grid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + + const bool hasSourceCrop = + layoutOptions_.sourceCropWidth >= 2 && + layoutOptions_.sourceCropHeight >= 2; + const int sourceCropX = hasSourceCrop + ? std::max(0, std::min(layoutOptions_.sourceCropX, srcWidth - 2)) & ~1 + : 0; + const int sourceCropY = hasSourceCrop + ? std::max(0, std::min(layoutOptions_.sourceCropY, srcHeight - 2)) & ~1 + : 0; + const int sourceCropWidth = hasSourceCrop + ? std::max(2, std::min(layoutOptions_.sourceCropWidth, srcWidth - sourceCropX)) & ~1 + : srcWidth; + const int sourceCropHeight = hasSourceCrop + ? std::max(2, std::min(layoutOptions_.sourceCropHeight, srcHeight - sourceCropY)) & ~1 + : srcHeight; + + // Fused composite + accumulate: the first sample replaces the target (the + // NVENC input buffer is not pre-zeroed), later samples saturate-accumulate + // the same (weight * value + 128) >> 8 math the previous two-pass + // fill/composite/accumulate pipeline produced. One pass per sample and no + // scratch buffer. The legacy path composites the full frame per sample; + // the background-precompose path below restricts per-sample work to the + // changing content region once the invariant background is accumulated. + int bboxLeft = width_; + int bboxTop = height_; + int bboxRight = 0; + int bboxBottom = 0; + bool anyContentVisible = false; + // Exact stationary shutter-window detection: when every sample resolves + // to a bit-identical camera transform (scale/x/y), every pixel selects + // the same source/layout value for every sample, so the weighted + // temporal accumulation can evaluate source + layout once per pixel and + // apply the existing fixed-point weights in order (see + // compositeStaticStationaryNv12Kernel). Any double inequality is + // enough to fall back to the per-sample paths; the comparison is exact + // so the fused path can never be chosen when per-sample transforms + // differ (even by one ULP). + bool stationaryWindow = !samples.empty(); + double stationaryScale = 0.0; + double stationaryX = 0.0; + double stationaryY = 0.0; + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + if (index == 0) { + stationaryScale = sample.scale; + stationaryX = sample.x; + stationaryY = sample.y; + } else if ( + sample.scale != stationaryScale || + sample.x != stationaryX || + sample.y != stationaryY) { + stationaryWindow = false; + } + const float safeZoomScale = std::max(0.01f, static_cast(sample.scale)); + const int transformedLeft = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentX * safeZoomScale + sample.x))); + const int transformedTop = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentY * safeZoomScale + sample.y))); + const int transformedRight = std::min( + width_, + static_cast(std::ceil( + (layoutOptions_.contentX + layoutOptions_.contentWidth) * safeZoomScale + + sample.x))); + const int transformedBottom = std::min( + height_, + static_cast(std::ceil( + (layoutOptions_.contentY + layoutOptions_.contentHeight) * safeZoomScale + + sample.y))); + if (transformedRight > transformedLeft && transformedBottom > transformedTop) { + anyContentVisible = true; + bboxLeft = std::min(bboxLeft, transformedLeft); + bboxTop = std::min(bboxTop, transformedTop); + bboxRight = std::max(bboxRight, transformedRight); + bboxBottom = std::max(bboxBottom, transformedBottom); + } + } + if (stationaryWindow) { + // Every sample applies the identical transform, so sample 0's + // transform (captured by the detection loop) is the per-sample + // transform the original loop used for every sample. The fused + // kernel reproduces the replace-then-accumulate chain exactly while + // evaluating source + layout once. + ensureTemporalWeightsDevice(); + const bool sampleZoomEnabled = zoomTrack_ && stationaryScale > 0.01; + compositeStaticStationaryNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + 0, + 0, + width_, + height_, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + sampleZoomEnabled, + static_cast(stationaryScale), + static_cast(stationaryX), + static_cast(stationaryY), + temporalWeightsDevice_, + static_cast(samples.size())); + checkCuda(cudaGetLastError(), "compositeStaticStationaryNv12Kernel"); + temporalBlurSamplesTotal_ += static_cast(samples.size()); + ++temporalBlurFrames_; + ++temporalBlurStationaryFrames_; + return; + } + + // The region passed to the per-sample composite must cover the UV corner + // pixels: a UV block at even x is decided by the corner (x + 1, y + 1), + // which can map inside the content rect one pixel beyond the luma bbox + // (e.g. at an odd bbox edge). Expand the bounding box by one pixel on + // every side (clamped to the frame) so corner-driven chroma blocks get + // the same per-sample content/bg selection the full-frame kernel makes. + // Pixels inside the expansion that are background for every sample are + // recomputed identically by the region kernel, so the expansion is + // exact and only adds one boundary row/column of work. + const int regionLeft = std::max(0, bboxLeft - 1); + const int regionTop = std::max(0, bboxTop - 1); + const int regionRight = std::min(width_, bboxRight + 1); + const int regionBottom = std::min(height_, bboxBottom + 1); + const int regionWidth = regionRight - regionLeft; + const int regionHeight = regionBottom - regionTop; + const bool useBackgroundPrecompose = + !samples.empty() && + layoutOptions_.shadowIntensityPct == 0 && + (!anyContentVisible || + (regionWidth > 0 && regionHeight > 0 && + // Sample-count-aware break-even gate: the precompose path costs one + // full-frame background pass plus sampleCount region passes, while + // the per-sample path costs sampleCount full-frame passes, so + // precompose wins when 1 + N*r < N with r the region/frame area + // ratio, i.e. regionArea * N < frameArea * (N - 1). Both paths are + // bit-identical; the gate only trades GPU work. + static_cast(regionWidth) * regionHeight * + static_cast(samples.size()) < + static_cast(width_) * height_ * + static_cast(samples.size() - 1))); + if (useBackgroundPrecompose) { + compositeTemporalBlurSamplesWithBackgroundPrecompose( + target, + targetPitch, + targetChromaOffset, + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + samples, + block, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + regionLeft, + regionTop, + regionWidth, + regionHeight, + anyContentVisible, + outputFrameTimeMs); + ++temporalBlurFrames_; + ++temporalBlurBgPrecomposedFrames_; + return; + } + + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + const bool sampleZoomEnabled = zoomTrack_ && sample.scale > 0.01; + const unsigned int weightFixed = static_cast( + std::lround(samples[index].weight * 256.0)); + compositeStaticNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + 0, + 0, + width_, + height_, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + false, + false, + 0, + 0, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + sampleZoomEnabled, + static_cast(sample.scale), + static_cast(sample.x), + static_cast(sample.y), + weightFixed, + index == 0 ? 1 : 2); + checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel temporal sample"); + temporalBlurSamplesTotal_ += 1; + } + ++temporalBlurFrames_; + } + + // Fast temporal-blur path: the invariant weighted background is accumulated + // once per export into a persistent NV12 cache (see + // ensureTemporalBackgroundCache); each output frame copies that cache into + // the target and then composites only the changing content bounding box per + // temporal sample. Pixels outside the bbox are outside the content rect for + // every sample, so their per-sample value is always the background and the + // cached accumulation is exact. Inside the bbox, sample 0 replaces the + // cached background (the target is not pre-zeroed) and later samples + // saturate-accumulate, preserving the exact replace-then-accumulate + // contract of the per-sample full-frame composites term-for-term. The + // cache copy is bit-identical to the previous per-frame accumulate because + // accumulateBackgroundNv12Kernel replaces (never accumulates into) each + // destination pixel. + void compositeTemporalBlurSamplesWithBackgroundPrecompose( + unsigned char* target, + int targetPitch, + int targetChromaOffset, + const unsigned char* srcFrame, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, + const std::vector& samples, + const dim3& block, + int sourceCropX, + int sourceCropY, + int sourceCropWidth, + int sourceCropHeight, + int regionLeft, + int regionTop, + int regionWidth, + int regionHeight, + bool anyContentVisible, + double outputFrameTimeMs) { + ensureTemporalWeightsDevice(); + ensureTemporalBackgroundCache(); + checkCuda( + cudaMemcpy2DAsync( + target, + static_cast(targetPitch), + temporalBgCacheDevice_, + static_cast(width_), + static_cast(width_), + static_cast(height_), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync temporal background cache Y"); + checkCuda( + cudaMemcpy2DAsync( + target + targetChromaOffset, + static_cast(targetPitch), + temporalBgCacheDevice_ + static_cast(width_) * static_cast(height_), + static_cast(width_), + static_cast(width_), + static_cast(height_ / 2), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync temporal background cache UV"); + ++temporalBgCacheHits_; + + if (!anyContentVisible || regionWidth <= 0 || regionHeight <= 0) { + return; + } + const dim3 contentGrid( + (regionWidth + block.x - 1) / block.x, + (regionHeight + block.y - 1) / block.y); + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + const bool sampleZoomEnabled = zoomTrack_ && sample.scale > 0.01; + const unsigned int weightFixed = static_cast( + std::lround(samples[index].weight * 256.0)); + compositeStaticNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + regionLeft, + regionTop, + regionWidth, + regionHeight, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + false, + false, + 0, + 0, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + sampleZoomEnabled, + static_cast(sample.scale), + static_cast(sample.x), + static_cast(sample.y), + weightFixed, + index == 0 ? 1 : 2); + checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel temporal sample region"); + temporalBlurSamplesTotal_ += 1; + } + } + + // Builds the persistent NV12 cache of the invariant weighted background + // accumulation once per export. The accumulated background depends only on + // the fixed sample weight plan and the fixed background (color or NV12 + // sidecar), so it is identical for every temporal-blur output frame; the + // first precomposed frame fills the cache with the same + // accumulateBackgroundNv12Kernel pass the previous code ran per frame, and + // later frames copy the cache into the target instead of re-accumulating. + // The cache is tightly packed (pitch == width), so the per-frame copy is + // two ordered cudaMemcpy2DAsync calls on the compositor stream. The copy is + // term-for-term identical to the per-frame accumulate because + // accumulateBackgroundNv12Kernel replaces (never accumulates into) each + // destination pixel. + void ensureTemporalBackgroundCache() { + if (temporalBgCacheDevice_) { + return; + } + const size_t requiredBytes = + static_cast(width_) * static_cast(height_) * 3 / 2; + checkCuda(cudaMalloc(&temporalBgCacheDevice_, requiredBytes), "cudaMalloc temporalBgCacheDevice"); + const dim3 block(16, 16); + const dim3 fullGrid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + accumulateBackgroundNv12Kernel<<>>( + temporalBgCacheDevice_, + width_, + static_cast(static_cast(width_) * static_cast(height_)), + width_, + height_, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + temporalWeightsDevice_, + static_cast(temporalSamplePlan_.size())); + checkCuda(cudaGetLastError(), "accumulateBackgroundNv12Kernel cache build"); + ++temporalBgCacheBuilds_; + } + + // Uploads the cos-tapered temporal sample weights to a device buffer once; + // accumulateBackgroundNv12Kernel needs the whole plan resident for the + // invariant-background pass and the stationary fused kernel needs it for + // the in-order fixed-point accumulation. + void ensureTemporalWeightsDevice() { + const size_t count = temporalSamplePlan_.size(); + if (count == 0 || (temporalWeightsDevice_ && temporalWeightsDeviceCount_ == count)) { + return; + } + if (temporalWeightsDevice_) { + checkCuda(cudaFree(temporalWeightsDevice_), "cudaFree temporalWeightsDevice"); + temporalWeightsDevice_ = nullptr; + } + std::vector weights(count); + for (size_t index = 0; index < count; ++index) { + weights[index] = static_cast( + std::lround(temporalSamplePlan_[index].weight * 256.0)); + } + checkCuda( + cudaMalloc(&temporalWeightsDevice_, count * sizeof(unsigned int)), + "cudaMalloc temporalWeightsDevice"); + checkCuda( + cudaMemcpy( + temporalWeightsDevice_, + weights.data(), + count * sizeof(unsigned int), + cudaMemcpyHostToDevice), + "cudaMemcpy temporalWeightsDevice"); + temporalWeightsDeviceCount_ = count; + } + + void applySharpOverlays( + unsigned char* frame, + int framePitch, + int frameChromaOffset, + const unsigned char* webcamFrame, + const CursorPosition& cursorPosition, + int cursorX, + int cursorY, + int cursorWidth, + int cursorHeight, + bool useCursorAtlas, + const CursorAtlasEntry* cursorEntry, + const dim3& block) { + if (webcamFrame && layoutOptions_.webcamSize > 0) { + const int webcamRegionX = std::max(0, layoutOptions_.webcamX - 1); + const int webcamRegionY = std::max(0, layoutOptions_.webcamY - 1); + const int webcamRegionRight = + std::min(width_, layoutOptions_.webcamX + layoutOptions_.webcamSize); + const int webcamRegionBottom = + std::min(height_, layoutOptions_.webcamY + layoutOptions_.webcamSize); + const int webcamRegionWidth = webcamRegionRight - webcamRegionX; + const int webcamRegionHeight = webcamRegionBottom - webcamRegionY; + if (webcamRegionWidth > 0 && webcamRegionHeight > 0) { + const dim3 webcamGrid( + (webcamRegionWidth + block.x - 1) / block.x, + (webcamRegionHeight + block.y - 1) / block.y); + overlayWebcamNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + webcamFrame, + webcamRegionX, + webcamRegionY, + webcamRegionWidth, + webcamRegionHeight, + layoutOptions_.webcamX, + layoutOptions_.webcamY, + layoutOptions_.webcamSize, + webcamFrameWidth(), + webcamFrameHeight(), + layoutOptions_.webcamRadius, + layoutOptions_.webcamMirror); + checkCuda(cudaGetLastError(), "overlayWebcamNv12Kernel sharp"); + } + } + + if (cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0) { + const int cursorPadding = useCursorAtlas ? 4 : 2; + const int regionX = std::max(0, cursorX - cursorPadding); + const int regionY = std::max(0, cursorY - cursorPadding); + const int regionRight = std::min(width_, cursorX + cursorWidth + cursorPadding); + const int regionBottom = std::min(height_, cursorY + cursorHeight + cursorPadding); + const int regionWidth = regionRight - regionX; + const int regionHeight = regionBottom - regionY; + if (regionWidth > 0 && regionHeight > 0) { + const dim3 cursorGrid( + (regionWidth + block.x - 1) / block.x, + (regionHeight + block.y - 1) / block.y); + overlayCursorNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + regionX, + regionY, + regionWidth, + regionHeight, + (cursorPosition.visible && !hasOverlayLayers_), + cursorX, + cursorY, + cursorWidth, + cursorHeight, + useCursorAtlas ? cursorAtlasDevice_ : nullptr, + cursorAtlasWidth_, + cursorAtlasHeight_, + useCursorAtlas ? cursorEntry->x : 0, + useCursorAtlas ? cursorEntry->y : 0, + useCursorAtlas ? cursorEntry->width : 0, + useCursorAtlas ? cursorEntry->height : 0); + checkCuda(cudaGetLastError(), "overlayCursorNv12Kernel sharp"); + } + } + } + + void applyZoomBlurFrame( + unsigned char* frame, + int framePitch, + int frameChromaOffset, + int regionLeft, + int regionTop, + int regionRight, + int regionBottom, + float centerX, + float centerY, + float strength) { + const size_t requiredBytes = + static_cast(framePitch) * static_cast(height_) + + static_cast(framePitch) * static_cast(height_ / 2); + if (!zoomBlurScratch_ || zoomBlurScratchBytes_ < requiredBytes) { + if (zoomBlurScratch_) { + checkCuda(cudaFree(zoomBlurScratch_), "cudaFree zoomBlurScratch"); + zoomBlurScratch_ = nullptr; + zoomBlurScratchBytes_ = 0; + } + checkCuda(cudaMalloc(&zoomBlurScratch_, requiredBytes), "cudaMalloc zoomBlurScratch"); + zoomBlurScratchBytes_ = requiredBytes; + } + + const dim3 block(16, 16); + const dim3 grid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + zoomBlurNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + zoomBlurScratch_, + framePitch, + frameChromaOffset, + regionLeft, + regionTop, + regionRight, + regionBottom, + centerX, + centerY, + strength); + checkCuda(cudaGetLastError(), "zoomBlurNv12Kernel"); + + checkCuda( + cudaMemcpy2DAsync( + frame, + static_cast(framePitch), + zoomBlurScratch_, + static_cast(framePitch), + static_cast(width_), + static_cast(height_), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync zoom blur Y"); + checkCuda( + cudaMemcpy2DAsync( + frame + frameChromaOffset, + static_cast(framePitch), + zoomBlurScratch_ + static_cast(framePitch) * static_cast(height_), + static_cast(framePitch), + static_cast(width_), + static_cast(height_ / 2), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync zoom blur UV"); + ++zoomBlurFrames_; + } + bool canUseFastRoiComposite(bool zoomChangesLayout) const { return hasStaticLayout(layoutOptions_) && layoutOptions_.contentX == 0 && @@ -2772,7 +6738,30 @@ private: int roiCompositeFrames_ = 0; int monolithicCompositeFrames_ = 0; int copyCompositeFrames_ = 0; - bool streamSync_ = false; + int zoomBlurFrames_ = 0; + int overlayBlendFrames_ = 0; + int tiledOverlayBlendFrames_ = 0; + int temporalBlurFrames_ = 0; + int temporalBlurBgPrecomposedFrames_ = 0; + int temporalBlurStationaryFrames_ = 0; + int temporalBgCacheBuilds_ = 0; + int64_t temporalBgCacheHits_ = 0; + int64_t overlayStaticRegionBlends_ = 0; + int64_t temporalBlurSamplesTotal_ = 0; + bool zoomBlurRecorded_ = false; + bool overlayRecorded_ = false; + double compositeGpuMs_ = 0.0; + double zoomBlurGpuMs_ = 0.0; + double overlayBlendGpuMs_ = 0.0; + NvencCapabilityProbe capabilityProbe_; + NvencConfigUsed nvencConfigUsed_; + int temporalBlurSampleCount_ = 0; + double temporalBlurShutterFraction_ = 0.0; + double temporalBlurWeightPower_ = 1.0; + std::vector temporalSamplePlan_; + unsigned int* temporalWeightsDevice_ = nullptr; + size_t temporalWeightsDeviceCount_ = 0; + unsigned char* temporalBgCacheDevice_ = nullptr; Options layoutOptions_; unsigned char* backgroundDevice_ = nullptr; unsigned char* webcamDevice_ = nullptr; @@ -2783,7 +6772,19 @@ private: const WebcamFrameCache* webcamCache_ = nullptr; const CursorTrack* cursorTrack_ = nullptr; const ZoomTrack* zoomTrack_ = nullptr; + OverlayFrameSource* overlaySource_ = nullptr; + TiledOverlayFrameSource* tiledOverlaySource_ = nullptr; + std::vector compositeLayers_; + bool hasOverlayLayers_ = false; cudaStream_t copyStream_ = nullptr; + cudaEvent_t compositeStartEvent_ = nullptr; + cudaEvent_t compositeEndEvent_ = nullptr; + cudaEvent_t blurStartEvent_ = nullptr; + cudaEvent_t blurEndEvent_ = nullptr; + cudaEvent_t overlayStartEvent_ = nullptr; + cudaEvent_t overlayEndEvent_ = nullptr; + unsigned char* zoomBlurScratch_ = nullptr; + size_t zoomBlurScratchBytes_ = 0; }; struct CallbackEncodeState { @@ -2798,6 +6799,9 @@ struct CallbackEncodeState { const WebcamFrameCache* webcamCache = nullptr; const CursorTrack* cursorTrack = nullptr; const ZoomTrack* zoomTrack = nullptr; + OverlayFrameSource* overlaySource = nullptr; + TiledOverlayFrameSource* tiledOverlaySource = nullptr; + const NvencCapabilityProbe* capabilityProbe = nullptr; ProgressReportState* progress = nullptr; bool oneFramePerMappedDisplayFrame = false; int mappedFrames = 0; @@ -2815,11 +6819,35 @@ ProgressCounters collectProgressCounters( counters.encodeMs = encodeMs; if (sink) { counters.compositeMs = sink->compositeMs(); + counters.compositeGpuMs = sink->compositeGpuMs(); + counters.zoomBlurGpuMs = sink->zoomBlurGpuMs(); + counters.overlayBlendGpuMs = sink->overlayBlendGpuMs(); + counters.overlayUploadMs = sink->overlayUploadMs(); counters.nvencMs = sink->nvencMs(); counters.packetWriteMs = sink->packetWriteMs(); counters.roiCompositeFrames = sink->roiCompositeFrames(); counters.monolithicCompositeFrames = sink->monolithicCompositeFrames(); counters.copyCompositeFrames = sink->copyCompositeFrames(); + counters.zoomBlurFrames = sink->zoomBlurFrames(); + counters.overlayBlendFrames = sink->overlayBlendFrames(); + counters.temporalBlurFrames = sink->temporalBlurFrames(); + counters.temporalBlurSamplesTotal = sink->temporalBlurSamplesTotal(); + counters.temporalBlurBgPrecomposedFrames = sink->temporalBlurBgPrecomposedFrames(); + counters.temporalBlurStationaryFrames = sink->temporalBlurStationaryFrames(); + counters.temporalBgCacheBuilds = sink->temporalBgCacheBuilds(); + counters.temporalBgCacheHits = sink->temporalBgCacheHits(); + counters.overlayStaticRegionBlends = sink->overlayStaticRegionBlends(); + counters.overlayFileLoads = sink->overlayFileLoads(); + counters.overlayCacheHits = sink->overlayCacheHits(); + counters.overlayPinnedHits = sink->overlayPinnedHits(); + counters.overlayReadWaits = sink->overlayReadWaits(); + counters.overlayPendingReadsPeak = sink->overlayPendingReadsPeak(); + counters.overlayHostReadMs = sink->overlayHostReadMs(); + counters.overlayH2DEnqueueMs = sink->overlayH2DEnqueueMs(); + counters.tiledOverlayLayers = sink->tiledOverlayLayerCount(); + counters.changedTileCount = sink->tiledChangedTileCount(); + counters.uploadedTileBytes = sink->tiledUploadedTileBytes(); + counters.cachedTileCount = sink->tiledCachedTileCount(); } if (webcamCache) { counters.webcamDecodeMs = webcamCache->decodeMs; @@ -2884,6 +6912,14 @@ void encodeMappedDisplayFrame( } if (!*state->sink) { + validateOverlayBounds( + *state->options, + outputWidthForSource(*state->options, width), + outputHeightForSource(*state->options, height)); + validateTiledOverlayBounds( + *state->options, + outputWidthForSource(*state->options, width), + outputHeightForSource(*state->options, height)); *state->sink = std::make_unique( state->context, outputWidthForSource(*state->options, width), @@ -2891,11 +6927,13 @@ void encodeMappedDisplayFrame( state->options->fps, state->bitrate, state->options->outputPath, - state->options->streamSync, *state->options, state->webcamCache, state->cursorTrack, - state->zoomTrack); + state->zoomTrack, + state->overlaySource, + state->tiledOverlaySource, + state->capabilityProbe ? *state->capabilityProbe : NvencCapabilityProbe{}); } while (*state->encodedFrames < expectedOutputFrames && *state->encodedFrames < maxOutputFrames) { @@ -2953,6 +6991,16 @@ void reportEncodingProgress( intervalMs > 0.0 && intervalFrames > 0 ? static_cast(intervalFrames) * 1000.0 / intervalMs : 0.0; const double intervalEncodeMs = std::max(0.0, counters.encodeMs - state.lastCounters.encodeMs); const double intervalCompositeMs = std::max(0.0, counters.compositeMs - state.lastCounters.compositeMs); + const double intervalCompositeGpuMs = std::max(0.0, counters.compositeGpuMs - state.lastCounters.compositeGpuMs); + const double intervalZoomBlurGpuMs = std::max(0.0, counters.zoomBlurGpuMs - state.lastCounters.zoomBlurGpuMs); + const double intervalOverlayBlendGpuMs = std::max(0.0, counters.overlayBlendGpuMs - state.lastCounters.overlayBlendGpuMs); + const double intervalOverlayUploadMs = std::max(0.0, counters.overlayUploadMs - state.lastCounters.overlayUploadMs); + const double intervalOverlayHostReadMs = std::max( + 0.0, + counters.overlayHostReadMs - state.lastCounters.overlayHostReadMs); + const double intervalOverlayH2DEnqueueMs = std::max( + 0.0, + counters.overlayH2DEnqueueMs - state.lastCounters.overlayH2DEnqueueMs); const double intervalNvencMs = std::max(0.0, counters.nvencMs - state.lastCounters.nvencMs); const double intervalPacketWriteMs = std::max(0.0, counters.packetWriteMs - state.lastCounters.packetWriteMs); const double intervalWebcamDecodeMs = std::max(0.0, counters.webcamDecodeMs - state.lastCounters.webcamDecodeMs); @@ -2965,8 +7013,47 @@ void reportEncodingProgress( std::max(0, counters.monolithicCompositeFrames - state.lastCounters.monolithicCompositeFrames); const int intervalCopyCompositeFrames = std::max(0, counters.copyCompositeFrames - state.lastCounters.copyCompositeFrames); + const int intervalZoomBlurFrames = + std::max(0, counters.zoomBlurFrames - state.lastCounters.zoomBlurFrames); + const int intervalOverlayBlendFrames = + std::max(0, counters.overlayBlendFrames - state.lastCounters.overlayBlendFrames); + const int intervalTemporalBlurFrames = + std::max(0, counters.temporalBlurFrames - state.lastCounters.temporalBlurFrames); + const int64_t intervalTemporalBlurSamples = + std::max(0, counters.temporalBlurSamplesTotal - state.lastCounters.temporalBlurSamplesTotal); + const int intervalTemporalBlurBgPrecomposedFrames = std::max( + 0, + counters.temporalBlurBgPrecomposedFrames - state.lastCounters.temporalBlurBgPrecomposedFrames); + const int intervalTemporalBlurStationaryFrames = std::max( + 0, + counters.temporalBlurStationaryFrames - state.lastCounters.temporalBlurStationaryFrames); + const int intervalTemporalBgCacheBuilds = std::max( + 0, + counters.temporalBgCacheBuilds - state.lastCounters.temporalBgCacheBuilds); + const int64_t intervalTemporalBgCacheHits = std::max( + 0, + counters.temporalBgCacheHits - state.lastCounters.temporalBgCacheHits); + const int64_t intervalOverlayStaticRegionBlends = std::max( + 0, + counters.overlayStaticRegionBlends - state.lastCounters.overlayStaticRegionBlends); + const int64_t intervalOverlayFileLoads = + std::max(0, counters.overlayFileLoads - state.lastCounters.overlayFileLoads); + const int64_t intervalOverlayCacheHits = + std::max(0, counters.overlayCacheHits - state.lastCounters.overlayCacheHits); + const int64_t intervalOverlayPinnedHits = + std::max(0, counters.overlayPinnedHits - state.lastCounters.overlayPinnedHits); + const int64_t intervalOverlayReadWaits = + std::max(0, counters.overlayReadWaits - state.lastCounters.overlayReadWaits); + const int64_t intervalChangedTileCount = + std::max(0, counters.changedTileCount - state.lastCounters.changedTileCount); + const int64_t intervalUploadedTileBytes = std::max( + 0, + counters.uploadedTileBytes - state.lastCounters.uploadedTileBytes); + const int64_t intervalCachedTileCount = + std::max(0, counters.cachedTileCount - state.lastCounters.cachedTileCount); std::cerr << std::fixed << std::setprecision(2) - << "PROGRESS {\"currentFrame\":" << encodedFrames + << "PROGRESS {\"outputCodec\":\"" << state.outputCodec + << "\",\"currentFrame\":" << encodedFrames << ",\"totalFrames\":" << totalFrames << ",\"percentage\":" << percentage << ",\"averageFps\":" << averageFps @@ -2977,6 +7064,12 @@ void reportEncodingProgress( << ",\"intervalEncodeMs\":" << intervalEncodeMs << ",\"intervalPipelineWaitMs\":" << intervalPipelineWaitMs << ",\"intervalCompositeMs\":" << intervalCompositeMs + << ",\"intervalCompositeGpuMs\":" << intervalCompositeGpuMs + << ",\"intervalZoomBlurGpuMs\":" << intervalZoomBlurGpuMs + << ",\"intervalOverlayBlendGpuMs\":" << intervalOverlayBlendGpuMs + << ",\"intervalOverlayUploadMs\":" << intervalOverlayUploadMs + << ",\"intervalOverlayHostReadMs\":" << intervalOverlayHostReadMs + << ",\"intervalOverlayH2DEnqueueMs\":" << intervalOverlayH2DEnqueueMs << ",\"intervalNvencMs\":" << intervalNvencMs << ",\"intervalPacketWriteMs\":" << intervalPacketWriteMs << ",\"intervalWebcamDecodeMs\":" << intervalWebcamDecodeMs @@ -2984,6 +7077,24 @@ void reportEncodingProgress( << ",\"intervalRoiCompositeFrames\":" << intervalRoiCompositeFrames << ",\"intervalMonolithicCompositeFrames\":" << intervalMonolithicCompositeFrames << ",\"intervalCopyCompositeFrames\":" << intervalCopyCompositeFrames + << ",\"intervalZoomBlurFrames\":" << intervalZoomBlurFrames + << ",\"intervalOverlayBlendFrames\":" << intervalOverlayBlendFrames + << ",\"intervalTemporalBlurFrames\":" << intervalTemporalBlurFrames + << ",\"intervalTemporalBlurSamples\":" << intervalTemporalBlurSamples + << ",\"intervalTemporalBlurBgPrecomposedFrames\":" << intervalTemporalBlurBgPrecomposedFrames + << ",\"intervalTemporalBlurStationaryFrames\":" << intervalTemporalBlurStationaryFrames + << ",\"intervalTemporalBgCacheBuilds\":" << intervalTemporalBgCacheBuilds + << ",\"intervalTemporalBgCacheHits\":" << intervalTemporalBgCacheHits + << ",\"intervalOverlayStaticRegionBlends\":" << intervalOverlayStaticRegionBlends + << ",\"intervalOverlayFileLoads\":" << intervalOverlayFileLoads + << ",\"intervalOverlayCacheHits\":" << intervalOverlayCacheHits + << ",\"intervalOverlayPinnedHits\":" << intervalOverlayPinnedHits + << ",\"intervalOverlayReadWaits\":" << intervalOverlayReadWaits + << ",\"intervalChangedTileCount\":" << intervalChangedTileCount + << ",\"intervalUploadedTileBytes\":" << intervalUploadedTileBytes + << ",\"intervalCachedTileCount\":" << intervalCachedTileCount + << ",\"tiledOverlayLayers\":" << counters.tiledOverlayLayers + << ",\"overlayPendingReadsPeak\":" << counters.overlayPendingReadsPeak << "}" << std::endl; state.lastReportAt = now; state.lastReportedFrame = encodedFrames; @@ -2993,8 +7104,15 @@ void reportEncodingProgress( } // namespace int main(int argc, char** argv) { + const char* requestedOutputCodec = "h264"; + NvencCapabilityProbe capabilityProbe; + // Declared outside the try so the failure summary can report the validated + // tiled overlay descriptor facts (layer count + derived bookkeeping) when + // an encode-stage error aborts before the runtime counters exist. + Options options; try { - Options options = parseOptions(argc, argv); + options = parseOptions(argc, argv); + requestedOutputCodec = outputCodecName(options.outputCodec); options.timelineSegments = loadTimelineMap(options.timelineMapPath); if (!options.timelineSegments.empty() && !options.callbackEncode) { fail("Timeline-map CUDA export requires --callback-encode"); @@ -3005,10 +7123,30 @@ int main(int argc, char** argv) { checkCu(cuInit(0), "cuInit"); CUdevice device = 0; checkCu(cuDeviceGet(&device, 0), "cuDeviceGet"); + // Use the primary context (shared with the CUDA runtime API used for + // buffer allocation) rather than a separate cuCtxCreate context. NVENC + // capability queries and the runtime allocations must see the same + // primary context; a detached context causes caps queries to fail with + // NV_ENC_ERR_ENCODER_NOT_INITIALIZED style errors on current drivers. CUcontext context = nullptr; - checkCu(cuCtxCreate(&context, 0, device), "cuCtxCreate"); + checkCu(cuDevicePrimaryCtxRetain(&context, device), "cuDevicePrimaryCtxRetain"); checkCu(cuCtxSetCurrent(context), "cuCtxSetCurrent"); + // Run the NVENC capability probe before any runtime-API prewarm work so + // the caps query happens on a freshly current primary context. The probe + // opens a real session and queries the caps for the requested output + // codec, so the config decisions below consume real capability reads. + capabilityProbe = probeNvencCapabilities( + context, + options.outputCodec == OutputCodec::HEVC ? NV_ENC_CODEC_HEVC_GUID : NV_ENC_CODEC_H264_GUID); prewarmCuda(options.prewarmMs); + if (!capabilityProbe.apiLoaded || !capabilityProbe.sessionOpened) { + std::cerr << "{\"success\":false,\"outputCodec\":\"" + << requestedOutputCodec + << "\",\"backend\":\"nvidia-nvenc\",\"error\":\"NVENC capability probe failed: " + << capabilityProbe.error + << "\",\"noCpuFallback\":true}" << std::endl; + return 1; + } std::ifstream input(options.inputPath, std::ios::binary); if (!input) { @@ -3021,6 +7159,21 @@ int main(int argc, char** argv) { const CursorTrack* cursorTrackPtr = cursorTrack.get(); std::unique_ptr zoomTrack = loadZoomTrack(options); const ZoomTrack* zoomTrackPtr = zoomTrack.get(); + std::unique_ptr overlaySource = + options.overlayLayers.empty() + ? nullptr + : std::make_unique(options.overlayLayers); + OverlayFrameSource* overlaySourcePtr = overlaySource.get(); + // Tiled/delta overlay stream: the descriptor was validated at parse time + // and the device tile cache is staged here (before decoding), so a + // truncated payload or CUDA allocation failure surfaces before any + // encode work. Raw and tiled sources coexist in a single global z-order + // (ascending `order`) rather than being grouped raw-first-then-tiled. + std::unique_ptr tiledOverlaySource = + options.tiledOverlayLayers.empty() + ? nullptr + : std::make_unique(options.tiledOverlayLayers); + TiledOverlayFrameSource* tiledOverlaySourcePtr = tiledOverlaySource.get(); const std::vector sourcePts = loadFramePts(options.sourcePtsPath); const bool useSourcePts = options.inputFrames > 0 && @@ -3060,6 +7213,7 @@ int main(int argc, char** argv) { double encodeMs = 0.0; ProgressReportState progressState; progressState.startedAt = std::chrono::steady_clock::now(); + progressState.outputCodec = outputCodecName(options.outputCodec); progressState.lastReportAt = progressState.startedAt; const int progressTotalFrames = maxCallbackOutputFrames(options); reportEncodingProgress(0, progressTotalFrames, progressState, ProgressCounters{}, true); @@ -3075,6 +7229,9 @@ int main(int argc, char** argv) { webcamCachePtr, cursorTrackPtr, zoomTrackPtr, + overlaySourcePtr, + tiledOverlaySourcePtr, + &capabilityProbe, &progressState, useDecoderFramePolicy, 0, @@ -3124,6 +7281,14 @@ int main(int argc, char** argv) { continue; } if (!sink) { + validateOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); + validateTiledOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); sink = std::make_unique( context, outputWidthForSource(options, decoder->GetWidth()), @@ -3131,11 +7296,13 @@ int main(int argc, char** argv) { options.fps, bitrate, options.outputPath, - options.streamSync, options, webcamCachePtr, cursorTrackPtr, - zoomTrackPtr); + zoomTrackPtr, + overlaySourcePtr, + tiledOverlaySourcePtr, + capabilityProbe); } const auto encodeStart = std::chrono::steady_clock::now(); sink->encodeFrame( @@ -3170,6 +7337,14 @@ int main(int argc, char** argv) { continue; } if (!sink) { + validateOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); + validateTiledOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); sink = std::make_unique( context, outputWidthForSource(options, decoder->GetWidth()), @@ -3177,11 +7352,13 @@ int main(int argc, char** argv) { options.fps, bitrate, options.outputPath, - options.streamSync, options, webcamCachePtr, cursorTrackPtr, - zoomTrackPtr); + zoomTrackPtr, + overlaySourcePtr, + tiledOverlaySourcePtr, + capabilityProbe); } const auto encodeStart = std::chrono::steady_clock::now(); sink->encodeFrame( @@ -3234,6 +7411,9 @@ int main(int argc, char** argv) { << "{" << "\"success\":true," << "\"mode\":\"nvdec-cuda-nvenc-annexb\"," + << "\"outputCodec\":\"" << outputCodecName(options.outputCodec) << "\"," + << "\"elementaryStreamFormat\":\"" + << outputCodecName(options.outputCodec) << "\"," << "\"selectionStage\":\"" << (options.callbackEncode ? (useDecoderFramePolicy ? "decoder-policy-mapped-callback" : "mapped-callback") @@ -3242,7 +7422,7 @@ int main(int argc, char** argv) { << "\"sourceTimestampMode\":\"" << (useSourcePts ? "pts" : "ordinal") << "\"," << "\"timelineMap\":" << (!options.timelineSegments.empty() ? "true" : "false") << "," << "\"timelineSegments\":" << options.timelineSegments.size() << "," - << "\"syncMode\":\"" << (options.streamSync ? "stream" : "device") << "\"," + << "\"syncMode\":\"stream\"," << "\"prewarmMs\":" << options.prewarmMs << "," << "\"chunkMb\":" << options.chunkMb << "," << "\"width\":" << outputWidthForSource(options, decoder->GetWidth()) << "," @@ -3279,6 +7459,48 @@ int main(int argc, char** argv) { << "\"cursorAtlas\":" << (!options.cursorAtlasRgbaPath.empty() ? "true" : "false") << "," << "\"zoomOverlay\":" << (zoomTrackPtr ? "true" : "false") << "," << "\"zoomSamples\":" << (zoomTrackPtr ? zoomTrackPtr->samples.size() : 0) << "," + << "\"zoomBlurFrames\":" << (sink ? sink->zoomBlurFrames() : 0) << "," + << "\"overlayLayers\":" << options.overlayLayers.size() << "," + << "\"overlayBlendFrames\":" << (sink ? sink->overlayBlendFrames() : 0) << "," + << "\"tiledOverlayLayers\":" << (sink ? sink->tiledOverlayLayerCount() : 0) << "," + << "\"tiledOverlayBlendFrames\":" << (sink ? sink->tiledOverlayBlendFrames() : 0) << "," + << "\"changedTileCount\":" << (sink ? sink->tiledChangedTileCount() : 0) << "," + << "\"uploadedTileBytes\":" << (sink ? sink->tiledUploadedTileBytes() : 0) << "," + << "\"cachedTileCount\":" << (sink ? sink->tiledCachedTileCount() : 0) << "," + << "\"rawFallbackReason\":\"" + << (sink ? sink->tiledRawFallbackReason() : "") << "\"," + << "\"temporalBlurFrames\":" << (sink ? sink->temporalBlurFrames() : 0) << "," + << "\"temporalBlurSamplesTotal\":" << (sink ? sink->temporalBlurSamplesTotal() : 0) << "," + << "\"temporalBlurBgPrecomposedFrames\":" << (sink ? sink->temporalBlurBgPrecomposedFrames() : 0) << "," + << "\"temporalBlurStationaryFrames\":" << (sink ? sink->temporalBlurStationaryFrames() : 0) << "," + << "\"temporalBgCacheBuilds\":" << (sink ? sink->temporalBgCacheBuilds() : 0) << "," + << "\"temporalBgCacheHits\":" << (sink ? sink->temporalBgCacheHits() : 0) << "," + << "\"overlayStaticRegionBlends\":" << (sink ? sink->overlayStaticRegionBlends() : 0) << "," + << "\"overlayFileLoads\":" << (sink ? sink->overlayFileLoads() : 0) << "," + << "\"overlayCacheHits\":" << (sink ? sink->overlayCacheHits() : 0) << "," + << "\"overlayPinnedHits\":" << (sink ? sink->overlayPinnedHits() : 0) << "," + << "\"overlayReadWaits\":" << (sink ? sink->overlayReadWaits() : 0) << "," + << "\"overlayPendingReadsPeak\":" << (sink ? sink->overlayPendingReadsPeak() : 0) << "," + << "\"nvencDiagnostics\":{" + << "\"deviceName\":\"" << (sink ? sink->capabilityProbe().deviceName : "") << "\"," + << "\"cudaDriverVersion\":" << (sink ? sink->capabilityProbe().cudaDriverVersion : 0) << "," + << "\"cudaComputeMajor\":" << (sink ? sink->capabilityProbe().cudaComputeMajor : 0) << "," + << "\"cudaComputeMinor\":" << (sink ? sink->capabilityProbe().cudaComputeMinor : 0) << "," + << "\"sdkApiVersion\":" << (sink ? sink->capabilityProbe().sdkApiVersion : 0) << "," + << "\"driverMaxApiVersion\":" << (sink ? sink->capabilityProbe().driverMaxApiVersion : 0) << "," + << "\"h264Supported\":" << (sink && sink->capabilityProbe().h264Supported ? "true" : "false") << "," + << "\"hevcSupported\":" << (sink && sink->capabilityProbe().hevcSupported ? "true" : "false") << "," + << "\"supportedRateControlModes\":" << (sink ? sink->capabilityProbe().supportedRateControlModes : 0) << "," + << "\"customVbvSupported\":" << (sink && sink->capabilityProbe().customVbvBufferSizeSupported ? "true" : "false") << "," + << "\"asyncEncodeSupported\":" << (sink && sink->capabilityProbe().asyncEncodeSupported ? "true" : "false") << "," + << "\"temporalAqSupported\":" << (sink && sink->capabilityProbe().temporalAqSupported ? "true" : "false") << "," + << "\"widthMax\":" << (sink ? sink->capabilityProbe().widthMax : 0) << "," + << "\"heightMax\":" << (sink ? sink->capabilityProbe().heightMax : 0) << "," + << "\"mbPerSecMax\":" << (sink ? sink->capabilityProbe().mbPerSecMax : 0) << "," + << "\"probeError\":\"" << (sink ? sink->capabilityProbe().error : "") << "\"," + << "\"rcModeUsed\":\"" << (sink ? sink->nvencConfigUsed().rcMode : "") << "\"," + << "\"customVbvUsed\":" << (sink && sink->nvencConfigUsed().customVbv ? "true" : "false") << "," + << "\"aqUsed\":" << (sink && sink->nvencConfigUsed().aq ? "true" : "false") << "}," << "\"sourceFrames\":" << reportedSourceFrames << "," << "\"mappedDisplayFrames\":" << mappedDisplayFrames << "," << "\"selectedDisplayFrames\":" << selectedDisplayFrames << "," @@ -3289,6 +7511,12 @@ int main(int argc, char** argv) { << "\"decodeWallMs\":" << decodeMs << "," << "\"encodeMs\":" << encodeMs << "," << "\"compositeMs\":" << sink->compositeMs() << "," + << "\"compositeGpuMs\":" << sink->compositeGpuMs() << "," + << "\"zoomBlurGpuMs\":" << sink->zoomBlurGpuMs() << "," + << "\"overlayBlendGpuMs\":" << sink->overlayBlendGpuMs() << "," + << "\"overlayUploadMs\":" << sink->overlayUploadMs() << "," + << "\"overlayHostReadMs\":" << sink->overlayHostReadMs() << "," + << "\"overlayH2DEnqueueMs\":" << sink->overlayH2DEnqueueMs() << "," << "\"roiCompositeFrames\":" << sink->roiCompositeFrames() << "," << "\"monolithicCompositeFrames\":" << sink->monolithicCompositeFrames() << "," << "\"copyCompositeFrames\":" << sink->copyCompositeFrames() << "," @@ -3304,10 +7532,42 @@ int main(int argc, char** argv) { sink.reset(); decoder.reset(); webcamStream.reset(); - checkCu(cuCtxDestroy(context), "cuCtxDestroy"); + // OverlayFrameSource and TiledOverlayFrameSource own device/pinned + // buffers (cudaFree/cudaFreeHost in their destructors); they must be + // destroyed while the primary context is still current, before the + // context is released. + overlaySource.reset(); + tiledOverlaySource.reset(); + // The primary context is released, not destroyed. + checkCu(cuDevicePrimaryCtxRelease(device), "cuDevicePrimaryCtxRelease"); return 0; } catch (const std::exception& error) { - std::cerr << "{\"success\":false,\"error\":\"" << error.what() << "\"}" << std::endl; + const TiledOverlayDerivedMetrics tiledMetrics = + computeTiledOverlayDerivedMetrics(options.tiledOverlayLayers); + std::cerr << "{\"success\":false,\"outputCodec\":\"" + << requestedOutputCodec + << "\",\"backend\":\"nvidia-nvenc\",\"error\":\"" + << error.what() + << "\",\"tiledOverlayLayers\":" << options.tiledOverlayLayers.size() + << ",\"changedTileCount\":" << tiledMetrics.changedTileCount + << ",\"uploadedTileBytes\":" << tiledMetrics.uploadedTileBytes + << ",\"cachedTileCount\":" << tiledMetrics.cachedTileCount + << ",\"rawFallbackReason\":\"" << tiledMetrics.rawFallbackReason + << "\",\"nvencDiagnostics\":{" + << "\"deviceName\":\"" << capabilityProbe.deviceName << "\"," + << "\"cudaDriverVersion\":" << capabilityProbe.cudaDriverVersion << "," + << "\"cudaComputeMajor\":" << capabilityProbe.cudaComputeMajor << "," + << "\"cudaComputeMinor\":" << capabilityProbe.cudaComputeMinor << "," + << "\"sdkApiVersion\":" << capabilityProbe.sdkApiVersion << "," + << "\"driverMaxApiVersion\":" << capabilityProbe.driverMaxApiVersion << "," + << "\"h264Supported\":" << (capabilityProbe.h264Supported ? "true" : "false") << "," + << "\"hevcSupported\":" << (capabilityProbe.hevcSupported ? "true" : "false") << "," + << "\"supportedRateControlModes\":" << capabilityProbe.supportedRateControlModes << "," + << "\"customVbvSupported\":" << (capabilityProbe.customVbvBufferSizeSupported ? "true" : "false") << "," + << "\"asyncEncodeSupported\":" << (capabilityProbe.asyncEncodeSupported ? "true" : "false") << "," + << "\"temporalAqSupported\":" << (capabilityProbe.temporalAqSupported ? "true" : "false") << "," + << "\"probeError\":\"" << capabilityProbe.error << "\"}," + << "\"noCpuFallback\":true}" << std::endl; return 1; } } diff --git a/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs b/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs new file mode 100644 index 000000000..41c02cde4 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs @@ -0,0 +1,529 @@ +import { describe, expect, it } from "vitest"; + +// Contract test for the fused temporal composite + accumulate in main.cu +// (compositeStaticNv12Kernel temporalWeightFixed/temporalAccumulateMode). The +// native kernel replaced the two-pass fill/composite/accumulate pipeline with a +// single fused pass: sample 0 replaces the target with (w0 * v + 128) >> 8 and +// later samples saturate-accumulate (w * v + 128) >> 8. This JS mirror verifies +// the fused order reproduces the previous zero-fill + saturating adds exactly, +// including the fixed-point weight plan used by buildTemporalSamplePlan. +// +// Module 3 contracts (constants synchronized with current CUDA math in +// electron/native/nvidia-cuda-compositor/src/main.cu): +// - stationary-transform fixed-point accumulation equivalence: with an +// invariant per-sample transform, the fused full-frame path, the legacy +// zero-fill path, and the background-precompose path (accumulateBackgroundNv12Kernel +// one-pass term chain + region re-composite) are bit-identical. +// - sample-count-aware precompose threshold: the cost-model break-even region +// ratio is (sampleCount - 1) / sampleCount, while the CUDA decision keeps a +// fixed 19/20 region gate (regionPixels * 20 < framePixels * 19) plus the +// empty-sample and shadowIntensityPct == 0 gates. +// - a stationary-window predicate mirror for the invariant-transform property +// the compositor exploits implicitly (per-sample content rects collapse to +// one rect, so the UV-expanded region is identical every sample). + +function buildTemporalSamplePlan(sampleCount, shutterFraction, weightCurvePower, frameDurationUs) { + const safeSampleCount = Math.max(1, sampleCount); + if (safeSampleCount <= 1) { + return [{ offsetUs: 0.0, weight: 1.0 }]; + } + const shutterWindowUs = + Math.max(1.0, frameDurationUs) * Math.max(0.0, Math.min(3.0, shutterFraction)); + const startOffsetUs = -shutterWindowUs / 2.0; + const stepUs = shutterWindowUs / (safeSampleCount - 1); + const offsetsUs = []; + for (let index = 0; index < safeSampleCount; index += 1) { + offsetsUs.push(startOffsetUs + stepUs * index); + } + const kWeightFloor = 0.22; + const centerIndex = (safeSampleCount - 1) / 2; + const rawWeights = []; + let totalWeight = 0; + for (let index = 0; index < safeSampleCount; index += 1) { + const normalizedDistance = Math.abs(index - centerIndex) / Math.max(1, centerIndex); + const taperedWeight = Math.cos(normalizedDistance * (Math.PI / 2)); + const rawWeight = + kWeightFloor + + (1 - kWeightFloor) * Math.pow(Math.max(0, taperedWeight), weightCurvePower); + rawWeights.push(rawWeight); + totalWeight += rawWeight; + } + return offsetsUs.map((offsetUs, index) => ({ + offsetUs, + weight: totalWeight > 0 ? rawWeights[index] / totalWeight : 1 / safeSampleCount, + })); +} + +function temporalAccumulateByte(current, value, weightFixed, mode) { + if (mode === 0) { + return value; + } + const weighted = (weightFixed * value + 128) >> 8; + if (mode === 1) { + return Math.min(255, weighted); + } + return Math.min(255, current + weighted); +} + +// Old two-pass: zero-fill target, then per sample add (w * scratch + 128) >> 8. +function oldAccumulate(values, weights) { + let target = 0; + for (let index = 0; index < values.length; index += 1) { + const weightFixed = Math.round(weights[index] * 256); + target = Math.min(255, target + ((weightFixed * values[index] + 128) >> 8)); + } + return target; +} + +// New fused: sample 0 replaces with (w0 * v + 128) >> 8, rest saturate-accumulate. +function fusedAccumulate(values, weights) { + let target = 0; + for (let index = 0; index < values.length; index += 1) { + const weightFixed = Math.round(weights[index] * 256); + target = temporalAccumulateByte(target, values[index], weightFixed, index === 0 ? 1 : 2); + } + return target; +} + +// Mirrors ensureTemporalWeightsDevice in main.cu: weights are rounded to 8-bit +// fixed point once (std::lround(weight * 256.0)) and reused by every +// accumulate pass, so per-sample rounding is identical across paths. +function fixedWeights(weights) { + return weights.map((weight) => Math.round(weight * 256)); +} + +// Mirrors accumulateBackgroundNv12Kernel in main.cu: sample 0 seeds the +// accumulator with (w0 * v + 128) >> 8 and every later sample saturate-adds +// (w * v + 128) >> 8 (min(255, acc + term)). This is the kernel's one-pass +// invariant-background accumulation; for a stationary transform it is exactly +// what the fused and legacy full-frame paths produce per pixel. +function saturatingAccumulate(value, weights) { + if (weights.length === 0) { + return 0; + } + const fixed = fixedWeights(weights); + let acc = (fixed[0] * value + 128) >> 8; + for (let index = 1; index < fixed.length; index += 1) { + acc = Math.min(255, acc + ((fixed[index] * value + 128) >> 8)); + } + return acc; +} + +// Mirrors the useBackgroundPrecompose decision in compositeTemporalBlurSamples: +// precompose requires at least one sample, no shadow compositing +// (shadowIntensityPct == 0), and either no visible content or a positive +// UV-expanded region strictly smaller than 19/20 of the frame +// (regionPixels * 20 < framePixels * 19). The 20/19 constants are the current +// CUDA math; the sample-count-aware cost model is mirrored separately below. +function shouldUseBackgroundPrecompose({ + sampleCount, + shadowIntensityPct, + anyContentVisible, + regionWidth, + regionHeight, + frameWidth, + frameHeight, +}) { + return ( + sampleCount > 0 && + shadowIntensityPct === 0 && + (!anyContentVisible || + (regionWidth > 0 && + regionHeight > 0 && + regionWidth * regionHeight * 20 < frameWidth * frameHeight * 19)) + ); +} + +// Cost model behind the precompose choice: the legacy path composites the full +// frame per sample (sampleCount * framePixels); precompose runs one full-frame +// background pass plus one region pass per sample +// (framePixels + sampleCount * regionPixels). Precompose wins strictly when +// regionPixels < framePixels * (sampleCount - 1) / sampleCount. The integer +// form keeps the strict-inequality boundary exact for every supported sample +// count (the CUDA option gate accepts 3..61 samples). +function precomposeWinsCostModel(sampleCount, regionPixels, framePixels) { + return regionPixels * sampleCount < framePixels * (sampleCount - 1); +} + +function precomposeBreakEvenRegionRatio(sampleCount) { + return (sampleCount - 1) / sampleCount; +} + +// Stationary-window predicate mirror. The CUDA compositor has no dedicated +// stationary predicate: invariance is implicit because every sample composites +// the same transform, so the per-sample content bounding boxes collapse to one +// rect and the UV-expanded region is identical for every sample. This mirror +// models that property (scale/x/y unchanged within epsilon) for the +// stationary-equivalence tests. blurStrength/blurCenter are intentionally +// ignored: the temporal path replaces spatial blur for those frames. +function isStationarySampleWindow(samples) { + if (samples.length === 0) { + return false; + } + const first = samples[0]; + if ( + !first || + typeof first.scale !== "number" || + typeof first.x !== "number" || + typeof first.y !== "number" + ) { + return false; + } + const epsilon = 1e-9; + return samples.every( + (sample) => + Math.abs(sample.scale - first.scale) <= epsilon && + Math.abs(sample.x - first.x) <= epsilon && + Math.abs(sample.y - first.y) <= epsilon, + ); +} + +// Models one NV12 plane (luma or chroma) for a stationary transform: every +// sample composites the same invariant per-pixel value (content inside the +// region, background outside), so the whole-frame fused path, the legacy +// zero-fill path, and the background-precompose path must agree bit-for-bit. +// Returns the three results as arrays indexed by y * width + x. +function stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues, + backgroundValue, + weights, +}) { + const fullFrameFused = []; + const fullFrameOld = []; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const invariantValue = + x < regionWidth && y < regionHeight + ? contentValues[y * width + x] + : backgroundValue; + fullFrameFused.push( + fusedAccumulate(new Array(weights.length).fill(invariantValue), weights), + ); + fullFrameOld.push( + oldAccumulate(new Array(weights.length).fill(invariantValue), weights), + ); + } + } + + // Background precompose: accumulate the invariant background once per pixel + // over the full frame (the kernel mirror), then re-composite only the region + // per sample: sample 0 replaces the precomposed background (the target is + // not pre-zeroed) and later samples saturate-accumulate. + const backgroundAcc = saturatingAccumulate(backgroundValue, weights); + const backgroundPrecompose = new Array(width * height).fill(backgroundAcc); + for (let y = 0; y < regionHeight; y += 1) { + for (let x = 0; x < regionWidth; x += 1) { + const index = y * width + x; + backgroundPrecompose[index] = fusedAccumulate( + new Array(weights.length).fill(contentValues[index]), + weights, + ); + } + } + + return { fullFrameFused, fullFrameOld, backgroundPrecompose }; +} + +function planWeights(sampleCount, shutterFraction = 0.5, weightCurvePower = 2) { + return buildTemporalSamplePlan( + sampleCount, + shutterFraction, + weightCurvePower, + 1000000 / 30, + ).map((sample) => sample.weight); +} + +describe("fused temporal accumulate contract", () => { + it("fused replace-then-accumulate equals zero-fill + saturating adds", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + const fixedSum = weights.reduce((sum, w) => sum + Math.round(w * 256), 0); + expect(fixedSum).toBeGreaterThan(250); + + // Deterministic pseudo-random values covering the full byte range. + let seed = 12345; + const values = []; + for (let i = 0; i < 13; i += 1) { + seed = (seed * 1664525 + 1013904223) >>> 0; + values.push((seed >> 16) & 0xff); + } + expect(fusedAccumulate(values, weights)).toBe(oldAccumulate(values, weights)); + }); + + it("constant luma/background values accumulate to the renderer result", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + // Constant 128 (neutral chroma) must land on the documented 131 for this plan. + expect(fusedAccumulate(new Array(13).fill(128), weights)).toBe(131); + }); + + it("keeps saturating semantics per step (no wraparound near 255)", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + const result = fusedAccumulate(new Array(13).fill(255), weights); + // Saturating adds must never wrap below 250 for an all-255 frame. + expect(result).toBeGreaterThan(250); + expect(result).toBe(oldAccumulate(new Array(13).fill(255), weights)); + }); + + it("matches the old pipeline for a range of shutter fractions and sample counts", () => { + for (const sampleCount of [3, 5, 9, 13, 17]) { + for (const shutter of [0.25, 0.5, 1.0]) { + const plan = buildTemporalSamplePlan(sampleCount, shutter, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + for (const base of [0, 1, 16, 64, 128, 200, 254, 255]) { + const values = new Array(sampleCount).fill(base); + expect(fusedAccumulate(values, weights)).toBe(oldAccumulate(values, weights)); + } + } + } + }); +}); + +describe("stationary-transform fixed-point accumulation equivalence", () => { + it("fused full-frame, legacy zero-fill, and background precompose agree for a stationary window", () => { + const width = 64; + const height = 36; + const regionWidth = 40; + const regionHeight = 24; + let seed = 987654321; + const contentValues = []; + for (let index = 0; index < regionWidth * regionHeight; index += 1) { + seed = (seed * 1664525 + 1013904223) >>> 0; + contentValues.push((seed >> 16) & 0xff); + } + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + const { fullFrameFused, fullFrameOld, backgroundPrecompose } = + stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues, + backgroundValue: 16, + weights, + }); + expect(fullFrameFused).toEqual(fullFrameOld); + expect(backgroundPrecompose).toEqual(fullFrameFused); + } + }); + + it("the background accumulate kernel mirror equals the fused and legacy paths for any constant value", () => { + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + for (const backgroundValue of [0, 1, 16, 64, 128, 200, 254, 255]) { + expect(saturatingAccumulate(backgroundValue, weights)).toBe( + fusedAccumulate(new Array(sampleCount).fill(backgroundValue), weights), + ); + expect(saturatingAccumulate(backgroundValue, weights)).toBe( + oldAccumulate(new Array(sampleCount).fill(backgroundValue), weights), + ); + } + } + }); + + it("matches on chroma planes and at saturation boundaries across sample counts", () => { + const width = 32; + const height = 18; + const regionWidth = 20; + const regionHeight = 12; + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + const chroma = stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues: new Array(regionWidth * regionHeight).fill(128), + backgroundValue: 128, + weights, + }); + const luma = stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues: new Array(regionWidth * regionHeight).fill(255), + backgroundValue: 16, + weights, + }); + expect(chroma.backgroundPrecompose).toEqual(chroma.fullFrameFused); + expect(luma.backgroundPrecompose).toEqual(luma.fullFrameOld); + // Saturating adds must never wrap; neutral chroma stays <= 255. + expect(Math.max(...chroma.fullFrameFused)).toBeLessThanOrEqual(255); + expect(Math.max(...luma.backgroundPrecompose)).toBeLessThanOrEqual(255); + } + }); +}); + +describe("sample-count-aware precompose threshold", () => { + it("break-even region ratio is (sampleCount - 1) / sampleCount", () => { + expect(precomposeBreakEvenRegionRatio(3)).toBe(2 / 3); + expect(precomposeBreakEvenRegionRatio(5)).toBe(4 / 5); + expect(precomposeBreakEvenRegionRatio(13)).toBe(12 / 13); + expect(precomposeBreakEvenRegionRatio(61)).toBe(60 / 61); + }); + + it("strictly favors precompose only below the break-even region for 3, 5, 13, and 61 samples", () => { + const framePixels = 1920 * 1080; + for (const sampleCount of [3, 5, 13, 61]) { + // Largest integer region strictly below the real break-even + // B = framePixels * (sampleCount - 1) / sampleCount is ceil(B) - 1; + // floor(B) is one too large when B is not an integer. + const largestBelow = Math.ceil((framePixels * (sampleCount - 1)) / sampleCount) - 1; + expect(precomposeWinsCostModel(sampleCount, largestBelow, framePixels)).toBe(true); + expect(precomposeWinsCostModel(sampleCount, largestBelow + 1, framePixels)).toBe(false); + expect(precomposeWinsCostModel(sampleCount, 0, framePixels)).toBe(true); + expect(precomposeWinsCostModel(sampleCount, framePixels, framePixels)).toBe(false); + } + }); + + it("mirrors the current CUDA 20/19 fixed region threshold", () => { + const base = { + sampleCount: 13, + shadowIntensityPct: 0, + anyContentVisible: true, + frameWidth: 1000, + frameHeight: 1000, + }; + // 949000 < 950000 (19/20 of 1000x1000): precompose. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 949, regionHeight: 1000 }), + ).toBe(true); + // Exactly 19/20 is not strictly smaller: legacy full-frame path. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 950, regionHeight: 1000 }), + ).toBe(false); + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 951, regionHeight: 1000 }), + ).toBe(false); + }); + + it("keeps the empty-sample and shadow gates from the CUDA decision", () => { + const base = { + sampleCount: 13, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 100, + regionHeight: 100, + frameWidth: 1000, + frameHeight: 1000, + }; + expect(shouldUseBackgroundPrecompose({ ...base, sampleCount: 0 })).toBe(false); + expect(shouldUseBackgroundPrecompose({ ...base, shadowIntensityPct: 40 })).toBe(false); + expect(shouldUseBackgroundPrecompose(base)).toBe(true); + // No visible content: precompose regardless of the region size. + expect( + shouldUseBackgroundPrecompose({ + ...base, + anyContentVisible: false, + regionWidth: 0, + regionHeight: 0, + }), + ).toBe(true); + // Degenerate region with visible content: legacy. + expect( + shouldUseBackgroundPrecompose({ + ...base, + anyContentVisible: true, + regionWidth: 0, + regionHeight: 0, + }), + ).toBe(false); + }); + + it("applies the fixed threshold at every supported sample count", () => { + for (const sampleCount of [3, 5, 13, 61]) { + const base = { + sampleCount, + shadowIntensityPct: 0, + anyContentVisible: true, + frameWidth: 1920, + frameHeight: 1080, + }; + // 19/20 of 1920x1080 is exactly 1920x1026. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 1920, regionHeight: 1026 }), + ).toBe(false); + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 1920, regionHeight: 1025 }), + ).toBe(true); + } + }); + + it("pins the band where the fixed threshold and the cost model diverge by sample count", () => { + const framePixels = 1920 * 1080; + // 3 samples: break-even is 2/3. A 90%-of-frame region is cheaper via the + // legacy path, but the current CUDA threshold still selects precompose + // (exact output, just more region work than legacy). + expect(precomposeWinsCostModel(3, 1920 * 972, framePixels)).toBe(false); + expect( + shouldUseBackgroundPrecompose({ + sampleCount: 3, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 1920, + regionHeight: 972, + frameWidth: 1920, + frameHeight: 1080, + }), + ).toBe(true); + // 61 samples: break-even is 60/61 (~98.4%). A 96%-of-frame region is + // cheaper via precompose, but the fixed threshold keeps the legacy path. + expect(precomposeWinsCostModel(61, 1920 * 1037, framePixels)).toBe(true); + expect( + shouldUseBackgroundPrecompose({ + sampleCount: 61, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 1920, + regionHeight: 1037, + frameWidth: 1920, + frameHeight: 1080, + }), + ).toBe(false); + }); +}); + +describe("stationary-window predicate mirror", () => { + it("accepts a window where every sample shares the same transform", () => { + const samples = [ + { offsetUs: -16666.666666666668, weight: 0.02918, scale: 1.25, x: 40, y: 20 }, + { offsetUs: 0, weight: 0.081, scale: 1.25, x: 40, y: 20 }, + { offsetUs: 16666.666666666668, weight: 0.02918, scale: 1.25, x: 40, y: 20 }, + ]; + expect(isStationarySampleWindow(samples)).toBe(true); + }); + + it("rejects windows with zoom or pan motion between samples", () => { + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.1, x: 0, y: 0 }, + ]), + ).toBe(false); + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.0, x: 3, y: 0 }, + ]), + ).toBe(false); + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.0, x: 0, y: -2 }, + ]), + ).toBe(false); + }); + + it("accepts a single-sample plan and rejects empty or malformed windows", () => { + expect(isStationarySampleWindow([])).toBe(false); + expect(isStationarySampleWindow([{ scale: 1.0, x: 0, y: 0 }])).toBe(true); + expect(isStationarySampleWindow([{ weight: 1.0 }])).toBe(false); + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.mjs b/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.mjs new file mode 100644 index 000000000..576e57461 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.mjs @@ -0,0 +1,324 @@ +// Renderer-prepared tiled/delta transparent RGBA overlay manifest reader for +// the NVIDIA CUDA compositor wrapper (run-mp4-pipeline.mjs). +// +// Mirrors the TS contract in src/lib/exporter/nativeStaticLayoutOverlays.ts +// (validated independently so the CUDA module never trusts an opaque blob). +// Fixed 128x128 lossless raw RGBA tiles: staticTiles define the full initial +// layer state emitted once; frameDeltas carry per-frame changed tile payloads. +// The payload stream is bounded (payloadPath/payloadByteLength); every payload +// region is written exactly once and unchanged tiles are referenced, so sparse +// 4K overlays never duplicate unchanged pixels. Sidecar/session data produced +// from this contract is never persisted. + +import { existsSync, readFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +export const TILED_OVERLAY_STORAGE_VERSION = 1; +export const TILED_OVERLAY_TILE_SIZE = 128; +export const TILED_OVERLAY_PIXEL_FORMAT = "rgba"; +export const TILED_OVERLAY_TILE_BYTE_SIZE = TILED_OVERLAY_TILE_SIZE * TILED_OVERLAY_TILE_SIZE * 4; + +// Conservative tiled-vs-raw density/size heuristics mirrored from the TS side. +const TILED_OVERLAY_MIN_TILE_COUNT = 4; +const TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION = 0.5; +const TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION = 0.7; + +function fail(message) { + throw new Error(message); +} + +function isSafeInteger(value) { + return Number.isSafeInteger(value); +} + +function isValidFrameRate(value, expected) { + return ( + typeof value === "number" && + Number.isFinite(value) && + value > 0 && + Math.abs(value - expected) <= 0.01 + ); +} + +export function tileCountForSize(width, height) { + const columns = Math.max(1, Math.ceil(width / TILED_OVERLAY_TILE_SIZE)); + const rows = Math.max(1, Math.ceil(height / TILED_OVERLAY_TILE_SIZE)); + return columns * rows; +} + +function validateTileRecord(record, layerId, tileCount, payloadByteLength) { + if ( + !isSafeInteger(record?.tileIndex) || + record.tileIndex < 0 || + record.tileIndex >= tileCount + ) { + return `Tiled overlay layer ${layerId} references an out-of-bounds tile: ${JSON.stringify(record)}`; + } + if ( + !isSafeInteger(record?.byteOffset) || + record.byteOffset < 0 || + !isSafeInteger(record?.byteLength) || + record.byteLength !== TILED_OVERLAY_TILE_BYTE_SIZE || + record.byteOffset + record.byteLength > payloadByteLength + ) { + return `Tiled overlay layer ${layerId} has an invalid tile payload range: ${JSON.stringify(record)}`; + } + return null; +} + +export function readTiledOverlayManifest(manifestPath, outputSize) { + if (!manifestPath) { + return []; + } + const { outputWidth, outputHeight, frameRate, durationSec } = outputSize; + const resolvedPath = resolve(manifestPath); + if (!existsSync(resolvedPath)) { + fail(`Tiled overlay manifest does not exist: ${resolvedPath}`); + } + + let manifest; + try { + manifest = JSON.parse(readFileSync(resolvedPath, "utf8")); + } catch (error) { + fail(`Invalid tiled overlay manifest ${resolvedPath}: ${error.message}`); + } + if (manifest?.version !== TILED_OVERLAY_STORAGE_VERSION) { + fail(`Unsupported tiled overlay storage version ${manifest?.version}: ${resolvedPath}`); + } + if (manifest.outputWidth !== outputWidth || manifest.outputHeight !== outputHeight) { + fail(`Tiled overlay storage dimensions do not match the output: ${resolvedPath}`); + } + if (!isValidFrameRate(manifest.frameRate, frameRate)) { + fail(`Tiled overlay storage frame rate does not match the output: ${resolvedPath}`); + } + if ( + !Number.isFinite(manifest.durationSec) || + manifest.durationSec <= 0 || + Math.abs(manifest.durationSec - durationSec) > 1 / frameRate + ) { + fail(`Tiled overlay storage duration does not match the output: ${resolvedPath}`); + } + if (!Array.isArray(manifest.layers)) { + fail(`Tiled overlay manifest requires a layers array: ${resolvedPath}`); + } + + const layers = []; + let previousOrder = -1; + let previousId = ""; + for (const rawLayer of manifest.layers) { + const id = typeof rawLayer?.id === "string" ? rawLayer.id : ""; + const layerPath = typeof rawLayer?.payloadPath === "string" ? rawLayer.payloadPath : ""; + if (!id || !layerPath) { + fail(`Tiled overlay layer requires an id and payload path: ${resolvedPath}`); + } + const order = Number(rawLayer?.order); + const x = Number(rawLayer?.x); + const y = Number(rawLayer?.y); + const width = Number(rawLayer?.width); + const height = Number(rawLayer?.height); + const layerFrameRate = Number(rawLayer?.frameRate); + const layerDurationSec = Number(rawLayer?.durationSec); + const frameCount = Number(rawLayer?.frameCount); + const tileSize = Number(rawLayer?.tileSize); + const pixelFormat = rawLayer?.pixelFormat; + const payloadByteLength = Number(rawLayer?.payloadByteLength); + const staticTiles = Array.isArray(rawLayer?.staticTiles) ? rawLayer.staticTiles : null; + const frameDeltas = Array.isArray(rawLayer?.frameDeltas) ? rawLayer.frameDeltas : null; + + if (pixelFormat !== TILED_OVERLAY_PIXEL_FORMAT) { + fail(`Tiled overlay layer ${id} must use RGBA tiles: ${resolvedPath}`); + } + if (tileSize !== TILED_OVERLAY_TILE_SIZE) { + fail( + `Tiled overlay layer ${id} must use ${TILED_OVERLAY_TILE_SIZE}px tiles: ${resolvedPath}`, + ); + } + if ( + ![order, x, y, width, height, frameCount].every(isSafeInteger) || + order < 0 || + width <= 0 || + height <= 0 || + frameCount <= 0 || + x < 0 || + y < 0 + ) { + fail(`Invalid tiled overlay layer ${id}: ${resolvedPath}`); + } + if (x + width > outputWidth || y + height > outputHeight) { + fail(`Tiled overlay layer ${id} exceeds the output canvas: ${resolvedPath}`); + } + if (!isValidFrameRate(layerFrameRate, frameRate)) { + fail(`Tiled overlay layer ${id} has an incompatible frame rate: ${resolvedPath}`); + } + if ( + !Number.isFinite(layerDurationSec) || + layerDurationSec <= 0 || + Math.abs(layerDurationSec - durationSec) > 1 / frameRate + ) { + fail(`Tiled overlay layer ${id} has an incompatible duration: ${resolvedPath}`); + } + const expectedFrameCount = Math.ceil(layerDurationSec * layerFrameRate); + if (!isSafeInteger(frameCount) || frameCount < expectedFrameCount) { + fail(`Tiled overlay layer ${id} does not contain enough frames: ${resolvedPath}`); + } + if (!isSafeInteger(payloadByteLength) || payloadByteLength < 0) { + fail(`Tiled overlay layer ${id} has an invalid payload byte length: ${resolvedPath}`); + } + if (!Array.isArray(staticTiles)) { + fail(`Tiled overlay layer ${id} requires a static tile base: ${resolvedPath}`); + } + if (!Array.isArray(frameDeltas)) { + fail(`Tiled overlay layer ${id} requires frame delta records: ${resolvedPath}`); + } + if (order < previousOrder || (order === previousOrder && id <= previousId)) { + fail(`Tiled overlay layers must be sorted by order then id: ${resolvedPath}`); + } + previousOrder = order; + previousId = id; + + const tileCount = tileCountForSize(width, height); + const seenStaticTiles = new Set(); + const seenPayloadRanges = new Set(); + const checkPayloadRange = (record) => { + const rangeKey = `${record.byteOffset}:${record.byteLength}`; + if (seenPayloadRanges.has(rangeKey)) { + return `Tiled overlay layer ${id} duplicates tile payload bytes: ${resolvedPath}`; + } + seenPayloadRanges.add(rangeKey); + return null; + }; + for (const record of staticTiles) { + const issue = validateTileRecord(record, id, tileCount, payloadByteLength); + if (issue) { + fail(issue); + } + if (seenStaticTiles.has(record.tileIndex)) { + fail( + `Tiled overlay layer ${id} emits duplicate static tile ${record.tileIndex}: ${resolvedPath}`, + ); + } + seenStaticTiles.add(record.tileIndex); + const rangeIssue = checkPayloadRange(record); + if (rangeIssue) { + fail(rangeIssue); + } + } + if (seenStaticTiles.size !== tileCount) { + fail( + `Tiled overlay layer ${id} does not fully define the static tile base: ${resolvedPath}`, + ); + } + let previousFrameIndex = -1; + for (const delta of frameDeltas) { + if ( + !isSafeInteger(delta?.frameIndex) || + delta.frameIndex < 0 || + delta.frameIndex >= frameCount + ) { + fail(`Tiled overlay layer ${id} has an invalid delta frame index: ${resolvedPath}`); + } + if (delta.frameIndex <= previousFrameIndex) { + fail( + `Tiled overlay layer ${id} has unsorted or duplicate delta frame indices: ${resolvedPath}`, + ); + } + previousFrameIndex = delta.frameIndex; + const seenDeltaTiles = new Set(); + for (const record of delta.changedTiles) { + const issue = validateTileRecord(record, id, tileCount, payloadByteLength); + if (issue) { + fail(issue); + } + if (seenDeltaTiles.has(record.tileIndex)) { + fail( + `Tiled overlay layer ${id} repeats tile ${record.tileIndex} within a frame delta: ${resolvedPath}`, + ); + } + seenDeltaTiles.add(record.tileIndex); + const rangeIssue = checkPayloadRange(record); + if (rangeIssue) { + fail(rangeIssue); + } + } + } + if (1 + frameDeltas.length > frameCount) { + fail(`Tiled overlay layer ${id} has more state versions than frames: ${resolvedPath}`); + } + + const resolvedLayerPath = resolve(layerPath); + if (!existsSync(resolvedLayerPath)) { + fail(`Tiled overlay layer ${id} does not exist: ${resolvedLayerPath}`); + } + const stat = statSync(resolvedLayerPath); + if (stat.size < payloadByteLength) { + fail( + `Tiled overlay layer ${id} payload is truncated: expected at least ${payloadByteLength} bytes, received ${stat.size}`, + ); + } + layers.push({ + id, + order, + x, + y, + width, + height, + frameRate: layerFrameRate, + durationSec: layerDurationSec, + frameCount, + tileSize, + pixelFormat, + payloadPath: resolvedLayerPath, + payloadByteLength, + staticTiles, + frameDeltas, + }); + } + return layers; +} + +/** + * Additive renderer-derived throughput bookkeeping for a validated tiled layer. + * Diagnostic only; cachedTileCount is reference bookkeeping, never a zero-copy + * claim. Mirrors resolveNativeTiledOverlayMetrics on the TS side. + */ +export function resolveTiledOverlayLayerMetrics(layer) { + const changedTileCount = layer.frameDeltas.reduce( + (total, delta) => total + delta.changedTiles.length, + 0, + ); + const uploadedTileCount = layer.staticTiles.length + changedTileCount; + const tileCount = tileCountForSize(layer.width, layer.height); + return { + effectiveFrameCount: 1 + layer.frameDeltas.length, + changedTileCount, + uploadedTileCount, + uploadedTileBytes: uploadedTileCount * TILED_OVERLAY_TILE_BYTE_SIZE, + cachedTileCount: Math.max(0, tileCount * layer.frameCount - uploadedTileCount), + }; +} + +/** + * Conservative tiled-vs-raw eligibility decision. Returns null when eligible or + * a reason string (small-layer | dense-frame-delta | payload-bytes-exceed-raw) + * when the layer must keep the raw full-frame fallback. Mirrors + * resolveNativeTiledOverlayRawFallbackReason on the TS side. + */ +export function resolveTiledOverlayRawFallbackReason( + layer, + metrics = resolveTiledOverlayLayerMetrics(layer), +) { + const tileCount = tileCountForSize(layer.width, layer.height); + if (tileCount < TILED_OVERLAY_MIN_TILE_COUNT) { + return "small-layer"; + } + for (const delta of layer.frameDeltas) { + if (delta.changedTiles.length > tileCount * TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION) { + return "dense-frame-delta"; + } + } + const rawPhysicalBytes = layer.width * layer.height * 4 * layer.frameCount; + if (metrics.uploadedTileBytes >= rawPhysicalBytes * TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION) { + return "payload-bytes-exceed-raw"; + } + return null; +} diff --git a/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.test.mjs b/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.test.mjs new file mode 100644 index 000000000..e7b6ea567 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/tiledOverlayManifest.test.mjs @@ -0,0 +1,458 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + readTiledOverlayManifest, + resolveTiledOverlayLayerMetrics, + resolveTiledOverlayRawFallbackReason, + TILED_OVERLAY_STORAGE_VERSION, + TILED_OVERLAY_TILE_BYTE_SIZE, + TILED_OVERLAY_TILE_SIZE, + tileCountForSize, +} from "./tiledOverlayManifest.mjs"; + +// Contract for the versioned tiled/delta overlay descriptor: +// - version 1, fixed 128x128 lossless raw RGBA tiles. +// - staticTiles define the full initial layer state emitted once; frameDeltas +// carry per-frame changed tile payloads (ascending, unique frame indices). +// - Every payload region is written exactly once; the payload stream is bounded +// by payloadPath/payloadByteLength and validated against the actual file. +// - Mirrors the TS validator so the CUDA module never trusts an opaque blob. + +const OUTPUT_SIZE = { outputWidth: 1920, outputHeight: 1080, frameRate: 30, durationSec: 2 }; +const LAYER_WIDTH = 384; +const LAYER_HEIGHT = 256; +const TILE_COUNT = 6; // 3 columns * 2 rows + +function makeTempDir(prefix = "recordly-tiled-overlay-") { + return mkdtempSync(join(tmpdir(), prefix)); +} + +function tileRecord(tileIndex, byteOffset) { + return { tileIndex, byteOffset, byteLength: TILED_OVERLAY_TILE_BYTE_SIZE }; +} + +function staticTiles() { + return Array.from({ length: TILE_COUNT }, (_, tileIndex) => + tileRecord(tileIndex, tileIndex * TILED_OVERLAY_TILE_BYTE_SIZE), + ); +} + +function writePayload(dir, byteLength, name = "overlay-tiles.bin") { + const payloadPath = join(dir, name); + writeFileSync(payloadPath, Buffer.alloc(byteLength, 0x7f)); + return payloadPath; +} + +function writeManifest(dir, manifest, name = "tiled-overlay-manifest.json") { + const manifestPath = join(dir, name); + writeFileSync(manifestPath, JSON.stringify(manifest)); + return manifestPath; +} + +function tiledLayer(overrides = {}) { + return { + id: "tiled-effects", + order: 0, + x: 0, + y: 0, + width: LAYER_WIDTH, + height: LAYER_HEIGHT, + frameRate: 30, + durationSec: 2, + frameCount: 60, + tileSize: TILED_OVERLAY_TILE_SIZE, + pixelFormat: "rgba", + payloadPath: "", + payloadByteLength: TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE, + staticTiles: staticTiles(), + frameDeltas: [], + ...overrides, + }; +} + +function tiledManifest(overrides = {}) { + return { + version: TILED_OVERLAY_STORAGE_VERSION, + outputWidth: 1920, + outputHeight: 1080, + frameRate: 30, + durationSec: 2, + layers: [tiledLayer()], + ...overrides, + }; +} + +describe("readTiledOverlayManifest", () => { + it("returns an empty array when no manifest path is provided", () => { + expect(readTiledOverlayManifest("", OUTPUT_SIZE)).toEqual([]); + expect(readTiledOverlayManifest(null, OUTPUT_SIZE)).toEqual([]); + }); + + it("rejects a missing manifest file and invalid JSON", () => { + expect(() => readTiledOverlayManifest("/missing/tiled.json", OUTPUT_SIZE)).toThrow( + "Tiled overlay manifest does not exist:", + ); + const dir = makeTempDir(); + try { + const badJson = join(dir, "bad.json"); + writeFileSync(badJson, "{not json"); + expect(() => readTiledOverlayManifest(badJson, OUTPUT_SIZE)).toThrow( + /Invalid tiled overlay manifest/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects unsupported versions and storage/output mismatches", () => { + const dir = makeTempDir(); + try { + const payload = writePayload(dir, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE); + const withPath = (manifest) => + writeManifest(dir, { + ...manifest, + layers: Array.isArray(manifest.layers) + ? manifest.layers.map((layer) => ({ ...layer, payloadPath: payload })) + : manifest.layers, + }); + expect(() => + readTiledOverlayManifest(withPath(tiledManifest({ version: 2 })), OUTPUT_SIZE), + ).toThrow("Unsupported tiled overlay storage version 2:"); + expect(() => + readTiledOverlayManifest( + withPath(tiledManifest({ outputWidth: 1280 })), + OUTPUT_SIZE, + ), + ).toThrow("Tiled overlay storage dimensions do not match the output:"); + expect(() => + readTiledOverlayManifest(withPath(tiledManifest({ frameRate: 24 })), OUTPUT_SIZE), + ).toThrow("Tiled overlay storage frame rate does not match the output:"); + expect(() => + readTiledOverlayManifest(withPath(tiledManifest({ durationSec: 4 })), OUTPUT_SIZE), + ).toThrow("Tiled overlay storage duration does not match the output:"); + expect(() => + readTiledOverlayManifest(withPath(tiledManifest({ layers: null })), OUTPUT_SIZE), + ).toThrow("Tiled overlay manifest requires a layers array:"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts an identical-frames layer with a static base only", () => { + const dir = makeTempDir(); + try { + const payload = writePayload(dir, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE); + const manifestPath = writeManifest(dir, tiledManifest()); + const manifest = JSON.parse(require("node:fs").readFileSync(manifestPath, "utf8")); + manifest.layers[0].payloadPath = payload; + writeFileSync(manifestPath, JSON.stringify(manifest)); + const layers = readTiledOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(1); + expect(layers[0]).toMatchObject({ + id: "tiled-effects", + width: LAYER_WIDTH, + height: LAYER_HEIGHT, + frameCount: 60, + tileSize: TILED_OVERLAY_TILE_SIZE, + pixelFormat: "rgba", + payloadPath: payload, + payloadByteLength: TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE, + }); + expect(layers[0].staticTiles).toHaveLength(TILE_COUNT); + expect(layers[0].frameDeltas).toEqual([]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts moving/dynamic content described by ascending changed tile deltas", () => { + const dir = makeTempDir(); + try { + const payload = writePayload(dir, (TILE_COUNT + 4) * TILED_OVERLAY_TILE_BYTE_SIZE); + const layer = tiledLayer({ + payloadPath: payload, + payloadByteLength: (TILE_COUNT + 4) * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 10, + changedTiles: [tileRecord(1, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE)], + }, + { + frameIndex: 20, + changedTiles: [ + tileRecord(4, (TILE_COUNT + 1) * TILED_OVERLAY_TILE_BYTE_SIZE), + tileRecord(5, (TILE_COUNT + 2) * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 30, + changedTiles: [ + tileRecord(2, (TILE_COUNT + 3) * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + ], + }); + const manifestPath = writeManifest(dir, tiledManifest({ layers: [layer] })); + const layers = readTiledOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].frameDeltas).toHaveLength(3); + expect(resolveTiledOverlayLayerMetrics(layers[0])).toEqual({ + effectiveFrameCount: 4, + changedTileCount: 4, + uploadedTileCount: 10, + uploadedTileBytes: 10 * TILED_OVERLAY_TILE_BYTE_SIZE, + cachedTileCount: TILE_COUNT * 60 - 10, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects malformed layers, tile ranges, deltas, and unsorted ordering", () => { + const dir = makeTempDir(); + try { + const payload = writePayload(dir, (TILE_COUNT + 4) * TILED_OVERLAY_TILE_BYTE_SIZE); + const payloadLarge = writePayload( + dir, + (TILE_COUNT + 60) * TILED_OVERLAY_TILE_BYTE_SIZE, + "large.bin", + ); + const run = (layer, name) => { + const manifestPath = writeManifest(dir, tiledManifest({ layers: [layer] }), name); + return readTiledOverlayManifest(manifestPath, OUTPUT_SIZE); + }; + const base = (overrides = {}) => tiledLayer({ payloadPath: payload, ...overrides }); + + expect(() => run(base({ pixelFormat: "yuv420p" }), "a.json")).toThrow( + "must use RGBA tiles", + ); + expect(() => run(base({ tileSize: 64 }), "b.json")).toThrow("must use 128px tiles"); + expect(() => run(base({ width: 0 }), "c.json")).toThrow( + "Invalid tiled overlay layer tiled-effects:", + ); + expect(() => run(base({ width: 1921 }), "d.json")).toThrow("exceeds the output canvas"); + expect(() => run(base({ frameCount: 59 }), "e.json")).toThrow( + "does not contain enough frames", + ); + expect(() => run(base({ staticTiles: null }), "f.json")).toThrow( + "requires a static tile base", + ); + expect(() => run(base({ frameDeltas: null }), "g.json")).toThrow( + "requires frame delta records", + ); + expect(() => + run( + base({ + staticTiles: [ + tileRecord(0, 0), + ...staticTiles().slice(1, -1), + tileRecord(0, TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }), + "h.json", + ), + ).toThrow("emits duplicate static tile 0:"); + expect(() => + run( + base({ + staticTiles: staticTiles().slice(0, 5), + }), + "i.json", + ), + ).toThrow("does not fully define the static tile base:"); + expect(() => + run( + base({ + payloadByteLength: TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tileRecord(0, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + ], + }), + "j.json", + ), + ).toThrow("has an invalid tile payload range:"); + expect(() => + run( + base({ + payloadByteLength: (TILE_COUNT + 2) * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tileRecord(6, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + ], + }), + "k.json", + ), + ).toThrow("references an out-of-bounds tile:"); + expect(() => + run( + base({ + payloadByteLength: (TILE_COUNT + 1) * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 20, + changedTiles: [ + tileRecord(0, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 10, + changedTiles: [ + tileRecord(1, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + ], + }), + "l.json", + ), + ).toThrow("unsorted or duplicate delta frame indices:"); + expect(() => + run( + base({ + payloadByteLength: (TILE_COUNT + 1) * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tileRecord(0, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 1, + changedTiles: [ + tileRecord(0, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE), + ], + }, + ], + }), + "m.json", + ), + ).toThrow("duplicates tile payload bytes:"); + expect(() => + run( + base({ + payloadPath: payloadLarge, + payloadByteLength: (TILE_COUNT + 60) * TILED_OVERLAY_TILE_BYTE_SIZE, + frameDeltas: Array.from({ length: 60 }, (_, frameIndex) => ({ + frameIndex, + changedTiles: [ + tileRecord( + 0, + (TILE_COUNT + frameIndex) * TILED_OVERLAY_TILE_BYTE_SIZE, + ), + ], + })), + }), + "n.json", + ), + ).toThrow("more state versions than frames:"); + + const unsorted = writeManifest( + dir, + tiledManifest({ + layers: [ + tiledLayer({ id: "b", order: 1, payloadPath: payload }), + tiledLayer({ id: "a", order: 0, payloadPath: payload }), + ], + }), + "o.json", + ); + expect(() => readTiledOverlayManifest(unsorted, OUTPUT_SIZE)).toThrow( + "Tiled overlay layers must be sorted by order then id:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fails a payload truncated below the declared byte length", () => { + const dir = makeTempDir(); + try { + const payload = writePayload(dir, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE - 1); + const manifestPath = writeManifest( + dir, + tiledManifest({ layers: [tiledLayer({ payloadPath: payload })] }), + ); + expect(() => readTiledOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Tiled overlay layer tiled-effects payload is truncated: expected at least ${ + TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE + } bytes, received ${TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE - 1}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a missing payload file", () => { + const dir = makeTempDir(); + try { + const manifestPath = writeManifest( + dir, + tiledManifest({ layers: [tiledLayer({ payloadPath: join(dir, "nope.bin") })] }), + ); + expect(() => readTiledOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Tiled overlay layer tiled-effects does not exist:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("tiled overlay wrapper metrics and raw fallback", () => { + it("computes tile geometry and additive metrics", () => { + expect(tileCountForSize(384, 256)).toBe(6); + expect(tileCountForSize(1920, 1080)).toBe(15 * 9); + expect(TILED_OVERLAY_TILE_BYTE_SIZE).toBe(128 * 128 * 4); + }); + + it("keeps sparse layers eligible and surfaces dense/small fallback reasons", () => { + const sparse = { + width: LAYER_WIDTH, + height: LAYER_HEIGHT, + frameCount: 60, + staticTiles: staticTiles(), + frameDeltas: [ + { + frameIndex: 10, + changedTiles: [tileRecord(1, TILE_COUNT * TILED_OVERLAY_TILE_BYTE_SIZE)], + }, + ], + }; + expect(resolveTiledOverlayRawFallbackReason(sparse)).toBeNull(); + + const small = { + width: 256, + height: 128, + frameCount: 60, + staticTiles: [tileRecord(0, 0), tileRecord(1, TILED_OVERLAY_TILE_BYTE_SIZE)], + frameDeltas: [], + }; + expect(resolveTiledOverlayRawFallbackReason(small)).toBe("small-layer"); + + const dense = { + width: LAYER_WIDTH, + height: LAYER_HEIGHT, + frameCount: 60, + staticTiles: staticTiles(), + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [0, 1, 2, 3].map((tileIndex, index) => + tileRecord(tileIndex, (TILE_COUNT + index) * TILED_OVERLAY_TILE_BYTE_SIZE), + ), + }, + ], + }; + expect(resolveTiledOverlayRawFallbackReason(dense)).toBe("dense-frame-delta"); + }); +}); diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..242b61cfa 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,7 +1,53 @@ import { contextBridge, ipcRenderer } from "electron"; import type { RecordingSessionData } from "./ipc/types"; -type NativeVideoExportWriteResult = { success: boolean; error?: string }; +type NativeVideoExportWriteResult = { + success: boolean; + error?: string; + fallbackAvailable?: boolean; +}; +type NativeVideoExportFrameChannelResult = NativeVideoExportWriteResult; +type NativeVideoExportFramePortResponse = + | { + type: "ready"; + protocol: 1; + sessionId: string; + transferable: true; + transferProbe: ArrayBuffer; + } + | { + type: "ack"; + protocol: 1; + sessionId: string; + requestId: number; + sequence: number; + success: true; + } + | { + type: "error"; + protocol: 1; + sessionId: string; + requestId?: number; + sequence?: number; + success: false; + error: string; + fallbackAvailable: boolean; + }; +type NativeVideoExportFrameChannelState = { + sessionId: string; + port: MessagePort; + nextSequence: number; + pending: Map< + number, + { sequence: number; resolve: (result: NativeVideoExportWriteResult) => void } + >; + ready: Promise; + resolveReady: () => void; + rejectReady: (error: Error) => void; + readySettled: boolean; + closed: boolean; + handshakeTimeout: ReturnType; +}; type NativeVideoAudioMuxMetrics = { tempVideoWriteMs?: number; tempEditedAudioWriteMs?: number; @@ -56,6 +102,35 @@ type NativeStaticLayoutChunkMetric = { outputBytes: number; fallbackReason?: string; windowsGpuSummary?: WindowsGpuExportSummary; + nvidiaCudaSummary?: { + success?: boolean; + outputCodec?: "h264" | "hevc"; + }; +}; +type NativeTiledOverlayTileRecord = { + tileIndex: number; + byteOffset: number; + byteLength: number; +}; +type NativeTiledOverlayLayerDescriptor = { + id: string; + order: number; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + tileSize: 128; + pixelFormat: "rgba"; + payloadPath: string; + payloadByteLength: number; + staticTiles: readonly NativeTiledOverlayTileRecord[]; + frameDeltas: readonly { + frameIndex: number; + changedTiles: readonly NativeTiledOverlayTileRecord[]; + }[]; }; type NativeStaticLayoutMetrics = NativeVideoAudioMuxMetrics & { chunkCount: number; @@ -66,6 +141,15 @@ type NativeStaticLayoutMetrics = NativeVideoAudioMuxMetrics & { fallbackChunkCount: number; videoOnlyBytes?: number; chunks: NativeStaticLayoutChunkMetric[]; + tiledOverlayLayers?: number; + tiledOverlayBlendFrames?: number; + changedTileCount?: number; + uploadedTileBytes?: number; + cachedTileCount?: number; + rawFallbackReason?: string; + overlayHostReadMs?: number; + overlayH2DEnqueueMs?: number; + overlayCacheHits?: number; }; type NativeStaticLayoutProgress = { sessionId?: string; @@ -73,6 +157,8 @@ type NativeStaticLayoutProgress = { stage?: "preparing" | "finalizing"; elapsedMs?: number; averageFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; currentFrame: number; totalFrames: number; percentage: number; @@ -113,6 +199,7 @@ const nativeVideoExportWriteRequests = new Map< let nextNativeVideoExportWriteRequestId = 1; let nativeVideoExportWriteResultListenerAttached = false; +const nativeVideoExportFrameChannels = new Map(); function ensureNativeVideoExportWriteResultListener() { if (nativeVideoExportWriteResultListenerAttached) { @@ -163,6 +250,309 @@ function settleNativeVideoExportPendingRequests( } } +function writeNativeVideoExportFramesLegacy( + sessionId: string, + frameDataList: Uint8Array[], +): Promise { + ensureNativeVideoExportWriteResultListener(); + + return new Promise((resolve) => { + const requestId = nextNativeVideoExportWriteRequestId++; + nativeVideoExportWriteRequests.set(requestId, { + sessionId, + resolve, + }); + + ipcRenderer.send("native-video-export-write-frames-async", { + sessionId, + requestId, + frameDataList, + }); + }); +} + +function isArrayBuffer(value: unknown): value is ArrayBuffer { + return value instanceof ArrayBuffer; +} + +function isNativeVideoExportFramePortResponse( + value: unknown, +): value is NativeVideoExportFramePortResponse { + if (!value || typeof value !== "object") { + return false; + } + + const payload = value as Record; + return payload.protocol === 1 && typeof payload.type === "string"; +} + +function settleNativeVideoExportFrameChannelState( + state: NativeVideoExportFrameChannelState, + error: string, +) { + if (state.closed) { + return; + } + + state.closed = true; + clearTimeout(state.handshakeTimeout); + if (!state.readySettled) { + state.readySettled = true; + state.rejectReady(new Error(error)); + } + for (const pendingRequest of state.pending.values()) { + pendingRequest.resolve({ success: false, error }); + } + state.pending.clear(); + if (nativeVideoExportFrameChannels.get(state.sessionId) === state) { + nativeVideoExportFrameChannels.delete(state.sessionId); + } + try { + state.port.close(); + } catch { + // The main process may already have closed the port. + } +} + +function handleNativeVideoExportFramePortResponse( + state: NativeVideoExportFrameChannelState, + value: unknown, +) { + if (!isNativeVideoExportFramePortResponse(value) || value.sessionId !== state.sessionId) { + return; + } + + if (value.type === "ready") { + if ( + value.transferable !== true || + !isArrayBuffer(value.transferProbe) || + value.transferProbe.byteLength !== 1 || + state.readySettled + ) { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel returned an invalid handshake", + ); + return; + } + state.readySettled = true; + state.resolveReady(); + return; + } + + if (value.type === "ack") { + const pendingRequest = state.pending.get(value.requestId); + if (!pendingRequest) { + return; + } + state.pending.delete(value.requestId); + pendingRequest.resolve( + value.sequence === pendingRequest.sequence + ? { success: true } + : { + success: false, + error: "Native export frame acknowledgement sequence mismatch", + }, + ); + return; + } + + if (value.type === "error") { + if (typeof value.requestId === "number") { + const pendingRequest = state.pending.get(value.requestId); + if (!pendingRequest) { + return; + } + state.pending.delete(value.requestId); + pendingRequest.resolve({ + success: false, + error: value.error, + fallbackAvailable: value.fallbackAvailable, + }); + return; + } + settleNativeVideoExportFrameChannelState(state, value.error); + } +} + +function openNativeVideoExportFrameChannel( + sessionId: string, +): Promise { + const existing = nativeVideoExportFrameChannels.get(sessionId); + if (existing) { + return existing.ready + .then(() => ({ success: true })) + .catch((error: unknown) => ({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + })); + } + + if (typeof MessageChannel === "undefined" || typeof ipcRenderer.postMessage !== "function") { + return Promise.resolve({ + success: false, + error: "Native export transferable frame channels are unavailable", + fallbackAvailable: true, + }); + } + + let channel: MessageChannel; + try { + channel = new MessageChannel(); + } catch (error) { + return Promise.resolve({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + }); + } + + let resolveReady!: () => void; + let rejectReady!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + const state: NativeVideoExportFrameChannelState = { + sessionId, + port: channel.port1, + nextSequence: 0, + pending: new Map(), + ready, + resolveReady, + rejectReady, + readySettled: false, + closed: false, + handshakeTimeout: setTimeout(() => { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel handshake timed out", + ); + }, 2_000), + }; + nativeVideoExportFrameChannels.set(sessionId, state); + channel.port1.onmessage = (event: MessageEvent) => { + handleNativeVideoExportFramePortResponse(state, event.data); + }; + channel.port1.onmessageerror = () => { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel delivery failed", + ); + }; + channel.port1.start(); + + try { + ipcRenderer.postMessage("native-video-export-frame-channel", { sessionId }, [ + channel.port2, + ]); + const capabilityProbe = new ArrayBuffer(1); + channel.port1.postMessage( + { + type: "hello", + protocol: 1, + sessionId, + capabilityProbe, + }, + [capabilityProbe], + ); + if (capabilityProbe.byteLength !== 0) { + settleNativeVideoExportFrameChannelState( + state, + "Native export transferable ArrayBuffer delivery is unavailable", + ); + } + } catch (error) { + settleNativeVideoExportFrameChannelState( + state, + error instanceof Error ? error.message : String(error), + ); + } + + return ready + .then(() => ({ success: true })) + .catch((error: unknown) => ({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + })); +} + +function getNativeVideoExportTransferBuffer(frameData: Uint8Array): ArrayBuffer { + if ( + frameData.buffer instanceof ArrayBuffer && + frameData.byteOffset === 0 && + frameData.byteLength === frameData.buffer.byteLength + ) { + return frameData.buffer; + } + return frameData.slice().buffer as ArrayBuffer; +} + +async function writeNativeVideoExportFramesViaChannel( + sessionId: string, + frameDataList: Uint8Array[], +): Promise { + const state = nativeVideoExportFrameChannels.get(sessionId); + if (!state) { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + + try { + await state.ready; + } catch { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + + const acknowledgements: Array> = []; + let postedFrameCount = 0; + for (const frameData of frameDataList) { + const requestId = nextNativeVideoExportWriteRequestId++; + const sequence = state.nextSequence++; + const frame = getNativeVideoExportTransferBuffer(frameData); + const acknowledgement = new Promise((resolve) => { + state.pending.set(requestId, { sequence, resolve }); + }); + acknowledgements.push(acknowledgement); + try { + state.port.postMessage( + { + type: "frame", + protocol: 1, + sessionId, + requestId, + sequence, + frame, + }, + [frame], + ); + postedFrameCount += 1; + if (frame.byteLength !== 0) { + throw new Error("Native export transferable ArrayBuffer delivery is unavailable"); + } + } catch (error) { + state.pending.delete(requestId); + const message = error instanceof Error ? error.message : String(error); + settleNativeVideoExportFrameChannelState(state, message); + if (postedFrameCount === 0) { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + return { success: false, error: message, fallbackAvailable: false }; + } + } + + const results = await Promise.all(acknowledgements); + return results.find((result) => !result.success) ?? { success: true }; +} + +function closeNativeVideoExportFrameChannel(sessionId: string, error: string) { + const state = nativeVideoExportFrameChannels.get(sessionId); + if (state) { + settleNativeVideoExportFrameChannelState(state, error); + } +} + contextBridge.exposeInMainWorld("electronAPI", { hudOverlaySetIgnoreMouse: (ignore: boolean) => { ipcRenderer.send("hud-overlay-set-ignore-mouse", ignore); @@ -223,12 +613,28 @@ contextBridge.exposeInMainWorld("electronAPI", { nativeStaticLayoutExport: (options: { sessionId?: string; inputPath: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; width: number; height: number; frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; durationSec: number; + overlayLayers?: Array<{ + id: string; + order: number; + path: string; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + pixelFormat: "rgba"; + }>; + tiledOverlayLayers?: NativeTiledOverlayLayerDescriptor[]; contentWidth: number; contentHeight: number; offsetX: number; @@ -270,7 +676,15 @@ contextBridge.exposeInMainWorld("electronAPI", { anchorY: number; aspectRatio: number; }>; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; timelineSegments?: Array<{ sourceStartMs: number; sourceEndMs: number; @@ -297,6 +711,14 @@ contextBridge.exposeInMainWorld("electronAPI", { return ipcRenderer.invoke("native-static-layout-export", options) as Promise<{ success: boolean; tempPath?: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; + route?: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; encoderName?: string; error?: string; metrics?: NativeStaticLayoutMetrics; @@ -322,10 +744,21 @@ contextBridge.exposeInMainWorld("electronAPI", { bitrate: number; encodingMode: "fast" | "balanced" | "quality"; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; }) => { return ipcRenderer.invoke("native-video-export-start", options); }, + nativeVideoExportOpenFrameChannel: (sessionId: string) => + openNativeVideoExportFrameChannel(sessionId), + nativeVideoExportWriteFrameViaChannel: (sessionId: string, frameData: Uint8Array) => + writeNativeVideoExportFramesViaChannel(sessionId, [frameData]), + nativeVideoExportWriteFramesViaChannel: (sessionId: string, frameDataList: Uint8Array[]) => + writeNativeVideoExportFramesViaChannel(sessionId, frameDataList), nativeVideoExportWriteFrame: (sessionId: string, frameData: Uint8Array) => { + if (nativeVideoExportFrameChannels.has(sessionId)) { + return writeNativeVideoExportFramesViaChannel(sessionId, [frameData]); + } ensureNativeVideoExportWriteResultListener(); return new Promise((resolve) => { @@ -343,6 +776,9 @@ contextBridge.exposeInMainWorld("electronAPI", { }); }, nativeVideoExportWriteFrames: (sessionId: string, frameDataList: Uint8Array[]) => { + if (nativeVideoExportFrameChannels.has(sessionId)) { + return writeNativeVideoExportFramesViaChannel(sessionId, frameDataList); + } ensureNativeVideoExportWriteResultListener(); return new Promise((resolve) => { @@ -390,7 +826,20 @@ contextBridge.exposeInMainWorld("electronAPI", { }, ); + closeNativeVideoExportFrameChannel( + sessionId, + result?.success + ? "Native video export session finished" + : "Native video export session failed", + ); return result; + }) + .catch((error: unknown) => { + closeNativeVideoExportFrameChannel( + sessionId, + "Native video export finish request failed", + ); + throw error; }) as Promise<{ success: boolean; data?: Uint8Array; @@ -401,6 +850,10 @@ contextBridge.exposeInMainWorld("electronAPI", { }, nativeVideoExportCancel: (sessionId: string) => { return ipcRenderer.invoke("native-video-export-cancel", sessionId).finally(() => { + closeNativeVideoExportFrameChannel( + sessionId, + "Native video export session was cancelled", + ); settleNativeVideoExportPendingRequests(sessionId, { success: false, error: "Native video export session was cancelled", @@ -765,7 +1218,7 @@ contextBridge.exposeInMainWorld("electronAPI", { }, getLocalMediaUrl: (filePath: string) => { return ipcRenderer.invoke("get-local-media-url", filePath) as Promise< - { success: true; url: string } | { success: false } + { success: true; url: string; pending?: boolean } | { success: false } >; }, saveProjectFile: ( diff --git a/scripts/benchmark-cuda4k.mjs b/scripts/benchmark-cuda4k.mjs new file mode 100644 index 000000000..a225a85da --- /dev/null +++ b/scripts/benchmark-cuda4k.mjs @@ -0,0 +1,188 @@ +// 4K CUDA compositor benchmark runner. Generates nothing; expects a prepared +// .tmp/cuda4k workspace with source-4k.mp4, overlay-4k.rgba, overlay-manifest.json, +// cursor-telemetry.json, zoom-telemetry.csv. +// +// Usage: node scripts/benchmark-cuda4k.mjs [--tag name] [--frames N] [--temporal N] [--overlay 0|1] [--cursor 0|1] [--codec h264|hevc] +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, ".."); +const workDir = path.join(repoRoot, ".tmp", "cuda4k"); +const pipeline = path.join( + repoRoot, + "electron", + "native", + "nvidia-cuda-compositor", + "run-mp4-pipeline.mjs", +); +const ffmpeg = path.join(repoRoot, "node_modules", "ffmpeg-static", "ffmpeg.exe"); +const ffprobe = path.join( + repoRoot, + "node_modules", + "ffprobe-static", + "bin", + "win32", + "x64", + "ffprobe.exe", +); + +function arg(name, fallback) { + const index = process.argv.indexOf(name); + return index >= 0 && index + 1 < process.argv.length ? process.argv[index + 1] : fallback; +} + +const tag = arg("--tag", "baseline"); +const durationSec = Number(arg("--duration", "6")); +const fps = Number(arg("--fps", "30")); +const temporal = Number(arg("--temporal", "13")); +const withOverlay = arg("--overlay", "1") === "1"; +const withCursor = arg("--cursor", "0") === "1"; +const codec = arg("--codec", "hevc"); +const encodingMode = arg("--mode", "balanced"); + +if (!existsSync(path.join(workDir, "source-4k.mp4"))) { + console.error("Missing 4K source; run generation first"); + process.exit(2); +} + +const targetFrames = Math.ceil(durationSec * fps); +const outputPath = path.join(workDir, `out-${tag}-${codec}.mp4`); +const args = [ + pipeline, + "--input", + path.join(workDir, "source-4k.mp4"), + "--output", + outputPath, + "--output-codec", + codec, + "--width", + "3840", + "--height", + "2160", + "--fps", + String(fps), + "--bitrate-mbps", + "40", + "--encoding-mode", + encodingMode, + "--duration-sec", + String(durationSec), + "--stream-sync", + "--prewarm-ms", + "300", + "--content-x", + "0", + "--content-y", + "0", + "--content-width", + "3840", + "--content-height", + "2160", + "--radius", + "0", + "--background-y", + "16", + "--background-u", + "128", + "--background-v", + "128", + "--zoom-telemetry", + path.join(workDir, "zoom-telemetry.csv"), +]; +if (withOverlay) { + const manifestPath = path.join(workDir, "overlay-manifest.json"); + const absoluteManifest = { + layers: [ + { + id: "bench-overlay", + path: path.join(workDir, "overlay-4k.rgba"), + x: 800, + y: 500, + width: 900, + height: 560, + frameCount: targetFrames, + }, + ], + }; + writeFileSync(manifestPath, JSON.stringify(absoluteManifest)); + args.push("--overlay-manifest", manifestPath); +} +if (withCursor) { + args.push( + "--cursor-json", + path.join(workDir, "cursor-telemetry.json"), + "--cursor-height", + "84", + ); +} +if (temporal >= 3) { + args.push( + "--temporal-blur-sample-count", + String(temporal), + "--temporal-blur-shutter-fraction", + "0.5", + "--temporal-blur-weight-power", + "2", + ); +} + +const env = { + ...process.env, + RECORDLY_FFMPEG_EXE: ffmpeg, + RECORDLY_FFPROBE_EXE: ffprobe, + RECORDLY_NVIDIA_CUDA_EXPORT_HIGH_PRIORITY: "1", +}; + +const startedAt = performance.now(); +const result = spawnSync("node", args, { env, encoding: "utf8", maxBuffer: 512 * 1024 * 1024 }); +const elapsedMs = performance.now() - startedAt; +if (result.status !== 0) { + console.error("Benchmark run failed", result.status); + console.error(result.stderr.slice(-4000)); + process.exit(1); +} + +const stdout = result.stdout; +const lines = stdout.split(/\r?\n/); +const summaryStart = lines.findIndex((line) => line.trim() === "{"); +if (summaryStart === -1) { + console.error("No summary JSON found"); + process.exit(1); +} +const summary = JSON.parse(lines.slice(summaryStart).join("\n")); +const ns = summary.nativeSummary ?? {}; +const out = { + tag, + codec: summary.outputCodec, + temporal, + withOverlay: Boolean(ns.overlayLayers), + withCursor: Boolean(ns.cursorOverlay), + frames: ns.frames, + targetFrames: summary.targetFrames, + measuredFps: ns.measuredFps, + realtimeMultiplier: ns.realtimeMultiplier, + totalMs: ns.totalMs, + decodeMs: ns.decodeMs, + encodeMs: ns.encodeMs, + compositeMs: ns.compositeMs, + compositeGpuMs: ns.compositeGpuMs, + zoomBlurGpuMs: ns.zoomBlurGpuMs, + overlayBlendGpuMs: ns.overlayBlendGpuMs, + overlayUploadMs: ns.overlayUploadMs, + nvencMs: ns.nvencMs, + packetWriteMs: ns.packetWriteMs, + flushMs: ns.flushMs, + temporalBlurFrames: ns.temporalBlurFrames, + temporalBlurSamplesTotal: ns.temporalBlurSamplesTotal, + roiCompositeFrames: ns.roiCompositeFrames, + monolithicCompositeFrames: ns.monolithicCompositeFrames, + copyCompositeFrames: ns.copyCompositeFrames, + rcMode: ns.nvencDiagnostics?.rcModeUsed, + outputBytes: ns.outputBytes, + wallMs: Number(elapsedMs.toFixed(2)), +}; +console.log(JSON.stringify(out, null, 1)); +writeFileSync(path.join(workDir, `result-${tag}.json`), JSON.stringify(out, null, 1) + "\n"); diff --git a/scripts/benchmark-native-frame-transport.mjs b/scripts/benchmark-native-frame-transport.mjs new file mode 100644 index 000000000..5176fbe24 --- /dev/null +++ b/scripts/benchmark-native-frame-transport.mjs @@ -0,0 +1,669 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; + +const require = createRequire(import.meta.url); +const reportPrefix = "RECORDLY_NATIVE_FRAME_TRANSPORT_BENCHMARK "; +const oneMiB = 1024 * 1024; +const payloadSizes = [oneMiB, 33 * oneMiB]; +const iterations = parsePositiveInteger( + process.env.RECORDLY_IPC_BENCH_ITERATIONS ?? "4", + "RECORDLY_IPC_BENCH_ITERATIONS", +); +const windowSize = parsePositiveInteger( + process.env.RECORDLY_IPC_BENCH_WINDOW ?? "2", + "RECORDLY_IPC_BENCH_WINDOW", +); +const timeoutMs = parsePositiveInteger( + process.env.RECORDLY_IPC_BENCH_TIMEOUT_MS ?? "120000", + "RECORDLY_IPC_BENCH_TIMEOUT_MS", +); +const keepFixture = process.env.RECORDLY_IPC_BENCH_KEEP_TEMP === "1"; + +function parsePositiveInteger(value, label) { + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer`); + } + + return parsed; +} + +function createFixtureSources() { + const mainSource = ` +const { app, BrowserWindow, ipcMain, MessageChannelMain } = require("electron"); +const path = require("node:path"); + +let benchmarkWindow = null; +let benchmarkPort = null; +let didReport = false; + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +function finishWithError(message) { + if (didReport) { + return; + } + + didReport = true; + process.stdout.write( + ${JSON.stringify(reportPrefix)} + + JSON.stringify({ type: "error", message: String(message) }) + + "\\n", + ); + setImmediate(() => { + if (benchmarkWindow && !benchmarkWindow.isDestroyed()) { + benchmarkWindow.destroy(); + } + if (app.isReady()) { + app.quit(); + } + }); +} + +function finishWithResult(result) { + if (didReport) { + return; + } + + didReport = true; + process.stdout.write( + ${JSON.stringify(reportPrefix)} + + JSON.stringify({ type: "result", result }) + + "\\n", + ); + setImmediate(() => { + if (benchmarkWindow && !benchmarkWindow.isDestroyed()) { + benchmarkWindow.destroy(); + } + app.quit(); + }); +} + +function validateFrame(message) { + if (!message || typeof message !== "object") { + throw new Error("Main received a non-object frame message"); + } + if (!(message.payload instanceof ArrayBuffer)) { + throw new Error("Main received a frame without an ArrayBuffer payload"); + } + + const bytes = new Uint8Array(message.payload); + const expectedFirst = message.seq % 251; + const expectedLast = (message.seq + 1) % 251; + if (bytes.length === 0 || bytes[0] !== expectedFirst || bytes.at(-1) !== expectedLast) { + throw new Error(\`Main received corrupt payload for sequence \${message.seq}\`); + } + + return bytes.byteLength; +} + +function installMessagePort(port) { + benchmarkPort = port; + benchmarkPort.on("message", (event) => { + try { + const message = event.data; + if (message?.type === "probe") { + const receivedBytes = validateFrame({ ...message, seq: 0 }); + benchmarkPort.postMessage({ type: "probe-ack", receivedBytes }); + return; + } + if (message === null || message === undefined) { + throw new Error("MessagePortMain delivered no data for the transferable ArrayBuffer probe; this runtime does not support the tested renderer-to-main transfer"); + } + if (message?.type !== "frame") { + throw new Error("Main received an unknown MessagePort message"); + } + + const receivedBytes = validateFrame(message); + benchmarkPort.postMessage({ + type: "ack", + runId: message.runId, + seq: message.seq, + receivedBytes, + }); + } catch (error) { + finishWithError(\`MessagePortMain could not receive a transferable frame: \${errorMessage(error)}\`); + } + }); + benchmarkPort.start(); +} + +ipcMain.on("legacy-frame", (event, message) => { + try { + const receivedBytes = validateFrame(message); + event.sender.send("legacy-ack", { + runId: message.runId, + seq: message.seq, + receivedBytes, + }); + } catch (error) { + finishWithError(\`ipcRenderer.send could not receive a frame: \${errorMessage(error)}\`); + } +}); + +ipcMain.on("benchmark-failed", (_event, message) => { + finishWithError(\`Renderer benchmark failed: \${String(message?.message ?? message)}\`); +}); + +ipcMain.on("benchmark-ready", (event) => { + try { + if (typeof MessageChannelMain !== "function") { + throw new Error("MessageChannelMain is unavailable in this Electron runtime"); + } + + const channel = new MessageChannelMain(); + if (!channel.port1 || !channel.port2 || typeof channel.port1.postMessage !== "function") { + throw new Error("Electron did not provide usable MessagePortMain endpoints"); + } + installMessagePort(channel.port1); + event.sender.postMessage("transferable-port", null, [channel.port2]); + event.sender.send("benchmark-start", { + payloadSizes: ${JSON.stringify(payloadSizes)}, + iterations: ${iterations}, + windowSize: ${windowSize}, + timeoutMs: ${timeoutMs}, + }); + } catch (error) { + finishWithError(\`Electron could not establish transferable resources: \${errorMessage(error)}\`); + } +}); + +ipcMain.on("benchmark-result", (_event, result) => { + finishWithResult({ + electronVersion: process.versions.electron ?? "unknown", + nodeVersion: process.versions.node ?? "unknown", + platform: process.platform, + arch: process.arch, + results: result, + }); +}); + +app.disableHardwareAcceleration(); +app.setPath("userData", path.join(__dirname, "user-data")); +app.setPath("sessionData", path.join(__dirname, "session-data")); +app.commandLine.appendSwitch("disable-gpu"); + +app.whenReady() + .then(async () => { + benchmarkWindow = new BrowserWindow({ + show: false, + webPreferences: { + contextIsolation: false, + nodeIntegration: true, + sandbox: false, + }, + }); + benchmarkWindow.webContents.on("did-fail-load", (_event, errorCode, description) => { + finishWithError(\`Electron could not load the benchmark renderer (\${errorCode}): \${description}\`); + }); + benchmarkWindow.webContents.on("render-process-gone", (_event, details) => { + finishWithError(\`Electron renderer exited before completing the benchmark: \${details.reason}\`); + }); + await benchmarkWindow.loadFile(path.join(__dirname, "renderer.html")); + }) + .catch((error) => finishWithError(\`Electron could not launch the benchmark fixture: \${errorMessage(error)}\`)); +`; + + const rendererSource = ` +const { ipcRenderer } = require("electron"); + +let messagePort = null; +let benchmarkConfig = null; +let probeDetached = false; +let probeComplete = false; +let hasStarted = false; +let activeReject = null; +const legacyRuns = new Map(); +const messagePortRuns = new Map(); + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +function reportFailure(error) { + const message = errorMessage(error); + if (activeReject) { + activeReject(new Error(message)); + return; + } + + try { + ipcRenderer.send("benchmark-failed", { message }); + } catch (sendError) { + console.error(\`Unable to report benchmark failure: \${errorMessage(sendError)}\`); + } +} + +function makePayload(size, sequence) { + const payload = new ArrayBuffer(size); + const bytes = new Uint8Array(payload); + bytes[0] = sequence % 251; + bytes[size - 1] = (sequence + 1) % 251; + return payload; +} + +function handleAck(runs, message) { + const run = runs.get(message?.runId); + if (run) { + run.ack(message); + } +} + +ipcRenderer.on("legacy-ack", (_event, message) => handleAck(legacyRuns, message)); + +ipcRenderer.on("transferable-port", (event) => { + try { + const candidate = event.ports?.[0]; + if (!candidate || typeof candidate.postMessage !== "function") { + throw new Error("Electron transferred no usable renderer MessagePort"); + } + if (typeof candidate.start !== "function") { + throw new Error("The transferred renderer MessagePort has no start() method"); + } + + messagePort = candidate; + messagePort.onmessage = (messageEvent) => { + const message = messageEvent.data; + if (message?.type === "probe-ack") { + if (message.receivedBytes !== 2 || !probeDetached) { + reportFailure( + "Electron established a MessagePort but did not transfer the probe ArrayBuffer", + ); + return; + } + probeComplete = true; + maybeStart(); + return; + } + if (message?.type === "ack") { + handleAck(messagePortRuns, message); + } + }; + messagePort.start(); + + const probe = makePayload(2, 0); + messagePort.postMessage({ type: "probe", payload: probe }, [probe]); + probeDetached = probe.byteLength === 0; + if (!probeDetached) { + throw new Error("MessagePort.postMessage did not detach the transferred ArrayBuffer"); + } + } catch (error) { + reportFailure(\`Electron could not establish transferable ArrayBuffers: \${errorMessage(error)}\`); + } +}); + +ipcRenderer.on("benchmark-start", (_event, config) => { + benchmarkConfig = config; + maybeStart(); +}); + +function maybeStart() { + if (hasStarted || !benchmarkConfig || !probeComplete || !messagePort) { + return; + } + + hasStarted = true; + runBenchmark() + .then((results) => ipcRenderer.send("benchmark-result", results)) + .catch((error) => reportFailure(error)); +} + +function summarize(values) { + const sorted = [...values].sort((left, right) => left - right); + const percentile = (fraction) => { + const index = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction)); + return sorted[index]; + }; + const total = values.reduce((sum, value) => sum + value, 0); + return { + averageMs: total / values.length, + medianMs: percentile(0.5), + p95Ms: percentile(0.95), + }; +} + +function runTransport(route, size, port) { + return new Promise((resolve, reject) => { + const runId = \`\${route}-\${size}-\${Date.now()}-\${Math.random()}\`; + const pending = new Map(); + const ackLatencies = []; + const detachmentSamples = []; + let nextSequence = 0; + let completed = 0; + let inFlightBytes = 0; + let peakInFlightBytes = 0; + let settled = false; + const startedAt = performance.now(); + const totalBytes = size * benchmarkConfig.iterations; + const runs = route === "legacy-ipc-send" ? legacyRuns : messagePortRuns; + const timeout = setTimeout(() => fail(new Error( + \`\${route} timed out waiting for ACKs at \${formatBytes(size)}\`, + )), benchmarkConfig.timeoutMs); + + function finish() { + if (settled) { + return; + } + settled = true; + activeReject = null; + clearTimeout(timeout); + runs.delete(runId); + const durationMs = performance.now() - startedAt; + resolve({ + route, + payloadBytes: size, + payloadMiB: size / (1024 * 1024), + iterations: benchmarkConfig.iterations, + windowSize: benchmarkConfig.windowSize, + totalBytes, + durationMs, + throughputMiBPerSec: totalBytes / (1024 * 1024) / (durationMs / 1000), + ackLatencyMs: summarize(ackLatencies), + peakInFlightPayloadBytes: peakInFlightBytes, + peakInFlightPayloadMiB: peakInFlightBytes / (1024 * 1024), + receivedBytes: totalBytes, + bufferOwnership: { + semantics: + route === "message-port-transfer" + ? "sender buffer detached after transfer-list post" + : "sender buffer remained attached; no transfer list", + detachedAfterPostCount: detachmentSamples.filter(Boolean).length, + samples: detachmentSamples.length, + allDetachedAfterPost: detachmentSamples.every(Boolean), + }, + physicalZeroCopy: "not measured; ownership transfer does not prove physical zero-copy", + }); + } + + function fail(error) { + if (settled) { + return; + } + settled = true; + activeReject = null; + clearTimeout(timeout); + runs.delete(runId); + reject(error instanceof Error ? error : new Error(String(error))); + } + + function ack(message) { + if (settled) { + return; + } + const sent = pending.get(message?.seq); + if (!sent) { + fail(new Error(\`\${route} received an unexpected ACK\`)); + return; + } + if (message.receivedBytes !== size) { + fail(new Error(\`\${route} ACK reported \${message.receivedBytes} bytes, expected \${size}\`)); + return; + } + pending.delete(message.seq); + completed += 1; + inFlightBytes -= sent.bytes; + ackLatencies.push(performance.now() - sent.sentAt); + if (completed === benchmarkConfig.iterations) { + finish(); + return; + } + sendAvailable(); + } + + function sendAvailable() { + while (!settled && + nextSequence < benchmarkConfig.iterations && + pending.size < benchmarkConfig.windowSize) { + const sequence = nextSequence; + nextSequence += 1; + const payload = makePayload(size, sequence); + pending.set(sequence, { bytes: size, sentAt: performance.now() }); + inFlightBytes += size; + peakInFlightBytes = Math.max(peakInFlightBytes, inFlightBytes); + try { + if (route === "legacy-ipc-send") { + ipcRenderer.send("legacy-frame", { runId, seq: sequence, payload }); + } else { + port.postMessage( + { type: "frame", runId, seq: sequence, payload }, + [payload], + ); + } + detachmentSamples.push(payload.byteLength === 0); + } catch (error) { + fail(new Error(\`\${route} could not send an ArrayBuffer: \${errorMessage(error)}\`)); + } + } + } + + runs.set(runId, { ack }); + activeReject = fail; + sendAvailable(); + }); +} + +function formatBytes(bytes) { + return \`\${bytes / (1024 * 1024)} MiB\`; +} + +async function runBenchmark() { + const results = []; + for (const size of benchmarkConfig.payloadSizes) { + results.push(await runTransport("legacy-ipc-send", size, null)); + results.push(await runTransport("message-port-transfer", size, messagePort)); + } + activeReject = null; + return results; +} + +window.addEventListener("error", (event) => reportFailure(event.error ?? new Error(event.message))); +window.addEventListener("unhandledrejection", (event) => reportFailure(event.reason)); +ipcRenderer.send("benchmark-ready"); +`; + + const htmlSource = ` + +Native frame transport benchmark + + +`; + + return { mainSource, htmlSource }; +} + +async function createFixture() { + const fixtureDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), "recordly-native-frame-transport-"), + ); + const sources = createFixtureSources(); + await Promise.all([ + fs.writeFile(path.join(fixtureDirectory, "main.cjs"), sources.mainSource), + fs.writeFile(path.join(fixtureDirectory, "renderer.html"), sources.htmlSource), + ]); + return fixtureDirectory; +} + +function resolveElectronPath() { + let electronPath; + try { + electronPath = require("electron"); + } catch (error) { + throw new Error( + `Electron is not installed or could not be resolved: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (typeof electronPath !== "string" || electronPath.length === 0) { + throw new Error("The Electron package did not expose an executable path"); + } + + return electronPath; +} + +function parseReportLine(line) { + if (!line.startsWith(reportPrefix)) { + return null; + } + + try { + return JSON.parse(line.slice(reportPrefix.length)); + } catch (error) { + throw new Error( + `Electron emitted an invalid benchmark report: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function launchFixture(electronPath, fixtureDirectory) { + return new Promise((resolve, reject) => { + let child; + try { + child = spawn( + electronPath, + ["--no-sandbox", "--disable-gpu", path.join(fixtureDirectory, "main.cjs")], + { + cwd: fixtureDirectory, + env: { + ...process.env, + RECORDLY_IPC_BENCH_ITERATIONS: String(iterations), + RECORDLY_IPC_BENCH_WINDOW: String(windowSize), + RECORDLY_IPC_BENCH_TIMEOUT_MS: String(timeoutMs), + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + } catch (error) { + reject( + new Error( + `Electron could not be launched: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + return; + } + + let outputBuffer = ""; + let report = null; + let timedOut = false; + let settled = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, timeoutMs + 5000); + + const settle = (callback) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + callback(); + }; + + child.on("error", (error) => { + settle(() => reject(new Error(`Electron could not be launched: ${error.message}`))); + }); + child.stdout.on("data", (chunk) => { + outputBuffer += chunk.toString(); + let newlineIndex = outputBuffer.indexOf("\n"); + while (newlineIndex >= 0) { + const line = outputBuffer.slice(0, newlineIndex).trim(); + outputBuffer = outputBuffer.slice(newlineIndex + 1); + if (line.length > 0) { + const parsed = parseReportLine(line); + if (parsed) { + report = parsed; + } + } + newlineIndex = outputBuffer.indexOf("\n"); + } + }); + child.stderr.on("data", (chunk) => process.stderr.write(chunk)); + child.on("close", (code, signal) => { + settle(() => { + if (timedOut) { + reject(new Error(`Electron benchmark timed out after ${timeoutMs} ms`)); + return; + } + if (!report) { + reject( + new Error( + `Electron exited without a benchmark report (code ${code ?? "unknown"}, signal ${signal ?? "none"})`, + ), + ); + return; + } + if (report.type === "error") { + reject(new Error(report.message)); + return; + } + if (code !== 0) { + reject(new Error(`Electron benchmark exited with code ${code}`)); + return; + } + resolve(report.result); + }); + }); + }); +} + +function formatNumber(value, digits = 2) { + return Number.isFinite(value) ? value.toFixed(digits) : "n/a"; +} + +function printReport(report) { + console.log("Native frame transport benchmark"); + console.log( + JSON.stringify({ + electronVersion: report.electronVersion, + nodeVersion: report.nodeVersion, + platform: report.platform, + arch: report.arch, + payloadSizesBytes: payloadSizes, + iterations, + windowSize, + peakInFlightDefinition: "unacknowledged logical payload bytes", + physicalZeroCopy: "not measured or guaranteed by this benchmark", + }), + ); + for (const result of report.results) { + console.log( + `${result.route} ${formatNumber(result.payloadMiB, 0)} MiB: ` + + `throughput=${formatNumber(result.throughputMiBPerSec)} MiB/s, ` + + `ACK p50=${formatNumber(result.ackLatencyMs.medianMs)} ms, ` + + `ACK p95=${formatNumber(result.ackLatencyMs.p95Ms)} ms, ` + + `peakInFlight=${formatNumber(result.peakInFlightPayloadMiB, 0)} MiB, ` + + `detached=${result.bufferOwnership.detachedAfterPostCount}/${result.bufferOwnership.samples}`, + ); + } + console.log(JSON.stringify(report.results)); +} + +async function main() { + let fixtureDirectory = null; + try { + const electronPath = resolveElectronPath(); + fixtureDirectory = await createFixture(); + const report = await launchFixture(electronPath, fixtureDirectory); + printReport(report); + } catch (error) { + console.error( + `[benchmark-native-frame-transport] unavailable: ${error instanceof Error ? error.message : String(error)}`, + ); + console.error( + "No Recordly project data was used; the benchmark only creates a temporary Electron fixture.", + ); + process.exitCode = 1; + } finally { + if (fixtureDirectory && !keepFixture) { + await fs.rm(fixtureDirectory, { recursive: true, force: true }); + } else if (fixtureDirectory) { + console.log(`Temporary fixture retained at ${fixtureDirectory}`); + } + } +} + +await main(); diff --git a/scripts/build-nvidia-cuda-compositor.mjs b/scripts/build-nvidia-cuda-compositor.mjs index f1447c4b7..2d3e94dd8 100644 --- a/scripts/build-nvidia-cuda-compositor.mjs +++ b/scripts/build-nvidia-cuda-compositor.mjs @@ -1,5 +1,13 @@ import { execSync } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import path from "node:path"; import { @@ -28,6 +36,15 @@ const generatorArch = process.arch === "arm64" ? "ARM64" : "x64"; const videoCodecSdkRoot = process.env.RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT?.trim() || path.join(projectRoot, ".tmp", "video-sdk-samples"); +const nvEncHeadersRoot = + process.env.RECORDLY_NVENC_HEADERS_ROOT?.trim() || + path.join(projectRoot, ".tmp", "nv-codec-headers"); +// nvEncodeAPI.h 13.x is required for Blackwell-era NVENC; the public +// Video Codec SDK samples repo still ships the legacy 8.1 header which fails +// with NV_ENC_ERR_INVALID_PARAM (error 8) on current drivers. Pin the FFmpeg +// nv-codec-headers release that provides API 13.0. +const NVENC_HEADERS_TAG = "n13.0.19.1"; +const NVENC_HEADERS_INCLUDE = path.join(nvEncHeadersRoot, "include", "ffnvcodec"); if (process.platform !== "win32") { console.log("[build-nvidia-cuda-compositor] Skipping NVIDIA CUDA compositor build."); @@ -115,6 +132,68 @@ function findCmake() { return null; } +function findCudaToolkitRoot() { + const candidates = [ + process.env.CUDA_PATH, + ...Object.entries(process.env) + .filter(([name]) => /^CUDA_PATH_V\d+_\d+$/.test(name)) + .map(([, value]) => value), + ]; + const cudaInstallRoot = path.join( + "C:", + "Program Files", + "NVIDIA GPU Computing Toolkit", + "CUDA", + ); + if (existsSync(cudaInstallRoot)) { + candidates.push( + ...readdirSync(cudaInstallRoot) + .sort() + .reverse() + .map((version) => path.join(cudaInstallRoot, version)), + ); + } + + return ( + candidates + .filter((candidate) => typeof candidate === "string" && candidate.length > 0) + .map((candidate) => path.normalize(candidate)) + .find((candidate) => existsSync(path.join(candidate, "bin", "nvcc.exe"))) ?? null + ); +} + +function ensureNvEncHeaders() { + if (existsSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"))) { + return; + } + console.log(`[build-nvidia-cuda-compositor] Cloning nv-codec-headers ${NVENC_HEADERS_TAG}...`); + execSync( + `git clone --depth 1 --branch ${NVENC_HEADERS_TAG} https://github.com/FFmpeg/nv-codec-headers.git "${nvEncHeadersRoot}"`, + { stdio: "inherit" }, + ); + if (!existsSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"))) { + fallbackToBundledHelperOrExit( + `nv-codec-headers ${NVENC_HEADERS_TAG} could not be staged; a Blackwell-compatible nvEncodeAPI.h is required.`, + ); + } + // The legacy samples checkout ships nvEncodeAPI.h 8.1; the compiler picks the + // quoted include from the NvEncoder directory first, so the 13.0 header must + // replace it to build the encoder library against the current API. + const samplesHeader = path.join( + videoCodecSdkRoot, + "Samples", + "NvCodec", + "NvEncoder", + "nvEncodeAPI.h", + ); + const versionLine = readFileSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"), "utf8") + .split(/\r?\n/) + .find((line) => line.includes("NVENCAPI_MAJOR_VERSION")); + if (existsSync(samplesHeader) && !/NVENCAPI_MAJOR_VERSION 13/.test(versionLine ?? "")) { + copyFileSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"), samplesHeader); + } +} + if (!existsSync(path.join(videoCodecSdkRoot, "Samples", "NvCodec"))) { fallbackToBundledHelperOrExit( `NVIDIA Video Codec SDK samples not found at ${videoCodecSdkRoot}. Set RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT to build from source.`, @@ -217,6 +296,69 @@ using RecordlyDisplayFramePolicy = bool (*)(int, void*); } } +try { + ensureNvEncHeaders(); +} catch (error) { + fallbackToBundledHelperOrExit( + `Failed to stage nv-codec-headers: ${error instanceof Error ? error.message : String(error)}`, + ); +} + +// The samples NvEncoder library predates nvEncodeAPI 13.x; patch the few API +// incompatibilities so it compiles against the staged header. +function patchNvEncoderForNvEnc13Headers() { + const nvEncoderDir = path.join(videoCodecSdkRoot, "Samples", "NvCodec", "NvEncoder"); + const sourcePath = path.join(nvEncoderDir, "NvEncoder.cpp"); + let source = readFileSync(sourcePath, "utf8"); + + if (!source.includes("nvEncEncodePicture API failed: ")) { + source = replaceOrThrow( + sourcePath, + source, + /if \(pIntializeParams->presetGUID != NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID\r?\n(?:\s+)&& pIntializeParams->presetGUID != NV_ENC_PRESET_LOSSLESS_HP_GUID\)\r?\n(?:\s+)\{\r?\n(?:\s+)pIntializeParams->encodeConfig->rcParams\.constQP = \{ 28, 31, 25 \};\r?\n(?:\s+)\}/, + " pIntializeParams->encodeConfig->rcParams.constQP = { 28, 31, 25 };", + "NVENC 13 lossless preset GUID check", + ); + source = replaceOrThrow( + sourcePath, + source, + /pIntializeParams->encodeConfig->encodeCodecConfig\.hevcConfig\.pixelBitDepthMinus8 =\r?\n(?:\s+)\(m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV420_10BIT \|\| m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV444_10BIT \) \? 2 : 0;\r?\n/, + "", + "NVENC 13 HEVC bit depth field removal", + ); + source = replaceOrThrow( + sourcePath, + source, + /bool yuv10BitFormat = \(m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV420_10BIT \|\| m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV444_10BIT\) \? true : false;\r?\n(?:\s+)if \(yuv10BitFormat && pEncoderParams->encodeConfig->encodeCodecConfig\.hevcConfig\.pixelBitDepthMinus8 != 2\)\r?\n(?:\s+)\{\r?\n(?:\s+)NVENC_THROW_ERROR\("Invalid PixelBitdepth", NV_ENC_ERR_INVALID_PARAM\);\r?\n(?:\s+)\}\r?\n\r?\n/, + "", + "NVENC 13 HEVC bit depth check removal", + ); + source = replaceOrThrow( + sourcePath, + source, + /NV_ENC_PRESET_DEFAULT_GUID/, + "NV_ENC_PRESET_P4_GUID", + "NVENC 13 default preset GUID", + ); + source = replaceOrThrow( + sourcePath, + source, + /"nvEncEncodePicture API failed"/, + '"nvEncEncodePicture API failed: " + std::to_string(nvStatus)', + "NVENC encode-picture error detail", + ); + writeFileSync(sourcePath, source); + } +} + +try { + patchNvEncoderForNvEnc13Headers(); +} catch (error) { + fallbackToBundledHelperOrExit( + `Failed to patch NVIDIA NvEncoder for NVENC 13 headers: ${error instanceof Error ? error.message : String(error)}`, + ); +} + try { patchNvDecoderForRecordlyCallbacks(); } catch (error) { @@ -232,6 +374,13 @@ if (!cmake) { ); } +const cudaToolkitRoot = findCudaToolkitRoot(); +if (!cudaToolkitRoot) { + fallbackToBundledHelperOrExit( + "CUDA Toolkit not found. Install CUDA Toolkit or set CUDA_PATH before building.", + ); +} + mkdirSync(buildDir, { recursive: true }); function clearCmakeCache() { @@ -246,7 +395,15 @@ try { clearCache: clearCmakeCache, configure: (generator, toolset) => execSync( - `${cmake} .. -G "${generator}" -A ${generatorArch}${toolset ? ` -T ${toolset}` : ""} -DRECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT="${videoCodecSdkRoot}"`, + `${cmake} .. -G "${generator}" -A ${generatorArch} -T "${[ + toolset, + `cuda=${cudaToolkitRoot}`, + "host=x64", + ] + .filter(Boolean) + .join( + ",", + )}" -DCMAKE_CUDA_COMPILER="${path.join(cudaToolkitRoot, "bin", "nvcc.exe")}" -DCUDAToolkit_ROOT="${cudaToolkitRoot}" -DRECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT="${videoCodecSdkRoot}"`, { cwd: buildDir, stdio: "inherit", diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 09cca63ef..9f0a1b3af 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -83,6 +83,7 @@ function LaunchWindowContent() { const { elapsed, formatTime } = useRecordingTimer(recording, paused); const hudContentRef = useRef(null); const hudBarRef = useRef(null); + const pendingAutoStartRef = useRef(false); const { selectedSource, @@ -162,6 +163,12 @@ function LaunchWindowContent() { }; }, []); + useEffect(() => { + if (openId !== "sources") { + pendingAutoStartRef.current = false; + } + }, [openId]); + const { recordingHudOffset, isHudDragging, @@ -226,7 +233,13 @@ function LaunchWindowContent() { <> { + await handleSourceSelect(source); + if (pendingAutoStartRef.current) { + pendingAutoStartRef.current = false; + void toggleRecording(); + } + }} onOpen={beginInteractiveHudAction} trigger={ + ); + })} + +
+ + {tSettings("export.codecTitle", "Video codec")} + + + + +
+
+ {( + [ + { value: "h264", label: tSettings("export.codec.h264", "H.264") }, + { value: "hevc", label: tSettings("export.codec.hevc", "H.265") }, + ] as const + ).map((option) => { + const isActive = exportVideoCodec === option.value; + return ( + ); @@ -281,35 +420,35 @@ export function ExportSettingsMenu({
- {tSettings("export.pipelineTitle", "Pipeline")} + {tSettings("export.bitrateTitle", "Bitrate")}
-
+
{( [ { - value: "legacy", - label: tSettings("export.pipeline.legacy", "Legacy"), + value: "auto", + label: tSettings("export.bitrate.auto", "Auto"), }, { - value: "modern", - label: tSettings("export.pipeline.modern", "Lightning (Beta)"), + value: "custom", + label: tSettings("export.bitrate.custom", "Custom"), }, ] as const ).map((option) => { - const isActive = exportPipelineModel === option.value; + const isActive = exportBitrateMode === option.value; return ( + ); + })} +
+
+
- {tSettings("export.nvidiaCuda.title", "NVIDIA CUDA")} - - - {tSettings("export.nvidiaCuda.badge", "Experimental")} + {tSettings( + "export.nvidiaCuda.compositorTitle", + "NVIDIA CUDA compositor", + )} + {nvidiaCudaCompositorRequired ? ( + + {tSettings( + "export.nvidiaCuda.requiredBadge", + "Required", + )} + + ) : experimentalNvidiaCudaExport ? ( + + {tSettings( + "export.nvidiaCuda.selectedBadge", + "Selected", + )} + + ) : nvidiaCudaExportAvailable ? ( + + {tSettings( + "export.nvidiaCuda.availableBadge", + "Available", + )} + + ) : ( + + {tSettings( + "export.nvidiaCuda.unavailableBadge", + "Unavailable", + )} + + )}

- {tSettings( - "export.nvidiaCuda.hint", - "Try GPU export on this Windows device.", - )} + {tSettings("export.nvidiaCuda.backendLabel", "Backend")} + {": "} + {nvidiaCudaCompositorRequired || experimentalNvidiaCudaExport + ? tSettings( + "export.nvidiaCuda.backendSelected", + "NVIDIA CUDA compositor", + ) + : tSettings("export.backend.auto", "Auto")}

+

+ {nvidiaCudaCompositorRequired + ? tSettings( + "export.nvidiaCuda.hintRequired", + "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + ) + : experimentalNvidiaCudaExport + ? tSettings( + "export.nvidiaCuda.hintSelected", + "Exports will use the NVIDIA CUDA compositor on this device.", + ) + : nvidiaCudaExportAvailable + ? tSettings( + "export.nvidiaCuda.hint", + "Compose and encode on the NVIDIA GPU for fast exports.", + ) + : nvidiaCudaExportSkipReason + ? tSettings( + "export.nvidiaCuda.unavailableReason", + `CUDA compositor is unavailable (${nvidiaCudaExportSkipReason}).`, + { + reason: nvidiaCudaExportSkipReason, + }, + ) + : tSettings( + "export.nvidiaCuda.unavailableGeneric", + "CUDA compositor is unavailable on this device.", + )} +

+ {nvidiaCudaCompositorRequired && !nvidiaCudaExportAvailable ? ( +

+ {tSettings( + "export.nvidiaCuda.unavailableRequired", + "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto.", + )} +

+ ) : null}
- + {nvidiaCudaCompositorRequired ? ( + + ) : nvidiaCudaExportAvailable ? ( + + ) : null}
- ) : null} +
{showCaptionSidecarOption ? (

- {tSettings("export.captionSidecar.title", "Export captions file")} + {tSettings( + "export.captionSidecar.title", + "Export captions file", + )}

{tSettings( @@ -414,7 +714,7 @@ export function ExportSettingsMenu({ {isActive ? ( ( initialEditorPreferences.exportPipelineModel, ); + const [exportVideoCodec, setExportVideoCodec] = useState( + initialEditorPreferences.exportVideoCodec ?? "h264", + ); + const [exportEncoderPreference, setExportEncoderPreference] = useState( + initialEditorPreferences.exportEncoderPreference ?? "auto", + ); + const [exportBitrateMode, setExportBitrateMode] = useState( + initialEditorPreferences.exportBitrateMode ?? "auto", + ); + const [exportBitrateMbps, setExportBitrateMbps] = useState( + initialEditorPreferences.exportBitrateMbps ?? EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + ); const enableModernExportPipeline = useCallback(() => { setExportPipelineModel("modern"); }, []); const { nvidiaCudaExportAvailable, + nvidiaCudaExportSkipReason, experimentalNvidiaCudaExport, setExperimentalNvidiaCudaExport, } = useNvidiaCudaExportOptIn({ @@ -810,6 +828,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, autoCaptionSettings: { ...autoCaptionSettings }, whisperExecutablePath, whisperModelPath, @@ -867,6 +889,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, autoCaptionSettings, whisperExecutablePath, whisperModelPath, @@ -962,6 +988,10 @@ export default function VideoEditor() { setExportQuality(snapshot.exportQuality); setMp4FrameRate(snapshot.mp4FrameRate); setExportFormat(snapshot.exportFormat); + setExportVideoCodec(snapshot.exportVideoCodec ?? "h264"); + setExportEncoderPreference(snapshot.exportEncoderPreference ?? "auto"); + setExportBitrateMode(snapshot.exportBitrateMode ?? "auto"); + setExportBitrateMbps(snapshot.exportBitrateMbps ?? EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS); setGifFrameRate(snapshot.gifFrameRate); setGifLoop(snapshot.gifLoop); setGifSizePreset(snapshot.gifSizePreset); @@ -1755,6 +1785,10 @@ export default function VideoEditor() { gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; + exportBitrateMode: ExportBitrateMode; + exportBitrateMbps: number; sourceAudioTrackSettingsByClip: Record; defaultSourceAudioTrackSettings: SourceAudioTrackSettings; }>, @@ -1878,6 +1912,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, }), @@ -1944,6 +1982,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, frame, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, @@ -2143,6 +2185,12 @@ export default function VideoEditor() { setGifFrameRate(normalizedEditor.gifFrameRate); setGifLoop(normalizedEditor.gifLoop); setGifSizePreset(normalizedEditor.gifSizePreset); + setExportVideoCodec(normalizedEditor.exportVideoCodec ?? "h264"); + setExportEncoderPreference(normalizedEditor.exportEncoderPreference ?? "auto"); + setExportBitrateMode(normalizedEditor.exportBitrateMode ?? "auto"); + setExportBitrateMbps( + normalizedEditor.exportBitrateMbps ?? EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + ); setSelectedZoomId(null); setSelectedClipId(null); @@ -2674,6 +2722,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, whisperExecutablePath, whisperModelPath, }); @@ -2730,6 +2782,10 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, whisperExecutablePath, whisperModelPath, ]); @@ -4756,24 +4812,40 @@ export default function VideoEditor() { } } else { // MP4 Export - const { quality, encodingMode, selectedMp4FrameRate } = - resolveMp4ExportSettings({ - smokeExportConfig: { - enabled: smokeExportConfig.enabled, - quality: smokeExportConfig.quality, - encodingMode: smokeExportConfig.encodingMode, - fps: smokeExportConfig.fps, - }, - settings, - exportQuality, - exportEncodingMode, - mp4FrameRate, - }); + const { + quality, + encodingMode, + selectedMp4FrameRate, + exportVideoCodec: effectiveExportVideoCodec, + exportEncoderPreference: effectiveExportEncoderPreference, + exportBitrateMode: effectiveExportBitrateMode, + exportBitrateMbps: effectiveExportBitrateMbps, + } = resolveMp4ExportSettings({ + smokeExportConfig: { + enabled: smokeExportConfig.enabled, + quality: smokeExportConfig.quality, + encodingMode: smokeExportConfig.encodingMode, + fps: smokeExportConfig.fps, + videoCodec: smokeExportConfig.videoCodec, + encoderPreference: smokeExportConfig.encoderPreference, + bitrateMode: smokeExportConfig.bitrateMode, + bitrateMbps: smokeExportConfig.bitrateMbps, + }, + settings, + exportQuality, + exportEncodingMode, + mp4FrameRate, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, + }); const { pipelineModel, useExperimentalNativeExport, useExperimentalNvidiaCudaExport, backendPreference, + needsNativeRawFrame, } = resolveMp4ExportRouting({ smokeExportConfig: { enabled: smokeExportConfig.enabled, @@ -4784,24 +4856,53 @@ export default function VideoEditor() { settings, exportPipelineModel, exportBackendPreference, + exportVideoCodec: effectiveExportVideoCodec, + exportEncoderPreference: effectiveExportEncoderPreference, experimentalNvidiaCudaExport, nvidiaCudaExportAvailable, }); - const supportedSourceDimensions = - await ensureSupportedMp4SourceDimensions(selectedMp4FrameRate); - const { width: exportWidth, height: exportHeight } = - calculateMp4ExportDimensions( + // HEVC and any explicit Hardware/CPU encoder choice bypass the + // WebCodecs dimension probe and always route through the H.264- + // independent native FFmpeg raw-frame encoder. + const normalizeEvenDimension = (value: number) => + Math.max(2, Math.floor(value / 2) * 2); + const desiredRawOutput = calculateMp4ExportDimensions( + desiredMp4SourceDimensions.width, + desiredMp4SourceDimensions.height, + quality, + ); + let exportWidth: number; + let exportHeight: number; + let preferredEncoderPath: SupportedMp4EncoderPath | null | undefined; + if (needsNativeRawFrame) { + exportWidth = normalizeEvenDimension(desiredRawOutput.width); + exportHeight = normalizeEvenDimension(desiredRawOutput.height); + preferredEncoderPath = undefined; + } else { + const supportedSourceDimensions = + await ensureSupportedMp4SourceDimensions(selectedMp4FrameRate); + const output = calculateMp4ExportDimensions( supportedSourceDimensions.width, supportedSourceDimensions.height, quality, ); - const bitrate = getMp4ExportBitrate({ + exportWidth = output.width; + exportHeight = output.height; + preferredEncoderPath = supportedSourceDimensions.encoderPath; + } + // Auto bitrate keeps the existing heuristic (with static-layout + // floors/caps); Custom bitrate bypasses those floors/caps entirely and + // uses the user's explicit 1-200 Mbps value. + const bitrate = resolveExportBitrate({ + mode: effectiveExportBitrateMode, + customMbps: effectiveExportBitrateMbps, width: exportWidth, height: exportHeight, frameRate: selectedMp4FrameRate, quality, encodingMode, - useModernNativeStaticLayout: useExperimentalNativeExport, + useModernNativeStaticLayout: + useExperimentalNativeExport && !needsNativeRawFrame, }); const sourceAudioTrackSettingsForExport = selectedClipId !== null @@ -4815,8 +4916,12 @@ export default function VideoEditor() { frameRate: selectedMp4FrameRate, bitrate, codec: DEFAULT_MP4_CODEC, + exportVideoCodec: effectiveExportVideoCodec, + exportEncoderPreference: effectiveExportEncoderPreference, + exportBitrateMode: effectiveExportBitrateMode, + exportBitrateMbps: effectiveExportBitrateMbps, encodingMode, - preferredEncoderPath: supportedSourceDimensions.encoderPath, + preferredEncoderPath, preferredRenderBackend: smokeExportConfig.renderBackend, experimentalNativeExport: useExperimentalNativeExport, experimentalNvidiaCudaExport: useExperimentalNvidiaCudaExport, @@ -5152,6 +5257,11 @@ export default function VideoEditor() { exportEncodingMode, exportBackendPreference, exportPipelineModel, + desiredMp4SourceDimensions, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, experimentalNvidiaCudaExport, nvidiaCudaExportAvailable, borderRadius, @@ -5187,6 +5297,10 @@ export default function VideoEditor() { smokeExportConfig.encodingMode, smokeExportConfig.fps, smokeExportConfig.quality, + smokeExportConfig.videoCodec, + smokeExportConfig.encoderPreference, + smokeExportConfig.bitrateMode, + smokeExportConfig.bitrateMbps, saveBlobExport, ], ); @@ -5326,6 +5440,10 @@ export default function VideoEditor() { mp4FrameRate, exportBackendPreference, exportPipelineModel, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, gifFrameRate, gifLoop, gifSizePreset, @@ -5348,6 +5466,10 @@ export default function VideoEditor() { includeCaptionSidecar, exportBackendPreference, exportPipelineModel, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, handleExport, ]); @@ -6231,15 +6353,26 @@ export default function VideoEditor() { onMp4FrameRateChange={setMp4FrameRate} exportPipelineModel={exportPipelineModel} onExportPipelineModelChange={setExportPipelineModel} - experimentalNvidiaCudaExport={ - experimentalNvidiaCudaExport && nvidiaCudaExportAvailable - } + experimentalNvidiaCudaExport={experimentalNvidiaCudaExport} onExperimentalNvidiaCudaExportChange={ setExperimentalNvidiaCudaExport } nvidiaCudaExportAvailable={nvidiaCudaExportAvailable} + nvidiaCudaExportSkipReason={nvidiaCudaExportSkipReason} + nvidiaCudaCompositorRequired={ + exportVideoCodec === "hevc" && + exportEncoderPreference === "hardware" + } exportQuality={exportQuality} onExportQualityChange={setExportQuality} + exportVideoCodec={exportVideoCodec} + onExportVideoCodecChange={setExportVideoCodec} + exportEncoderPreference={exportEncoderPreference} + onExportEncoderPreferenceChange={setExportEncoderPreference} + exportBitrateMode={exportBitrateMode} + onExportBitrateModeChange={setExportBitrateMode} + exportBitrateMbps={exportBitrateMbps} + onExportBitrateMbpsChange={setExportBitrateMbps} gifFrameRate={gifFrameRate} onGifFrameRateChange={setGifFrameRate} gifLoop={gifLoop} diff --git a/src/components/video-editor/editorPreferences.test.ts b/src/components/video-editor/editorPreferences.test.ts index 59566e62d..be05e77a6 100644 --- a/src/components/video-editor/editorPreferences.test.ts +++ b/src/components/video-editor/editorPreferences.test.ts @@ -472,4 +472,91 @@ describe("editorPreferences", () => { expect(saveEditorPresets([])).toBe(false); }); + + it("hydrates missing H.265/bitrate preferences to defaults", () => { + vi.stubGlobal( + "localStorage", + createStorageMock({ + [EDITOR_PREFERENCES_STORAGE_KEY]: JSON.stringify({ + wallpaper: "#123456", + }), + }), + ); + + const loaded = loadEditorPreferences(); + expect(loaded.exportVideoCodec).toBe("h264"); + expect(loaded.exportEncoderPreference).toBe("auto"); + expect(loaded.exportBitrateMode).toBe("auto"); + expect(loaded.exportBitrateMbps).toBe(20); + }); + + it("normalizes invalid persisted H.265/bitrate preferences", () => { + const normalized = normalizeEditorPreferences({ + exportVideoCodec: "vp9", + exportEncoderPreference: "turbo", + exportBitrateMode: "ultra", + exportBitrateMbps: 5000, + }); + + expect(normalized.exportVideoCodec).toBe("h264"); + expect(normalized.exportEncoderPreference).toBe("auto"); + expect(normalized.exportBitrateMode).toBe("auto"); + expect(normalized.exportBitrateMbps).toBe(105); + + expect(normalizeEditorPreferences({ exportBitrateMbps: -3 }).exportBitrateMbps).toBe(1); + expect( + normalizeEditorPreferences({ exportBitrateMbps: Number.NaN }).exportBitrateMbps, + ).toBe(20); + }); + + it("survives an editor preferences round trip for H.265/bitrate values", () => { + const localStorage = createStorageMock(); + vi.stubGlobal("localStorage", localStorage); + + saveEditorPreferences({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + exportBitrateMode: "custom", + exportBitrateMbps: 64, + }); + + expect(loadEditorPreferences()).toMatchObject({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + exportBitrateMode: "custom", + exportBitrateMbps: 64, + }); + }); + + it("survives an editor preset snapshot round trip for H.265/bitrate values", () => { + const localStorage = createStorageMock(); + vi.stubGlobal("localStorage", localStorage); + + expect( + saveEditorPresets([ + { + id: "preset-bitrate", + name: "HEVC Bitrate", + createdAt: "2026-05-01T00:00:00.000Z", + updatedAt: "2026-05-01T00:00:00.000Z", + snapshot: { + ...DEFAULT_EDITOR_PREFERENCES, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 70, + cropRegion: DEFAULT_CROP_REGION, + autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, + }, + }, + ]), + ).toBe(true); + + expect(loadEditorPresets()[0]?.snapshot).toMatchObject({ + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 70, + }); + }); }); diff --git a/src/components/video-editor/editorPreferences.ts b/src/components/video-editor/editorPreferences.ts index fbf354ab7..6734bac20 100644 --- a/src/components/video-editor/editorPreferences.ts +++ b/src/components/video-editor/editorPreferences.ts @@ -1,67 +1,83 @@ import { loadAppSetting, saveAppSetting } from "../../lib/appSettings"; import { normalizeExportBackendPreference, + normalizeExportBitrateMbps, + normalizeExportBitrateMode, + normalizeExportEncoderPreference, normalizeExportMp4FrameRate, normalizeExportPipelineModel, + normalizeExportVideoCodec, normalizeProjectEditor, type ProjectEditorState, stripPersistedDevMotionBlurSettings, } from "./projectPersistence"; -type PersistedEditorControls = Pick< - ProjectEditorState, - | "wallpaper" - | "shadowIntensity" - | "backgroundBlur" - | "zoomMotionBlur" - | "zoomMotionBlurTuning" - | "zoomTemporalMotionBlur" - | "zoomMotionBlurSampleCount" - | "zoomMotionBlurShutterFraction" - | "connectZooms" - | "zoomInDurationMs" - | "zoomInOverlapMs" - | "zoomOutDurationMs" - | "connectedZoomGapMs" - | "connectedZoomDurationMs" - | "zoomInEasing" - | "zoomOutEasing" - | "connectedZoomEasing" - | "showCursor" - | "loopCursor" - | "cursorStyle" - | "cursorSize" - | "cursorSmoothing" - | "cursorSpringStiffnessMultiplier" - | "cursorSpringDampingMultiplier" - | "cursorSpringMassMultiplier" - | "cameraSpringStiffnessMultiplier" - | "cameraSpringDampingMultiplier" - | "cameraSpringMassMultiplier" - | "cursorMotionBlur" - | "cursorClickEffect" - | "cursorClickEffectColor" - | "cursorClickEffectScale" - | "cursorClickEffectOpacity" - | "cursorClickEffectDurationMs" - | "cursorClickBounce" - | "cursorClickBounceDuration" - | "cursorSway" - | "borderRadius" - | "padding" - | "frame" - | "webcam" - | "aspectRatio" - | "exportEncodingMode" - | "exportBackendPreference" - | "exportPipelineModel" - | "exportQuality" - | "mp4FrameRate" - | "exportFormat" - | "gifFrameRate" - | "gifLoop" - | "gifSizePreset" ->; +type PersistedEditorControls = Omit< + Pick< + ProjectEditorState, + | "wallpaper" + | "shadowIntensity" + | "backgroundBlur" + | "zoomMotionBlur" + | "zoomMotionBlurTuning" + | "zoomTemporalMotionBlur" + | "zoomMotionBlurSampleCount" + | "zoomMotionBlurShutterFraction" + | "connectZooms" + | "zoomInDurationMs" + | "zoomInOverlapMs" + | "zoomOutDurationMs" + | "connectedZoomGapMs" + | "connectedZoomDurationMs" + | "zoomInEasing" + | "zoomOutEasing" + | "connectedZoomEasing" + | "showCursor" + | "loopCursor" + | "cursorStyle" + | "cursorSize" + | "cursorSmoothing" + | "cursorSpringStiffnessMultiplier" + | "cursorSpringDampingMultiplier" + | "cursorSpringMassMultiplier" + | "cameraSpringStiffnessMultiplier" + | "cameraSpringDampingMultiplier" + | "cameraSpringMassMultiplier" + | "cursorMotionBlur" + | "cursorClickEffect" + | "cursorClickEffectColor" + | "cursorClickEffectScale" + | "cursorClickEffectOpacity" + | "cursorClickEffectDurationMs" + | "cursorClickBounce" + | "cursorClickBounceDuration" + | "cursorSway" + | "borderRadius" + | "padding" + | "frame" + | "webcam" + | "aspectRatio" + | "exportEncodingMode" + | "exportBackendPreference" + | "exportPipelineModel" + | "exportQuality" + | "mp4FrameRate" + | "exportVideoCodec" + | "exportEncoderPreference" + | "exportBitrateMode" + | "exportBitrateMbps" + | "exportFormat" + | "gifFrameRate" + | "gifLoop" + | "gifSizePreset" + >, + "exportVideoCodec" | "exportEncoderPreference" | "exportBitrateMode" | "exportBitrateMbps" +> & { + exportVideoCodec?: ProjectEditorState["exportVideoCodec"]; + exportEncoderPreference?: ProjectEditorState["exportEncoderPreference"]; + exportBitrateMode?: ProjectEditorState["exportBitrateMode"]; + exportBitrateMbps?: ProjectEditorState["exportBitrateMbps"]; +}; type PartialEditorControls = Partial; @@ -145,6 +161,10 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = { exportPipelineModel: DEFAULT_EDITOR_CONTROLS.exportPipelineModel, exportQuality: DEFAULT_EDITOR_CONTROLS.exportQuality, mp4FrameRate: DEFAULT_EDITOR_CONTROLS.mp4FrameRate, + exportVideoCodec: DEFAULT_EDITOR_CONTROLS.exportVideoCodec, + exportEncoderPreference: DEFAULT_EDITOR_CONTROLS.exportEncoderPreference, + exportBitrateMode: DEFAULT_EDITOR_CONTROLS.exportBitrateMode, + exportBitrateMbps: DEFAULT_EDITOR_CONTROLS.exportBitrateMbps, exportFormat: DEFAULT_EDITOR_CONTROLS.exportFormat, gifFrameRate: DEFAULT_EDITOR_CONTROLS.gifFrameRate, gifLoop: DEFAULT_EDITOR_CONTROLS.gifLoop, @@ -353,6 +373,22 @@ function normalizeEditorControls( sanitizedRaw.mp4FrameRate === undefined ? fallback.mp4FrameRate : normalizeExportMp4FrameRate(sanitizedRaw.mp4FrameRate), + exportVideoCodec: + sanitizedRaw.exportVideoCodec === undefined + ? fallback.exportVideoCodec + : normalizeExportVideoCodec(sanitizedRaw.exportVideoCodec), + exportEncoderPreference: + sanitizedRaw.exportEncoderPreference === undefined + ? fallback.exportEncoderPreference + : normalizeExportEncoderPreference(sanitizedRaw.exportEncoderPreference), + exportBitrateMode: + sanitizedRaw.exportBitrateMode === undefined + ? fallback.exportBitrateMode + : normalizeExportBitrateMode(sanitizedRaw.exportBitrateMode), + exportBitrateMbps: + sanitizedRaw.exportBitrateMbps === undefined + ? fallback.exportBitrateMbps + : normalizeExportBitrateMbps(sanitizedRaw.exportBitrateMbps), exportFormat: sanitizedRaw.exportFormat ?? fallback.exportFormat, gifFrameRate: sanitizedRaw.gifFrameRate ?? fallback.gifFrameRate, gifLoop: sanitizedRaw.gifLoop ?? fallback.gifLoop, @@ -409,6 +445,10 @@ function normalizeEditorControls( exportPipelineModel: normalized.exportPipelineModel, exportQuality: normalized.exportQuality, mp4FrameRate: normalized.mp4FrameRate, + exportVideoCodec: normalized.exportVideoCodec, + exportEncoderPreference: normalized.exportEncoderPreference, + exportBitrateMode: normalized.exportBitrateMode, + exportBitrateMbps: normalized.exportBitrateMbps, exportFormat: normalized.exportFormat, gifFrameRate: normalized.gifFrameRate, gifLoop: normalized.gifLoop, diff --git a/src/components/video-editor/exportStartSettings.test.ts b/src/components/video-editor/exportStartSettings.test.ts index 567807da6..e6c09adf2 100644 --- a/src/components/video-editor/exportStartSettings.test.ts +++ b/src/components/video-editor/exportStartSettings.test.ts @@ -11,6 +11,10 @@ const baseOptions = { mp4FrameRate: 30 as const, exportBackendPreference: "auto" as const, exportPipelineModel: "modern" as const, + exportVideoCodec: "h264" as const, + exportEncoderPreference: "auto" as const, + exportBitrateMode: "auto" as const, + exportBitrateMbps: 20, gifFrameRate: 20 as const, gifLoop: true, gifSizePreset: "medium" as const, @@ -25,11 +29,33 @@ describe("resolveExportStartSettings", () => { mp4FrameRate: 30, backendPreference: "auto", pipelineModel: "modern", + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + exportBitrateMode: "auto", + exportBitrateMbps: 20, quality: "good", gifConfig: undefined, }); }); + it("carries MP4-only codec/encoder/bitrate fields for MP4", () => { + expect( + resolveExportStartSettings({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 12.5, + }), + ).toMatchObject({ + format: "mp4", + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 12.5, + }); + }); + it("omits MP4-only fields and resolves GIF dimensions for GIF exports", () => { expect( resolveExportStartSettings({ @@ -37,6 +63,10 @@ describe("resolveExportStartSettings", () => { sourceWidth: 2560, sourceHeight: 1440, exportFormat: "gif", + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 40, gifFrameRate: 15, gifLoop: false, gifSizePreset: "medium", @@ -48,6 +78,10 @@ describe("resolveExportStartSettings", () => { mp4FrameRate: undefined, backendPreference: undefined, pipelineModel: undefined, + exportVideoCodec: undefined, + exportEncoderPreference: undefined, + exportBitrateMode: undefined, + exportBitrateMbps: undefined, quality: undefined, gifConfig: { frameRate: 15, diff --git a/src/components/video-editor/exportStartSettings.ts b/src/components/video-editor/exportStartSettings.ts index dc0aeab72..604aa6bd0 100644 --- a/src/components/video-editor/exportStartSettings.ts +++ b/src/components/video-editor/exportStartSettings.ts @@ -1,12 +1,15 @@ import { calculateOutputDimensions, type ExportBackendPreference, + type ExportBitrateMode, + type ExportEncoderPreference, type ExportEncodingMode, type ExportFormat, type ExportMp4FrameRate, type ExportPipelineModel, type ExportQuality, type ExportSettings, + type ExportVideoCodec, GIF_SIZE_PRESETS, type GifFrameRate, type GifSizePreset, @@ -22,6 +25,10 @@ export function resolveExportStartSettings({ mp4FrameRate, exportBackendPreference, exportPipelineModel, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, gifFrameRate, gifLoop, gifSizePreset, @@ -35,6 +42,10 @@ export function resolveExportStartSettings({ mp4FrameRate: ExportMp4FrameRate; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; + exportBitrateMode: ExportBitrateMode; + exportBitrateMbps: number; gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; @@ -51,6 +62,10 @@ export function resolveExportStartSettings({ mp4FrameRate: exportFormat === "mp4" ? mp4FrameRate : undefined, backendPreference: exportFormat === "mp4" ? exportBackendPreference : undefined, pipelineModel: exportFormat === "mp4" ? exportPipelineModel : undefined, + exportVideoCodec: exportFormat === "mp4" ? exportVideoCodec : undefined, + exportEncoderPreference: exportFormat === "mp4" ? exportEncoderPreference : undefined, + exportBitrateMode: exportFormat === "mp4" ? exportBitrateMode : undefined, + exportBitrateMbps: exportFormat === "mp4" ? exportBitrateMbps : undefined, quality: exportFormat === "mp4" ? exportQuality : undefined, gifConfig: exportFormat === "gif" && gifDimensions diff --git a/src/components/video-editor/exportStatusModel.test.ts b/src/components/video-editor/exportStatusModel.test.ts index 9cb3158ac..1076feed7 100644 --- a/src/components/video-editor/exportStatusModel.test.ts +++ b/src/components/video-editor/exportStatusModel.test.ts @@ -156,6 +156,28 @@ describe("resolveExportStatusModel", () => { expect(status.runtimeLabel).toBe("VideoEncoder"); }); + it("labels the NVIDIA CUDA compositor backend instead of Breeze", () => { + const status = resolveExportStatusModel({ + isExporting: true, + exportProgress: progress({ encoderName: "nvidia-cuda-compositor" }), + exportFormat: "mp4", + exportPipelineModel: "modern", + }); + + expect(status.runtimeLabel).toBe("NVIDIA CUDA compositor"); + }); + + it("labels the Windows D3D11 compositor backend distinctly", () => { + const status = resolveExportStatusModel({ + isExporting: true, + exportProgress: progress({ encoderName: "windows-d3d11-compositor" }), + exportFormat: "mp4", + exportPipelineModel: "modern", + }); + + expect(status.runtimeLabel).toBe("Windows D3D11 compositor"); + }); + it("prefers multiple native skip reasons over the single legacy reason", () => { const status = resolveExportStatusModel({ isExporting: true, @@ -171,6 +193,43 @@ describe("resolveExportStatusModel", () => { "timeline-edits-present", "unsupported-background", ]); - expect(status.nativeSkipLabel).toBe("Native skipped: timeline-edits-present (+1 more)"); + expect(status.nativeSkipLabel).toBe("Native skipped: timeline-edits-present; background"); + }); + + it("formats known native skip reasons without hiding additional blockers", () => { + const status = resolveExportStatusModel({ + isExporting: true, + exportProgress: progress({ + nativeStaticLayoutSkipReasons: [ + "unsupported-cursor-click-effect", + "unsupported-annotation-overlay", + "unsupported-audio-mode:edited-track", + ], + }), + exportFormat: "mp4", + exportPipelineModel: "modern", + }); + + expect(status.nativeSkipLabel).toBe( + "Native skipped: cursor click effect; annotation overlay; unsupported-audio-mode:edited-track", + ); + }); + + it("labels temporal zoom motion blur distinctly from the overlay-route rejection", () => { + const status = resolveExportStatusModel({ + isExporting: true, + exportProgress: progress({ + nativeStaticLayoutSkipReasons: [ + "unsupported-temporal-motion-blur", + "unsupported-motion-blur-on-overlay-route", + ], + }), + exportFormat: "mp4", + exportPipelineModel: "modern", + }); + + expect(status.nativeSkipLabel).toBe( + "Native skipped: temporal zoom motion blur; zoom motion blur over overlay layers", + ); }); }); diff --git a/src/components/video-editor/exportStatusModel.ts b/src/components/video-editor/exportStatusModel.ts index 7641717ab..d3e2e6078 100644 --- a/src/components/video-editor/exportStatusModel.ts +++ b/src/components/video-editor/exportStatusModel.ts @@ -18,6 +18,43 @@ export type ExportStatusModel = { nativeSkipLabel: string | null; }; +const NATIVE_SKIP_REASON_LABELS: Record = { + "native-static-api-unavailable": "native export API unavailable", + "odd-output-dimensions": "output dimensions are not even", + "unsupported-background-video": "video background", + "unsupported-cursor-click-effect": "cursor click effect", + "unsupported-cursor-motion-blur": "cursor motion blur", + "unsupported-extension-hook": "extension render hook", + "unsupported-annotation-overlay": "annotation overlay", + "unsupported-blur-annotation-overlay": "blur annotation overlay", + "unsupported-caption-overlay": "caption overlay", + "unsupported-frame-overlay": "frame overlay", + "unsupported-webcam-source": "webcam source", + "unsupported-rectangular-webcam-overlay": "rectangular webcam overlay", + "unsupported-motion-blur": "zoom motion blur", + "unsupported-temporal-motion-blur": "temporal zoom motion blur", + "unsupported-motion-blur-on-overlay-route": "zoom motion blur over overlay layers", + "unsupported-native-speed-timeline": "speed timeline", + "unsupported-native-trim-timeline": "trim timeline", + "native-timeline-requires-windows-gpu": "timeline requires Windows GPU export", + "native-zoom-requires-windows-gpu": "zoom requires Windows GPU export", + "overlay-layers-do-not-support-native-timeline": "overlay timeline mapping", + "native-overlay-preparation-failed": "native overlay preparation", + "invalid-crop-region": "invalid crop region", + "missing-source-path": "source path unavailable", + "missing-audio-options": "audio options unavailable", + "unsupported-background": "background", + "cursor-atlas-unavailable": "cursor atlas unavailable", + "invalid-native-speed-timeline": "invalid speed timeline", + "invalid-native-trim-timeline": "invalid trim timeline", + "invalid-layout-or-duration": "invalid layout or duration", +}; + +export function formatNativeSkipReason(reason: string): string { + const baseReason = reason.split(":", 1)[0]; + return NATIVE_SKIP_REASON_LABELS[baseReason] ?? reason; +} + export function resolveExportStatusModel({ isExporting, exportProgress, @@ -84,9 +121,7 @@ export function resolveExportStatusModel({ : []; const nativeSkipLabel = nativeSkipReasons.length > 0 - ? `Native skipped: ${nativeSkipReasons[0]}${ - nativeSkipReasons.length > 1 ? ` (+${nativeSkipReasons.length - 1} more)` : "" - }` + ? `Native skipped: ${nativeSkipReasons.map(formatNativeSkipReason).join("; ")}` : null; return { @@ -117,6 +152,15 @@ function resolveRuntimeLabel(exportProgress: ExportProgress | null): string | nu return null; } + // The NVIDIA CUDA compositor is a distinct native backend; it must never be + // mislabeled as Breeze (the FFmpeg CLI encoder) in export status. + if (encoderName === "nvidia-cuda-compositor") { + return "NVIDIA CUDA compositor"; + } + if (encoderName === "windows-d3d11-compositor") { + return "Windows D3D11 compositor"; + } + const rendererLabel = renderBackend === "webgpu" ? "WebGPU" : renderBackend === "webgl" ? "WebGL" : null; const encoderLabel = diff --git a/src/components/video-editor/mp4ExportRouting.test.ts b/src/components/video-editor/mp4ExportRouting.test.ts index 72f943c9e..300dbbe3a 100644 --- a/src/components/video-editor/mp4ExportRouting.test.ts +++ b/src/components/video-editor/mp4ExportRouting.test.ts @@ -10,6 +10,8 @@ const baseOptions = { settings: {}, exportPipelineModel: "modern" as const, exportBackendPreference: "breeze" as const, + exportVideoCodec: "h264" as const, + exportEncoderPreference: "auto" as const, experimentalNvidiaCudaExport: false, nvidiaCudaExportAvailable: false, }; @@ -20,7 +22,9 @@ describe("resolveMp4ExportRouting", () => { pipelineModel: "modern", useExperimentalNativeExport: true, useExperimentalNvidiaCudaExport: false, + nvidiaCudaCompositorRequired: false, backendPreference: "auto", + needsNativeRawFrame: false, }); }); @@ -34,7 +38,9 @@ describe("resolveMp4ExportRouting", () => { pipelineModel: "legacy", useExperimentalNativeExport: false, useExperimentalNvidiaCudaExport: false, + nvidiaCudaCompositorRequired: false, backendPreference: "webcodecs", + needsNativeRawFrame: false, }); }); @@ -51,7 +57,9 @@ describe("resolveMp4ExportRouting", () => { pipelineModel: "modern", useExperimentalNativeExport: false, useExperimentalNvidiaCudaExport: false, + nvidiaCudaCompositorRequired: false, backendPreference: "webcodecs", + needsNativeRawFrame: false, }); expect( @@ -66,11 +74,17 @@ describe("resolveMp4ExportRouting", () => { pipelineModel: "modern", useExperimentalNativeExport: true, useExperimentalNvidiaCudaExport: false, + nvidiaCudaCompositorRequired: false, backendPreference: "breeze", + needsNativeRawFrame: false, }); }); - it("only enables NVIDIA CUDA when native export is active and the device is available", () => { + it("enables NVIDIA CUDA from the persisted opt-in without racing the async GPU probe", () => { + // The runtime static-layout attempt is the authoritative capability check; + // a stale/async availability probe must not disable a working route (the + // opt-in hook already forces the toggle off when the helper/GPU is truly + // unavailable). expect( resolveMp4ExportRouting({ ...baseOptions, @@ -79,6 +93,14 @@ describe("resolveMp4ExportRouting", () => { }).useExperimentalNvidiaCudaExport, ).toBe(true); + expect( + resolveMp4ExportRouting({ + ...baseOptions, + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: false, + }).useExperimentalNvidiaCudaExport, + ).toBe(true); + expect( resolveMp4ExportRouting({ ...baseOptions, @@ -91,9 +113,197 @@ describe("resolveMp4ExportRouting", () => { expect( resolveMp4ExportRouting({ ...baseOptions, - experimentalNvidiaCudaExport: true, - nvidiaCudaExportAvailable: false, + experimentalNvidiaCudaExport: false, + nvidiaCudaExportAvailable: true, }).useExperimentalNvidiaCudaExport, ).toBe(false); }); + + it("forces the modern native pipeline for HEVC output", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + settings: { pipelineModel: "legacy" }, + }); + expect(result.needsNativeRawFrame).toBe(true); + expect(result.pipelineModel).toBe("modern"); + expect(result.useExperimentalNativeExport).toBe(true); + expect(result.backendPreference).toBe("auto"); + }); + + it("selects the HEVC Auto GPU candidate without forcing raw dimensions", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + + expect(result).toMatchObject({ + pipelineModel: "modern", + useExperimentalNativeExport: true, + useExperimentalNvidiaCudaExport: true, + backendPreference: "auto", + needsNativeRawFrame: false, + }); + }); + + it("selects the HEVC Hardware GPU candidate without allowing CPU routing", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + + expect(result.useExperimentalNvidiaCudaExport).toBe(true); + expect(result.needsNativeRawFrame).toBe(false); + expect(result.pipelineModel).toBe("modern"); + }); + + it("keeps HEVC CPU on rawvideo and out of the GPU compositor", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + + expect(result.useExperimentalNvidiaCudaExport).toBe(false); + expect(result.needsNativeRawFrame).toBe(true); + }); + + it("forces the modern native pipeline when the encoder preference is cpu", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "h264", + exportEncoderPreference: "cpu", + }); + expect(result.needsNativeRawFrame).toBe(true); + expect(result.pipelineModel).toBe("modern"); + expect(result.useExperimentalNativeExport).toBe(true); + expect(result.backendPreference).toBe("auto"); + }); + + it("needs a native raw frame when the encoder preference is hardware", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "h264", + exportEncoderPreference: "hardware", + }); + expect(result.needsNativeRawFrame).toBe(true); + expect(result.pipelineModel).toBe("modern"); + expect(result.useExperimentalNativeExport).toBe(true); + expect(result.backendPreference).toBe("auto"); + }); + + it("routes H.264 Hardware to the native GPU compositor when RTX Rendering is on", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "h264", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + expect(result.useExperimentalNvidiaCudaExport).toBe(true); + expect(result.needsNativeRawFrame).toBe(false); + expect(result.pipelineModel).toBe("modern"); + }); + + it("keeps the existing legacy/auto route for H.264 with auto encoder preference", () => { + const legacy = resolveMp4ExportRouting({ + ...baseOptions, + settings: { pipelineModel: "legacy", backendPreference: "breeze" }, + }); + expect(legacy).toEqual({ + pipelineModel: "legacy", + useExperimentalNativeExport: false, + useExperimentalNvidiaCudaExport: false, + nvidiaCudaCompositorRequired: false, + backendPreference: "webcodecs", + needsNativeRawFrame: false, + }); + expect(legacy.needsNativeRawFrame).toBe(false); + }); + + it("forces the NVIDIA CUDA compositor for HEVC Hardware even when the opt-in toggle is off", () => { + // The user's HEVC + Hardware selection IS the opt-in: the CUDA compositor is + // mandatory and must never depend on a hidden experimental flag. + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: false, + nvidiaCudaExportAvailable: true, + }); + + expect(result.nvidiaCudaCompositorRequired).toBe(true); + expect(result.useExperimentalNvidiaCudaExport).toBe(true); + expect(result.needsNativeRawFrame).toBe(false); + expect(result.pipelineModel).toBe("modern"); + expect(result.backendPreference).toBe("auto"); + }); + + it("keeps HEVC Hardware on the mandatory CUDA route even when the async capability probe is stale", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: false, + nvidiaCudaExportAvailable: false, + }); + + expect(result.nvidiaCudaCompositorRequired).toBe(true); + expect(result.useExperimentalNvidiaCudaExport).toBe(true); + expect(result.needsNativeRawFrame).toBe(false); + }); + + it("does not mark H.264, HEVC Auto, or HEVC CPU as mandatory CUDA routes", () => { + const h264Auto = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + expect(h264Auto.nvidiaCudaCompositorRequired).toBe(false); + expect(h264Auto.useExperimentalNvidiaCudaExport).toBe(true); + expect(h264Auto.needsNativeRawFrame).toBe(false); + + const hevcAuto = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + expect(hevcAuto.nvidiaCudaCompositorRequired).toBe(false); + expect(hevcAuto.useExperimentalNvidiaCudaExport).toBe(true); + expect(hevcAuto.needsNativeRawFrame).toBe(false); + + const hevcCpu = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + expect(hevcCpu.nvidiaCudaCompositorRequired).toBe(false); + expect(hevcCpu.useExperimentalNvidiaCudaExport).toBe(false); + expect(hevcCpu.needsNativeRawFrame).toBe(true); + }); + + it("keeps HEVC Auto on the opt-in toggle: CUDA only when the user enables it", () => { + const toggledOff = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + experimentalNvidiaCudaExport: false, + nvidiaCudaExportAvailable: true, + }); + expect(toggledOff.useExperimentalNvidiaCudaExport).toBe(false); + expect(toggledOff.needsNativeRawFrame).toBe(true); + }); }); diff --git a/src/components/video-editor/mp4ExportRouting.ts b/src/components/video-editor/mp4ExportRouting.ts index 95b157de3..a474d2a31 100644 --- a/src/components/video-editor/mp4ExportRouting.ts +++ b/src/components/video-editor/mp4ExportRouting.ts @@ -1,7 +1,9 @@ import type { ExportBackendPreference, + ExportEncoderPreference, ExportPipelineModel, ExportSettings, + ExportVideoCodec, } from "@/lib/exporter"; import type { SmokeExportConfig } from "./smokeExportConfig"; @@ -9,7 +11,12 @@ export type Mp4ExportRouting = { pipelineModel: ExportPipelineModel; useExperimentalNativeExport: boolean; useExperimentalNvidiaCudaExport: boolean; + /** True when HEVC + Hardware makes the NVIDIA CUDA compositor mandatory. The + * CUDA route is forced regardless of the persisted opt-in toggle and the + * export hard-fails when the compositor cannot run. */ + nvidiaCudaCompositorRequired: boolean; backendPreference: ExportBackendPreference; + needsNativeRawFrame: boolean; }; export function resolveMp4ExportRouting({ @@ -17,8 +24,9 @@ export function resolveMp4ExportRouting({ settings, exportPipelineModel, exportBackendPreference, + exportVideoCodec = "h264", + exportEncoderPreference = "auto", experimentalNvidiaCudaExport, - nvidiaCudaExportAvailable, }: { smokeExportConfig: Pick< SmokeExportConfig, @@ -27,17 +35,60 @@ export function resolveMp4ExportRouting({ settings: Pick; exportPipelineModel: ExportPipelineModel; exportBackendPreference: ExportBackendPreference; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; experimentalNvidiaCudaExport: boolean; nvidiaCudaExportAvailable: boolean; }): Mp4ExportRouting { - const pipelineModel = smokeExportConfig.enabled - ? (smokeExportConfig.pipelineModel ?? "modern") - : (settings.pipelineModel ?? exportPipelineModel); + // HEVC and any explicit encoder preference stay on the modern pipeline. HEVC + // can use the native CUDA compositor for Auto/Hardware, with rawvideo reserved + // for unavailable GPU routes and unsupported effects. + const requiresModernCodecRoute = + exportVideoCodec === "hevc" || exportEncoderPreference !== "auto"; + const pipelineModel = requiresModernCodecRoute + ? "modern" + : smokeExportConfig.enabled + ? (smokeExportConfig.pipelineModel ?? "modern") + : (settings.pipelineModel ?? exportPipelineModel); const useExperimentalNativeExport = pipelineModel === "modern" && (smokeExportConfig.enabled ? smokeExportConfig.useNativeExport : true); + // Auto and explicit GPU/Hardware preferences may use the native NVIDIA + // compositor for BOTH codecs (H.264 and HEVC). CPU stays on the software + // encoder; the H.264 compatibility default (Auto) is unchanged. + const mayUseNativeGpuCompositor = + exportEncoderPreference === "auto" || exportEncoderPreference === "hardware"; + // HEVC + Hardware makes the NVIDIA CUDA compositor mandatory: the user's codec + // and encoder choice IS the opt-in, so the CUDA route no longer depends on a + // hidden experimental toggle. If the compositor cannot run (no helper, no + // NVIDIA GPU, overlay preparation failure), the exporter hard-fails instead of + // falling back to WebGPU/Breeze/raw or CPU. + const nvidiaCudaCompositorRequired = + exportVideoCodec === "hevc" && exportEncoderPreference === "hardware"; + // For non-mandatory routes the CUDA route stays driven by the user's persisted + // opt-in state (which the capability hook already forces off when the + // helper/GPU is unavailable). The runtime `native-static-layout-export` + // attempt remains the authoritative capability check; gating on the async + // `nvidiaCudaExportAvailable` probe here would race exports started before the + // probe resolves and would skip a working route on a flaky Electron GPU-info + // probe. const useExperimentalNvidiaCudaExport = - useExperimentalNativeExport && experimentalNvidiaCudaExport && nvidiaCudaExportAvailable; + useExperimentalNativeExport && + mayUseNativeGpuCompositor && + (nvidiaCudaCompositorRequired || experimentalNvidiaCudaExport); + const canUseHevcNativeGpuCompositor = + exportVideoCodec === "hevc" && + exportEncoderPreference !== "cpu" && + useExperimentalNvidiaCudaExport; + // H.264 Auto stays on the compatibility path (native layout with the + // automatic bitrate heuristic, WebCodecs/native/Breeze routing unchanged). + // H.264 Hardware uses the native GPU compositor when eligible, exactly like + // HEVC Hardware; only H.264 CPU forces the raw software frame path. + const needsNativeRawFrame = + exportVideoCodec === "hevc" + ? !canUseHevcNativeGpuCompositor + : exportEncoderPreference === "cpu" || + (exportEncoderPreference === "hardware" && !useExperimentalNvidiaCudaExport); const backendPreference = pipelineModel === "legacy" ? "webcodecs" @@ -52,6 +103,8 @@ export function resolveMp4ExportRouting({ pipelineModel, useExperimentalNativeExport, useExperimentalNvidiaCudaExport, + nvidiaCudaCompositorRequired, backendPreference, + needsNativeRawFrame, }; } diff --git a/src/components/video-editor/mp4ExportSettings.test.ts b/src/components/video-editor/mp4ExportSettings.test.ts index 6d4c04084..7e47a5a6c 100644 --- a/src/components/video-editor/mp4ExportSettings.test.ts +++ b/src/components/video-editor/mp4ExportSettings.test.ts @@ -10,6 +10,10 @@ const baseOptions = { exportQuality: "high" as const, exportEncodingMode: "balanced" as const, mp4FrameRate: 30 as const, + exportVideoCodec: "h264" as const, + exportEncoderPreference: "auto" as const, + exportBitrateMode: "auto" as const, + exportBitrateMbps: 20, }; describe("resolveMp4ExportSettings", () => { @@ -18,6 +22,10 @@ describe("resolveMp4ExportSettings", () => { quality: "high", encodingMode: "balanced", selectedMp4FrameRate: 30, + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + exportBitrateMode: "auto", + exportBitrateMbps: 20, }); }); @@ -29,12 +37,20 @@ describe("resolveMp4ExportSettings", () => { quality: "source", encodingMode: "quality", mp4FrameRate: 60, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 35, }, }), ).toEqual({ quality: "source", encodingMode: "quality", selectedMp4FrameRate: 60, + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + exportBitrateMode: "custom", + exportBitrateMbps: 35, }); }); @@ -47,20 +63,42 @@ describe("resolveMp4ExportSettings", () => { quality: "medium", encodingMode: "fast", fps: 24, + videoCodec: "hevc", + encoderPreference: "hardware", + bitrateMode: "custom", + bitrateMbps: 25, }, settings: { quality: "source", - encodingMode: "quality", - mp4FrameRate: 60, }, }), ).toEqual({ quality: "medium", encodingMode: "fast", selectedMp4FrameRate: 24, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + exportBitrateMode: "custom", + exportBitrateMbps: 25, }); }); + it("clamps custom bitrate to the 1-200 Mbps safety range", () => { + expect( + resolveMp4ExportSettings({ + ...baseOptions, + settings: { exportBitrateMode: "custom" }, + exportBitrateMbps: 500, + }).exportBitrateMbps, + ).toBe(200); + expect( + resolveMp4ExportSettings({ + ...baseOptions, + exportBitrateMbps: -3, + }).exportBitrateMbps, + ).toBe(1); + }); + it("falls back from incomplete smoke settings to menu settings and editor defaults", () => { expect( resolveMp4ExportSettings({ @@ -77,6 +115,10 @@ describe("resolveMp4ExportSettings", () => { quality: "good", encodingMode: "quality", selectedMp4FrameRate: 30, + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + exportBitrateMode: "auto", + exportBitrateMbps: 20, }); }); }); diff --git a/src/components/video-editor/mp4ExportSettings.ts b/src/components/video-editor/mp4ExportSettings.ts index 95017a2c0..a755ca5f2 100644 --- a/src/components/video-editor/mp4ExportSettings.ts +++ b/src/components/video-editor/mp4ExportSettings.ts @@ -1,8 +1,11 @@ import type { + ExportBitrateMode, + ExportEncoderPreference, ExportEncodingMode, ExportMp4FrameRate, ExportQuality, ExportSettings, + ExportVideoCodec, } from "@/lib/exporter"; import type { SmokeExportConfig } from "./smokeExportConfig"; @@ -10,23 +13,63 @@ export type ResolvedMp4ExportSettings = { quality: ExportQuality; encodingMode: ExportEncodingMode; selectedMp4FrameRate: ExportMp4FrameRate; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; + exportBitrateMode: ExportBitrateMode; + exportBitrateMbps: number; }; +const DEFAULT_VIDEO_CODEC: ExportVideoCodec = "h264"; +const DEFAULT_ENCODER_PREFERENCE: ExportEncoderPreference = "auto"; +const DEFAULT_BITRATE_MODE: ExportBitrateMode = "auto"; +const DEFAULT_CUSTOM_MBPS = 20; + +function normalizeBitrateMbps(value: number | undefined): number { + if (value === undefined || !Number.isFinite(value)) { + return DEFAULT_CUSTOM_MBPS; + } + return Math.min(200, Math.max(1, value)); +} + export function resolveMp4ExportSettings({ smokeExportConfig, settings, exportQuality, exportEncodingMode, mp4FrameRate, + exportVideoCodec, + exportEncoderPreference, + exportBitrateMode, + exportBitrateMbps, }: { smokeExportConfig: Pick< SmokeExportConfig, - "enabled" | "quality" | "encodingMode" | "fps" + | "enabled" + | "quality" + | "encodingMode" + | "fps" + | "videoCodec" + | "encoderPreference" + | "bitrateMode" + | "bitrateMbps" + >; + settings: Pick< + ExportSettings, + | "quality" + | "encodingMode" + | "mp4FrameRate" + | "exportVideoCodec" + | "exportEncoderPreference" + | "exportBitrateMode" + | "exportBitrateMbps" >; - settings: Pick; exportQuality: ExportQuality; exportEncodingMode: ExportEncodingMode; mp4FrameRate: ExportMp4FrameRate; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; + exportBitrateMode: ExportBitrateMode; + exportBitrateMbps: number; }): ResolvedMp4ExportSettings { return { quality: smokeExportConfig.enabled @@ -38,5 +81,30 @@ export function resolveMp4ExportSettings({ selectedMp4FrameRate: smokeExportConfig.enabled ? (smokeExportConfig.fps ?? settings.mp4FrameRate ?? mp4FrameRate) : (settings.mp4FrameRate ?? mp4FrameRate), + exportVideoCodec: smokeExportConfig.enabled + ? (smokeExportConfig.videoCodec ?? + settings.exportVideoCodec ?? + exportVideoCodec ?? + DEFAULT_VIDEO_CODEC) + : (settings.exportVideoCodec ?? exportVideoCodec ?? DEFAULT_VIDEO_CODEC), + exportEncoderPreference: smokeExportConfig.enabled + ? (smokeExportConfig.encoderPreference ?? + settings.exportEncoderPreference ?? + exportEncoderPreference ?? + DEFAULT_ENCODER_PREFERENCE) + : (settings.exportEncoderPreference ?? + exportEncoderPreference ?? + DEFAULT_ENCODER_PREFERENCE), + exportBitrateMode: smokeExportConfig.enabled + ? (smokeExportConfig.bitrateMode ?? + settings.exportBitrateMode ?? + exportBitrateMode ?? + DEFAULT_BITRATE_MODE) + : (settings.exportBitrateMode ?? exportBitrateMode ?? DEFAULT_BITRATE_MODE), + exportBitrateMbps: smokeExportConfig.enabled + ? (smokeExportConfig.bitrateMbps ?? + settings.exportBitrateMbps ?? + normalizeBitrateMbps(exportBitrateMbps)) + : (settings.exportBitrateMbps ?? normalizeBitrateMbps(exportBitrateMbps)), }; } diff --git a/src/components/video-editor/projectPersistence.test.ts b/src/components/video-editor/projectPersistence.test.ts index 575c3b676..edf843d9c 100644 --- a/src/components/video-editor/projectPersistence.test.ts +++ b/src/components/video-editor/projectPersistence.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { normalizeProjectEditor } from "./projectPersistence"; +import { + createProjectData, + normalizeProjectEditor, + validateProjectData, +} from "./projectPersistence"; import { ADVANCED_VERTICAL_PADDING_MAX } from "./types"; describe("normalizeProjectEditor", () => { @@ -43,4 +47,58 @@ describe("normalizeProjectEditor", () => { linked: true, }); }); + + it("defaults new H.265/bitrate fields to h264/auto/auto/20", () => { + const editor = normalizeProjectEditor({}); + + expect(editor.exportVideoCodec).toBe("h264"); + expect(editor.exportEncoderPreference).toBe("auto"); + expect(editor.exportBitrateMode).toBe("auto"); + expect(editor.exportBitrateMbps).toBe(20); + }); + + it("normalizes invalid H.265/bitrate persisted values safely", () => { + const editor = normalizeProjectEditor({ + exportVideoCodec: "vp9", + exportEncoderPreference: "turbo", + exportBitrateMode: "ultra", + exportBitrateMbps: 5000, + }); + + expect(editor.exportVideoCodec).toBe("h264"); + expect(editor.exportEncoderPreference).toBe("auto"); + expect(editor.exportBitrateMode).toBe("auto"); + expect(editor.exportBitrateMbps).toBe(105); + + const hevcClamp = normalizeProjectEditor({ + exportVideoCodec: "hevc", + exportBitrateMbps: 5000, + }); + expect(hevcClamp.exportVideoCodec).toBe("hevc"); + expect(hevcClamp.exportBitrateMbps).toBe(70); + + const lowClamp = normalizeProjectEditor({ exportBitrateMbps: -3 }); + expect(lowClamp.exportBitrateMbps).toBe(1); + + const nanFallback = normalizeProjectEditor({ exportBitrateMbps: Number.NaN }); + expect(nanFallback.exportBitrateMbps).toBe(20); + }); + + it("preserves valid H.265/bitrate values through a project round trip", () => { + const editor = normalizeProjectEditor({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + exportBitrateMode: "custom", + exportBitrateMbps: 48, + }); + + const data = createProjectData("/tmp/video.mp4", editor); + expect(validateProjectData(data)).toBe(true); + + const reloaded = normalizeProjectEditor(data.editor); + expect(reloaded.exportVideoCodec).toBe("hevc"); + expect(reloaded.exportEncoderPreference).toBe("hardware"); + expect(reloaded.exportBitrateMode).toBe("custom"); + expect(reloaded.exportBitrateMbps).toBe(48); + }); }); diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 9810d5fb3..be0129e16 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -1,15 +1,19 @@ import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; -import type { - ExportBackendPreference, - ExportEncodingMode, - ExportFormat, - ExportMp4FrameRate, - ExportPipelineModel, - ExportQuality, - GifFrameRate, - GifSizePreset, +import { + clampCustomBitrateMbps, + type ExportBackendPreference, + type ExportBitrateMode, + type ExportEncoderPreference, + type ExportEncodingMode, + type ExportFormat, + type ExportMp4FrameRate, + type ExportPipelineModel, + type ExportQuality, + type ExportVideoCodec, + type GifFrameRate, + type GifSizePreset, + isValidMp4FrameRate, } from "@/lib/exporter"; -import { isValidMp4FrameRate } from "@/lib/exporter"; import { TEMPORAL_MOTION_BLUR_DEFAULT_SAMPLE_COUNT, TEMPORAL_MOTION_BLUR_DEFAULT_SHUTTER_FRACTION, @@ -149,6 +153,10 @@ export interface ProjectEditorState { exportPipelineModel: ExportPipelineModel; exportQuality: ExportQuality; mp4FrameRate: ExportMp4FrameRate; + exportVideoCodec: ExportVideoCodec; + exportEncoderPreference: ExportEncoderPreference; + exportBitrateMode: ExportBitrateMode; + exportBitrateMbps: number; exportFormat: ExportFormat; gifFrameRate: GifFrameRate; gifLoop: boolean; @@ -210,6 +218,34 @@ export function normalizeExportMp4FrameRate(value: unknown): ExportMp4FrameRate return typeof value === "number" && isValidMp4FrameRate(value) ? value : 30; } +export function normalizeExportVideoCodec(value: unknown): ExportVideoCodec { + if (value === "h264" || value === "hevc") { + return value; + } + + return "h264"; +} + +export function normalizeExportEncoderPreference(value: unknown): ExportEncoderPreference { + if (value === "auto" || value === "hardware" || value === "cpu") { + return value; + } + + return "auto"; +} + +export function normalizeExportBitrateMode(value: unknown): ExportBitrateMode { + if (value === "auto" || value === "custom") { + return value; + } + + return "auto"; +} + +export function normalizeExportBitrateMbps(value: unknown, codec?: ExportVideoCodec): number { + return clampCustomBitrateMbps(typeof value === "number" ? value : Number.NaN, codec); +} + function normalizeZoomTransitionEasing( value: unknown, fallback: ZoomTransitionEasing, @@ -894,6 +930,7 @@ export function normalizeProjectEditor(editor: Partial): Pro ); const normalizedMotionPreset = CURSOR_MOTION_PRESETS[resolveCursorMotionPresetId(normalizedMotionValues)]; + const normalizedExportVideoCodec = normalizeExportVideoCodec(editor.exportVideoCodec); return { wallpaper: typeof editor.wallpaper === "string" ? editor.wallpaper : DEFAULT_WALLPAPER_PATH, @@ -1092,6 +1129,13 @@ export function normalizeProjectEditor(editor: Partial): Pro ? editor.exportQuality : "source", mp4FrameRate: normalizeExportMp4FrameRate(editor.mp4FrameRate), + exportVideoCodec: normalizedExportVideoCodec, + exportEncoderPreference: normalizeExportEncoderPreference(editor.exportEncoderPreference), + exportBitrateMode: normalizeExportBitrateMode(editor.exportBitrateMode), + exportBitrateMbps: normalizeExportBitrateMbps( + editor.exportBitrateMbps, + normalizedExportVideoCodec, + ), exportFormat: editor.exportFormat === "gif" ? "gif" : "mp4", gifFrameRate: editor.gifFrameRate === 15 || diff --git a/src/components/video-editor/smokeExportConfig.test.ts b/src/components/video-editor/smokeExportConfig.test.ts index c221e1e4f..e4aa14203 100644 --- a/src/components/video-editor/smokeExportConfig.test.ts +++ b/src/components/video-editor/smokeExportConfig.test.ts @@ -23,6 +23,10 @@ describe("getSmokeExportConfig", () => { projectPath: null, quality: undefined, fps: undefined, + videoCodec: undefined, + encoderPreference: undefined, + bitrateMode: undefined, + bitrateMbps: undefined, }); }); @@ -45,7 +49,11 @@ describe("getSmokeExportConfig", () => { "&smokeMaxPendingFrames=10" + "&smokeProject=/tmp/project.recordly" + "&smokeQuality=source" + - "&smokeFps=60", + "&smokeFps=60" + + "&smokeVideoCodec=hevc" + + "&smokeEncoderPreference=cpu" + + "&smokeBitrateMode=custom" + + "&smokeBitrateMbps=30", ); expect(config).toEqual({ @@ -67,6 +75,10 @@ describe("getSmokeExportConfig", () => { projectPath: "/tmp/project.recordly", quality: "source", fps: 60, + videoCodec: "hevc", + encoderPreference: "cpu", + bitrateMode: "custom", + bitrateMbps: 30, }); }); @@ -84,7 +96,11 @@ describe("getSmokeExportConfig", () => { "&smokeMaxDecodeQueue=-4" + "&smokeMaxPendingFrames=abc" + "&smokeQuality=ultra" + - "&smokeFps=25", + "&smokeFps=25" + + "&smokeVideoCodec=vp9" + + "&smokeEncoderPreference=quantum" + + "&smokeBitrateMode=smart" + + "&smokeBitrateMbps=-4", ); expect(config).toMatchObject({ @@ -102,6 +118,10 @@ describe("getSmokeExportConfig", () => { maxPendingFrames: undefined, quality: undefined, fps: undefined, + videoCodec: undefined, + encoderPreference: undefined, + bitrateMode: undefined, + bitrateMbps: undefined, }); }); }); @@ -109,9 +129,7 @@ describe("getSmokeExportConfig", () => { describe("getDevOpenRecordingConfig", () => { it("reads dev-open paths independently from smoke export", () => { expect( - getDevOpenRecordingConfig( - "?devOpenInput=/tmp/input.mp4&devOpenWebcam=/tmp/webcam.mp4", - ), + getDevOpenRecordingConfig("?devOpenInput=/tmp/input.mp4&devOpenWebcam=/tmp/webcam.mp4"), ).toEqual({ inputPath: "/tmp/input.mp4", webcamInputPath: "/tmp/webcam.mp4", diff --git a/src/components/video-editor/smokeExportConfig.ts b/src/components/video-editor/smokeExportConfig.ts index 0d71779bb..091184ff6 100644 --- a/src/components/video-editor/smokeExportConfig.ts +++ b/src/components/video-editor/smokeExportConfig.ts @@ -1,11 +1,14 @@ import { - isValidMp4FrameRate, type ExportBackendPreference, + type ExportBitrateMode, + type ExportEncoderPreference, type ExportEncodingMode, type ExportMp4FrameRate, type ExportPipelineModel, type ExportQuality, type ExportRenderBackend, + type ExportVideoCodec, + isValidMp4FrameRate, } from "@/lib/exporter/types"; export type SmokeExportConfig = { @@ -27,6 +30,10 @@ export type SmokeExportConfig = { projectPath?: string | null; quality?: ExportQuality; fps?: ExportMp4FrameRate; + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; + bitrateMode?: ExportBitrateMode; + bitrateMbps?: number; }; export type DevOpenRecordingConfig = { @@ -110,15 +117,44 @@ export function getSmokeExportConfig(search: string): SmokeExportConfig { : enabled && params.get("smokeBackendPreference") === "breeze" ? "breeze" : undefined, - renderBackend: enabled ? parseSmokeRenderBackend(params.get("smokeRenderBackend")) : undefined, - maxEncodeQueue: enabled ? parseSmokeExportNumber(params.get("smokeMaxEncodeQueue")) : undefined, - maxDecodeQueue: enabled ? parseSmokeExportNumber(params.get("smokeMaxDecodeQueue")) : undefined, + renderBackend: enabled + ? parseSmokeRenderBackend(params.get("smokeRenderBackend")) + : undefined, + maxEncodeQueue: enabled + ? parseSmokeExportNumber(params.get("smokeMaxEncodeQueue")) + : undefined, + maxDecodeQueue: enabled + ? parseSmokeExportNumber(params.get("smokeMaxDecodeQueue")) + : undefined, maxPendingFrames: enabled ? parseSmokeExportNumber(params.get("smokeMaxPendingFrames")) : undefined, projectPath: enabled ? params.get("smokeProject") : null, quality: enabled ? parseSmokeExportQuality(params.get("smokeQuality")) : undefined, fps: enabled ? parseSmokeExportFps(params.get("smokeFps")) : undefined, + videoCodec: + enabled && params.get("smokeVideoCodec") === "hevc" + ? "hevc" + : enabled && params.get("smokeVideoCodec") === "h264" + ? "h264" + : undefined, + encoderPreference: + enabled && params.get("smokeEncoderPreference") === "hardware" + ? "hardware" + : enabled && params.get("smokeEncoderPreference") === "cpu" + ? "cpu" + : enabled && params.get("smokeEncoderPreference") === "auto" + ? "auto" + : undefined, + bitrateMode: + enabled && params.get("smokeBitrateMode") === "custom" + ? "custom" + : enabled && params.get("smokeBitrateMode") === "auto" + ? "auto" + : undefined, + bitrateMbps: enabled + ? parseSmokeExportNonNegativeNumber(params.get("smokeBitrateMbps")) + : undefined, }; } diff --git a/src/components/video-editor/useNvidiaCudaExportOptIn.ts b/src/components/video-editor/useNvidiaCudaExportOptIn.ts index a674b4ae5..80264e36b 100644 --- a/src/components/video-editor/useNvidiaCudaExportOptIn.ts +++ b/src/components/video-editor/useNvidiaCudaExportOptIn.ts @@ -8,6 +8,7 @@ type NativeExportCapabilitiesResult = { capabilities?: { nvidiaCuda?: { available?: boolean; + skipReason?: string | null; }; }; } | null; @@ -18,6 +19,12 @@ export function isNvidiaCudaExportAvailable( return result?.capabilities?.nvidiaCuda?.available === true; } +export function getNvidiaCudaExportSkipReason( + result: NativeExportCapabilitiesResult | undefined, +): string | null { + return result?.capabilities?.nvidiaCuda?.skipReason ?? null; +} + export function resolveNvidiaCudaExportOptIn( requested: boolean, nvidiaCudaExportAvailable: boolean, @@ -39,6 +46,9 @@ export function useNvidiaCudaExportOptIn({ onEnabled?: () => void; } = {}) { const [nvidiaCudaExportAvailable, setNvidiaCudaExportAvailable] = useState(false); + const [nvidiaCudaExportSkipReason, setNvidiaCudaExportSkipReason] = useState( + null, + ); const [experimentalNvidiaCudaExport, setExperimentalNvidiaCudaExportState] = useState( loadInitialNvidiaCudaExportOptIn, ); @@ -55,6 +65,13 @@ export function useNvidiaCudaExportOptIn({ const available = isNvidiaCudaExportAvailable(result); setNvidiaCudaExportAvailable(available); + setNvidiaCudaExportSkipReason(getNvidiaCudaExportSkipReason(result)); + console.info("[export] NVIDIA CUDA export capability probe", { + available, + skipReason: getNvidiaCudaExportSkipReason(result), + nvidiaCuda: result?.capabilities?.nvidiaCuda, + storedOptIn: loadInitialNvidiaCudaExportOptIn(), + }); if (!available) { setExperimentalNvidiaCudaExportState(false); } @@ -90,6 +107,7 @@ export function useNvidiaCudaExportOptIn({ return { nvidiaCudaExportAvailable, + nvidiaCudaExportSkipReason, experimentalNvidiaCudaExport, setExperimentalNvidiaCudaExport, }; diff --git a/src/components/video-editor/videoPlayback/zoomTransform.test.ts b/src/components/video-editor/videoPlayback/zoomTransform.test.ts index 6031977a6..cc4d55bd3 100644 --- a/src/components/video-editor/videoPlayback/zoomTransform.test.ts +++ b/src/components/video-editor/videoPlayback/zoomTransform.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { applyZoomTransform, computeZoomTransform, createMotionBlurState } from "./zoomTransform"; +import { + analyzeZoomMotionBlurStep, + applyZoomTransform, + computeZoomTransform, + createMotionBlurState, +} from "./zoomTransform"; function createStubContainer() { return { @@ -122,3 +127,74 @@ describe("applyZoomTransform motion blur routing", () => { expect(zoomBlurFilter.radius).toBe(-1); }); }); + +describe("analyzeZoomMotionBlurStep", () => { + const baseMask = { x: 80, y: 60, width: 1120, height: 600 }; + const stageSize = { width: 1280, height: 720 }; + + it("reports zero strength when the camera step is static", () => { + const result = analyzeZoomMotionBlurStep({ + previousTransform: { scale: 1, x: 0, y: 0 }, + currentTransform: { scale: 1, x: 0, y: 0 }, + baseMask, + stageSize, + motionBlurAmount: 1, + deltaSeconds: 1 / 30, + }); + + expect(result.strength).toBe(0); + expect(result.centerX).toBeCloseTo(640, 0); + expect(result.centerY).toBeCloseTo(360, 0); + }); + + it("derives the same radial strength the renderer applies during zoom steps", () => { + const previous = { scale: 1, x: 0, y: 0 }; + const current = computeZoomTransform({ + stageSize, + baseMask, + zoomScale: 1.4, + zoomProgress: 1, + focusX: 0.5, + focusY: 0.5, + }); + + const result = analyzeZoomMotionBlurStep({ + previousTransform: previous, + currentTransform: current, + baseMask, + stageSize, + motionBlurAmount: 0.35, + deltaSeconds: 1 / 30, + }); + + expect(result.strength).toBeGreaterThan(0); + // A zoom about the stage center keeps the blur center at the stage center. + expect(result.centerX).toBeCloseTo(stageSize.width / 2, 0); + expect(result.centerY).toBeCloseTo(stageSize.height / 2, 0); + }); + + it("places the blur center at the zoom pivot for off-center zooms", () => { + const previous = { scale: 1, x: 0, y: 0 }; + const current = computeZoomTransform({ + stageSize, + baseMask, + zoomScale: 1.6, + zoomProgress: 1, + focusX: 0.25, + focusY: 0.75, + }); + + const result = analyzeZoomMotionBlurStep({ + previousTransform: previous, + currentTransform: current, + baseMask, + stageSize, + motionBlurAmount: 0.35, + deltaSeconds: 1 / 30, + }); + + expect(result.strength).toBeGreaterThan(0); + expect(result.centerX).toBeLessThan(stageSize.width / 2); + expect(result.centerY).toBeGreaterThan(stageSize.height / 2); + }); +}); diff --git a/src/components/video-editor/videoPlayback/zoomTransform.ts b/src/components/video-editor/videoPlayback/zoomTransform.ts index 65fe8ce7f..fa64e9633 100644 --- a/src/components/video-editor/videoPlayback/zoomTransform.ts +++ b/src/components/video-editor/videoPlayback/zoomTransform.ts @@ -25,6 +25,47 @@ export function createMotionBlurState(): MotionBlurState { }; } +// Shared camera-step zoom-blur analysis used by both the interactive renderer +// (applyZoomTransform) and native CUDA export telemetry. Returns the radial +// zoom-blur strength (0 when the step is not classified as zoom motion) and the +// output-space blur center, matching what the renderer feeds into +// ZoomBlurFilter. Keeping this here means native export derives the exact same +// values the on-screen renderer would use, instead of re-implementing a +// divergent approximation. +export function analyzeZoomMotionBlurStep({ + previousTransform, + currentTransform, + baseMask, + stageSize, + motionBlurAmount, + motionBlurTuning, + deltaSeconds, +}: { + previousTransform: { scale: number; x: number; y: number }; + currentTransform: { scale: number; x: number; y: number }; + baseMask: { x: number; y: number; width: number; height: number }; + stageSize: { width: number; height: number }; + motionBlurAmount: number; + motionBlurTuning?: ZoomMotionBlurTuning; + deltaSeconds: number; +}): { strength: number; centerX: number; centerY: number } { + const previousQuad = computeTransformQuad(baseMask, previousTransform); + const currentQuad = computeTransformQuad(baseMask, currentTransform); + const analysis = analyzeCameraStep({ + previousQuad, + currentQuad, + stageSize, + motionBlurAmount, + motionBlurTuning: resolveMotionBlurTuning(motionBlurTuning), + deltaSeconds, + }); + return { + strength: analysis.mode === "zoom" ? analysis.zoomStrength : 0, + centerX: analysis.zoomCenter.x, + centerY: analysis.zoomCenter.y, + }; +} + interface TransformParams { cameraContainer: Container; zoomBlurFilter?: ZoomBlurFilter | null; @@ -340,18 +381,9 @@ function analyzeCameraStep({ motionBlurTuning: ZoomMotionBlurTuning; deltaSeconds: number; }): CameraStepAnalysis { - const mode = classifyMotionMode( - previousQuad, - currentQuad, - motionBlurTuning, - deltaSeconds, - ); + const mode = classifyMotionMode(previousQuad, currentQuad, motionBlurTuning, deltaSeconds); const moveDelta = computeMoveDelta(previousQuad, currentQuad); - const blurChannels = resolveBlurChannels( - motionBlurAmount, - motionBlurTuning, - deltaSeconds, - ); + const blurChannels = resolveBlurChannels(motionBlurAmount, motionBlurTuning, deltaSeconds); const moveBlurVelocity = { x: moveDelta.x * blurChannels.motion, y: moveDelta.y * blurChannels.motion, diff --git a/src/i18n/locales/de/settings.json b/src/i18n/locales/de/settings.json index 706dcc0d8..0cf4003d8 100644 --- a/src/i18n/locales/de/settings.json +++ b/src/i18n/locales/de/settings.json @@ -232,6 +232,49 @@ "high": "Hoch", "original": "Original" }, + "codecTitle": "Videocodec", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Encoder", + "encoder": { + "auto": "Automatisch", + "hardware": "Hardware", + "cpu": "CPU" + }, + "bitrateTitle": "Bitrate", + "bitrate": { + "auto": "Automatisch", + "custom": "Benutzerdefiniert", + "mbpsInput": "Benutzerdefinierte Bitrate in Mbit/s", + "range": "1–105 Mbit/s (H.264) · 1–70 Mbit/s (HEVC)" + }, + "hevcHint": "HEVC (H.265) erzeugt kleinere Dateien, wird aber möglicherweise nicht von älteren oder Web-Playern unterstützt. Die Vorschauwiedergabe in Recordly bleibt unverändert.", + "hardwareUnavailable": "Kein nutzbarer Hardware-Encoder gefunden. Wählen Sie CPU oder Auto, oder aktualisieren Sie den GPU-Treiber.", + "errors": { + "hardwareUnavailable": "Kein nutzbarer Hardware-Encoder gefunden. Wählen Sie CPU oder Auto, oder aktualisieren Sie den GPU-Treiber." + }, + "backend": { + "auto": "Automatisch" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA-CUDA-Compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA-CUDA-Compositor", + "availableBadge": "Verfügbar", + "selectedBadge": "Ausgewählt", + "requiredBadge": "Erforderlich", + "unavailableBadge": "Nicht verfügbar", + "toggle": "NVIDIA-CUDA-Compositor für GPU-beschleunigte Exporte verwenden", + "requiredToggle": "Der NVIDIA-CUDA-Compositor ist für H.265-Hardware-Exporte erforderlich", + "hint": "Komposition und Encoding auf der NVIDIA-GPU für schnelle Exporte.", + "hintSelected": "Exporte verwenden auf diesem Gerät den NVIDIA-CUDA-Compositor.", + "hintRequired": "H.265-Hardware-Exporte verwenden den NVIDIA-CUDA-Compositor und fallen nie auf Renderer-Frames zurück.", + "unavailableReason": "CUDA-Compositor nicht verfügbar ({{reason}}).", + "unavailableGeneric": "CUDA-Compositor auf diesem Gerät nicht verfügbar.", + "unavailableRequired": "H.265- und Hardware-Exporte schlagen fehl, bis der CUDA-Compositor verfügbar ist. NVIDIA-Treiber installieren oder aktualisieren oder Encoder auf Auto umstellen." + }, "fpsTitle": "FPS", "loop": "Wiederholen", "outputDimensions": "Ausgabe: {{dimensions}}px", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 60aa86221..809481762 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -232,6 +232,48 @@ "high": "High", "original": "Original" }, + "codecTitle": "Video codec", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Encoder", + "encoder": { + "auto": "Auto", + "hardware": "GPU", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA RTX Rendering", + "backendLabel": "Backend", + "backendSelected": "NVIDIA RTX Rendering", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use NVIDIA RTX Rendering for GPU-accelerated exports", + "requiredToggle": "NVIDIA RTX Rendering is required for H.265 GPU exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use NVIDIA RTX Rendering on this device.", + "hintRequired": "H.265 GPU exports use NVIDIA RTX Rendering and never fall back to renderer frames.", + "unavailableReason": "RTX Rendering is unavailable ({{reason}}).", + "unavailableGeneric": "RTX Rendering is unavailable on this device.", + "unavailableRequired": "H.265 + GPU exports will fail until RTX Rendering is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "Bitrate", + "bitrate": { + "auto": "Auto", + "custom": "Custom", + "mbpsInput": "Custom bitrate in Mbps", + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" + }, + "hardwareUnavailable": "No usable hardware encoder was found. Pick CPU or Auto, or update your GPU driver.", + "errors": { + "hardwareUnavailable": "No usable hardware encoder was found. Pick CPU or Auto, or update your GPU driver." + }, "fpsTitle": "FPS", "loop": "Loop", "outputDimensions": "Output: {{dimensions}}px", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index d3545d595..a026397c8 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -232,6 +232,49 @@ "high": "Alta", "original": "Original" }, + "codecTitle": "Códec de vídeo", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Codificador", + "encoder": { + "auto": "Automático", + "hardware": "Hardware", + "cpu": "CPU" + }, + "bitrateTitle": "Tasa de bits", + "bitrate": { + "auto": "Automática", + "custom": "Personalizada", + "mbpsInput": "Tasa de bits personalizada en Mbps", + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" + }, + "hevcHint": "HEVC (H.265) crea archivos más pequeños pero puede no reproducirse en reproductores antiguos o web. La vista previa de Recordly no cambia.", + "hardwareUnavailable": "No se encontró ningún codificador de hardware utilizable. Elige CPU o Auto, o actualiza el controlador de tu GPU.", + "errors": { + "hardwareUnavailable": "No se encontró ningún codificador de hardware utilizable. Elige CPU o Auto, o actualiza el controlador de tu GPU." + }, + "backend": { + "auto": "Automático" + }, + "nvidiaCuda": { + "compositorTitle": "Compositor NVIDIA CUDA", + "backendLabel": "Backend", + "backendSelected": "Compositor NVIDIA CUDA", + "availableBadge": "Disponible", + "selectedBadge": "Seleccionado", + "requiredBadge": "Requerido", + "unavailableBadge": "No disponible", + "toggle": "Usar el compositor NVIDIA CUDA para exportaciones aceleradas por GPU", + "requiredToggle": "El compositor NVIDIA CUDA es obligatorio para exportaciones H.265 con hardware", + "hint": "Componer y codificar en la GPU NVIDIA para exportaciones rápidas.", + "hintSelected": "Las exportaciones usarán el compositor NVIDIA CUDA en este dispositivo.", + "hintRequired": "Las exportaciones H.265 con hardware usan el compositor NVIDIA CUDA y nunca recurren a fotogramas del renderizador.", + "unavailableReason": "El compositor CUDA no está disponible ({{reason}}).", + "unavailableGeneric": "El compositor CUDA no está disponible en este dispositivo.", + "unavailableRequired": "Las exportaciones H.265 + Hardware fallarán hasta que el compositor CUDA esté disponible. Instala o actualiza los controladores NVIDIA, o cambia el codificador a Auto." + }, "fpsTitle": "FPS", "loop": "Bucle", "outputDimensions": "Salida: {{dimensions}}px", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 03d17ba6b..4f2e64599 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -232,6 +232,49 @@ "high": "Élevée", "original": "Originale" }, + "codecTitle": "Codec vidéo", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Encodeur", + "encoder": { + "auto": "Auto", + "hardware": "Matériel", + "cpu": "CPU" + }, + "bitrateTitle": "Débit binaire", + "bitrate": { + "auto": "Auto", + "custom": "Personnalisé", + "mbpsInput": "Débit binaire personnalisé en Mbit/s", + "range": "1–105 Mbit/s (H.264) · 1–70 Mbit/s (HEVC)" + }, + "hevcHint": "HEVC (H.265) crée des fichiers plus petits mais peut ne pas être lu sur les lecteurs anciens ou web. L'aperçu de Recordly reste inchangé.", + "hardwareUnavailable": "Aucun encodeur matériel utilisable trouvé. Choisissez CPU ou Auto, ou mettez à jour le pilote de votre GPU.", + "errors": { + "hardwareUnavailable": "Aucun encodeur matériel utilisable trouvé. Choisissez CPU ou Auto, ou mettez à jour le pilote de votre GPU." + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "Compositor NVIDIA CUDA", + "backendLabel": "Backend", + "backendSelected": "Compositor NVIDIA CUDA", + "availableBadge": "Disponible", + "selectedBadge": "Sélectionné", + "requiredBadge": "Requis", + "unavailableBadge": "Indisponible", + "toggle": "Utiliser le compositor NVIDIA CUDA pour les exports accélérés par GPU", + "requiredToggle": "Le compositor NVIDIA CUDA est obligatoire pour les exports H.265 avec matériel", + "hint": "Composer et encoder sur le GPU NVIDIA pour des exports rapides.", + "hintSelected": "Les exports utiliseront le compositor NVIDIA CUDA sur cet appareil.", + "hintRequired": "Les exports H.265 avec matériel utilisent le compositor NVIDIA CUDA et ne reviennent jamais aux images du rendu.", + "unavailableReason": "Compositor CUDA indisponible ({{reason}}).", + "unavailableGeneric": "Compositor CUDA indisponible sur cet appareil.", + "unavailableRequired": "Les exports H.265 + Matériel échoueront tant que le compositor CUDA n'est pas disponible. Installez ou mettez à jour les pilotes NVIDIA, ou passez l'encodeur sur Auto." + }, "fpsTitle": "IPS", "loop": "Boucle", "outputDimensions": "Sortie : {{dimensions}} px", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 7158b0daf..917611c55 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -232,6 +232,49 @@ "high": "Alta", "original": "Originale" }, + "codecTitle": "Codec video", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Encoder", + "encoder": { + "auto": "Auto", + "hardware": "Hardware", + "cpu": "CPU" + }, + "bitrateTitle": "Bitrate", + "bitrate": { + "auto": "Auto", + "custom": "Personalizzato", + "mbpsInput": "Bitrate personalizzato in Mbps", + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" + }, + "hevcHint": "HEVC (H.265) produce file più piccoli ma potrebbe non essere riprodotto su lettori vecchi o web. L'anteprima di Recordly rimane invariata.", + "hardwareUnavailable": "Nessun encoder hardware utilizzabile trovato. Scegli CPU o Auto, oppure aggiorna il driver della GPU.", + "errors": { + "hardwareUnavailable": "Nessun encoder hardware utilizzabile trovato. Scegli CPU o Auto, oppure aggiorna il driver della GPU." + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "Compositor NVIDIA CUDA", + "backendLabel": "Backend", + "backendSelected": "Compositor NVIDIA CUDA", + "availableBadge": "Disponibile", + "selectedBadge": "Selezionato", + "requiredBadge": "Richiesto", + "unavailableBadge": "Non disponibile", + "toggle": "Usa il compositor NVIDIA CUDA per esportazioni accelerate via GPU", + "requiredToggle": "Il compositor NVIDIA CUDA è obbligatorio per le esportazioni H.265 con hardware", + "hint": "Componi e codifica sulla GPU NVIDIA per esportazioni veloci.", + "hintSelected": "Le esportazioni useranno il compositor NVIDIA CUDA su questo dispositivo.", + "hintRequired": "Le esportazioni H.265 con hardware usano il compositor NVIDIA CUDA e non ricadono mai sui fotogrammi del renderer.", + "unavailableReason": "Compositor CUDA non disponibile ({{reason}}).", + "unavailableGeneric": "Compositor CUDA non disponibile su questo dispositivo.", + "unavailableRequired": "Le esportazioni H.265 + Hardware falliranno finché il compositor CUDA non sarà disponibile. Installa o aggiorna i driver NVIDIA oppure imposta Encoder su Auto." + }, "fpsTitle": "FPS", "loop": "Loop", "outputDimensions": "Output: {{dimensions}}px", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index e33828da2..b816a8c12 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -232,6 +232,49 @@ "high": "높음", "original": "원본" }, + "codecTitle": "비디오 코덱", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "인코더", + "encoder": { + "auto": "자동", + "hardware": "하드웨어", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "비트레이트", + "bitrate": { + "auto": "자동", + "custom": "사용자 지정", + "mbpsInput": "Mbps 단위 사용자 지정 비트레이트", + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" + }, + "hevcHint": "HEVC(H.265)는 파일을 더 작게 만들지만 이전 또는 웹 플레이어에서 재생되지 않을 수 있습니다. Recordly 미리보기 재생은 변경되지 않습니다.", + "hardwareUnavailable": "사용 가능한 하드웨어 인코더를 찾을 수 없습니다. CPU 또는 자동을 선택하거나 GPU 드라이버를 업데이트하세요.", + "errors": { + "hardwareUnavailable": "사용 가능한 하드웨어 인코더를 찾을 수 없습니다. CPU 또는 자동을 선택하거나 GPU 드라이버를 업데이트하세요." + }, "fpsTitle": "FPS", "loop": "반복", "outputDimensions": "출력: {{dimensions}}px", diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 8d7269c7a..e1e3d6cad 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -232,6 +232,49 @@ "high": "Hoog", "original": "Origineel" }, + "codecTitle": "Videocodec", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Encoder", + "encoder": { + "auto": "Automatisch", + "hardware": "Hardware", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "Bitrate", + "bitrate": { + "auto": "Automatisch", + "custom": "Aangepast", + "mbpsInput": "Aangepaste bitrate in Mbps", + "range": "1–200 Mbps" + }, + "hevcHint": "HEVC (H.265) maakt kleinere bestanden, maar werkt mogelijk niet op oudere of webspelers. De preview-weergave van Recordly blijft ongewijzigd.", + "hardwareUnavailable": "Geen bruikbare hardware-encoder gevonden. Kies CPU of Automatisch, of werk je GPU-stuurprogramma bij.", + "errors": { + "hardwareUnavailable": "Geen bruikbare hardware-encoder gevonden. Kies CPU of Automatisch, of werk je GPU-stuurprogramma bij." + }, "fpsTitle": "FPS", "loop": "Herhalen", "outputDimensions": "Uitvoer: {{dimensions}}px", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 6cea132ea..d2caa838b 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -232,6 +232,49 @@ "high": "Alta", "original": "Original" }, + "codecTitle": "Codec de vídeo", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Codificador", + "encoder": { + "auto": "Automático", + "hardware": "Hardware", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "Taxa de bits", + "bitrate": { + "auto": "Automático", + "custom": "Personalizado", + "mbpsInput": "Taxa de bits personalizada em Mbps", + "range": "1–200 Mbps" + }, + "hevcHint": "HEVC (H.265) gera arquivos menores, mas pode não funcionar em players antigos ou da web. A reprodução da pré-visualização do Recordly permanece inalterada.", + "hardwareUnavailable": "Nenhum codificador de hardware utilizável foi encontrado. Escolha CPU ou Automático, ou atualize o driver da sua GPU.", + "errors": { + "hardwareUnavailable": "Nenhum codificador de hardware utilizável foi encontrado. Escolha CPU ou Automático, ou atualize o driver da sua GPU." + }, "fpsTitle": "FPS", "loop": "Repetir", "outputDimensions": "Saída: {{dimensions}}px", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 15197cab4..8914c91f5 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -232,6 +232,49 @@ "high": "Высокое", "original": "Исходное" }, + "codecTitle": "Видеокодек", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "Кодировщик", + "encoder": { + "auto": "Авто", + "hardware": "Аппаратный", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "Битрейт", + "bitrate": { + "auto": "Авто", + "custom": "Пользовательский", + "mbpsInput": "Пользовательский битрейт в Мбит/с", + "range": "1–200 Мбит/с" + }, + "hevcHint": "HEVC (H.265) создает файлы меньшего размера, но может не воспроизводиться на старых или веб-плеерах. Воспроизведение предпросмотра Recordly не меняется.", + "hardwareUnavailable": "Не найден подходящий аппаратный кодировщик. Выберите CPU или Авто либо обновите драйвер GPU.", + "errors": { + "hardwareUnavailable": "Не найден подходящий аппаратный кодировщик. Выберите CPU или Авто либо обновите драйвер GPU." + }, "fpsTitle": "FPS", "loop": "Зациклить", "outputDimensions": "Размер: {{dimensions}}px", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 424324e9f..69ebe639a 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -232,6 +232,49 @@ "high": "高", "original": "原始" }, + "codecTitle": "视频编解码器", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "编码器", + "encoder": { + "auto": "自动", + "hardware": "硬件", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "比特率", + "bitrate": { + "auto": "自动", + "custom": "自定义", + "mbpsInput": "自定义比特率（Mbps）", + "range": "1–200 Mbps" + }, + "hevcHint": "HEVC（H.265）生成的文件更小，但可能无法在较旧的或网页播放器中播放。Recordly 预览播放不受影响。", + "hardwareUnavailable": "未找到可用的硬件编码器。请选择 CPU 或自动，或更新你的 GPU 驱动程序。", + "errors": { + "hardwareUnavailable": "未找到可用的硬件编码器。请选择 CPU 或自动，或更新你的 GPU 驱动程序。" + }, "fpsTitle": "帧率", "loop": "循环", "outputDimensions": "输出：{{dimensions}}px", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 47fbbc381..7d459ba98 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -232,6 +232,49 @@ "high": "高", "original": "原始" }, + "codecTitle": "影片編解碼器", + "codec": { + "h264": "H.264", + "hevc": "H.265" + }, + "encoderTitle": "編碼器", + "encoder": { + "auto": "自動", + "hardware": "硬體", + "cpu": "CPU" + }, + "backend": { + "auto": "Auto" + }, + "nvidiaCuda": { + "compositorTitle": "NVIDIA CUDA compositor", + "backendLabel": "Backend", + "backendSelected": "NVIDIA CUDA compositor", + "availableBadge": "Available", + "selectedBadge": "Selected", + "requiredBadge": "Required", + "unavailableBadge": "Unavailable", + "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", + "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "hint": "Compose and encode on the NVIDIA GPU for fast exports.", + "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", + "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", + "unavailableGeneric": "CUDA compositor is unavailable on this device.", + "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + }, + "bitrateTitle": "位元率", + "bitrate": { + "auto": "自動", + "custom": "自訂", + "mbpsInput": "自訂位元率（Mbps）", + "range": "1–200 Mbps" + }, + "hevcHint": "HEVC（H.265）產生的檔案較小，但可能無法在較舊或網頁播放器中播放。Recordly 預覽播放不受影響。", + "hardwareUnavailable": "找不到可用的硬體編碼器。請選擇 CPU 或自動，或更新你的 GPU 驅動程式。", + "errors": { + "hardwareUnavailable": "找不到可用的硬體編碼器。請選擇 CPU 或自動，或更新你的 GPU 驅動程式。" + }, "fpsTitle": "FPS", "loop": "循環", "outputDimensions": "輸出：{{dimensions}}px", diff --git a/src/lib/exporter/exportBitrate.test.ts b/src/lib/exporter/exportBitrate.test.ts index 301689dd2..86ca414a3 100644 --- a/src/lib/exporter/exportBitrate.test.ts +++ b/src/lib/exporter/exportBitrate.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { getMp4ExportBitrate, getSourceQualityBitrate } from "./exportBitrate"; +import { + clampCustomBitrateMbps, + customBitrateMbpsToBps, + getMp4ExportBitrate, + getSourceQualityBitrate, + resolveExportBitrate, +} from "./exportBitrate"; describe("export bitrate policy", () => { it("keeps source-quality exports at a fuller screen-recording bitrate", () => { @@ -118,3 +124,162 @@ describe("export bitrate policy", () => { ).toBe(72_000_000); }); }); + +describe("export bitrate resolver", () => { + it("auto mode delegates identically to getMp4ExportBitrate", () => { + const first = { + width: 1920, + height: 1080, + frameRate: 30, + quality: "source" as const, + encodingMode: "quality" as const, + }; + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + ...first, + }), + ).toBe(getMp4ExportBitrate(first)); + + const second = { + width: 1280, + height: 720, + frameRate: 24, + quality: "good" as const, + encodingMode: "fast" as const, + useModernNativeStaticLayout: true, + }; + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + ...second, + }), + ).toBe(getMp4ExportBitrate(second)); + }); + + it("caps auto-mode HEVC bitrates at the HEVC maximum", () => { + const autoOptions = { + width: 3840, + height: 2160, + frameRate: 30, + quality: "source" as const, + encodingMode: "quality" as const, + useModernNativeStaticLayout: true, + }; + // The uncapped auto heuristic exceeds 70 Mbps for 4K source exports. + expect(getMp4ExportBitrate(autoOptions)).toBe(72_000_000); + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + codec: "hevc", + ...autoOptions, + }), + ).toBe(70_000_000); + // h264 auto keeps the existing heuristic (no codec cap applied). + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + codec: "h264", + ...autoOptions, + }), + ).toBe(72_000_000); + }); + + it("custom mode converts Mbps to whole-number bps", () => { + expect(customBitrateMbpsToBps(20)).toBe(20_000_000); + expect(customBitrateMbpsToBps(12.5)).toBe(12_500_000); + expect(customBitrateMbpsToBps(200, "h264")).toBe(105_000_000); + expect(customBitrateMbpsToBps(200, "hevc")).toBe(70_000_000); + }); + + it("custom mode clamps to the supported range and defaults non-finite input", () => { + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: 0.5, + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(1_000_000); + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: 500, + codec: "h264", + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(105_000_000); + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: 500, + codec: "hevc", + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(70_000_000); + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: NaN, + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(20_000_000); + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: -5, + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(1_000_000); + }); + + it("custom mode bypasses the native static-layout floor entirely", () => { + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: 2, + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + useModernNativeStaticLayout: true, + }), + ).toBe(2_000_000); + expect(customBitrateMbpsToBps(2)).toBe(2_000_000); + }); + + it("clampCustomBitrateMbps preserves fractional values within range", () => { + expect(clampCustomBitrateMbps(12.5)).toBe(12.5); + expect(clampCustomBitrateMbps(0.5)).toBe(1); + expect(clampCustomBitrateMbps(500)).toBe(200); + expect(clampCustomBitrateMbps(500, "h264")).toBe(105); + expect(clampCustomBitrateMbps(500, "hevc")).toBe(70); + expect(clampCustomBitrateMbps(12.5, "h264")).toBe(12.5); + expect(clampCustomBitrateMbps(70, "hevc")).toBe(70); + expect(clampCustomBitrateMbps(NaN)).toBe(20); + }); +}); diff --git a/src/lib/exporter/exportBitrate.ts b/src/lib/exporter/exportBitrate.ts index a82acb7ac..1d760b561 100644 --- a/src/lib/exporter/exportBitrate.ts +++ b/src/lib/exporter/exportBitrate.ts @@ -1,4 +1,15 @@ -import type { ExportEncodingMode, ExportMp4FrameRate, ExportQuality } from "./types"; +import { + EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_H264_MAX_MBPS, + EXPORT_BITRATE_HEVC_MAX_MBPS, + EXPORT_BITRATE_MAX_MBPS, + EXPORT_BITRATE_MIN_MBPS, + type ExportBitrateMode, + type ExportEncodingMode, + type ExportMp4FrameRate, + type ExportQuality, + type ExportVideoCodec, +} from "./types"; const MIN_MP4_BITRATE = 2_000_000; const REFERENCE_PIXEL_RATE = 1920 * 1080 * 30; @@ -125,3 +136,56 @@ export function getMp4ExportBitrate(options: { return Math.max(MIN_MP4_BITRATE, cappedBitrate); } + +function getCodecCustomBitrateCapMbps(codec: ExportVideoCodec | undefined): number { + switch (codec) { + case "h264": + return Math.min(EXPORT_BITRATE_H264_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS); + case "hevc": + return Math.min(EXPORT_BITRATE_HEVC_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS); + default: + return EXPORT_BITRATE_MAX_MBPS; + } +} + +function getCodecAutoBitrateCapBps(codec: ExportVideoCodec | undefined): number { + if (codec === "hevc") { + return EXPORT_BITRATE_HEVC_MAX_MBPS * 1_000_000; + } + // h264 and unknown codecs keep the existing auto heuristic unchanged. + return Number.POSITIVE_INFINITY; +} + +export function clampCustomBitrateMbps(mbps: number, codec?: ExportVideoCodec): number { + if (!Number.isFinite(mbps) || Number.isNaN(mbps)) { + return EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS; + } + if (mbps < EXPORT_BITRATE_MIN_MBPS) { + return EXPORT_BITRATE_MIN_MBPS; + } + if (mbps > getCodecCustomBitrateCapMbps(codec)) { + return getCodecCustomBitrateCapMbps(codec); + } + return mbps; +} + +export function customBitrateMbpsToBps(mbps: number, codec?: ExportVideoCodec): number { + return Math.floor(clampCustomBitrateMbps(mbps, codec) * 1_000_000); +} + +export function resolveExportBitrate(options: { + mode: ExportBitrateMode; + customMbps: number; + width: number; + height: number; + frameRate: ExportMp4FrameRate; + quality: ExportQuality; + encodingMode: ExportEncodingMode; + useModernNativeStaticLayout?: boolean; + codec?: ExportVideoCodec; +}): number { + if (options.mode === "custom") { + return customBitrateMbpsToBps(options.customMbps, options.codec); + } + return Math.min(getMp4ExportBitrate(options), getCodecAutoBitrateCapBps(options.codec)); +} diff --git a/src/lib/exporter/exportTuning.test.ts b/src/lib/exporter/exportTuning.test.ts index e2733916f..fbe0532c1 100644 --- a/src/lib/exporter/exportTuning.test.ts +++ b/src/lib/exporter/exportTuning.test.ts @@ -2,9 +2,12 @@ import { describe, expect, it } from "vitest"; import { getExportBackpressureProfile, + getNativeRawFrameBackpressureLimits, + getNativeRawFrameByteSize, getPreferredWebCodecsLatencyModes, getWebCodecsEncodeQueueLimit, getWebCodecsKeyFrameInterval, + NativeRawFrameBackpressureQueue, } from "./exportTuning"; describe("exportTuning", () => { @@ -53,6 +56,8 @@ describe("exportTuning", () => { expect(breezeProfile.maxDecodeQueue).toBe(14); expect(breezeProfile.maxPendingFrames).toBe(40); expect(breezeProfile.maxInFlightNativeWrites).toBe(8); + expect(breezeProfile.maxInFlightNativeRawFrames).toBe(4); + expect(breezeProfile.maxInFlightNativeRawBytes).toBe(1280 * 720 * 4 * 4); }); it("falls back to conservative native settings on low-core or very heavy workloads", () => { @@ -81,3 +86,57 @@ describe("exportTuning", () => { expect(breezeHeavyProfile.maxPendingFrames).toBe(16); }); }); + +describe("native raw-frame backpressure", () => { + it("keeps the transferable queue bounded by both frames and bytes", async () => { + const frameSize = getNativeRawFrameByteSize(2, 2); + const queue = new NativeRawFrameBackpressureQueue(frameSize * 2, 2); + + await queue.waitForCapacity(frameSize); + queue.reserve(frameSize); + queue.reserve(frameSize); + const waiter = queue.waitForCapacity(frameSize); + let waiterSettled = false; + void waiter.then(() => { + waiterSettled = true; + }); + + await Promise.resolve(); + expect(waiterSettled).toBe(false); + expect(queue.currentInFlightBytes).toBe(frameSize * 2); + queue.release(frameSize); + await waiter; + expect(queue.currentInFlightBytes).toBe(frameSize); + }); + + it("uses conservative frame and byte caps for cloned IPC", () => { + const profile = getExportBackpressureProfile({ + encodeBackend: "ffmpeg", + width: 1920, + height: 1080, + frameRate: 30, + hardwareConcurrency: 8, + }); + const limits = getNativeRawFrameBackpressureLimits({ + width: 1920, + height: 1080, + profile, + transportMode: "cloned-ipc", + }); + + expect(limits.maxInFlightFrames).toBe(2); + expect(limits.maxInFlightBytes).toBe(getNativeRawFrameByteSize(1920, 1080) * 2); + }); + + it("settles blocked waiters when the transport closes", async () => { + const queue = new NativeRawFrameBackpressureQueue(8, 1); + queue.reserve(8); + const waiter = queue.waitForCapacity(8); + const error = new Error("port closed"); + + queue.fail(error); + + await expect(waiter).rejects.toBe(error); + await expect(queue.waitForCapacity(8)).rejects.toBe(error); + }); +}); diff --git a/src/lib/exporter/exportTuning.ts b/src/lib/exporter/exportTuning.ts index 8e54343b0..a0d119849 100644 --- a/src/lib/exporter/exportTuning.ts +++ b/src/lib/exporter/exportTuning.ts @@ -1,8 +1,10 @@ -import type { ExportEncodeBackend, ExportEncodingMode } from "./types"; +import type { ExportEncodeBackend, ExportEncodingMode, ExportNativeTransportMode } from "./types"; const DEFAULT_ENCODING_MODE: ExportEncodingMode = "balanced"; type WebCodecsLatencyMode = "quality" | "realtime"; const BASELINE_PIXELS_PER_SECOND = 1280 * 720 * 60; +const RAW_FRAME_BYTES_PER_PIXEL = 4; +const CONSERVATIVE_RAW_FRAME_LIMIT = 2; const LATENCY_MODE_PREFERENCES: Record = { fast: ["realtime", "quality"], @@ -71,6 +73,8 @@ export interface ExportBackpressureProfile { maxDecodeQueue: number; maxPendingFrames: number; maxInFlightNativeWrites: number; + maxInFlightNativeRawFrames: number; + maxInFlightNativeRawBytes: number; } interface ExportBackpressureProfileOptions { @@ -82,6 +86,158 @@ interface ExportBackpressureProfileOptions { hardwareConcurrency?: number; } +export interface NativeRawFrameBackpressureLimits { + maxInFlightFrames: number; + maxInFlightBytes: number; +} + +export function getNativeRawFrameByteSize(width: number, height: number): number { + return ( + Math.max(1, Math.floor(width)) * Math.max(1, Math.floor(height)) * RAW_FRAME_BYTES_PER_PIXEL + ); +} + +export function getNativeRawFrameBackpressureLimits(options: { + width: number; + height: number; + profile: ExportBackpressureProfile; + transportMode: ExportNativeTransportMode; + maxInFlightFrames?: number; + maxInFlightBytes?: number; +}): NativeRawFrameBackpressureLimits { + const frameByteSize = getNativeRawFrameByteSize(options.width, options.height); + const requestedFrames = + typeof options.maxInFlightFrames === "number" && Number.isFinite(options.maxInFlightFrames) + ? Math.floor(options.maxInFlightFrames) + : options.profile.maxInFlightNativeRawFrames; + const requestedBytes = + typeof options.maxInFlightBytes === "number" && Number.isFinite(options.maxInFlightBytes) + ? Math.floor(options.maxInFlightBytes) + : options.profile.maxInFlightNativeRawBytes; + const transportFrameLimit = + options.transportMode === "cloned-ipc" + ? Math.min(requestedFrames, CONSERVATIVE_RAW_FRAME_LIMIT) + : requestedFrames; + const maxInFlightFrames = Math.max(1, transportFrameLimit); + const transportByteLimit = + options.transportMode === "cloned-ipc" + ? Math.min(requestedBytes, frameByteSize * CONSERVATIVE_RAW_FRAME_LIMIT) + : requestedBytes; + + return { + maxInFlightFrames, + maxInFlightBytes: Math.max( + frameByteSize, + Math.min( + Math.max(frameByteSize, transportByteLimit), + frameByteSize * maxInFlightFrames, + ), + ), + }; +} + +type NativeRawFrameWaiter = { + frameByteSize: number; + resolve: () => void; + reject: (error: Error) => void; +}; + +export class NativeRawFrameBackpressureQueue { + private inFlightBytes = 0; + private inFlightFrames = 0; + private closedError: Error | null = null; + private waiters = new Set(); + + constructor( + private readonly maxInFlightBytes: number, + private readonly maxInFlightFrames: number, + ) {} + + get currentInFlightBytes(): number { + return this.inFlightBytes; + } + + get currentInFlightFrames(): number { + return this.inFlightFrames; + } + + canAccept(frameByteSize: number): boolean { + return ( + frameByteSize > 0 && + this.inFlightFrames < this.maxInFlightFrames && + this.inFlightBytes + frameByteSize <= this.maxInFlightBytes + ); + } + + async waitForCapacity(frameByteSize: number): Promise { + this.validateFrameByteSize(frameByteSize); + if (this.closedError) { + throw this.closedError; + } + if (this.canAccept(frameByteSize)) { + return; + } + + await new Promise((resolve, reject) => { + this.waiters.add({ + frameByteSize, + resolve, + reject, + }); + }); + } + + reserve(frameByteSize: number): void { + this.validateFrameByteSize(frameByteSize); + if (this.closedError) { + throw this.closedError; + } + if (!this.canAccept(frameByteSize)) { + throw new Error("Native raw-frame backpressure capacity was not available"); + } + + this.inFlightBytes += frameByteSize; + this.inFlightFrames += 1; + } + + release(frameByteSize: number): void { + this.inFlightBytes = Math.max(0, this.inFlightBytes - frameByteSize); + this.inFlightFrames = Math.max(0, this.inFlightFrames - 1); + this.notifyWaiters(); + } + + fail(error: Error): void { + if (this.closedError) { + return; + } + this.closedError = error; + const waiters = [...this.waiters]; + this.waiters.clear(); + for (const waiter of waiters) { + waiter.reject(error); + } + } + + private validateFrameByteSize(frameByteSize: number): void { + if (!Number.isFinite(frameByteSize) || frameByteSize <= 0) { + throw new Error("Native raw-frame byte size must be positive"); + } + if (frameByteSize > this.maxInFlightBytes) { + throw new Error("Native raw-frame byte size exceeds the configured byte cap"); + } + } + + private notifyWaiters(): void { + for (const waiter of [...this.waiters]) { + if (!this.canAccept(waiter.frameByteSize)) { + continue; + } + this.waiters.delete(waiter); + waiter.resolve(); + } + } +} + export function getPreferredWebCodecsLatencyModes( encodingMode?: ExportEncodingMode, ): readonly WebCodecsLatencyMode[] { @@ -110,6 +266,29 @@ export function getWebCodecsKeyFrameInterval( return Math.max(1, Math.round(frameRate * KEYFRAME_INTERVAL_SECONDS[resolvedEncodingMode])); } +function createExportBackpressureProfile(options: { + name: string; + maxEncodeQueue: number; + maxDecodeQueue: number; + maxPendingFrames: number; + maxInFlightNativeWrites: number; + maxInFlightNativeRawFrames: number; + width: number; + height: number; +}): ExportBackpressureProfile { + return { + name: options.name, + maxEncodeQueue: options.maxEncodeQueue, + maxDecodeQueue: options.maxDecodeQueue, + maxPendingFrames: options.maxPendingFrames, + maxInFlightNativeWrites: options.maxInFlightNativeWrites, + maxInFlightNativeRawFrames: options.maxInFlightNativeRawFrames, + maxInFlightNativeRawBytes: + getNativeRawFrameByteSize(options.width, options.height) * + options.maxInFlightNativeRawFrames, + }; +} + export function getExportBackpressureProfile( options: ExportBackpressureProfileOptions, ): ExportBackpressureProfile { @@ -127,59 +306,77 @@ export function getExportBackpressureProfile( if (options.encodeBackend === "ffmpeg") { if (isLowCoreSystem || isExtremeWorkload) { - return { + return createExportBackpressureProfile({ name: "breeze-conservative", maxEncodeQueue, maxDecodeQueue: 8, maxPendingFrames: 16, maxInFlightNativeWrites: 2, - }; + maxInFlightNativeRawFrames: 2, + width: options.width, + height: options.height, + }); } if (isHighCoreSystem && !isHeavyWorkload) { - return { + return createExportBackpressureProfile({ name: "breeze-balanced-plus", maxEncodeQueue, maxDecodeQueue: 14, maxPendingFrames: 40, maxInFlightNativeWrites: 8, - }; + maxInFlightNativeRawFrames: 4, + width: options.width, + height: options.height, + }); } - return { + return createExportBackpressureProfile({ name: "breeze-balanced", maxEncodeQueue, maxDecodeQueue: 12, maxPendingFrames: 28, maxInFlightNativeWrites: 4, - }; + maxInFlightNativeRawFrames: 4, + width: options.width, + height: options.height, + }); } if (isLowCoreSystem || isExtremeWorkload) { - return { + return createExportBackpressureProfile({ name: "webcodecs-conservative", maxEncodeQueue, maxDecodeQueue: 8, maxPendingFrames: 20, maxInFlightNativeWrites: 1, - }; + maxInFlightNativeRawFrames: 1, + width: options.width, + height: options.height, + }); } if (isHighCoreSystem && !isHeavyWorkload) { - return { + return createExportBackpressureProfile({ name: "webcodecs-balanced-plus", maxEncodeQueue, maxDecodeQueue: 12, maxPendingFrames: 32, maxInFlightNativeWrites: 1, - }; + maxInFlightNativeRawFrames: 1, + width: options.width, + height: options.height, + }); } - return { + return createExportBackpressureProfile({ name: "webcodecs-balanced", maxEncodeQueue, maxDecodeQueue: 10, maxPendingFrames: 24, maxInFlightNativeWrites: 1, - }; + maxInFlightNativeRawFrames: 1, + width: options.width, + height: options.height, + }); } diff --git a/src/lib/exporter/index.ts b/src/lib/exporter/index.ts index 3561cb3a1..e004a596d 100644 --- a/src/lib/exporter/index.ts +++ b/src/lib/exporter/index.ts @@ -1,3 +1,26 @@ +export type { + CursorRect, + CursorSpriteCaptureResult, + CursorSpriteCapturerOptions, + CursorSpriteExpansion, + CursorSpriteRenderer, + CursorSpriteRoiResult, + CursorSpriteStripData, +} from "./cursorSpriteOverlay"; +export { + buildCursorSpriteRenderTransform, + CursorSpriteCapturer, + clampCursorRoiToCanvas, + DEFAULT_CURSOR_SPRITE_EXPANSION, + expandCursorBounds, + isValidCursorBounds, + resolveCursorRoi, +} from "./cursorSpriteOverlay"; +export { + clampCustomBitrateMbps, + customBitrateMbpsToBps, + resolveExportBitrate, +} from "./exportBitrate"; export { FrameRenderer } from "./frameRenderer"; export { calculateOutputDimensions, GifExporter } from "./gifExporter"; export { ModernVideoExporter } from "./modernVideoExporter"; @@ -12,11 +35,26 @@ export { resolveSupportedMp4EncoderPath, } from "./mp4Support"; export { VideoMuxer } from "./muxer"; +export type { + NativeCursorSpriteOverlayLayer, + NativeCursorSpritePosition, + NativeStaticLayoutOverlayLayer, +} from "./nativeStaticLayoutOverlays"; +export { + clampNativeCursorSpritePosition, + getNativeStaticLayoutOverlayFrameByteSize, + isNativeCursorSpriteOverlayLayer, + sortNativeStaticLayoutOverlayLayers, + validateNativeCursorSpriteOverlayLayer, + validateNativeStaticLayoutOverlayLayer, +} from "./nativeStaticLayoutOverlays"; export { StreamingVideoDecoder } from "./streamingDecoder"; export type { ExportBackendPreference, + ExportBitrateMode, ExportConfig, ExportEncodeBackend, + ExportEncoderPreference, ExportEncodingMode, ExportFormat, ExportMetrics, @@ -27,12 +65,18 @@ export type { ExportRenderBackend, ExportResult, ExportSettings, + ExportVideoCodec, GifExportConfig, GifFrameRate, GifSizePreset, VideoFrameData, } from "./types"; export { + EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_H264_MAX_MBPS, + EXPORT_BITRATE_HEVC_MAX_MBPS, + EXPORT_BITRATE_MAX_MBPS, + EXPORT_BITRATE_MIN_MBPS, GIF_FRAME_RATES, GIF_SIZE_PRESETS, isValidGifFrameRate, diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index 612a58985..8df18fb27 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -218,7 +218,9 @@ describe("ModernFrameRenderer Pixi lifecycle", () => { }; renderer.config.preferredRenderBackend = "webgpu"; - await expect(renderer.createPixiApplication({} as HTMLCanvasElement)).resolves.toMatchObject({ + await expect( + renderer.createPixiApplication({} as HTMLCanvasElement), + ).resolves.toMatchObject({ backend: "webgl", }); @@ -289,6 +291,40 @@ describe("ModernFrameRenderer blur export path", () => { expect(renderer.capturePixelsForNativeExport()).not.toBeNull(); }); + it("builds the overlay layout cache lazily so a fresh overlay renderer can render frames", async () => { + vi.clearAllMocks(); + const renderer = createRenderer() as any; + const canvas = createMockCanvas(); + renderer.app = { canvas, render: vi.fn() }; + renderer.cameraContainer = { visible: true }; + renderer.videoEffectsContainer = { visible: true }; + renderer.frameContainer = { visible: true }; + renderer.cursorContainer = { visible: true }; + renderer.annotationContainer = { visible: true }; + renderer.overlayContainer = { visible: true }; + renderer.captionContainer = { visible: true }; + renderer.backgroundContainer = { visible: true }; + renderer.webcamRootContainer = { visible: true }; + renderer.webcamMaskGraphics = {}; + renderer.annotationSprites = []; + renderer.layoutCache = null; + + // A freshly created overlay renderer never stages source-video frames, so + // its layout cache must be built from the export config instead of from the + // video-sprite layout path. Before the fix, renderOverlayFrame threw + // "Overlay renderer is not initialized" here and the native overlay sidecar + // preparation failed (native-overlay-preparation-failed). + await expect(renderer.renderOverlayFrame(0)).resolves.toBeUndefined(); + expect(renderer.layoutCache).not.toBeNull(); + expect(renderer.layoutCache.maskRect).toMatchObject({ + x: 0, + y: 0, + width: 1920, + height: 1080, + }); + expect(renderer.app.render).toHaveBeenCalledTimes(1); + }); + it("uses the sampled scene transform for blur annotations during temporal blur", async () => { const renderer = createRenderer() as any; renderer.config.zoomTemporalMotionBlur = 1; @@ -639,14 +675,14 @@ describe("ModernFrameRenderer webcam export fallback", () => { }; renderer.config.webcamUrl = "file:///tmp/webcam.webm"; - await renderer.setupWebcamSource(); - const syncPromise = renderer.syncWebcamFrame(1); + await renderer.setupWebcamSource(); + const syncPromise = renderer.syncWebcamFrame(1); await vi.advanceTimersByTimeAsync(5_001); - await expect(syncPromise).resolves.toBeUndefined(); + await expect(syncPromise).resolves.toBeUndefined(); - expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); - expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); + expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); + expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); expect(revoke).toHaveBeenCalled(); expect(renderer.webcamForwardFrameSource).toBeNull(); expect(renderer.webcamVideoElement).toBeNull(); diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 33aad0dde..b9c4b9961 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -166,6 +166,13 @@ interface FrameRenderConfig { zoomClassicMode?: boolean; frame?: string | null; nativeReadbackMode?: "pixels" | "canvas"; + /** + * When true the cursor is owned by the native CUDA cursor atlas path and + * must not be baked into the transparent overlay sidecar. Set only for the + * native static-layout overlay renderer; the full render path never sets it, + * so cursor rendering there is unchanged. + */ + excludeCursorOverlay?: boolean; } interface AnimationState { @@ -594,7 +601,7 @@ export class FrameRenderer { this.webcamContainer.addChild(this.webcamMaskGraphics); this.webcamContainer.mask = this.webcamMaskGraphics; - if (cursorOverlayEnabled) { + if (cursorOverlayEnabled && this.config.excludeCursorOverlay !== true) { this.cursorOverlay = new PixiCursorOverlay({ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), style: this.config.cursorStyle ?? "tahoe", @@ -3315,6 +3322,91 @@ export class FrameRenderer { ); } + /** Render only transparent UI/effect layers for native CUDA composition. */ + async renderOverlayFrame( + timestamp: number, + cursorTimestamp = timestamp, + backgroundTimelineTimestamp = timestamp, + ): Promise { + if ( + !this.app || + !this.cameraContainer || + !this.videoEffectsContainer || + !this.frameContainer || + !this.cursorContainer || + !this.annotationContainer || + !this.overlayContainer || + !this.captionContainer + ) { + throw new Error("Overlay renderer is not initialized"); + } + // The overlay renderer never stages source-video frames, so its layout + // cache is not populated by the video-sprite layout path. Build the stage + // mask/layout from the export config so cursor/caption/annotation/webcam + // positioning matches the native CUDA compositor's padded layout. + this.ensureOverlayLayoutCache(); + if (!this.layoutCache) { + throw new Error("Overlay renderer layout is unavailable"); + } + + this.currentVideoTime = timestamp / 1_000_000; + const webcamTimeSeconds = Math.max(0, backgroundTimelineTimestamp / 1_000_000); + if (this.webcamForwardFrameSource || this.webcamVideoElement) { + await this.syncWebcamFrame(webcamTimeSeconds); + } + + const timeMs = this.currentVideoTime * 1000; + const cursorTimeMs = cursorTimestamp / 1000; + if (this.cursorOverlay) { + this.cursorOverlay.update( + this.config.cursorTelemetry ?? [], + cursorTimeMs, + this.layoutCache.maskRect, + this.config.showCursor ?? true, + false, + ); + } + + this.updateAnimationState(timeMs); + applyZoomTransform({ + cameraContainer: this.cameraContainer, + zoomBlurFilter: this.zoomBlurFilter, + motionBlurFilter: this.motionBlurFilter, + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: this.animationState.scale, + zoomProgress: this.animationState.progress, + focusX: this.animationState.focusX, + focusY: this.animationState.focusY, + isPlaying: true, + motionBlurAmount: 0, + motionBlurTuning: this.config.zoomMotionBlurTuning, + transformOverride: { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, + motionBlurState: this.motionBlurState, + frameTimeMs: timeMs, + }); + + this.updateAnnotationLayer(timeMs); + this.updateCaptionLayer(timeMs); + this.updateWebcamOverlay(webcamTimeSeconds); + + if (this.backgroundContainer) { + this.backgroundContainer.visible = false; + } + this.videoEffectsContainer.visible = false; + this.frameContainer.visible = true; + this.cursorContainer.visible = true; + this.annotationContainer.visible = true; + this.overlayContainer.visible = true; + this.captionContainer.visible = true; + this.app.render(); + this.outputCanvasOverride = null; + } + private compositeExtensions( timeMs: number, cursorTimeMs: number, @@ -3526,18 +3618,60 @@ export class FrameRenderer { } } - private updateLayout(): void { - if (!this.app || !this.videoSprite || !this.videoMaskGraphics) return; + private buildLayoutCacheFromConfig(): LayoutCache | null { + const { width, height, cropRegion, padding = 0, videoWidth, videoHeight } = this.config; + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + !cropRegion || + !Number.isFinite(videoWidth) || + !Number.isFinite(videoHeight) || + videoWidth <= 0 || + videoHeight <= 0 + ) { + return null; + } - const { + const layout = computePaddedLayout({ width, height, + padding, + frameInsets: this.frameInsets, cropRegion, - borderRadius = 0, - padding = 0, videoWidth, videoHeight, - } = this.config; + }); + + return { + stageSize: { width, height }, + videoSize: { + width: videoWidth * cropRegion.width, + height: videoHeight * cropRegion.height, + }, + baseScale: layout.scale, + baseOffset: { x: layout.spriteX, y: layout.spriteY }, + maskRect: { + x: layout.centerOffsetX, + y: layout.centerOffsetY, + width: layout.croppedDisplayWidth, + height: layout.croppedDisplayHeight, + sourceCrop: cropRegion, + }, + }; + } + + private ensureOverlayLayoutCache(): void { + if (this.layoutCache) { + return; + } + this.layoutCache = this.buildLayoutCacheFromConfig(); + this.updateFrameLayout(); + } + + private updateLayout(): void { + if (!this.app || !this.videoSprite || !this.videoMaskGraphics) return; + + const { width, height, cropRegion, borderRadius = 0, padding = 0 } = this.config; const layout = computePaddedLayout({ width, @@ -3545,8 +3679,8 @@ export class FrameRenderer { padding, frameInsets: this.frameInsets, cropRegion, - videoWidth, - videoHeight, + videoWidth: this.config.videoWidth, + videoHeight: this.config.videoHeight, }); this.videoSprite.scale.set(layout.scale); @@ -3572,22 +3706,7 @@ export class FrameRenderer { maskRadius: scaledBorderRadius, }); - this.layoutCache = { - stageSize: { width, height }, - videoSize: { - width: videoWidth * cropRegion.width, - height: videoHeight * cropRegion.height, - }, - baseScale: layout.scale, - baseOffset: { x: layout.spriteX, y: layout.spriteY }, - maskRect: { - x: layout.centerOffsetX, - y: layout.centerOffsetY, - width: layout.croppedDisplayWidth, - height: layout.croppedDisplayHeight, - sourceCrop: cropRegion, - }, - }; + this.layoutCache = this.buildLayoutCacheFromConfig(); this.updateFrameLayout(); } diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.ts index e1afc68c2..a491efd42 100644 --- a/src/lib/exporter/modernVideoExporter.fallback.test.ts +++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import type { ExportMetrics } from "./types"; const mocks = vi.hoisted(() => { const videoInfo = { @@ -79,6 +80,140 @@ describe("ModernVideoExporter native fallback routing", () => { vi.unstubAllGlobals(); }); + it("preserves the H.264 Auto non-raw route", async () => { + const exporter = new ModernVideoExporter({ + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + } as never) as unknown as { + shouldForceNativeRawFrame: () => boolean; + }; + + expect(exporter.shouldForceNativeRawFrame()).toBe(false); + }); + + it("allows eligible HEVC Auto and Hardware exports to try the GPU compositor", () => { + const autoExporter = new ModernVideoExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + } as never) as unknown as { + canUseNativeGpuStaticLayout: () => boolean; + shouldForceNativeRawFrame: () => boolean; + }; + const hardwareExporter = new ModernVideoExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + } as never) as unknown as { + canUseNativeGpuStaticLayout: () => boolean; + shouldForceNativeRawFrame: () => boolean; + }; + + expect(autoExporter.canUseNativeGpuStaticLayout()).toBe(true); + expect(autoExporter.shouldForceNativeRawFrame()).toBe(false); + expect(hardwareExporter.canUseNativeGpuStaticLayout()).toBe(true); + expect(hardwareExporter.shouldForceNativeRawFrame()).toBe(false); + }); + + it("keeps HEVC CPU exports on rawvideo and out of the GPU compositor", () => { + const exporter = new ModernVideoExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "cpu", + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + } as never) as unknown as { + canUseNativeGpuStaticLayout: () => boolean; + shouldForceNativeRawFrame: () => boolean; + }; + + expect(exporter.canUseNativeGpuStaticLayout()).toBe(false); + expect(exporter.shouldForceNativeRawFrame()).toBe(true); + }); + + it("passes high-level HEVC preference to rawvideo without deriving encoder names", async () => { + vi.stubGlobal("window", { + electronAPI: { + nativeVideoExportStart: vi.fn().mockResolvedValue({ + success: true, + sessionId: "hevc-raw-session", + encoderName: "hevc_nvenc", + }), + }, + }); + const exporter = new ModernVideoExporter({ + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + } as never) as unknown as { + tryStartNativeVideoExportRawFrame: () => Promise; + encoderName: string | null; + }; + + await expect(exporter.tryStartNativeVideoExportRawFrame()).resolves.toBe(true); + expect(window.electronAPI.nativeVideoExportStart).toHaveBeenCalledWith( + expect.objectContaining({ + inputMode: "rawvideo", + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ); + expect(exporter.encoderName).toBe("hevc_nvenc"); + }); + it("records negotiated raw transport and ACK metrics", async () => { + const exporter = new ModernVideoExporter({} as never) as unknown as { + buildExportMetrics: () => ExportMetrics; + nativeTransportMode: "transferable-stream"; + nativeRawBytesSubmitted: number; + nativeRawFramesSubmitted: number; + nativeWriteTimeMs: number; + nativeWriteAckTimeMs: number; + nativeFrameTransportTimeMs: number; + peakNativeWriteInFlightBytes: number; + }; + exporter.nativeTransportMode = "transferable-stream"; + exporter.nativeRawBytesSubmitted = 16; + exporter.nativeRawFramesSubmitted = 2; + exporter.nativeWriteTimeMs = 12; + exporter.nativeWriteAckTimeMs = 12; + exporter.nativeFrameTransportTimeMs = 12; + exporter.peakNativeWriteInFlightBytes = 8; + + const metrics = exporter.buildExportMetrics(); + + expect(metrics.nativeTransportMode).toBe("transferable-stream"); + expect(metrics.nativeRawBytesSubmitted).toBe(16); + expect(metrics.nativeWriteMs).toBe(12); + expect(metrics.nativeWriteAckMs).toBe(12); + expect(metrics.averageNativeFrameTransportMs).toBe(6); + expect(metrics.peakNativeWriteInFlightBytes).toBe(8); + }); + + it("falls back to cloned IPC when raw channel negotiation is unavailable", async () => { + vi.stubGlobal("window", { + electronAPI: { + nativeVideoExportOpenFrameChannel: vi + .fn() + .mockResolvedValue({ success: false, error: "probe failed" }), + nativeVideoExportWriteFrameViaChannel: vi.fn(), + }, + }); + const exporter = new ModernVideoExporter({} as never) as unknown as { + negotiateNativeRawFrameTransport: (sessionId: string) => Promise; + nativeTransportMode: string | null; + nativeTransportFallbackReason: string | null; + }; + + await exporter.negotiateNativeRawFrameTransport("session"); + + expect(exporter.nativeTransportMode).toBe("cloned-ipc"); + expect(exporter.nativeTransportFallbackReason).toBe("probe failed"); + }); + it("falls back to WebCodecs instead of surfacing a native error when Breeze is unavailable", async () => { const exporter = new ModernVideoExporter({ videoUrl: "file:///recording.mp4", @@ -124,6 +259,172 @@ describe("ModernVideoExporter native fallback routing", () => { expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1); }, 15_000); + it("finishes eligible HEVC GPU exports before creating the canvas renderer", async () => { + const staticLayoutResult = { + success: true, + tempFilePath: "C:/Temp/hevc-gpu.mp4", + }; + const exporter = new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + backendPreference: "auto", + } as never) as unknown as { + export: () => Promise<{ success: boolean; tempFilePath?: string }>; + loadNativeStaticLayoutVideoInfo: () => Promise; + tryExportNativeStaticLayout: () => Promise; + tryStartNativeVideoExportRawFrame: () => Promise; + }; + + const loadNativeStaticLayoutVideoInfo = vi + .spyOn(exporter, "loadNativeStaticLayoutVideoInfo") + .mockResolvedValue(mocks.videoInfo); + const tryExportNativeStaticLayout = vi + .spyOn(exporter, "tryExportNativeStaticLayout") + .mockResolvedValue(staticLayoutResult); + const tryStartNativeVideoExportRawFrame = vi + .spyOn(exporter, "tryStartNativeVideoExportRawFrame") + .mockResolvedValue(true); + + const result = await exporter.export(); + + expect(result).toEqual(staticLayoutResult); + expect(loadNativeStaticLayoutVideoInfo).toHaveBeenCalledTimes(1); + expect(tryExportNativeStaticLayout).toHaveBeenCalledTimes(1); + expect(tryStartNativeVideoExportRawFrame).not.toHaveBeenCalled(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.streamingDecoderLoadMetadata).not.toHaveBeenCalled(); + }); + it("tries native CUDA static layout first for HEVC Hardware", async () => { + const staticLayoutResult = { + success: true, + tempFilePath: "C:/Temp/hevc-hardware-gpu.mp4", + }; + const exporter = new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + backendPreference: "auto", + } as never) as unknown as { + export: () => Promise<{ success: boolean; tempFilePath?: string }>; + loadNativeStaticLayoutVideoInfo: () => Promise; + tryExportNativeStaticLayout: () => Promise; + tryStartNativeVideoExportRawFrame: () => Promise; + }; + + const loadNativeStaticLayoutVideoInfo = vi + .spyOn(exporter, "loadNativeStaticLayoutVideoInfo") + .mockResolvedValue(mocks.videoInfo); + const tryExportNativeStaticLayout = vi + .spyOn(exporter, "tryExportNativeStaticLayout") + .mockResolvedValue(staticLayoutResult); + const tryStartNativeVideoExportRawFrame = vi + .spyOn(exporter, "tryStartNativeVideoExportRawFrame") + .mockResolvedValue(true); + + const result = await exporter.export(); + + expect(result).toEqual(staticLayoutResult); + expect(loadNativeStaticLayoutVideoInfo).toHaveBeenCalledTimes(1); + expect(tryExportNativeStaticLayout).toHaveBeenCalledTimes(1); + expect(tryStartNativeVideoExportRawFrame).not.toHaveBeenCalled(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + }); + + it("falls back to HEVC rawvideo for unsupported canvas-only features", async () => { + vi.stubGlobal("window", { + electronAPI: { + nativeStaticLayoutExport: vi.fn(), + nativeStaticLayoutExportCancel: vi.fn(), + }, + }); + mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1); + const exporter = new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + annotationRegions: [{ id: "annotation-1", startMs: 0, endMs: 500 }], + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + backendPreference: "auto", + } as never) as unknown as { + export: () => Promise<{ success: boolean; tempFilePath?: string }>; + loadNativeStaticLayoutVideoInfo: () => Promise; + tryStartNativeVideoExportRawFrame: () => Promise; + finishNativeVideoExport: () => Promise; + nativeStaticLayoutSkipReasons: string[]; + nativeRawFrameMode: boolean; + configureNativeRawFrameBackpressure: () => void; + }; + + const loadNativeStaticLayoutVideoInfo = vi + .spyOn(exporter, "loadNativeStaticLayoutVideoInfo") + .mockResolvedValue(mocks.videoInfo); + const tryStartNativeVideoExportRawFrame = vi + .spyOn(exporter, "tryStartNativeVideoExportRawFrame") + .mockImplementation(async () => { + exporter.nativeRawFrameMode = true; + return true; + }); + const configureNativeRawFrameBackpressure = vi.spyOn( + exporter, + "configureNativeRawFrameBackpressure", + ); + vi.spyOn(exporter, "finishNativeVideoExport").mockResolvedValue({ + success: true, + tempFilePath: "C:/Temp/hevc-raw.mp4", + }); + + const result = await exporter.export(); + + expect(result.success).toBe(true); + expect(loadNativeStaticLayoutVideoInfo).toHaveBeenCalledTimes(1); + expect(window.electronAPI.nativeStaticLayoutExport).not.toHaveBeenCalled(); + expect(tryStartNativeVideoExportRawFrame).toHaveBeenCalledTimes(1); + expect(exporter.nativeStaticLayoutSkipReasons).toContain( + "native-overlay-preparation-failed", + ); + expect(configureNativeRawFrameBackpressure).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + }); + it("keeps Windows auto exports on the streaming native route before static layout", async () => { vi.stubGlobal("navigator", { platform: "Win32", diff --git a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts index f0e57d13d..aac003e50 100644 --- a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts +++ b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts @@ -1,6 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AudioRegion, SpeedRegion } from "@/components/video-editor/types"; -import { ModernVideoExporter } from "./modernVideoExporter"; +import { + ModernVideoExporter, + shouldRejectNativeStaticLayoutResultForEffectPreservation, + shouldSkipForMissingCursorAtlas, +} from "./modernVideoExporter"; import type { DecodedVideoInfo } from "./streamingDecoder"; const videoInfo: DecodedVideoInfo = { @@ -20,6 +24,7 @@ function createExporter(overrides: Record = {}) { electronAPI: { nativeStaticLayoutExport: vi.fn(), nativeStaticLayoutExportCancel: vi.fn(), + discardExportedTemp: vi.fn(async () => ({ success: true })), }, }); @@ -40,6 +45,12 @@ function createExporter(overrides: Record = {}) { ...overrides, } as never) as unknown as { buildNativeAudioPlan: (videoInfo: DecodedVideoInfo) => unknown; + tryExportNativeStaticLayout: ( + videoInfo: DecodedVideoInfo, + audioPlan: unknown, + effectiveDurationSec: number, + totalFrames: number, + ) => Promise; buildNativeStaticLayoutVideoTimelineSegments: (videoInfo: DecodedVideoInfo) => Array<{ sourceStartMs: number; sourceEndMs: number; @@ -58,6 +69,9 @@ function createExporter(overrides: Record = {}) { videoInfo: DecodedVideoInfo, effectiveDurationSec: number, ) => string[]; + shouldForceNativeRawFrame: () => boolean; + requiresStrictNativeCudaRoute: () => boolean; + buildStrictNativeCudaHardwareError: (reason: string) => Error; getNativeStaticLayoutSourceCrop: (videoInfo: DecodedVideoInfo) => { x: number; y: number; @@ -70,6 +84,26 @@ function createExporter(overrides: Record = {}) { wallpaper: string, ) => CanvasGradient | null; getNativeStaticLayoutCursorSize: (contentWidth: number) => number; + getNativeStaticLayoutZoomTelemetry: ( + layout: { + centerOffsetX: number; + centerOffsetY: number; + croppedDisplayWidth: number; + croppedDisplayHeight: number; + }, + totalFrames: number, + cursorTelemetry?: unknown, + ) => + | Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength: number; + blurCenterX: number; + blurCenterY: number; + }> + | undefined; }; } @@ -109,6 +143,101 @@ describe("ModernVideoExporter native static-layout eligibility", () => { }); }); + it("passes HEVC codec and encoder preference to native static-layout", async () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + }); + const nativeStaticLayoutExport = window.electronAPI.nativeStaticLayoutExport; + vi.mocked(nativeStaticLayoutExport).mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + }); + + await expect( + exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), + ).resolves.toMatchObject({ success: true, tempFilePath: "C:/Temp/hevc-static.mp4" }); + expect(nativeStaticLayoutExport).toHaveBeenCalledWith( + expect.objectContaining({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ); + }); + + it.each([ + "cuda-overlay", + "cuda-scale-cpu-pad", + "cuda-static-composite", + ] as const)("accepts the FFmpeg CUDA HEVC route %s", async (route) => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + experimentalNvidiaCudaExport: true, + }); + vi.mocked(window.electronAPI.nativeStaticLayoutExport).mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-cuda.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route, + }); + + await expect( + exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), + ).resolves.toMatchObject({ success: true, tempFilePath: "C:/Temp/hevc-cuda.mp4" }); + }); + + it("rejects FFmpeg CUDA routes for strict HEVC Hardware", async () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + }); + vi.mocked(window.electronAPI.nativeStaticLayoutExport).mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-cuda.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "cuda-overlay", + }); + + await expect( + exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), + ).resolves.toBeNull(); + // The produced temp video must not stay on disk for the session when the + // native result is rejected because it cannot satisfy the strict route. + expect(window.electronAPI.discardExportedTemp).toHaveBeenCalledWith( + "C:/Temp/hevc-cuda.mp4", + ); + }); + + it("rejects the H.264-only Windows GPU route for HEVC", async () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + experimentalNvidiaCudaExport: true, + }); + vi.mocked(window.electronAPI.nativeStaticLayoutExport).mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-windows-gpu.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "windows-d3d11-compositor", + }); + + await expect( + exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), + ).resolves.toBeNull(); + // The produced temp video must not stay on disk for the session when the + // native result is rejected because it cannot satisfy the requested codec. + expect(window.electronAPI.discardExportedTemp).toHaveBeenCalledWith( + "C:/Temp/hevc-windows-gpu.mp4", + ); + }); + it("allows native static-layout for H.264 source metadata", () => { const exporter = createExporter(); @@ -244,7 +373,7 @@ describe("ModernVideoExporter native static-layout eligibility", () => { expect(exporter.getNativeStaticLayoutCursorSize(480)).toBeCloseTo(46.2, 6); }); - it("skips native static-layout when cursor click effects are enabled", () => { + it("allows native static-layout when cursor click effects are enabled", () => { const exporter = createExporter({ showCursor: true, cursorClickEffect: "echo", @@ -263,10 +392,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 60, ), - ).toBe("unsupported-cursor-click-effect"); + ).toBeNull(); }); - it("skips native static-layout when click effects are enabled and cursor is hidden", () => { + it("does not require a cursor overlay when click effects are enabled but cursor is hidden", () => { const exporter = createExporter({ showCursor: false, cursorClickEffect: "echo", @@ -285,10 +414,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 60, ), - ).toBe("unsupported-cursor-click-effect"); + ).toBeNull(); }); - it("reports frame overlays as the remaining native overlay blocker", () => { + it("allows native static-layout with a frame overlay", () => { const exporter = createExporter({ frame: "macbook" }); expect( @@ -300,7 +429,7 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 60, ), - ).toBe("unsupported-frame-overlay"); + ).toBeNull(); }); it("allows native static-layout with background blur", () => { @@ -401,13 +530,193 @@ describe("ModernVideoExporter native static-layout eligibility", () => { ).toEqual([ "odd-output-dimensions", "unsupported-background-video", - "unsupported-annotation-overlay", - "unsupported-caption-overlay", + "overlay-layers-do-not-support-native-timeline", "unsupported-webcam-source", - "unsupported-frame-overlay", ]); }); + it("routes spatial zoom motion blur to the native CUDA path instead of skipping", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + zoomMotionBlur: 0.35, + }); + + const reasons = exporter.getNativeStaticLayoutSkipReasons( + { audioMode: "copy-source" }, + videoInfo, + 60, + ); + + expect(reasons).not.toContain("unsupported-motion-blur"); + }); + + it("allows spatial zoom motion blur over overlay sidecars because the CUDA route composites both", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + zoomMotionBlur: 0.35, + annotationRegions: [{ id: "annotation-1", startMs: 0, endMs: 1_000 }], + }); + + const reasons = exporter.getNativeStaticLayoutSkipReasons( + { audioMode: "copy-source" }, + videoInfo, + 60, + ); + + // The generalized CUDA compositor applies the spatial zoom blur and then + // alpha-composites the transparent overlay sidecar, so the renderer no + // longer skips the whole static-layout attempt. Non-CUDA result routes are + // rejected after the export instead of dropping the effect silently. + expect(reasons).not.toContain("unsupported-motion-blur"); + }); + + it("routes temporal zoom motion blur to the CUDA compositor when enabled", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + zoomTemporalMotionBlur: 0.5, + zoomMotionBlurSampleCount: 5, + zoomMotionBlurShutterFraction: 0.5, + }); + + const reasons = exporter.getNativeStaticLayoutSkipReasons( + { audioMode: "copy-source" }, + videoInfo, + 60, + ); + + // The generalized CUDA compositor implements the temporal sample plan + // natively, so the static-layout route is no longer skipped for it. + expect(reasons).not.toContain("unsupported-temporal-motion-blur"); + }); + + it("keeps temporal zoom motion blur an explicit faithful fallback without the CUDA route", () => { + const exporter = createExporter({ + zoomTemporalMotionBlur: 0.5, + zoomMotionBlurSampleCount: 5, + zoomMotionBlurShutterFraction: 0.5, + }); + + const reasons = exporter.getNativeStaticLayoutSkipReasons( + { audioMode: "copy-source" }, + videoInfo, + 60, + ); + + expect(reasons).toContain("unsupported-temporal-motion-blur"); + // The reason must not be duplicated by other checks and must not share the + // spatial-blur reason string. + expect( + reasons.filter((reason) => reason === "unsupported-temporal-motion-blur"), + ).toHaveLength(1); + expect(reasons).not.toContain("unsupported-motion-blur"); + }); + + it("rejects non-CUDA result routes when zoom blur and overlay sidecars must both be preserved", () => { + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: true, + hasTemporalMotionBlur: false, + hasOverlayContent: true, + route: "cuda-overlay", + }), + ).toBe(true); + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: true, + hasTemporalMotionBlur: false, + hasOverlayContent: true, + route: "windows-d3d11-compositor", + }), + ).toBe(true); + }); + + it("rejects non-CUDA routes when temporal zoom motion blur is configured", () => { + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: false, + hasTemporalMotionBlur: true, + hasOverlayContent: false, + route: "cuda-scale-cpu-pad", + }), + ).toBe(true); + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: false, + hasTemporalMotionBlur: true, + hasOverlayContent: false, + route: "nvidia-cuda-compositor", + }), + ).toBe(false); + }); + + it("accepts the generalized CUDA route for zoom blur over overlay sidecars", () => { + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: true, + hasTemporalMotionBlur: false, + hasOverlayContent: true, + route: "nvidia-cuda-compositor", + }), + ).toBe(false); + }); + + it("does not reject overlay results without spatial zoom motion blur", () => { + expect( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur: false, + hasTemporalMotionBlur: false, + hasOverlayContent: true, + route: "cuda-overlay", + }), + ).toBe(false); + }); + + it("emits renderer-equivalent zoom blur telemetry for the native compositor", () => { + const exporter = createExporter({ + zoomRegions: [ + { + id: "zoom-1", + startMs: 0, + endMs: 2_000, + depth: 2, + focus: { cx: 0.5, cy: 0.5 }, + mode: "manual", + }, + ], + zoomMotionBlur: 0.35, + }); + + const telemetry = exporter.getNativeStaticLayoutZoomTelemetry( + { + centerOffsetX: 0, + centerOffsetY: 0, + croppedDisplayWidth: 1920, + croppedDisplayHeight: 1080, + }, + 60, + undefined, + ); + + expect(telemetry).toBeDefined(); + expect(telemetry?.length).toBe(60); + expect(telemetry?.[0].blurStrength).toBe(0); + const activeBlurSamples = (telemetry ?? []).filter( + (sample) => sample.blurStrength > 0.0005, + ); + expect(activeBlurSamples.length).toBeGreaterThan(0); + for (const sample of telemetry ?? []) { + expect(Number.isFinite(sample.blurCenterX)).toBe(true); + expect(Number.isFinite(sample.blurCenterY)).toBe(true); + expect(sample.blurStrength).toBeGreaterThanOrEqual(0); + } + }); + it("reports invalid crop geometry instead of passing native export bad coordinates", () => { const exporter = createExporter({ cropRegion: { x: 0, y: 0, width: 0, height: 1 }, @@ -418,6 +727,102 @@ describe("ModernVideoExporter native static-layout eligibility", () => { ); }); + it("does not require a native cursor atlas when the cursor is baked into the overlay sidecar", () => { + // Regression: the previous guard skipped the whole native static-layout + // route with "cursor-atlas-unavailable" for every cursor export (the atlas + // was intentionally null when overlay layers are used), forcing the slow + // renderer raw path. With overlay layers the cursor is baked into the + // sidecar so a missing atlas must not skip. + expect( + shouldSkipForMissingCursorAtlas({ + needsOverlayLayers: true, + hasCursorTelemetry: true, + hasCursorAtlas: false, + }), + ).toBe(false); + expect( + shouldSkipForMissingCursorAtlas({ + needsOverlayLayers: false, + hasCursorTelemetry: true, + hasCursorAtlas: false, + }), + ).toBe(true); + expect( + shouldSkipForMissingCursorAtlas({ + needsOverlayLayers: false, + hasCursorTelemetry: true, + hasCursorAtlas: true, + }), + ).toBe(false); + }); + + describe("strict HEVC Hardware CUDA policy", () => { + it("never forces the renderer raw frame path for HEVC Hardware", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + }); + + expect(exporter.requiresStrictNativeCudaRoute()).toBe(true); + // Even without the CUDA eligibility flags, the raw fallback must not be + // selected; the export hard-fails instead. + expect(exporter.shouldForceNativeRawFrame()).toBe(false); + }); + + it("keeps HEVC Auto and H.264 Auto free of the strict raw-frame ban", () => { + const hevcAuto = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "auto", + }); + expect(hevcAuto.requiresStrictNativeCudaRoute()).toBe(false); + + const h264Auto = createExporter({ + exportVideoCodec: "h264", + exportEncoderPreference: "auto", + }); + expect(h264Auto.requiresStrictNativeCudaRoute()).toBe(false); + }); + + it("builds a hard-fail error with the first skip reason and noCpuFallback", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + }); + + const error = exporter.buildStrictNativeCudaHardwareError("cursor-atlas-unavailable"); + expect(error.message).toContain("cursor-atlas-unavailable"); + expect(error.message).toContain("noCpuFallback:true"); + expect(error.message).toContain("requires the NVIDIA CUDA compositor"); + // The message must be actionable and must not point users at a hidden + // "experimental" toggle that no longer gates the mandatory route. + expect(error.message).toContain("switch the encoder preference to Auto"); + expect(error.message).not.toContain("experimental"); + expect((error as Error & { noCpuFallback?: boolean }).noCpuFallback).toBe(true); + }); + + it("surfaces the precise overlay preparation stage in the strict hard-fail error", () => { + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + }) as unknown as { + buildStrictNativeCudaHardwareError: (reason: string) => Error; + nativeStaticLayoutOverlayFailure: { stage: string; message: string } | null; + }; + exporter.nativeStaticLayoutOverlayFailure = { + stage: "overlay-renderer-frame", + message: "overlay-renderer-frame: Overlay renderer is not initialized", + }; + + const error = exporter.buildStrictNativeCudaHardwareError( + "native-overlay-preparation-failed", + ); + expect(error.message).toContain("native-overlay-preparation-failed"); + expect(error.message).toContain("overlay-renderer-frame"); + expect(error.message).toContain("Overlay renderer is not initialized"); + expect(error.message).toContain("noCpuFallback:true"); + }); + }); + it("materializes uploaded data-url image backgrounds for native static-layout", async () => { const jpegBytes = new Uint8Array([0xff, 0xd8, 0xff, 0xd9]); const dataUrl = `data:image/jpeg;base64,${Buffer.from(jpegBytes).toString("base64")}`; @@ -693,7 +1098,7 @@ describe("ModernVideoExporter native static-layout eligibility", () => { ).toBeNull(); }); - it("allows slow-speed webcam timelines through native source-time mapping", () => { + it("falls back for slow-speed webcam timelines with overlay layers", () => { const speedRegions: SpeedRegion[] = [ { id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 0.5 }, ]; @@ -714,10 +1119,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 63, ), - ).toBeNull(); + ).toBe("overlay-layers-do-not-support-native-timeline"); }); - it("skips native static layout for rectangular webcam overlays", () => { + it("allows native static layout for rectangular webcam overlays", () => { const exporter = createExporter({ webcam: { enabled: true, @@ -736,10 +1141,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 60, ), - ).toBe("unsupported-rectangular-webcam-overlay"); + ).toBeNull(); }); - it("allows native speed timelines with a resolvable webcam source", () => { + it("falls back for native speed timelines with overlay layers", () => { const speedRegions: SpeedRegion[] = [ { id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 1.5 }, ]; @@ -767,6 +1172,6 @@ describe("ModernVideoExporter native static-layout eligibility", () => { videoInfo, 59, ), - ).toBeNull(); + ).toBe("overlay-layers-do-not-support-native-timeline"); }); }); diff --git a/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts new file mode 100644 index 000000000..4e8b1f3ab --- /dev/null +++ b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts @@ -0,0 +1,1461 @@ +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import { NATIVE_TILED_OVERLAY_TILE_SIZE } from "./nativeStaticLayoutOverlays"; +import type { DecodedVideoInfo } from "./streamingDecoder"; + +const FRAME_BYTE_SIZE = 1920 * 1080 * 4; +const DEFAULT_FRAME_VALUE = 0xaa; +const TILE_BYTE_SIZE = NATIVE_TILED_OVERLAY_TILE_SIZE * NATIVE_TILED_OVERLAY_TILE_SIZE * 4; +const TILE_COLUMNS = Math.ceil(1920 / NATIVE_TILED_OVERLAY_TILE_SIZE); +const TILE_ROWS = Math.ceil(1080 / NATIVE_TILED_OVERLAY_TILE_SIZE); +const TILE_COUNT = TILE_COLUMNS * TILE_ROWS; +const STATIC_TILED_PAYLOAD_BYTES = TILE_COUNT * TILE_BYTE_SIZE; + +const frameSource = vi.hoisted(() => { + return { + values: [] as number[], + videoFrameCall: 0, + readbackCall: 0, + fill: (frameIndex: number, buffer: Uint8Array | Uint8ClampedArray) => { + const value = frameSource.values[frameIndex] ?? DEFAULT_FRAME_VALUE; + buffer.fill(value); + }, + }; +}); + +const mocks = vi.hoisted(() => { + const rendererCanvas = { + width: 1920, + height: 1080, + }; + const framesRendered: number[] = []; + // Cursor-sprite capture defaults to unavailable so the baked-cursor + // full-canvas sidecar fallback is the default behavior. Dedicated + // cursor-sprite tests override these to exercise the sprite success path. + // The defaults are re-applied in afterEach because mockReset() clears the + // implementations installed here by the hoisted factory. + const cursorSpriteDefaults = { + start: () => false, + capture: () => ({ + captured: false, + unavailableReason: "cursor sprite unavailable (mock)", + }), + finish: () => null, + cancel: () => {}, + }; + + return { + framesRendered, + rendererCanvas, + frameRendererDestroy: vi.fn(), + frameRendererGetCanvas: vi.fn(() => rendererCanvas), + frameRendererInitialize: vi.fn(async () => {}), + frameRendererRenderOverlayFrame: vi.fn(async (timestampUs: number) => { + framesRendered.push(timestampUs); + }), + frameRendererStartCursorSpriteCapture: vi.fn(cursorSpriteDefaults.start), + frameRendererCaptureCursorSpriteFrame: vi.fn(cursorSpriteDefaults.capture), + frameRendererFinishCursorSpriteCapture: vi.fn(cursorSpriteDefaults.finish), + frameRendererCancelCursorSpriteCapture: vi.fn(cursorSpriteDefaults.cancel), + cursorSpriteDefaults, + }; +}); + +vi.mock("./modernFrameRenderer", () => ({ + FrameRenderer: vi.fn().mockImplementation(function () { + return { + destroy: mocks.frameRendererDestroy, + getCanvas: mocks.frameRendererGetCanvas, + initialize: mocks.frameRendererInitialize, + renderOverlayFrame: mocks.frameRendererRenderOverlayFrame, + startCursorSpriteCapture: mocks.frameRendererStartCursorSpriteCapture, + captureCursorSpriteFrame: mocks.frameRendererCaptureCursorSpriteFrame, + finishCursorSpriteCapture: mocks.frameRendererFinishCursorSpriteCapture, + cancelCursorSpriteCapture: mocks.frameRendererCancelCursorSpriteCapture, + }; + }), +})); + +class FakeVideoFrame { + constructor( + public readonly source: unknown, + public readonly init: { timestamp?: number } = {}, + ) {} + + async copyTo( + buffer: Uint8Array, + options: { format?: string; layout?: Array<{ offset: number; stride: number }> }, + ): Promise { + frameSource.fill(frameSource.videoFrameCall, buffer); + frameSource.videoFrameCall += 1; + void options; + } + + close(): void { + // no-op + } +} + +class FakeOffscreenCanvas { + width = 1920; + height = 1080; + + getContext(): { + clearRect: () => void; + drawImage: () => void; + getImageData: () => { data: Uint8ClampedArray }; + } { + return { + clearRect: () => undefined, + drawImage: () => undefined, + getImageData: () => { + const data = new Uint8ClampedArray(FRAME_BYTE_SIZE); + frameSource.fill(frameSource.readbackCall, data); + frameSource.readbackCall += 1; + return { data }; + }, + }; + } +} + +function fillRect( + buffer: Uint8Array | Uint8ClampedArray, + x: number, + y: number, + width: number, + height: number, + color: number, +): void { + const endX = Math.min(1920, x + width); + const endY = Math.min(1080, y + height); + for (let rowY = y; rowY < endY; rowY += 1) { + const rowOffset = rowY * 1920 * 4; + for (let colX = x; colX < endX; colX += 1) { + const pixelOffset = rowOffset + colX * 4; + buffer[pixelOffset] = color; + buffer[pixelOffset + 1] = color; + buffer[pixelOffset + 2] = color; + buffer[pixelOffset + 3] = 0xff; + } + } +} + +function createWindowStub() { + const streamBytes: Record = {}; + const electronAPI = { + openExportStream: vi.fn(async ({ extension }: { extension: string }) => { + const streamId = `overlay-${extension}`; + const tempPath = `C:/Temp/overlay.${extension}`; + streamBytes[streamId] = 0; + return { success: true, streamId, tempPath }; + }), + writeExportStreamChunk: vi.fn( + async (streamId: string, offset: number, chunk: Uint8Array) => { + streamBytes[streamId] = Math.max( + streamBytes[streamId] ?? 0, + offset + chunk.byteLength, + ); + return { success: true }; + }, + ), + closeExportStream: vi.fn(async (streamId: string, options?: { abort?: boolean }) => { + const tempPath = `C:/Temp/overlay.${String(streamId).replace("overlay-", "")}`; + if (options?.abort) { + return { success: true, tempPath, bytesWritten: 0 }; + } + return { success: true, tempPath, bytesWritten: streamBytes[streamId] ?? 0 }; + }), + discardExportedTemp: vi.fn(async () => ({ success: true })), + nativeStaticLayoutExport: vi.fn(), + nativeStaticLayoutExportCancel: vi.fn(), + }; + vi.stubGlobal("window", { electronAPI }); + return electronAPI; +} + +function createExporter(overrides: Record = {}) { + return new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + cursorTelemetry: [{ timeMs: 0, cx: 0.25, cy: 0.35 }], + showCursor: true, + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + backendPreference: "auto", + ...overrides, + } as never) as unknown as { + prepareNativeStaticLayoutOverlay: ( + videoInfo: DecodedVideoInfo, + durationSec: number, + totalFrames: number, + cursorExcluded?: boolean, + onPreparationProgress?: (renderProgress: number) => void, + ) => Promise<{ + overlayLayers: Array>; + tiledOverlayLayers: Array>; + rawFallbackReason: string | null; + } | null>; + nativeStaticLayoutOverlayFailure: { stage: string; message: string } | null; + nativeStaticLayoutSkipReason: string | null; + nativeStaticLayoutSkipReasons: string[]; + hasNativeStaticLayoutOverlayContent: () => boolean; + tryExportNativeStaticLayout: ( + videoInfo: DecodedVideoInfo, + audioPlan: unknown, + effectiveDurationSec: number, + totalFrames: number, + ) => Promise<{ success: boolean; tempFilePath?: string; error?: string } | null>; + }; +} + +const videoInfo: DecodedVideoInfo = { + width: 1920, + height: 1080, + duration: 1, + streamDuration: 1, + frameRate: 30, + codec: "h264", + hasAudio: false, + audioCodec: null, + audioSampleRate: null, +}; + +let ModernVideoExporter: typeof ModernVideoExporterClass; + +describe("ModernVideoExporter native overlay preparation", () => { + beforeAll(async () => { + ({ ModernVideoExporter } = await import("./modernVideoExporter")); + }, 30_000); + + afterEach(() => { + frameSource.values = []; + frameSource.videoFrameCall = 0; + frameSource.readbackCall = 0; + frameSource.fill = (frameIndex, buffer) => { + const value = frameSource.values[frameIndex] ?? DEFAULT_FRAME_VALUE; + buffer.fill(value); + }; + vi.clearAllMocks(); + // Reset cursor-sprite mock implementations so the default (unavailable) + // fallback applies unless a test explicitly opts into the sprite path. + // mockReset() clears the defaults installed by the hoisted factory, so + // re-apply them here for tests that rely on the unavailable fallback. + mocks.frameRendererStartCursorSpriteCapture.mockReset(); + mocks.frameRendererCaptureCursorSpriteFrame.mockReset(); + mocks.frameRendererFinishCursorSpriteCapture.mockReset(); + mocks.frameRendererCancelCursorSpriteCapture.mockReset(); + mocks.frameRendererStartCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.start, + ); + mocks.frameRendererCaptureCursorSpriteFrame.mockImplementation( + mocks.cursorSpriteDefaults.capture, + ); + mocks.frameRendererFinishCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.finish, + ); + mocks.frameRendererCancelCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.cancel, + ); + vi.unstubAllGlobals(); + }); + + it("prepares a tiled overlay sidecar for a fully static overlay", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers[0]).toMatchObject({ + id: "native-effects", + order: 0, + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 1, + frameCount: 30, + tileSize: NATIVE_TILED_OVERLAY_TILE_SIZE, + pixelFormat: "rgba", + payloadPath: "C:/Temp/overlay.tiledrgba", + payloadByteLength: STATIC_TILED_PAYLOAD_BYTES, + staticTiles: expect.any(Array), + frameDeltas: [], + }); + expect(result?.rawFallbackReason).toBeNull(); + expect(result?.tiledOverlayLayers[0]?.staticTiles).toHaveLength(TILE_COUNT); + // The cursor-sprite path is attempted first (atlas not actually owned) and + // falls back to the baked sidecar when sprite capture is unavailable, so + // the overlay renderer is initialized once for the sprite attempt and + // once for the baked full-canvas sidecar. + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(2); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(api.writeExportStreamChunk).toHaveBeenCalledTimes(TILE_COUNT + 1); + const lastTiledWrite = api.writeExportStreamChunk.mock.calls[ + api.writeExportStreamChunk.mock.calls.length - 1 + ] as [string, number, Uint8Array]; + expect(lastTiledWrite[0]).toBe("overlay-tiledrgba"); + expect(lastTiledWrite[1]).toBe((TILE_COUNT - 1) * TILE_BYTE_SIZE); + expect(lastTiledWrite[2]).toHaveLength(TILE_BYTE_SIZE); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); + expect(exporter.nativeStaticLayoutOverlayFailure).toBeNull(); + }); + + it("falls back to a raw sidecar for a fully dynamic overlay", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + frameSource.values = Array.from({ length: 30 }, (_, index) => index); + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.overlayLayers[0]).toMatchObject({ + id: "native-effects", + path: "C:/Temp/overlay.rgba", + frameCount: 30, + pixelFormat: "rgba", + }); + expect("effectiveFrameCount" in (result?.overlayLayers[0] ?? {})).toBe(false); + expect(result?.rawFallbackReason).toBe("dense-frame-delta"); + // sprite + json export streams (sprite attempt) plus the baked rgba stream. + expect(api.openExportStream).toHaveBeenCalledTimes(3); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.writeExportStreamChunk).toHaveBeenCalledTimes(30); + const lastWrite = api.writeExportStreamChunk.mock.calls[29] as [string, number, Uint8Array]; + expect(lastWrite[0]).toBe("overlay-rgba"); + expect(lastWrite[1]).toBe(29 * FRAME_BYTE_SIZE); + expect(lastWrite[2][0]).toBe(29); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba"); + }); + + it("trims the identical raw suffix and preserves the raw fallback reason", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + frameSource.values = [ + 0, 1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, + ]; + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers[0]).toMatchObject({ + frameCount: 30, + effectiveFrameCount: 5, + }); + expect(result?.rawFallbackReason).toBe("dense-frame-delta"); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.writeExportStreamChunk).toHaveBeenCalledTimes(5); + for (let index = 0; index < 5; index += 1) { + const write = api.writeExportStreamChunk.mock.calls[index] as [ + string, + number, + Uint8Array, + ]; + expect(write[0]).toBe("overlay-rgba"); + expect(write[1]).toBe(index * FRAME_BYTE_SIZE); + expect(write[2][0]).toBe(index); + expect(write[2]).toHaveLength(FRAME_BYTE_SIZE); + } + }); + + it("records an overlay-renderer-frame failure stage when the overlay frame render throws", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererRenderOverlayFrame.mockRejectedValueOnce( + new Error("Overlay renderer is not initialized"), + ); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 1); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutOverlayFailure).toMatchObject({ + stage: "overlay-renderer-frame", + message: expect.stringContaining("Overlay renderer is not initialized"), + }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba", { abort: true }); + // One renderer for the sprite attempt, one for the baked sidecar. + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); + }); + + it("records an overlay-stream-truncated failure stage when the sidecar byte count is short", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + // Only the baked rgba stream finalize returns the truncated byte count; + // the cursor-sprite abort closes (sprite/json) return their defaults. + api.closeExportStream.mockImplementation( + async (streamId: string, options?: { abort?: boolean }) => { + const tempPath = `C:/Temp/overlay.${String(streamId).replace("overlay-", "")}`; + if (options?.abort) { + return { success: true, tempPath, bytesWritten: 0 }; + } + return { + success: true, + tempPath, + bytesWritten: streamId === "overlay-rgba" ? FRAME_BYTE_SIZE - 1 : 0, + }; + }, + ); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutOverlayFailure).toMatchObject({ + stage: "overlay-stream-truncated", + message: expect.stringContaining(`expected ${FRAME_BYTE_SIZE} bytes`), + }); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); + }); + + it("records an overlay-canvas-capture failure stage when canvas readback is unavailable", async () => { + vi.stubGlobal("VideoFrame", undefined); + vi.stubGlobal("OffscreenCanvas", undefined); + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 1); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutOverlayFailure).toMatchObject({ + stage: "overlay-canvas-capture", + }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba", { abort: true }); + }); + + it("prepares a sparse tiled sidecar for a small moving-region overlay", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + frameSource.fill = (frameIndex, buffer) => { + buffer.fill(0); + fillRect( + buffer, + 256, + 256, + NATIVE_TILED_OVERLAY_TILE_SIZE, + NATIVE_TILED_OVERLAY_TILE_SIZE, + frameIndex + 1, + ); + }; + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBeNull(); + const tileLayer = result?.tiledOverlayLayers[0] as Record; + expect(tileLayer.payloadByteLength).toBe(STATIC_TILED_PAYLOAD_BYTES + 29 * TILE_BYTE_SIZE); + expect(tileLayer.frameDeltas).toHaveLength(29); + for (let index = 0; index < 29; index += 1) { + const delta = ( + tileLayer.frameDeltas as Array<{ + frameIndex: number; + changedTiles: Array<{ tileIndex: number }>; + }> + )[index]; + expect(delta.frameIndex).toBe(index + 1); + expect(delta.changedTiles).toHaveLength(1); + expect(delta.changedTiles[0]?.tileIndex).toBe(2 * TILE_COLUMNS + 2); + } + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); + }); + + it("prepares a static tiled base when the overlay is fully transparent", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + frameSource.values = Array.from({ length: 30 }, () => 0); + createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(1); + const tileLayer = result?.tiledOverlayLayers[0] as Record; + expect(tileLayer.frameDeltas).toHaveLength(0); + expect(tileLayer.payloadByteLength).toBe(STATIC_TILED_PAYLOAD_BYTES); + expect(result?.rawFallbackReason).toBeNull(); + }); + + it("falls back to raw when a sparse overlay becomes dense mid-timeline", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + frameSource.fill = (frameIndex, buffer) => { + buffer.fill(0); + if (frameIndex <= 4) { + fillRect( + buffer, + 256, + 256, + NATIVE_TILED_OVERLAY_TILE_SIZE, + NATIVE_TILED_OVERLAY_TILE_SIZE, + frameIndex + 1, + ); + } else { + buffer.fill(frameIndex + 1); + } + }; + createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBe("dense-frame-delta"); + expect(result?.overlayLayers[0]).toMatchObject({ + path: "C:/Temp/overlay.rgba", + frameCount: 30, + }); + expect("effectiveFrameCount" in (result?.overlayLayers[0] ?? {})).toBe(false); + }); + + it("discards the raw sidecar and closes streams when the tiled sidecar is chosen", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).not.toBeNull(); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba"); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-tiledrgba"); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); + }); + + it("routes HEVC Hardware with overlay content to the native CUDA compositor with the tiled sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { + chunkCount: 1, + chunkDurationSec: 120, + chunkExecMs: 0, + chunks: [], + }, + }); + + // Cursor motion blur is a browser-rendered effect, so the cursor is baked + // into the transparent overlay sidecar (cursor-sidecar) rather than owned + // natively. This keeps the overlay-content path using existing preparation + // even when the deterministic no-browser-overlay fast lane is otherwise + // eligible. + const exporter = createExporter({ cursorMotionBlur: 1 }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true, tempFilePath: "C:/Temp/hevc-static.mp4" }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + const overlayLayers = exportCall[0].overlayLayers as Array>; + const tiledOverlayLayers = exportCall[0].tiledOverlayLayers as Array< + Record + >; + expect(overlayLayers ?? []).toHaveLength(0); + expect(tiledOverlayLayers).toHaveLength(1); + expect(tiledOverlayLayers[0]).toMatchObject({ + id: "native-effects", + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 1, + frameCount: 30, + tileSize: NATIVE_TILED_OVERLAY_TILE_SIZE, + pixelFormat: "rgba", + }); + expect(exportCall[0]).toMatchObject({ + videoCodec: "hevc", + encoderPreference: "hardware", + }); + }); + + it("returns no overlay sidecar when native CUDA owns the cursor and there are no browser-rendered pixels", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // cursorTelemetry + showCursor are set, but cursor render is excluded from + // the sidecar (native ownership) and there are no captions/annotations/ + // webcam/frame pixels, so the empty validated representation is correct. + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + expect(mocks.frameRendererDestroy).not.toHaveBeenCalled(); + }); + + it("still prepares the overlay sidecar when browser-rendered pixels coexist with native cursor ownership", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter({ frame: { enabled: true, width: 400, height: 300 } }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBeNull(); + }); + + it("coalesces and throttles preparation progress during tiled sidecar generation", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + createWindowStub(); + + const prepProgress: number[] = []; + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + false, + false, + (renderProgress) => prepProgress.push(renderProgress), + ); + + expect(result).not.toBeNull(); + // Every frame is identical, so all 30 frames would emit a raw per-frame + // update without coalescing. The bounded cadence must stay far below that. + expect(prepProgress.length).toBeGreaterThan(0); + expect(prepProgress.length).toBeLessThan(30); + for (const value of prepProgress) { + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThanOrEqual(100); + } + }); + + it("records a cancellation failure and aborts the overlay stream when cancelled mid-preparation", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter(); + (exporter as unknown as { cancelled: boolean }).cancelled = true; + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutOverlayFailure).toMatchObject({ + stage: "overlay-preparation", + }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba", { abort: true }); + // One renderer for the sprite attempt, one for the baked sidecar. + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); + }); + + it("reports an initial preparing route progress that identifies the CUDA compositor first", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { + chunkCount: 1, + chunkDurationSec: 120, + chunkExecMs: 0, + chunks: [], + }, + }); + + const emitted: Array> = []; + const exporter = createExporter({ + onProgress: (progress: Record) => emitted.push(progress), + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + expect(emitted.length).toBeGreaterThan(0); + expect(emitted[0]).toMatchObject({ + phase: "preparing", + currentFrame: 0, + encoderName: "nvidia-cuda-compositor", + encodeBackend: "ffmpeg", + }); + }); + + it("discards the produced temp video when the native route cannot preserve zoom motion blur", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-blur-route.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Spatial zoom motion blur over overlay content requires the generalized + // CUDA compositor. The FFmpeg effectful overlay route cannot preserve it, + // so the successful native result is rejected; the produced temp video + // (potentially GBs for HEVC) must not be left on disk for the session. + const exporter = createExporter({ + exportEncoderPreference: "auto", + zoomMotionBlur: 0.35, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe( + "unsupported-motion-blur-on-overlay-route", + ); + expect(exporter.nativeStaticLayoutSkipReasons).toContain( + "unsupported-motion-blur-on-overlay-route", + ); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/hevc-blur-route.mp4"); + }); + + it("prepares a cursor-sprite overlay layer for a renderer-baked cursor on the CUDA route", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + const width = 32; + const height = 32; + const frameCount = 30; + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width, + height, + frameCount, + frames: new Uint8Array(width * height * 4 * frameCount), + positions: Array.from({ length: frameCount }, (_, index) => ({ + x: 10 + index, + y: 20, + })), + }); + + // Cursor motion blur disables native atlas ownership, so the cursor is a + // renderer-baked ROI sprite instead of a full transparent canvas sidecar. + const exporter = createExporter({ cursorMotionBlur: 1 }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + frameCount, + false, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + x: 0, + y: 0, + width, + height, + frameRate: 30, + durationSec: 1, + frameCount, + pixelFormat: "rgba", + }); + expect(layer.positions).toHaveLength(frameCount); + expect(mocks.frameRendererStartCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererCaptureCursorSpriteFrame).toHaveBeenCalledTimes(frameCount); + expect(mocks.frameRendererFinishCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererCancelCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "json" }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-sprite"); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-json"); + }); + + it("uses the cursor-sprite path when the native atlas is eligible but not actually owned", async () => { + vi.stubGlobal("navigator", { platform: "Win32", userAgent: "node" }); + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // With the Win32 CUDA route the native atlas is *eligible*, but the atlas + // was not actually built/owned (cursorExcluded === false). The sprite path + // must run as the pixel-preserving fallback instead of forcing the + // expensive full-canvas tiled sidecar. Before the fix the atlas-eligibility + // gate blocked this and baked a full 4K sidecar per frame. + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("falls back to the baked cursor overlay sidecar when cursor-sprite capture is unavailable", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + // startCursorSpriteCapture defaults to false, so the cursor stays baked in + // the full transparent canvas sidecar (the preserved golden path). + + const exporter = createExporter({ cursorMotionBlur: 1 }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("rejects a non-CUDA native route that cannot compose the cursor sprite", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 1, y: 1 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, () => ({ x: 1, y: 1 })), + }); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "cuda-overlay", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + const exporter = createExporter({ cursorMotionBlur: 1, exportEncoderPreference: "auto" }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-cursor-sprite-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-cursor-sprite-route"); + }); + + it("uses the cursor-sprite ROI for a cursor-only H.264 CUDA export instead of baking a full 4K sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // H.264 CUDA-opt-in export with a cursor-only overlay (no browser pixels + // and no native atlas ownership). The generalized NVIDIA CUDA compositor + // consumes the cursor-sprite contract regardless of output codec, so the + // cheap ROI strip must be selected instead of baking the cursor into a full + // transparent 4K canvas per frame. Regression: the sprite path was gated on + // the HEVC-only canUseNativeGpuStaticLayout(), forcing H.264 CUDA cursor + // exports onto the ~1 minute full-canvas tiled sidecar this case reproduced. + const exporter = createExporter({ + exportVideoCodec: "h264", + exportEncoderPreference: "hardware", + }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("coalesces a long identical raw overlay run into bounded contiguous IPC writes", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + // 29 identical frames followed by one different frame produce a 29-frame + // identical middle run that must be coalesced into bounded contiguous IPC + // chunks instead of one writeExportStreamChunk call per frame. + frameSource.fill = (frameIndex: number, buffer: Uint8Array | Uint8ClampedArray) => { + buffer.fill(frameIndex >= 29 ? 0x11 : 0xaa); + }; + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + // The dense full-frame delta on the last frame keeps this on the raw sidecar. + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBe("dense-frame-delta"); + const rgbaWrites = api.writeExportStreamChunk.mock.calls.filter( + (call: [string, number, Uint8Array]) => call[0] === "overlay-rgba", + ) as Array<[string, number, Uint8Array]>; + // The 29 identical frames are coalesced; strictly fewer IPC calls than the + // 30 frames they represent. + expect(rgbaWrites.length).toBeLessThan(30); + // The coalesced batches must cover every overlay frame byte exactly once, + // preserving offsets and content (no dropped or duplicated pixels). + const totalBytes = rgbaWrites.reduce((sum, call) => sum + call[2].byteLength, 0); + expect(totalBytes).toBe(30 * FRAME_BYTE_SIZE); + let cursor = 0; + for (const [, offset, chunk] of rgbaWrites) { + expect(offset % FRAME_BYTE_SIZE).toBe(0); + expect(chunk.byteLength % FRAME_BYTE_SIZE).toBe(0); + expect(offset).toBe(cursor); + cursor += chunk.byteLength; + } + // The final (different) frame is present at the correct byte offset with the + // correct value. + expect(rgbaWrites[rgbaWrites.length - 1]?.[1]).toBe(29 * FRAME_BYTE_SIZE); + expect(rgbaWrites[rgbaWrites.length - 1]?.[2][0]).toBe(0x11); + }); + + it("uses the cursor-sprite ROI for the exact HEVC Hardware cursor-only CUDA case instead of a tiled sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // The user's reported case: HEVC Hardware with a cursor-only overlay (zoom + // is native, so it is not browser pixel content) on the CUDA route. The + // cursor-sprite ROI strip must be selected instead of baking a full + // transparent 4K canvas sidecar (which surfaces as tiledOverlayLayers: 1). + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("emits one-shot preparation stage diagnostics (overlay + IPC handoff) with codec/preference/route and elapsedMs", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { + chunkCount: 1, + chunkDurationSec: 120, + chunkExecMs: 0, + chunks: [], + }, + }); + + const infoMessages: Array<{ message: string; payload: Record }> = []; + const infoSpy = vi + .spyOn(console, "info") + .mockImplementation((first?: unknown, second?: unknown, third?: unknown) => { + const text = String(second ?? ""); + if (text.includes("Native static layout preparation stage")) { + infoMessages.push({ + message: text, + payload: (third ?? {}) as Record, + }); + } + }); + + const exporter = createExporter(); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const overlayStage = infoMessages.find((entry) => entry.payload.stage === "overlay"); + const ipcStage = infoMessages.find((entry) => entry.payload.stage === "ipc-handoff"); + expect(overlayStage).toBeDefined(); + expect(ipcStage).toBeDefined(); + expect(overlayStage?.payload).toMatchObject({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + route: "nvidia-cuda-compositor", + }); + expect(typeof overlayStage?.payload.elapsedMs).toBe("number"); + expect((overlayStage?.payload.elapsedMs as number) ?? 0).toBeGreaterThanOrEqual(0); + expect(ipcStage?.payload).toMatchObject({ + route: "nvidia-cuda-compositor", + success: true, + }); + infoSpy.mockRestore(); + }); + + it("makes the tiled route reason explicit when browser-rendered pixels coexist with the cursor", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + createWindowStub(); + + const infoPayloads: Array> = []; + const infoSpy = vi + .spyOn(console, "info") + .mockImplementation((message?: unknown, payload?: unknown) => { + if (String(message ?? "").includes("Cursor-sprite overlay path skipped")) { + infoPayloads.push((payload ?? {}) as Record); + } + }); + + // Native cursor ownership is active (Win32 CUDA route, no cursor effects) + // but a browser-rendered frame forces the baked full-canvas sidecar, which + // resolves to the tiled representation. This is necessary, not a sprite + // failure, and must be surfaced explicitly. + const exporter = createExporter({ frame: { enabled: true, width: 400, height: 300 } }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(infoPayloads).toHaveLength(1); + expect(infoPayloads[0]).toMatchObject({ + reason: "browser-overlay-pixels", + bakedSidecarRequired: true, + browserPixelSources: ["frame"], + }); + infoSpy.mockRestore(); + }); + + it("returns no overlay sidecar when the CUDA compositor owns a webcam-only overlay (webcam+zoom fast path)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // HEVC Hardware CUDA route with the webcam as the ONLY browser-rendered + // pixel (zoom is native, cursor is atlas-owned). The webcam is excluded + // from the renderer sidecar and the CUDA compositor draws it from + // webcamInputPath, so the sidecar is provably empty and must not render or + // read back a full 4K canvas per frame. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + true, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + expect(mocks.frameRendererDestroy).not.toHaveBeenCalled(); + }); + + it("still renders the baked webcam sidecar when captions coexist with the webcam (mixed overlay fallback)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // Captions are browser-rendered pixels, so the webcam cannot be owned + // natively: the existing baked full-canvas sidecar path must run unchanged + // (webcam stays in the sidecar; webcamInputPath is never sent alongside + // baked pixels, which prevents double-draw). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + autoCaptions: [{ startMs: 0, endMs: 1000, text: "Hi", lang: "en" }], + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + false, + ); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBeNull(); + }); + + it("keeps the webcam baked when a webcam shadow would be lost on the CUDA route", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // The CUDA compositor webcam overlay has no shadow support, so a shadowed + // webcam must stay on the baked sidecar path even when it is the only + // browser pixel. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4", shadow: 0.5 }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + false, + ); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("uses the cursor-sprite ROI for a webcam-native export and never bakes the webcam into the sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // Webcam-only browser pixels with a renderer-baked cursor: the webcam is + // excluded from the sidecar (native CUDA ownership) and the cursor is + // captured as the cheap ROI sprite, so no full 4K canvas is rendered per + // frame and the webcam is never double-drawn. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + false, + true, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("passes webcamInputPath and webcamNativeOwned for a webcam+zoom HEVC Hardware CUDA export", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Webcam is the only browser pixel and the cursor is disabled, so the + // deterministic fast lane selects an empty sidecar while the CUDA + // compositor owns the webcam natively: no renderer init, no per-frame + // canvas readback, and the webcam must reach the CUDA wrapper. + const exporter = createExporter({ + showCursor: false, + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].overlayLayers).toBeUndefined(); + expect(exportCall[0].tiledOverlayLayers).toBeUndefined(); + expect(exportCall[0].webcamInputPath).toBe("C:/webcam.mp4"); + expect(exportCall[0].webcamNativeOwned).toBe(true); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + }); + + it("passes webcamInputPath alongside a cursor-sprite overlay without double-drawing the webcam", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // Webcam native ownership + a renderer-baked cursor: the sidecar holds + // only the cursor-sprite ROI and the webcam still reaches the CUDA + // compositor, so the webcam is drawn exactly once (never also baked). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].overlayLayers).toHaveLength(1); + expect((exportCall[0].overlayLayers as Array>)[0]).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + }); + expect(exportCall[0].webcamInputPath).toBe("C:/webcam.mp4"); + expect(exportCall[0].webcamNativeOwned).toBe(true); + }); + + it("keeps webcamNativeOwned undefined and webcamInputPath null when captions coexist (baked path)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Captions are browser-rendered pixels, so the webcam stays baked in the + // sidecar and webcamInputPath must NOT be sent: sending it would make the + // CUDA compositor draw the webcam a second time (double-draw). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + autoCaptions: [{ startMs: 0, endMs: 1000, text: "Hi", lang: "en" }], + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].webcamInputPath).toBeNull(); + expect(exportCall[0].webcamNativeOwned).toBeUndefined(); + // The baked sidecar resolves to the tiled representation for static + // content: overlayLayers stays undefined and tiledOverlayLayers carries + // the sidecar pixels (cursor + captions + webcam baked together). + expect(exportCall[0].overlayLayers).toBeUndefined(); + expect(Array.isArray(exportCall[0].tiledOverlayLayers)).toBe(true); + expect((exportCall[0].tiledOverlayLayers as unknown[]).length).toBeGreaterThan(0); + expect(mocks.frameRendererInitialize).toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + }); + + it("keeps the webcam baked for HEVC Auto (non-strict) even when it is the only browser pixel", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-auto.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // HEVC Auto is not the strict CUDA-only route: a fallback could still + // render the webcam from the baked sidecar, so the webcam stays baked and + // is never excluded from the sidecar. + const exporter = createExporter({ + showCursor: false, + exportEncoderPreference: "auto", + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].webcamNativeOwned).toBeUndefined(); + expect(exportCall[0].webcamInputPath).toBeNull(); + expect(api.openExportStream).toHaveBeenCalled(); + expect(mocks.frameRendererInitialize).toHaveBeenCalled(); + }); +}); diff --git a/src/lib/exporter/modernVideoExporter.progressDedup.test.ts b/src/lib/exporter/modernVideoExporter.progressDedup.test.ts new file mode 100644 index 000000000..504900b21 --- /dev/null +++ b/src/lib/exporter/modernVideoExporter.progressDedup.test.ts @@ -0,0 +1,150 @@ +import { beforeAll, describe, expect, it, vi } from "vitest"; +import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import type { ExportProgress } from "./types"; + +describe("ModernVideoExporter reportProgress preparing dedup", () => { + let ModernVideoExporter: typeof ModernVideoExporterClass; + + beforeAll(async () => { + ({ ModernVideoExporter } = await import("./modernVideoExporter")); + }, 30_000); + + it("delivers the first preparing signal and suppresses identical repeats for one export", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + + expect(emitted).toHaveLength(1); + expect(emitted[0]).toMatchObject({ + currentFrame: 0, + totalFrames: 100, + phase: "preparing", + percentage: 0, + }); + }); + + it("does not suppress progress that carries render or audio progress", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing", undefined, 0.5); + reporter(0, 100, "preparing", undefined, 0.6); + + // The first plain preparing is suppressed for the second, but the two + // audio-progress-bearing signals each still carry their distinct values. + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => p.audioProgress)).toEqual([undefined, 0.5, 0.6]); + }); + + it("emits again once the export moves on and a different total frame count starts", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(50, 100, "extracting"); + reporter(0, 120, "preparing"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.totalFrames, p.phase])).toEqual([ + [0, 100, "preparing"], + [50, 100, "extracting"], + [0, 120, "preparing"], + ]); + }); + + it("re-delivers the preparing signal after a non-preparing phase reuses the same total frame count", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + // A non-preparing event ends the preparing phase, so the watermark must not + // suppress a later preparing phase that reuses the same total frame count. + reporter(50, 100, "extracting"); + reporter(0, 100, "preparing"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.totalFrames, p.phase])).toEqual([ + [0, 100, "preparing"], + [50, 100, "extracting"], + [0, 100, "preparing"], + ]); + }); + + it("keeps distributing later phase progress after the preparing signal", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(10, 100, "extracting"); + reporter(20, 100, "extracting"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.phase])).toEqual([ + [0, "preparing"], + [10, "extracting"], + [20, "extracting"], + ]); + }); +}); diff --git a/src/lib/exporter/modernVideoExporter.routeRejection.test.ts b/src/lib/exporter/modernVideoExporter.routeRejection.test.ts new file mode 100644 index 000000000..3ac22b8df --- /dev/null +++ b/src/lib/exporter/modernVideoExporter.routeRejection.test.ts @@ -0,0 +1,277 @@ +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import type { DecodedVideoInfo } from "./streamingDecoder"; + +const FRAME_BYTE_SIZE = 1920 * 1080 * 4; +const DEFAULT_FRAME_VALUE = 0xaa; + +const frameSource = vi.hoisted(() => { + return { + values: [] as number[], + videoFrameCall: 0, + readbackCall: 0, + fill: (frameIndex: number, buffer: Uint8Array | Uint8ClampedArray) => { + const value = frameSource.values[frameIndex] ?? DEFAULT_FRAME_VALUE; + buffer.fill(value); + }, + }; +}); + +const mocks = vi.hoisted(() => { + const rendererCanvas = { + width: 1920, + height: 1080, + }; + + return { + rendererCanvas, + frameRendererDestroy: vi.fn(), + frameRendererGetCanvas: vi.fn(() => rendererCanvas), + frameRendererInitialize: vi.fn(async () => {}), + frameRendererRenderOverlayFrame: vi.fn(async () => {}), + // Cursor-sprite capture defaults to unavailable so the baked-cursor + // full-canvas sidecar fallback is the default behavior. Dedicated + // cursor-sprite tests override these to exercise the sprite success path. + frameRendererStartCursorSpriteCapture: vi.fn(() => false), + frameRendererCaptureCursorSpriteFrame: vi.fn(() => ({ + captured: false, + unavailableReason: "cursor sprite unavailable (mock)", + })), + frameRendererFinishCursorSpriteCapture: vi.fn(() => null), + frameRendererCancelCursorSpriteCapture: vi.fn(() => {}), + }; +}); + +vi.mock("./modernFrameRenderer", () => ({ + FrameRenderer: vi.fn().mockImplementation(function () { + return { + destroy: mocks.frameRendererDestroy, + getCanvas: mocks.frameRendererGetCanvas, + initialize: mocks.frameRendererInitialize, + renderOverlayFrame: mocks.frameRendererRenderOverlayFrame, + startCursorSpriteCapture: mocks.frameRendererStartCursorSpriteCapture, + captureCursorSpriteFrame: mocks.frameRendererCaptureCursorSpriteFrame, + finishCursorSpriteCapture: mocks.frameRendererFinishCursorSpriteCapture, + cancelCursorSpriteCapture: mocks.frameRendererCancelCursorSpriteCapture, + }; + }), +})); + +class FakeVideoFrame { + constructor( + public readonly source: unknown, + public readonly init: { timestamp?: number } = {}, + ) {} + + async copyTo( + buffer: Uint8Array, + options: { format?: string; layout?: Array<{ offset: number; stride: number }> }, + ): Promise { + frameSource.fill(frameSource.videoFrameCall, buffer); + frameSource.videoFrameCall += 1; + void options; + } + + close(): void { + // no-op + } +} + +class FakeOffscreenCanvas { + width = 1920; + height = 1080; + + getContext(): { + clearRect: () => void; + drawImage: () => void; + getImageData: () => { data: Uint8ClampedArray }; + } { + return { + clearRect: () => undefined, + drawImage: () => undefined, + getImageData: () => { + const data = new Uint8ClampedArray(FRAME_BYTE_SIZE); + frameSource.fill(frameSource.readbackCall, data); + frameSource.readbackCall += 1; + return { data }; + }, + }; + } +} + +function createWindowStub() { + const streamBytes: Record = {}; + const electronAPI = { + openExportStream: vi.fn(async ({ extension }: { extension: string }) => { + const streamId = `overlay-${extension}`; + const tempPath = `C:/Temp/overlay.${extension}`; + streamBytes[streamId] = 0; + return { success: true, streamId, tempPath }; + }), + writeExportStreamChunk: vi.fn( + async (streamId: string, offset: number, chunk: Uint8Array) => { + streamBytes[streamId] = Math.max( + streamBytes[streamId] ?? 0, + offset + chunk.byteLength, + ); + return { success: true }; + }, + ), + closeExportStream: vi.fn(async (streamId: string) => { + const tempPath = `C:/Temp/overlay.${String(streamId).replace("overlay-", "")}`; + return { success: true, tempPath, bytesWritten: streamBytes[streamId] ?? 0 }; + }), + discardExportedTemp: vi.fn(async () => ({ success: true })), + nativeStaticLayoutExport: vi.fn(), + nativeStaticLayoutExportCancel: vi.fn(), + }; + vi.stubGlobal("window", { electronAPI }); + return electronAPI; +} + +function createExporter(overrides: Record = {}) { + return new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + ...overrides, + } as never) as unknown as { + tryExportNativeStaticLayout: ( + videoInfo: DecodedVideoInfo, + audioPlan: unknown, + effectiveDurationSec: number, + totalFrames: number, + ) => Promise<{ success: boolean; tempFilePath?: string; error?: string } | null>; + nativeStaticLayoutSkipReason: string | null; + nativeStaticLayoutSkipReasons: string[]; + canUseNativeWebcamOwnership: () => boolean; + }; +} + +const videoInfo: DecodedVideoInfo = { + width: 1920, + height: 1080, + duration: 1, + streamDuration: 1, + frameRate: 30, + codec: "h264", + hasAudio: false, + audioCodec: null, + audioSampleRate: null, +}; + +let ModernVideoExporter: typeof ModernVideoExporterClass; + +describe("ModernVideoExporter native static-layout route rejection cleanup", () => { + beforeAll(async () => { + ({ ModernVideoExporter } = await import("./modernVideoExporter")); + }, 30_000); + + afterEach(() => { + frameSource.values = []; + frameSource.videoFrameCall = 0; + frameSource.readbackCall = 0; + vi.clearAllMocks(); + // Reset cursor-sprite mock implementations so the default (unavailable) + // fallback applies unless a test explicitly opts into the sprite path. + mocks.frameRendererStartCursorSpriteCapture.mockReset(); + mocks.frameRendererCaptureCursorSpriteFrame.mockReset(); + mocks.frameRendererFinishCursorSpriteCapture.mockReset(); + mocks.frameRendererCancelCursorSpriteCapture.mockReset(); + vi.unstubAllGlobals(); + }); + + it("discards the produced temp video when the native route cannot compose the cursor sprite", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/h264-cursor-sprite.mp4", + videoCodec: "h264", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Cursor motion blur disables native atlas ownership, so the cursor is + // prepared as a cursor-sprite ROI layer. A non-CUDA result route cannot + // compose that contract, so the successful native result is rejected; the + // produced temp video (potentially GBs) must not be left on disk for the + // session. + const exporter = createExporter({ + showCursor: true, + cursorMotionBlur: 1, + cursorTelemetry: [ + { timeMs: 0, cx: 0.25, cy: 0.35 }, + { timeMs: 500, cx: 0.4, cy: 0.45 }, + ], + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-cursor-sprite-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-cursor-sprite-route"); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/h264-cursor-sprite.mp4"); + }); + + it("discards the produced temp video when the native route cannot draw the native-owned webcam", async () => { + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/h264-webcam-route.mp4", + videoCodec: "h264", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + const exporter = createExporter(); + // The public strict-HEVC policy already refuses non-CUDA routes earlier, + // so force native webcam ownership here to exercise the explicit + // webcam-route invariant on a codec/preference combination that reaches it. + exporter.canUseNativeWebcamOwnership = () => true; + + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-native-webcam-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-native-webcam-route"); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/h264-webcam-route.mp4"); + }); +}); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index c77ba9648..049061d62 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -37,11 +37,13 @@ import { } from "@/components/video-editor/videoPlayback/motionSmoothing"; import { getCursorStyleSizeMultiplier } from "@/components/video-editor/videoPlayback/uploadedCursorAssets"; import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; -import { computeZoomTransform } from "@/components/video-editor/videoPlayback/zoomTransform"; +import { + analyzeZoomMotionBlurStep, + computeZoomTransform, +} from "@/components/video-editor/videoPlayback/zoomTransform"; import { getWebcamOverlayPosition, getWebcamOverlaySizePx, - isWebcamCropRegionDefault, } from "@/components/video-editor/webcamOverlay"; import { extensionHost } from "@/lib/extensions"; import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; @@ -50,6 +52,7 @@ import { DEFAULT_WALLPAPER_RELATIVE_PATH, isVideoWallpaperSource, } from "@/lib/wallpapers"; +import { formatLogTs } from "../log"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { normalizeLightningRuntimePlatform, @@ -60,9 +63,12 @@ import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./e import { type ExportBackpressureProfile, getExportBackpressureProfile, + getNativeRawFrameBackpressureLimits, + getNativeRawFrameByteSize, getPreferredWebCodecsLatencyModes, getWebCodecsEncodeQueueLimit, getWebCodecsKeyFrameInterval, + NativeRawFrameBackpressureQueue, } from "./exportTuning"; import { advanceFinalizationProgress, @@ -78,16 +84,48 @@ import { type SupportedMp4EncoderPath, } from "./mp4Support"; import { VideoMuxer } from "./muxer"; +import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture"; import { roundNativeStaticLayoutContentSize } from "./nativeStaticLayoutGeometry"; +import type { + NativeCursorSpriteOverlayLayer, + NativeCursorSpritePosition, + NativeStaticLayoutOverlayLayer, + NativeTiledOverlayFrameDelta, + NativeTiledOverlayLayerDescriptor, + NativeTiledOverlayRawFallbackReason, + NativeTiledOverlayStaticTileRecord, + NativeTiledOverlayTileRecord, +} from "./nativeStaticLayoutOverlays"; +import { + areNativeStaticLayoutOverlayFramesEqual, + clampNativeCursorSpritePosition, + getNativeStaticLayoutOverlayFrameByteSize, + getNativeTiledOverlayTileColumns, + getNativeTiledOverlayTileCount, + getNativeTiledOverlayTileIndex, + getNativeTiledOverlayTileRows, + NATIVE_CURSOR_SPRITE_LAYER_KIND, + NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION, + NATIVE_TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION, + NATIVE_TILED_OVERLAY_PIXEL_FORMAT, + NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE, + NATIVE_TILED_OVERLAY_TILE_SIZE, + resolveNativeTiledOverlayRawFallbackReason, + sortNativeStaticLayoutOverlayLayers, + sortNativeTiledOverlayLayers, + validateNativeCursorSpriteOverlayLayer, +} from "./nativeStaticLayoutOverlays"; import { buildNativeStaticLayoutCursorTelemetry } from "./nativeStaticLayoutTelemetry"; import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder"; +import { getTemporalMotionBlurConfig } from "./temporalMotionBlur"; import type { ExportConfig, ExportEncodeBackend, ExportFfmpegAudioMuxBreakdown, ExportFinalizationStageMetrics, ExportMetrics, + ExportNativeTransportMode, ExportProgress, ExportRenderBackend, ExportResult, @@ -159,6 +197,36 @@ interface VideoExporterConfig extends ExportConfig { preferredEncoderPath?: SupportedMp4EncoderPath | null; } +/** + * Result shape for the native static-layout overlay sidecar. The renderer + * currently composites every overlay element (cursor, captions, annotations, + * webcam, frame) into a single transparent RGBA canvas in + * `ModernFrameRenderer.renderOverlayFrame`, so this result usually contains a + * single logical "native-effects" layer that is either tiled (sparse) or raw + * (dense/unsupported fallback). Both arrays are returned, sorted by order then + * id, and forwarded to `nativeStaticLayoutExport` so a future split renderer + * can emit mixed raw + tiled layers with preserved z-order; the native consumer + * in `electron/ipc/export/native-video.ts` already validates, sorts, and + * composites both lists. + */ + +type NativeStaticLayoutOverlayLayerUnion = + | NativeStaticLayoutOverlayLayer + | NativeCursorSpriteOverlayLayer; + +type NativeStaticLayoutOverlayPreparationResult = { + overlayLayers: NativeStaticLayoutOverlayLayerUnion[]; + tiledOverlayLayers: NativeTiledOverlayLayerDescriptor[]; + rawFallbackReason: NativeTiledOverlayRawFallbackReason | null; +}; + +/** Discriminates a cursor-sprite layer from a fixed-position rgba layer. */ +function isCursorSpriteOverlayLayer( + layer: NativeStaticLayoutOverlayLayerUnion, +): layer is NativeCursorSpriteOverlayLayer { + return (layer as { kind?: string }).kind === NATIVE_CURSOR_SPRITE_LAYER_KIND; +} + type NativeAudioPlan = | { audioMode: "none"; @@ -285,6 +353,16 @@ type NativeStaticLayoutZoomSample = { scale: number; x: number; y: number; + /** + * Renderer-equivalent radial zoom-blur strength for the step that ends at + * this sample (0 when the step is not zoom motion). Computed once per frame + * from the applied zoom telemetry so the CUDA compositor can reproduce the + * spatial ZoomBlurFilter without re-deriving camera-step analysis. + */ + blurStrength?: number; + /** Output-space zoom-blur center (pixels), matching the renderer's filter. */ + blurCenterX?: number; + blurCenterY?: number; }; const NATIVE_EXPORT_ENGINE_NAME = "Breeze"; @@ -300,6 +378,111 @@ const STATIC_LAYOUT_CHUNK_DURATION_SEC = 120; const MISSING_NATIVE_WALLPAPER_FALLBACK_COLOR = "#ffffff"; const NATIVE_STATIC_LAYOUT_MAX_EXTRACTING_PROGRESS = 95; const NATIVE_STATIC_LAYOUT_FRAME_COMPLETE_PROGRESS = 96; +const NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS = 300; + +// Upper bound in bytes for a single coalesced overlay sidecar IPC chunk. +// Pixel-identical consecutive frames in the raw sidecar are coalesced into one +// contiguous chunk (instead of one writeExportStreamChunk call per frame) to +// remove per-frame IPC round-trips for static overlay stretches, while capping +// the chunk so a long identical run never buffers the whole 4K sidecar at once. +const NATIVE_RAW_OVERLAY_RUN_BATCH_MAX_BYTES = 48 * 1024 * 1024; +const HEVC_NATIVE_STATIC_LAYOUT_ROUTES = new Set([ + "cuda-overlay", + "cuda-scale-cpu-pad", + "cuda-static-composite", + "nvidia-cuda-compositor", +]); + +/** + * The native cursor atlas is only required when the cursor is NOT baked into the + * transparent overlay sidecar. When overlay layers are prepared, the cursor is + * rendered into the sidecar, so a missing atlas must never skip the native + * static-layout route (previously this fell back to the slow renderer raw path + * for every cursor export). + */ +export function shouldSkipForMissingCursorAtlas(options: { + needsOverlayLayers: boolean; + hasCursorTelemetry: boolean; + hasCursorAtlas: boolean; +}): boolean { + return !options.needsOverlayLayers && options.hasCursorTelemetry && !options.hasCursorAtlas; +} + +/** + * A native static-layout result can only preserve zoom motion blur over + * transparent overlay sidecars, and temporal zoom motion blur in general, on + * the generalized CUDA compositor. Other routes (FFmpeg effectful overlay, + * D3D11 helper) would silently drop the effect, so the renderer rejects them + * and falls back to raw frames. + */ +export function shouldRejectNativeStaticLayoutResultForEffectPreservation(options: { + hasSpatialZoomMotionBlur: boolean; + hasTemporalMotionBlur: boolean; + hasOverlayContent: boolean; + route: string | null | undefined; +}): boolean { + const requiresCudaCompositor = + (options.hasSpatialZoomMotionBlur && options.hasOverlayContent) || + options.hasTemporalMotionBlur; + return requiresCudaCompositor && options.route !== "nvidia-cuda-compositor"; +} + +/** + * Detailed skip reason for the deterministic no-browser-overlay fast lane. + */ +export type NativeStaticLayoutFastLaneSkipReason = + | "not-native-cuda-route" + | "browser-overlay-pixels-present" + | "cursor-sidecar-required" + | "edited-audio-render-required" + | "native-source-not-authoritative"; + +export type NativeStaticLayoutFastLaneEligibility = { + eligible: boolean; + skipReasons: NativeStaticLayoutFastLaneSkipReason[]; +}; + +/** + * Deterministic no-browser-overlay fast lane for native CUDA static-layout + * export. When every visual input the browser would otherwise render is already + * authoritative natively (source video is a local file, no captions/annotations/ + * webcam/frame pixels, the cursor is either disabled or owned by the native CUDA + * compositor, and there is no edited-audio render), the export can skip renderer + * initialization, per-frame canvas capture, overlay sidecar creation, and cursor + * atlas generation and start the native export as early as safely allowed. + * + * The predicate is explicit and returns detailed skip reasons so callers can + * prove (and log) exactly why the fast lane was or was not selected. It never + * bypasses source validation/security, required audio muxing, timeline/zoom/ + * temporal native plans, cancellation/cleanup/progress settlement, or strict HEVC + * Hardware CUDA-only failure behavior. + */ +export function getNativeStaticLayoutFastLaneEligibility(options: { + canUseNativeGpuStaticLayout: boolean; + hasBrowserOverlayPixels: boolean; + cursorDisabled: boolean; + cursorNativeOwnershipActive: boolean; + requiresEditedAudioRender: boolean; + hasAuthoritativeNativeSource: boolean; +}): NativeStaticLayoutFastLaneEligibility { + const skipReasons: NativeStaticLayoutFastLaneSkipReason[] = []; + if (!options.canUseNativeGpuStaticLayout) { + skipReasons.push("not-native-cuda-route"); + } + if (options.hasBrowserOverlayPixels) { + skipReasons.push("browser-overlay-pixels-present"); + } + if (!options.cursorDisabled && !options.cursorNativeOwnershipActive) { + skipReasons.push("cursor-sidecar-required"); + } + if (options.requiresEditedAudioRender) { + skipReasons.push("edited-audio-render-required"); + } + if (!options.hasAuthoritativeNativeSource) { + skipReasons.push("native-source-not-authoritative"); + } + return { eligible: skipReasons.length === 0, skipReasons }; +} export class ModernVideoExporter { private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 64; @@ -329,17 +512,29 @@ export class ModernVideoExporter { private nativeExportSessionId: string | null = null; private nativeStaticLayoutSessionId: string | null = null; private nativeStaticLayoutAverageFps: number | null = null; + private nativeStaticLayoutFpsSource: "native" | "estimated" | null = null; private nativeWritePromises = new Set>(); + private nativeRawWritePromises = new Set>(); private nativeWriteError: Error | null = null; private pendingNativeWriteChunks: Uint8Array[] = []; private pendingNativeWriteBytes = 0; private maxNativeWriteInFlight = 1; + private nativeRawBackpressure: NativeRawFrameBackpressureQueue | null = null; + private maxNativeRawWriteFrames = 1; + private maxNativeRawWriteBytes = 0; + private nativeTransportMode: ExportNativeTransportMode | null = null; + private nativeTransportFallbackReason: string | null = null; private lastNativeExportError: string | null = null; private nativeStaticLayoutSkipReason: string | null = null; private nativeStaticLayoutSkipReasons: string[] = []; private nativeStaticLayoutBackgroundSkipReason: string | null = null; + private nativeStaticLayoutOverlayFailure: { + stage: string; + message: string; + } | null = null; private nativeH264Encoder: VideoEncoder | null = null; private nativeEncoderError: Error | null = null; + private nativeRawFrameMode = false; private effectiveDurationSec = 0; private totalExportStartTimeMs = 0; private metadataLoadTimeMs = 0; @@ -355,6 +550,11 @@ export class ModernVideoExporter { private peakNativeWriteInFlight = 0; private nativeCaptureTimeMs = 0; private nativeWriteTimeMs = 0; + private nativeWriteAckTimeMs = 0; + private nativeFrameTransportTimeMs = 0; + private nativeRawBytesSubmitted = 0; + private nativeRawFramesSubmitted = 0; + private peakNativeWriteInFlightBytes = 0; private finalizationTimeMs = 0; private finalizationStageMs: ExportFinalizationStageMetrics = {}; private processedFrameCount = 0; @@ -365,6 +565,7 @@ export class ModernVideoExporter { private lastProgressSampleTimeMs = 0; private lastProgressSampleFrame = 0; private displayedRenderFps = 0; + private lastPreparingTotalFrames: number | null = null; constructor(config: VideoExporterConfig) { this.config = config; @@ -387,22 +588,66 @@ export class ModernVideoExporter { this.totalExportStartTimeMs = this.getNowMs(); const backendPreference = this.config.backendPreference ?? "auto"; const runtimePlatform = this.getRuntimePlatform(); + // HEVC Auto/Hardware may use the NVIDIA CUDA compositor first. Rawvideo + // remains the strict fallback for unsupported effects and unavailable GPU + // routes; H.264 + Auto keeps its existing route selection unchanged. + const forceNativeRawFrame = this.shouldForceNativeRawFrame(); let useNativeEncoder = false; let triedNativeStaticLayoutWithProbe = false; const prefersNativeStaticLayoutBeforeBreeze = + !forceNativeRawFrame && shouldPreferNativeStaticLayoutBeforeBreeze(runtimePlatform, backendPreference); const shouldTryNativeStaticLayout = - backendPreference === "breeze" || - this.config.experimentalNvidiaCudaExport === true || - prefersNativeStaticLayoutBeforeBreeze; + !forceNativeRawFrame && + (this.canUseNativeGpuStaticLayout() || + backendPreference === "breeze" || + this.config.experimentalNvidiaCudaExport === true || + prefersNativeStaticLayoutBeforeBreeze); let shouldDeferNativeEncoderStart = - backendPreference === "breeze" || - this.config.experimentalNvidiaCudaExport === true || - prefersNativeStaticLayoutBeforeBreeze; + !forceNativeRawFrame && + (this.canUseNativeGpuStaticLayout() || + backendPreference === "breeze" || + this.config.experimentalNvidiaCudaExport === true || + prefersNativeStaticLayoutBeforeBreeze); this.lastNativeExportError = null; + // Strict HEVC Hardware: the CUDA static-layout route is mandatory. If it + // cannot even be selected (CUDA opt-in off / helper absent), fail now + // instead of falling through to the renderer raw/WebCodecs path. + if (this.requiresStrictNativeCudaRoute() && !this.canUseNativeGpuStaticLayout()) { + console.error( + formatLogTs(), + "[VideoExporter] Strict HEVC Hardware policy: CUDA compositor not eligible; refusing renderer raw fallback", + { + exportVideoCodec: this.config.exportVideoCodec, + exportEncoderPreference: this.config.exportEncoderPreference, + experimentalNativeExport: this.config.experimentalNativeExport === true, + experimentalNvidiaCudaExport: + this.config.experimentalNvidiaCudaExport === true, + skipReason: this.nativeStaticLayoutSkipReason, + skipReasons: this.nativeStaticLayoutSkipReasons, + }, + ); + throw this.buildStrictNativeCudaHardwareError( + this.nativeStaticLayoutSkipReason ?? "native-cuda-not-enabled", + ); + } + let stageStartedAt = this.getNowMs(); - if (shouldDeferNativeEncoderStart) { + if (forceNativeRawFrame) { + // Explicit per-request codec/encoder: start the native raw-frame encoder + // directly with no WebCodecs/static-layout fallback. If it cannot start + // (e.g. Hardware HEVC with no usable encoder), surface the error instead of + // silently switching codecs. + useNativeEncoder = await this.tryStartNativeVideoExportRawFrame(); + this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + if (!useNativeEncoder) { + throw new Error( + this.lastNativeExportError ?? + `${NATIVE_EXPORT_ENGINE_NAME} export could not start native ${this.config.exportVideoCodec?.toUpperCase() ?? "video"} encoding on this system.`, + ); + } + } else if (shouldDeferNativeEncoderStart) { // Defer the streaming native encoder until after metadata is known so // static-layout exports can use the fastest compatible compositor first. } else if ( @@ -469,6 +714,29 @@ export class ModernVideoExporter { frameRate: this.config.frameRate, encodingMode: this.config.encodingMode, }); + console.log(formatLogTs(), "[VideoExporter] Native static-layout decision", { + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", + experimentalNativeExport: this.config.experimentalNativeExport === true, + experimentalNvidiaCudaExport: this.config.experimentalNvidiaCudaExport === true, + backendPreference: this.config.backendPreference ?? "auto", + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + shouldForceNativeRawFrame: forceNativeRawFrame, + shouldTryNativeStaticLayout, + shouldDeferNativeEncoderStart, + useNativeEncoder, + zoomMotionBlur: this.config.zoomMotionBlur ?? 0, + zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur ?? 0, + hasOverlayContent: this.hasNativeStaticLayoutOverlayContent(), + hasBrowserOverlayPixels: this.hasNativeStaticLayoutBrowserOverlayPixels(), + showCursor: this.config.showCursor === true, + cursorTelemetrySamples: this.config.cursorTelemetry?.length ?? 0, + cursorMotionBlur: this.config.cursorMotionBlur ?? 0, + cursorSway: this.config.cursorSway ?? 0, + cursorAtlasOwnershipEligible: this.canUseNativeCursorAtlasOwnership(), + webcamOnlyBrowserPixels: this.hasNativeStaticLayoutWebcamOnlyBrowserPixels(), + webcamNativeOwnershipEligible: this.canUseNativeWebcamOwnership(), + }); this.maxNativeWriteInFlight = useNativeEncoder ? Math.max( 1, @@ -478,6 +746,9 @@ export class ModernVideoExporter { ), ) : 1; + if (useNativeEncoder && this.nativeRawFrameMode) { + this.configureNativeRawFrameBackpressure(); + } console.log("[VideoExporter] Backpressure profile", { profile: this.backpressureProfile.name, @@ -490,6 +761,12 @@ export class ModernVideoExporter { maxPendingFrames: this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames, maxInFlightNativeWrites: this.maxNativeWriteInFlight, + maxInFlightNativeRawFrames: this.nativeRawFrameMode + ? this.maxNativeRawWriteFrames + : undefined, + maxInFlightNativeRawBytes: this.nativeRawFrameMode + ? this.maxNativeRawWriteBytes + : undefined, }); if (shouldTryNativeStaticLayout && !useNativeEncoder) { @@ -511,6 +788,9 @@ export class ModernVideoExporter { this.disposeEncoder(); return staticLayoutResult; } + } else if (this.requiresStrictNativeCudaRoute()) { + this.nativeStaticLayoutSkipReason = "native-metadata-probe-unavailable"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; } } @@ -558,13 +838,65 @@ export class ModernVideoExporter { } if (shouldDeferNativeEncoderStart && !useNativeEncoder) { + if (this.requiresStrictNativeCudaRoute()) { + // Strict HEVC Hardware: the CUDA static-layout route did not produce + // video (skip, IPC failure, route mismatch, or post-validation + // failure). Never fall back to the renderer raw frame path, Breeze, + // WebGPU, or CPU; hard-fail with the first skip reason. + console.error( + formatLogTs(), + "[VideoExporter] Strict HEVC Hardware policy: CUDA static-layout route did not render; refusing raw renderer fallback", + { + skipReason: this.nativeStaticLayoutSkipReason, + skipReasons: this.nativeStaticLayoutSkipReasons, + lastNativeExportError: this.lastNativeExportError, + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + experimentalNvidiaCudaExport: + this.config.experimentalNvidiaCudaExport === true, + }, + ); + throw this.buildStrictNativeCudaHardwareError( + this.nativeStaticLayoutSkipReason ?? + this.lastNativeExportError ?? + "native-cuda-route-unavailable", + ); + } + if (this.requiresNativeRawFrame()) { + // Guaranteed observable reason when the native static-layout route + // did not produce video and HEVC/explicit-Hardware forces the raw + // renderer frame path. The skip reasons and last native error are + // surfaced here so a raw hevc_nvenc session can always be traced + // back to its cause. + console.warn( + formatLogTs(), + "[VideoExporter] Native static-layout route did not render; starting raw renderer frame path", + { + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: + this.config.exportEncoderPreference ?? "auto", + experimentalNativeExport: + this.config.experimentalNativeExport === true, + experimentalNvidiaCudaExport: + this.config.experimentalNvidiaCudaExport === true, + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + skipReason: this.nativeStaticLayoutSkipReason, + skipReasons: this.nativeStaticLayoutSkipReasons, + lastNativeExportError: this.lastNativeExportError, + }, + ); + } stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); + useNativeEncoder = this.requiresNativeRawFrame() + ? await this.tryStartNativeVideoExportRawFrame() + : await this.tryStartNativeVideoExport(); this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; if (!useNativeEncoder) { const nativeFailure = this.lastNativeExportError ?? `${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`; + if (this.requiresNativeRawFrame()) { + throw new Error(nativeFailure); + } console.warn( `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} native export unavailable after static-layout fallback; falling back to WebCodecs.`, nativeFailure, @@ -580,6 +912,9 @@ export class ModernVideoExporter { this.maxNativeWriteInFlight = 1; await this.initializeEncoder(); } + if (useNativeEncoder && this.nativeRawFrameMode) { + this.configureNativeRawFrameBackpressure(); + } } stageStartedAt = this.getNowMs(); @@ -747,7 +1082,7 @@ export class ModernVideoExporter { if (useNativeEncoder) { stageStartedAt = this.getNowMs(); this.reportFinalizingProgress(totalFrames, 99); - if (this.nativeH264Encoder) { + if (this.nativeH264Encoder && !this.nativeRawFrameMode) { await this.measureFinalizationStage("nativeEncoderFlushMs", async () => { await this.nativeH264Encoder!.flush(); }); @@ -1016,6 +1351,24 @@ export class ModernVideoExporter { return [...guidance]; } + private resolveRequestedBackendLabel(): string { + // The CUDA compositor is the active requested backend whenever the native + // CUDA route is selected (user opt-in for Auto, or mandatory for HEVC + + // Hardware). This is what the export will actually use for eligible jobs. + if (this.config.experimentalNvidiaCudaExport === true) { + return "NVIDIA CUDA compositor"; + } + + switch (this.config.backendPreference) { + case "webcodecs": + return "WebCodecs"; + case "breeze": + return "Breeze"; + default: + return "auto"; + } + } + private buildLightningExportError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); const resolvedEncodePath = @@ -1028,7 +1381,7 @@ export class ModernVideoExporter { `${LIGHTNING_PIPELINE_NAME} export failed.`, `Reason: ${message}`, `Platform: ${this.getPlatformLabel()}`, - `Requested backend mode: ${this.config.backendPreference ?? "auto"}`, + `Requested backend mode: ${this.resolveRequestedBackendLabel()}`, `Output: ${this.config.width}x${this.config.height} @ ${this.config.frameRate} FPS`, ]; @@ -1494,23 +1847,53 @@ export class ModernVideoExporter { } } - private hasUnsupportedNativeStaticLayoutWebcamShape(): boolean { - const webcam = this.config.webcam; - if (!webcam?.enabled) { - return false; + private getHevcNativeGpuFeatureSkipReasons(): string[] { + if (!this.canUseNativeGpuStaticLayout()) { + return []; } - const width = webcam.width ?? webcam.size ?? 40; - const height = webcam.height ?? webcam.size ?? 40; - return Math.abs(width - height) > 0.001 || !isWebcamCropRegionDefault(webcam.cropRegion); - } + const reasons: string[] = []; + + // Cursor motion blur is rendered into the transparent native overlay layer. + + const extensionHookPhases = [ + "background", + "post-video", + "post-zoom", + "post-cursor", + "post-webcam", + "post-annotations", + "final", + ] as const; + if ( + extensionHost.hasCursorEffects() || + extensionHookPhases.some((phase) => extensionHost.hasRenderHooks(phase)) + ) { + reasons.push("unsupported-extension-hook"); + } + return reasons; + } private getNativeStaticLayoutSkipReasons( audioPlan: NativeAudioPlan, videoInfo: DecodedVideoInfo, effectiveDurationSec: number, ): string[] { const reasons: string[] = []; + if ((this.config.zoomTemporalMotionBlur ?? 0) > 0.0005) { + const canUseNativeTemporalBlur = + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true; + if (!canUseNativeTemporalBlur) { + // Temporal zoom motion blur needs multi-frame shutter sampling. The + // generalized CUDA compositor implements it natively from the resolved + // temporal sample plan; without the CUDA route neither the FFmpeg + // effectful route nor the D3D11 helper can reproduce it, so keep an + // explicit, non-duplicated skip that surfaces in diagnostics instead of + // silently dropping the effect. + reasons.push("unsupported-temporal-motion-blur"); + } + } if ( typeof window === "undefined" || !window.electronAPI?.nativeStaticLayoutExport || @@ -1528,15 +1911,13 @@ export class ModernVideoExporter { } const speedRegions = this.config.speedRegions ?? []; - const hasCursorClickEffect = - (this.config.cursorTelemetry?.length ?? 0) > 0 && - (this.config.cursorClickEffect ?? "none") !== "none"; const configuredWallpaper = this.config.wallpaper?.trim() ?? ""; if (isVideoWallpaperSource(configuredWallpaper)) { reasons.push("unsupported-background-video"); } - if (hasCursorClickEffect) { - reasons.push("unsupported-cursor-click-effect"); + const unsupportedOverlayContent = this.hasUnsupportedNativeStaticLayoutOverlayContent(); + if (unsupportedOverlayContent) { + reasons.push(unsupportedOverlayContent); } const hasZoomRegions = (this.config.zoomRegions ?? []).length > 0; @@ -1547,6 +1928,18 @@ export class ModernVideoExporter { if (needsTimelineMap && this.config.experimentalNativeExport !== true) { reasons.push("native-timeline-requires-windows-gpu"); } + if ( + needsTimelineMap && + this.hasNativeStaticLayoutOverlayContent() && + !this.canUseNativeGpuStaticLayout() + ) { + // The generalized CUDA compositor maps output frames through the + // timeline AND alpha-composites the overlay sidecar (overlay frames are + // indexed by output frame), so timeline + overlay sidecars are supported + // on the CUDA route. Only the D3D11/FFmpeg fallback routes cannot + // preserve both, so keep the skip for those routes only. + reasons.push("overlay-layers-do-not-support-native-timeline"); + } if ( needsTimelineMap && this.buildNativeStaticLayoutVideoTimelineSegments(videoInfo).length === 0 @@ -1560,23 +1953,9 @@ export class ModernVideoExporter { if (hasZoomRegions && this.config.experimentalNativeExport !== true) { reasons.push("native-zoom-requires-windows-gpu"); } - if ((this.config.annotationRegions ?? []).length > 0) { - reasons.push("unsupported-annotation-overlay"); - } - if ((this.config.autoCaptions ?? []).length > 0) { - reasons.push("unsupported-caption-overlay"); - } - if (this.config.webcam?.enabled && !this.getNativeWebcamSourcePath()) { reasons.push("unsupported-webcam-source"); } - if (this.hasUnsupportedNativeStaticLayoutWebcamShape()) { - reasons.push("unsupported-rectangular-webcam-overlay"); - } - - if (this.config.frame) { - reasons.push("unsupported-frame-overlay"); - } const crop = this.config.cropRegion; if ( @@ -1590,6 +1969,7 @@ export class ModernVideoExporter { reasons.push("invalid-crop-region"); } + reasons.push(...this.getHevcNativeGpuFeatureSkipReasons()); return reasons; } @@ -2131,11 +2511,14 @@ export class ModernVideoExporter { const springY = createSpringState(0); const zoomSpringConfig = getZoomSpringConfig(this.config.zoomSmoothness); const frameDurationMs = 1000 / Math.max(1, this.config.frameRate); + const zoomBlurAmount = this.config.zoomMotionBlur ?? 0; + const zoomBlurTuning = this.config.zoomMotionBlurTuning; const samples: NativeStaticLayoutZoomSample[] = []; let lastContentTimeMs: number | null = null; let appliedScale = 1; let appliedX = 0; let appliedY = 0; + let previousAppliedTransform: { scale: number; x: number; y: number } | null = null; for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) { const timeMs = frameIndex * frameDurationMs; @@ -2211,157 +2594,1445 @@ export class ModernVideoExporter { ); } + const currentAppliedTransform = { scale: appliedScale, x: appliedX, y: appliedY }; + const blurStep = + zoomBlurAmount > 0 && previousAppliedTransform + ? analyzeZoomMotionBlurStep({ + previousTransform: previousAppliedTransform, + currentTransform: currentAppliedTransform, + baseMask, + stageSize, + motionBlurAmount: zoomBlurAmount, + motionBlurTuning: zoomBlurTuning, + deltaSeconds: Math.min(80, Math.max(1, deltaMs)) / 1000, + }) + : null; + previousAppliedTransform = currentAppliedTransform; + samples.push({ timeMs, scale: appliedScale, x: appliedX, y: appliedY, + blurStrength: blurStep?.strength ?? 0, + blurCenterX: blurStep?.centerX ?? stageSize.width / 2, + blurCenterY: blurStep?.centerY ?? stageSize.height / 2, }); } return samples; } - private async tryExportNativeStaticLayout( - videoInfo: DecodedVideoInfo, - audioPlan: NativeAudioPlan, - effectiveDuration: number, - totalFrames: number, - ): Promise { - const skipReason = this.getNativeStaticLayoutSkipReason( - audioPlan, - videoInfo, - effectiveDuration, + /** + * The generalized NVIDIA CUDA compositor can reproduce the cursor from the + * native atlas (sprite, position, type, click bounce, visibility) on top of + * the composed video. When eligible, cursor pixels are excluded from the + * transparent overlay sidecar and rendered natively instead, which keeps the + * cursor sharp and avoids baking it into the RGBA stream. Browser-only + * cursor effects (motion blur, sway, click effect rings), extension cursor + * visuals, or an unavailable atlas keep the baked-sidecar fallback. + */ + private canUseNativeCursorAtlasOwnership(): boolean { + if (this.config.showCursor !== true || (this.config.cursorTelemetry?.length ?? 0) === 0) { + return false; + } + if ((this.config.cursorMotionBlur ?? 0) > 0.0005) { + return false; + } + if ((this.config.cursorSway ?? 0) > 0.0005) { + return false; + } + const clickEffect = this.config.cursorClickEffect; + if (clickEffect !== undefined && clickEffect !== "none") { + return false; + } + if (this.hasNativeStaticLayoutExtensionCursorVisuals()) { + return false; + } + // Only the generalized NVIDIA CUDA compositor draws the atlas on top of + // the overlay sidecars; the FFmpeg overlay route and the D3D11 helper + // cannot, so native ownership requires the CUDA-opt-in Windows route. + return ( + this.getRuntimePlatform() === "win32" && + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true ); - const skipReasons = skipReason - ? this.getNativeStaticLayoutSkipReasons(audioPlan, videoInfo, effectiveDuration) - : []; - if (skipReason) { - this.nativeStaticLayoutSkipReason = skipReason; - this.nativeStaticLayoutSkipReasons = skipReasons; - console.info("[VideoExporter] Native static layout skipped", { - reason: skipReason, - reasons: skipReasons, - audioMode: audioPlan.audioMode, - zoomRegions: this.config.zoomRegions?.length ?? 0, - speedRegions: this.config.speedRegions?.length ?? 0, - audioRegions: this.config.audioRegions?.length ?? 0, - annotationRegions: this.config.annotationRegions?.length ?? 0, - hasFrame: Boolean(this.config.frame), - backgroundBlur: this.config.backgroundBlur, - hasCursorOverlay: - this.config.showCursor === true && - (this.config.cursorTelemetry?.length ?? 0) > 0, - experimentalNativeExport: this.config.experimentalNativeExport === true, - }); - return null; + } + + /** + * Whether the generalized NVIDIA CUDA compositor is an eligible consumer of + * the native `cursor-sprite` overlay contract. + * + * The cursor-sprite contract captures only the small cursor ROI strip + * instead of baking the cursor into a full transparent 4K canvas per frame. + * It is consumed solely by the generalized NVIDIA CUDA compositor, which is + * independent of the output codec: native-video.ts runs the same CUDA + * compositor for H.264 and HEVC overlay exports whenever the user opts into + * the CUDA route. This predicate therefore gates on the CUDA route, not on + * the (HEVC-only) canUseNativeGpuStaticLayout(). Gating the cheap ROI path + * on the codec wrongly forced H.264 CUDA exports with a cursor-only overlay + * to bake the full-canvas sidecar frame-by-frame (~1 min for 192 frames) + * instead of capturing the tiny cursor ROI. + * + * CPU encoder preference never reaches the CUDA compositor (it is the + * software-encoder route), so it must never attempt a sprite here. + */ + private canUseNativeCursorSpriteContract(): boolean { + if (this.config.exportEncoderPreference === "cpu") { + return false; } + return ( + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true + ); + } - const sourcePath = this.getNativeVideoSourcePath(); - const audioOptions = await this.getNativeStaticLayoutAudioOptions(audioPlan, totalFrames); - if (!sourcePath || !audioOptions) { - this.nativeStaticLayoutSkipReason = !sourcePath - ? "missing-source-path" - : "missing-audio-options"; - this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; - return null; + /** + * Whether extension cursor visuals / render hooks are active. Extension + * hooks draw into the full composite canvas outside the cursor container, so + * any path that captures only the cursor container (cursor-sprite ROI) would + * silently drop them. The baked full-canvas sidecar is required instead. + */ + private hasNativeStaticLayoutExtensionCursorVisuals(): boolean { + const extensionHookPhases = [ + "background", + "post-video", + "post-zoom", + "post-cursor", + "post-webcam", + "post-annotations", + "final", + ] as const; + return ( + extensionHost.hasCursorEffects() || + extensionHookPhases.some((phase) => extensionHost.hasRenderHooks(phase)) + ); + } + + private hasNativeStaticLayoutOverlayContent(): boolean { + return Boolean( + ((this.config.cursorTelemetry?.length ?? 0) > 0 && this.config.showCursor !== false) || + (this.config.annotationRegions?.length ?? 0) > 0 || + (this.config.autoCaptions?.length ?? 0) > 0 || + this.config.frame || + this.config.webcam?.enabled, + ); + } + + // Browser-rendered overlay pixels (everything the renderer draws into the + // transparent sidecar). When the native CUDA compositor owns the cursor atlas + // and none of these are present, the sidecar would be entirely transparent, + // so it can be skipped entirely without rendering/capturing a canvas per frame. + // When the webcam is owned natively by the CUDA compositor (webcamNativeOwned) + // the renderer must NOT bake it into the sidecar, so it is excluded from the + // browser-pixel check exactly like an atlas-owned cursor. + private hasNativeStaticLayoutBrowserOverlayPixels(webcamNativeOwned = false): boolean { + return Boolean( + (this.config.annotationRegions?.length ?? 0) > 0 || + (this.config.autoCaptions?.length ?? 0) > 0 || + Boolean(this.config.frame) || + (Boolean(this.config.webcam?.enabled) && !webcamNativeOwned), + ); + } + + /** + * Whether the webcam is the ONLY browser-rendered overlay pixel source and is + * fully representable by the generalized NVIDIA CUDA compositor's native + * webcam overlay contract. + * + * The CUDA compositor consumes the same resolved webcam geometry the renderer + * would bake (left/top/size/radius/mirror/time-offset via the native-video.ts + * webcam args), so a webcam-only export needs no renderer sidecar at all. + * Mixed browser content (captions, annotations, frame visuals) or extension + * render hooks keep the existing baked sidecar path, and a configured webcam + * shadow is not representable in the CUDA wrapper today, so a shadowed webcam + * must stay baked to preserve the golden visual. + */ + private hasNativeStaticLayoutWebcamOnlyBrowserPixels(): boolean { + const webcamOverlay = this.getNativeStaticLayoutWebcamOverlay(); + return ( + this.config.webcam?.enabled === true && + webcamOverlay !== null && + (webcamOverlay.shadowIntensity ?? 0) <= 0 && + (this.config.annotationRegions?.length ?? 0) === 0 && + (this.config.autoCaptions?.length ?? 0) === 0 && + !this.config.frame && + !this.hasNativeStaticLayoutExtensionCursorVisuals() + ); + } + + /** + * Whether the generalized NVIDIA CUDA compositor owns the webcam overlay + * natively for this export. + * + * Safe only on the strict HEVC Hardware CUDA route: that route guarantees the + * CUDA compositor runs (any fallback hard-fails with noCpuFallback:true), so + * excluding the webcam from the renderer sidecar can never silently drop it on + * an FFmpeg/D3D11 fallback that cannot draw a native webcam. HEVC Auto and + * H.264 keep the existing baked-webcam sidecar path unchanged. + */ + private canUseNativeWebcamOwnership(): boolean { + if (!this.requiresStrictNativeCudaRoute() || !this.canUseNativeGpuStaticLayout()) { + return false; } - const background = await this.resolveNativeStaticLayoutBackground(); - if (!background) { - this.nativeStaticLayoutSkipReason = - this.nativeStaticLayoutBackgroundSkipReason ?? "unsupported-background"; - this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; - return null; + return this.hasNativeStaticLayoutWebcamOnlyBrowserPixels(); + } + + private hasUnsupportedNativeStaticLayoutOverlayContent(): string | null { + if (this.config.annotationRegions?.some((annotation) => annotation.type === "blur")) { + return "unsupported-blur-annotation-overlay"; } + return null; + } - const layout = computePaddedLayout({ + private getNativeStaticLayoutFastLaneEligibility( + audioPlan: NativeAudioPlan, + cursorAtlasOwnedByNative: boolean, + webcamNativeOwned: boolean, + ): NativeStaticLayoutFastLaneEligibility { + const cursorDisabled = + this.config.showCursor !== true || (this.config.cursorTelemetry?.length ?? 0) === 0; + // Actual ownership, not eligibility: the empty sidecar fast lane is only + // safe when the cursor is disabled or the CUDA compositor will genuinely + // draw it from a successfully built atlas. An eligible-but-unbuilt atlas + // must not silently drop the cursor, so it keeps the sidecar preparation + // (cursor-sprite ROI or baked full-canvas) and never selects the fast lane. + const cursorNativeOwnershipActive = cursorAtlasOwnedByNative; + return getNativeStaticLayoutFastLaneEligibility({ + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + hasBrowserOverlayPixels: + this.hasNativeStaticLayoutBrowserOverlayPixels(webcamNativeOwned), + cursorDisabled, + cursorNativeOwnershipActive, + requiresEditedAudioRender: audioPlan.audioMode === "edited-track", + hasAuthoritativeNativeSource: Boolean(this.getNativeVideoSourcePath()), + }); + } + + private createNativeStaticLayoutOverlayRenderer( + videoInfo: DecodedVideoInfo, + excludeCursorOverlay = false, + excludeWebcamOverlay = false, + ) { + return new ModernFrameRenderer({ width: this.config.width, height: this.config.height, - padding: this.config.padding ?? 0, + preferredRenderBackend: undefined, + wallpaper: DEFAULT_WALLPAPER_PATH, + zoomRegions: this.config.zoomRegions, + showShadow: this.config.showShadow, + shadowIntensity: this.config.shadowIntensity, + backgroundBlur: 0, + zoomMotionBlur: 0, + connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, + borderRadius: this.config.borderRadius, + padding: this.config.padding, cropRegion: this.config.cropRegion, + webcam: excludeWebcamOverlay ? undefined : this.config.webcam, + webcamUrl: excludeWebcamOverlay ? null : this.config.webcamUrl, videoWidth: videoInfo.width, videoHeight: videoInfo.height, + annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, + speedRegions: this.config.speedRegions, + previewWidth: this.config.previewWidth, + previewHeight: this.config.previewHeight, + cursorTelemetry: this.config.cursorTelemetry, + showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, + cursorSize: this.config.cursorSize, + cursorSmoothing: this.config.cursorSmoothing, + cursorSpringStiffnessMultiplier: this.config.cursorSpringStiffnessMultiplier, + cursorSpringDampingMultiplier: this.config.cursorSpringDampingMultiplier, + cursorSpringMassMultiplier: this.config.cursorSpringMassMultiplier, + cameraSpringStiffnessMultiplier: this.config.cameraSpringStiffnessMultiplier, + cameraSpringDampingMultiplier: this.config.cameraSpringDampingMultiplier, + cameraSpringMassMultiplier: this.config.cameraSpringMassMultiplier, + cursorMotionBlur: this.config.cursorMotionBlur, + cursorClickEffect: this.config.cursorClickEffect, + cursorClickEffectColor: this.config.cursorClickEffectColor, + cursorClickEffectScale: this.config.cursorClickEffectScale, + cursorClickEffectOpacity: this.config.cursorClickEffectOpacity, + cursorClickEffectDurationMs: this.config.cursorClickEffectDurationMs, + cursorClickBounce: this.config.cursorClickBounce, + cursorClickBounceDuration: this.config.cursorClickBounceDuration, + cursorSway: this.config.cursorSway, + zoomSmoothness: this.config.zoomSmoothness, + zoomClassicMode: this.config.zoomClassicMode, + frame: this.config.frame, + excludeCursorOverlay, }); - const contentSize = roundNativeStaticLayoutContentSize({ - width: layout.croppedDisplayWidth, - height: layout.croppedDisplayHeight, - }); - const contentWidth = contentSize.width; - const contentHeight = contentSize.height; + } + + private extractNativeTiledOverlayTileInto( + target: Uint8Array, + source: Uint8Array, + sourceWidth: number, + sourceHeight: number, + tileX: number, + tileY: number, + ): void { + target.fill(0); + const startY = tileY * NATIVE_TILED_OVERLAY_TILE_SIZE; + const startX = tileX * NATIVE_TILED_OVERLAY_TILE_SIZE; + const endY = Math.min(sourceHeight, startY + NATIVE_TILED_OVERLAY_TILE_SIZE); + const endX = Math.min(sourceWidth, startX + NATIVE_TILED_OVERLAY_TILE_SIZE); + const copyRows = Math.max(0, endY - startY); + const copyCols = Math.max(0, endX - startX); + for (let row = 0; row < copyRows; row += 1) { + const sourceRowOffset = ((startY + row) * sourceWidth + startX) * 4; + const targetRowOffset = row * NATIVE_TILED_OVERLAY_TILE_SIZE * 4; + const rowBytes = copyCols * 4; + target.set( + source.subarray(sourceRowOffset, sourceRowOffset + rowBytes), + targetRowOffset, + ); + } + } + + /** + * Whether the cursor should be captured as a cursor-sprite ROI strip instead + * of being baked into a full transparent RGBA canvas sidecar. + * + * A cursor-sprite is only usable on the generalized NVIDIA CUDA compositor + * (the sole consumer of the native `cursor-sprite` contract) and only when + * the cursor is the entire overlay (no browser pixels) and is NOT actually + * owned by the native atlas (cursorExcluded === cursorAtlasOwnedByNative). + * When the atlas is eligible but was not successfully built, the sprite path + * is the pixel-preserving fallback: it renders the same Pixi cursor into the + * ROI instead of the expensive full-canvas tiled sidecar. Browser-only + * cursor effects (motion blur/sway/click) also use the sprite. Extension + * cursor visuals keep the baked full-canvas sidecar because extension hooks + * draw outside the cursor container (the sprite would drop them). When the + * sprite cannot be used the baked-cursor full-canvas sidecar path runs + * unchanged (the preserved golden path). + */ + private shouldUseNativeStaticLayoutCursorSprite( + cursorExcluded: boolean, + webcamExcluded = false, + ): boolean { + return ( + !cursorExcluded && + this.canUseNativeCursorSpriteContract() && + this.config.showCursor === true && + (this.config.cursorTelemetry?.length ?? 0) > 0 && + !this.hasNativeStaticLayoutExtensionCursorVisuals() && + !this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded) + ); + } + + /** + * Captures the cursor ROI as a fixed packed RGBA sprite strip plus per-frame + * top-left positions and returns a validated native `cursor-sprite` overlay + * layer. Returns null (recording an overlay failure) when the cursor-sprite + * contract cannot be prepared, in which case the caller falls back to the + * existing baked-cursor full-canvas sidecar path. + */ + private async prepareNativeStaticLayoutCursorSprite( + videoInfo: DecodedVideoInfo, + durationSec: number, + totalFrames: number, + webcamExcluded = false, + onPreparationProgress?: (renderProgress: number) => void, + ): Promise { + const api = typeof window === "undefined" ? null : window.electronAPI; if ( - contentWidth > this.config.width || - contentHeight > this.config.height || - !Number.isFinite(effectiveDuration) || - effectiveDuration <= 0 + !api?.openExportStream || + !api.writeExportStreamChunk || + !api.closeExportStream || + !api.discardExportedTemp ) { - this.nativeStaticLayoutSkipReason = "invalid-layout-or-duration"; - this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; - await this.cleanupNativeStaticLayoutBackground(background); + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-api-unavailable", + "Cursor-sprite export stream IPC is not available", + ); return null; } - - const offsetX = Math.round(layout.centerOffsetX); - const offsetY = Math.round(layout.centerOffsetY); - const sourceCrop = this.isDefaultCropRegion() - ? null - : this.getNativeStaticLayoutSourceCrop(videoInfo); - const borderRadius = scalePreviewBorderRadius( - this.config.width, - this.config.height, - this.config.borderRadius ?? 0, - ); - const shadowIntensity = this.config.showShadow - ? Math.min(1, Math.max(0, this.config.shadowIntensity)) - : 0; - const webcamOverlay = this.getNativeStaticLayoutWebcamOverlay(); - const cursorTelemetry = this.getNativeStaticLayoutCursorTelemetry(); - const zoomTelemetry = this.getNativeStaticLayoutZoomTelemetry( - layout, - totalFrames, - cursorTelemetry, - ); - const needsTimelineMap = this.shouldUseNativeStaticLayoutTimelineMap( + const renderer = this.createNativeStaticLayoutOverlayRenderer( videoInfo, - effectiveDuration, + false, + webcamExcluded, ); - const timelineSegments = needsTimelineMap - ? this.buildNativeStaticLayoutVideoTimelineSegments(videoInfo) - : undefined; - if (needsTimelineMap && !timelineSegments?.length) { - this.nativeStaticLayoutSkipReason = - (this.config.speedRegions ?? []).length > 0 - ? "invalid-native-speed-timeline" - : "invalid-native-trim-timeline"; - this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; - await this.cleanupNativeStaticLayoutBackground(background); - return null; - } - const cursorAtlas = - cursorTelemetry && cursorTelemetry.length > 0 - ? await buildNativeCursorAtlas(this.config.cursorStyle ?? "tahoe").catch( - (error) => { - console.warn("[VideoExporter] Native cursor atlas unavailable", error); - return null; - }, - ) - : null; - if (cursorTelemetry && cursorTelemetry.length > 0 && !cursorAtlas) { - this.nativeStaticLayoutSkipReason = "cursor-atlas-unavailable"; - this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; - await this.cleanupNativeStaticLayoutBackground(background); - return null; - } - const startedAt = this.getNowMs(); - const sessionId = `recordly-static-layout-${Date.now()}-${Math.random() - .toString(36) - .slice(2, 8)}`; - const previousEncodeBackend = this.encodeBackend; - const previousEncoderName = this.encoderName; - const restoreEncoderState = () => { - this.encodeBackend = previousEncodeBackend; - this.encoderName = previousEncoderName; - }; - + let spriteStreamId: string | null = null; + let positionsStreamId: string | null = null; + try { + const spriteStream = await api.openExportStream({ extension: "sprite" }); + if (!spriteStream.success || !spriteStream.streamId || !spriteStream.tempPath) { + this.recordNativeStaticLayoutOverlayFailure( + "open-cursor-sprite-stream", + spriteStream.error ?? "Cursor-sprite export stream could not be opened", + ); + return null; + } + spriteStreamId = spriteStream.streamId; + + const positionsStream = await api.openExportStream({ extension: "json" }); + if ( + !positionsStream.success || + !positionsStream.streamId || + !positionsStream.tempPath + ) { + this.recordNativeStaticLayoutOverlayFailure( + "open-cursor-positions-stream", + positionsStream.error ?? + "Cursor-sprite positions export stream could not be opened", + ); + return null; + } + positionsStreamId = positionsStream.streamId; + + await renderer.initialize(); + const started = renderer.startCursorSpriteCapture(); + if (!started) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-init", + "Cursor-sprite capture could not be initialized (no overlay renderer)", + ); + return null; + } + + let lastPreparationProgressMs = 0; + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) { + if (this.cancelled) { + throw new Error("Export cancelled"); + } + if (onPreparationProgress) { + const nowMs = this.getNowMs(); + if ( + nowMs - lastPreparationProgressMs >= + NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS || + frameIndex === totalFrames - 1 + ) { + lastPreparationProgressMs = nowMs; + onPreparationProgress( + totalFrames > 0 ? (frameIndex / totalFrames) * 100 : 0, + ); + } + } + const timestampUs = Math.round((frameIndex * 1_000_000) / this.config.frameRate); + try { + // The cursor-sprite path captures only the cursor ROI, so skip the + // full 4K canvas render that the baked full-canvas sidecar needs. + // All cursor state updates (sway spring, motion-blur velocity, + // click rings, zoom transform) still run; only the expensive + // full-canvas rasterization is skipped. + await renderer.renderOverlayFrame(timestampUs, timestampUs, timestampUs, true); + } catch (error) { + throw new Error( + `overlay-renderer-frame: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const capture = renderer.captureCursorSpriteFrame(); + if (!capture.captured) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-frame", + capture.unavailableReason ?? "Cursor-sprite frame could not be captured", + ); + return null; + } + } + + const strip = renderer.finishCursorSpriteCapture(); + if (!strip || strip.frameCount === 0) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-finish", + "Cursor-sprite capture produced no frames", + ); + return null; + } + + const spriteWrite = await api.writeExportStreamChunk(spriteStreamId, 0, strip.frames); + if (!spriteWrite.success) { + throw new Error( + `cursor-sprite-stream-write: ${spriteWrite.error ?? "Failed to write cursor-sprite strip"}`, + ); + } + const clampedPositions: NativeCursorSpritePosition[] = strip.positions.map((position) => + clampNativeCursorSpritePosition( + position, + strip.width, + strip.height, + this.config.width, + this.config.height, + ), + ); + const positionsBytes = new TextEncoder().encode(JSON.stringify(clampedPositions)); + const positionsWrite = await api.writeExportStreamChunk( + positionsStreamId, + 0, + positionsBytes, + ); + if (!positionsWrite.success) { + throw new Error( + `cursor-positions-stream-write: ${positionsWrite.error ?? "Failed to write cursor-sprite positions"}`, + ); + } + + const spriteClosed = await api.closeExportStream(spriteStreamId); + spriteStreamId = null; + if (!spriteClosed.success || !spriteClosed.tempPath) { + throw new Error( + `cursor-sprite-stream-close: ${spriteClosed.error ?? "Cursor-sprite stream did not finalize"}`, + ); + } + const positionsClosed = await api.closeExportStream(positionsStreamId); + positionsStreamId = null; + if (!positionsClosed.success || !positionsClosed.tempPath) { + throw new Error( + `cursor-positions-stream-close: ${positionsClosed.error ?? "Cursor-sprite positions stream did not finalize"}`, + ); + } + + const layer: NativeCursorSpriteOverlayLayer = { + id: "cursor-sprite", + order: 1, + kind: NATIVE_CURSOR_SPRITE_LAYER_KIND, + path: spriteClosed.tempPath, + positionsPath: positionsClosed.tempPath, + x: 0, + y: 0, + width: strip.width, + height: strip.height, + frameRate: this.config.frameRate, + durationSec, + frameCount: strip.frameCount, + positions: clampedPositions, + pixelFormat: "rgba", + }; + const validationError = validateNativeCursorSpriteOverlayLayer(layer, { + outputWidth: this.config.width, + outputHeight: this.config.height, + durationSec, + frameRate: this.config.frameRate, + }); + if (validationError) { + throw new Error(`cursor-sprite-layer-invalid: ${validationError}`); + } + console.info("[VideoExporter] Native static layout cursor-sprite selected", { + route: "nvidia-cuda-compositor", + cursorStyle: this.config.cursorStyle ?? "tahoe", + spriteWidth: strip.width, + spriteHeight: strip.height, + frameCount: strip.frameCount, + }); + return layer; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const stage = message.startsWith("cursor-") + ? message.split(":", 1)[0] + : "cursor-sprite-preparation"; + this.recordNativeStaticLayoutOverlayFailure(stage, message); + console.warn( + "[VideoExporter] Cursor-sprite preparation failed; falling back to baked cursor overlay sidecar", + { stage, message, totalFrames, cancelled: this.cancelled }, + ); + return null; + } finally { + // Abort any export streams still open on both the thrown-error path and + // the early-return-null paths (a finalized stream already nulls its id). + if (spriteStreamId) { + try { + await api.closeExportStream(spriteStreamId, { abort: true }); + } catch { + // Best-effort cleanup. + } + } + if (positionsStreamId) { + try { + await api.closeExportStream(positionsStreamId, { abort: true }); + } catch { + // Best-effort cleanup. + } + } + try { + renderer.cancelCursorSpriteCapture(); + } catch { + // Best-effort cleanup. + } + try { + renderer.destroy(); + } catch { + // Cleanup is best-effort after a failed cursor-sprite attempt. + } + } + } + + private async prepareNativeStaticLayoutOverlay( + videoInfo: DecodedVideoInfo, + durationSec: number, + totalFrames: number, + cursorExcluded = false, + webcamExcluded = false, + onPreparationProgress?: (renderProgress: number) => void, + ): Promise { + this.nativeStaticLayoutOverlayFailure = null; + if (!this.hasNativeStaticLayoutOverlayContent()) { + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + } + if (this.hasUnsupportedNativeStaticLayoutOverlayContent()) { + this.recordNativeStaticLayoutOverlayFailure( + "unsupported-overlay-content", + this.hasUnsupportedNativeStaticLayoutOverlayContent() ?? + "unsupported overlay content", + ); + return null; + } + // Safe empty-work fast path: the native CUDA compositor owns the cursor + // atlas and there are no browser-rendered overlay pixels (captions, + // annotations, webcam, frame, or other). Rendering/capturing a full + // transparent canvas for every output frame would be pure waste, so return + // the validated empty overlay representation instead of a sidecar. Zoom and + // temporal motion-blur effects are preserved natively on the GPU and are + // unaffected by omitting an empty sidecar. + if (cursorExcluded && !this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded)) { + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + } + // Cursor-sprite path: when the cursor is the only overlay content (no + // browser pixels) and it cannot be owned by the native atlas, capture the + // cursor ROI as a packed RGBA strip + per-frame positions instead of + // writing a full transparent canvas sidecar for every output frame. Only + // the generalized NVIDIA CUDA compositor consumes the cursor-sprite + // contract. When the sprite cannot be prepared the existing baked-cursor + // full-canvas sidecar path below runs unchanged (the preserved golden + // path) and carries a clear diagnostic note. + const cursorSpriteEligible = this.shouldUseNativeStaticLayoutCursorSprite( + cursorExcluded, + webcamExcluded, + ); + const cursorSpriteLayer = cursorSpriteEligible + ? await this.prepareNativeStaticLayoutCursorSprite( + videoInfo, + durationSec, + totalFrames, + webcamExcluded, + onPreparationProgress, + ) + : null; + if (cursorSpriteLayer) { + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([cursorSpriteLayer]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + } + // Surface why the cursor-sprite path was not taken. When the path was + // eligible but preparation failed, prepareNativeStaticLayoutCursorSprite + // already logged the stage/message; only the eligibility miss needs an + // explicit note here (the baked sidecar is the preserved golden path). + if (!cursorSpriteEligible) { + const hasBrowserPixels = this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded); + const browserPixelSources: string[] = []; + if ((this.config.annotationRegions?.length ?? 0) > 0) { + browserPixelSources.push("annotations"); + } + if ((this.config.autoCaptions?.length ?? 0) > 0) { + browserPixelSources.push("captions"); + } + if (this.config.frame) { + browserPixelSources.push("frame"); + } + if (this.config.webcam?.enabled === true && !webcamExcluded) { + browserPixelSources.push("webcam"); + } + const spriteAvailable = + this.canUseNativeCursorSpriteContract() && + this.config.showCursor === true && + (this.config.cursorTelemetry?.length ?? 0) > 0; + const reason = hasBrowserPixels + ? "browser-overlay-pixels" + : this.hasNativeStaticLayoutExtensionCursorVisuals() + ? "extension-cursor-visuals" + : !spriteAvailable + ? "cursor-sprite-contract-unavailable" + : "cursor-excluded-by-native-atlas"; + console.info("[VideoExporter] Cursor-sprite overlay path skipped", { + route: "nvidia-cuda-compositor", + reason, + bakedSidecarRequired: reason !== "cursor-excluded-by-native-atlas", + browserPixelSources, + hasExtensionCursorVisuals: this.hasNativeStaticLayoutExtensionCursorVisuals(), + cursorExcluded, + showCursor: this.config.showCursor === true, + cursorTelemetrySamples: this.config.cursorTelemetry?.length ?? 0, + cursorAtlasOwnershipEligible: this.canUseNativeCursorAtlasOwnership(), + hasBrowserOverlayPixels: hasBrowserPixels, + annotationRegions: this.config.annotationRegions?.length ?? 0, + autoCaptions: this.config.autoCaptions?.length ?? 0, + frame: Boolean(this.config.frame), + webcamEnabled: this.config.webcam?.enabled === true, + }); + } + // Falling back to the baked-cursor full-canvas sidecar; clear any + // cursor-sprite preparation failure so a successful sidecar is not + // misreported as an overlay failure. + this.nativeStaticLayoutOverlayFailure = null; + const api = typeof window === "undefined" ? null : window.electronAPI; + if ( + !api?.openExportStream || + !api.writeExportStreamChunk || + !api.closeExportStream || + !api.discardExportedTemp + ) { + this.recordNativeStaticLayoutOverlayFailure( + "export-stream-api-unavailable", + "Export stream IPC is not available for the native overlay sidecar", + ); + return null; + } + + let rawStream: Awaited> | null = null; + let rawStreamId: string | null = null; + try { + rawStream = await api.openExportStream({ extension: "rgba" }); + if (!rawStream.success || !rawStream.streamId || !rawStream.tempPath) { + this.recordNativeStaticLayoutOverlayFailure( + "open-export-stream", + rawStream.error ?? "Native overlay export stream could not be opened", + ); + return null; + } + rawStreamId = rawStream.streamId; + } catch (error) { + this.recordNativeStaticLayoutOverlayFailure( + "open-export-stream", + error instanceof Error ? error.message : String(error), + ); + return null; + } + + const renderer = this.createNativeStaticLayoutOverlayRenderer( + videoInfo, + cursorExcluded, + webcamExcluded, + ); + const frameByteSize = getNativeStaticLayoutOverlayFrameByteSize( + this.config.width, + this.config.height, + ); + const tileColumns = getNativeTiledOverlayTileColumns(this.config.width); + const tileRows = getNativeTiledOverlayTileRows(this.config.height); + const tileCount = getNativeTiledOverlayTileCount(this.config.width, this.config.height); + + const scratchTile = new Uint8Array(NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE); + const previousTiles: (Uint8Array | null)[] = new Array(tileCount).fill(null); + const staticTiles: NativeTiledOverlayStaticTileRecord[] = []; + const frameDeltas: NativeTiledOverlayFrameDelta[] = []; + const tiledPayloadBuffers: Uint8Array[] = []; + let tiledPayloadOffset = 0; + let tiledAbandoned = false; + let rawFallbackReason: NativeTiledOverlayRawFallbackReason | null = null; + const rawPhysicalBytes = this.config.width * this.config.height * 4 * totalFrames; + const maxTiledPayloadBytes = + rawPhysicalBytes * NATIVE_TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION; + + let rawWrittenFrameCount = 0; + let runStartFrameIndex = 0; + let runFrame: Uint8Array | null = null; + if (rawStreamId === null) { + this.recordNativeStaticLayoutOverlayFailure( + "open-export-stream", + "Native overlay export stream id was not set", + ); + return null; + } + const activeRawStreamId: string = rawStreamId; + const writeRawOverlayChunk = async ( + frameIndex: number, + frameCount: number, + chunk: Uint8Array, + ): Promise => { + try { + const result = await api.writeExportStreamChunk( + activeRawStreamId, + frameIndex * frameByteSize, + chunk, + ); + if (!result.success) { + throw new Error(result.error ?? "Failed to write native overlay frame"); + } + } catch (error) { + throw new Error( + `overlay-stream-write: ${error instanceof Error ? error.message : String(error)}`, + ); + } + rawWrittenFrameCount += frameCount; + }; + const flushRawIdenticalRun = async (untilFrameIndex: number): Promise => { + if (runFrame === null || untilFrameIndex <= runStartFrameIndex) { + return; + } + const runLength = untilFrameIndex - runStartFrameIndex; + const framesPerBatch = Math.max( + 1, + Math.floor(NATIVE_RAW_OVERLAY_RUN_BATCH_MAX_BYTES / frameByteSize), + ); + let batchStartFrameIndex = runStartFrameIndex; + while (batchStartFrameIndex < untilFrameIndex) { + const batchFrameCount = Math.min( + runLength - (batchStartFrameIndex - runStartFrameIndex), + framesPerBatch, + ); + if (batchFrameCount === 1) { + await writeRawOverlayChunk(batchStartFrameIndex, 1, runFrame); + } else { + const batchBytes = batchFrameCount * frameByteSize; + const batch = new Uint8Array(batchBytes); + for (let i = 0; i < batchFrameCount; i += 1) { + batch.set(runFrame, i * frameByteSize); + } + await writeRawOverlayChunk(batchStartFrameIndex, batchFrameCount, batch); + } + batchStartFrameIndex += batchFrameCount; + } + }; + + let rawTempPath: string | null = null; + let lastPreparationProgressMs = 0; + try { + await renderer.initialize(); + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) { + if (this.cancelled) { + throw new Error("Export cancelled"); + } + // Coalesced preparation heartbeat: report at most once per throttle + // interval (plus a final report on the last frame) so the UI stays + // responsive during long sidecar generation without one React update + // per frame. This is preparation progress only; render FPS is never + // faked here (currentFrame stays 0 in the preparing phase). + if (onPreparationProgress) { + const nowMs = this.getNowMs(); + if ( + nowMs - lastPreparationProgressMs >= + NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS || + frameIndex === totalFrames - 1 + ) { + lastPreparationProgressMs = nowMs; + onPreparationProgress( + totalFrames > 0 ? (frameIndex / totalFrames) * 100 : 0, + ); + } + } + const timestampUs = Math.round((frameIndex * 1_000_000) / this.config.frameRate); + try { + await renderer.renderOverlayFrame(timestampUs); + } catch (error) { + throw new Error( + `overlay-renderer-frame: ${error instanceof Error ? error.message : String(error)}`, + ); + } + let frame: Uint8Array; + try { + frame = await captureCanvasFrameForNativeExport( + renderer.getCanvas(), + timestampUs, + ); + } catch (error) { + throw new Error( + `overlay-canvas-capture: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (frame.byteLength !== frameByteSize) { + throw new Error( + `overlay-invalid-frame-size: expected ${frameByteSize} bytes, received ${frame.byteLength}`, + ); + } + + if (runFrame === null) { + runFrame = frame; + runStartFrameIndex = frameIndex; + } else if (!areNativeStaticLayoutOverlayFramesEqual(frame, runFrame)) { + await flushRawIdenticalRun(frameIndex); + runFrame = frame; + runStartFrameIndex = frameIndex; + } + + if (tiledAbandoned) { + continue; + } + + const changedTiles: NativeTiledOverlayTileRecord[] = []; + for (let tileY = 0; tileY < tileRows; tileY += 1) { + for (let tileX = 0; tileX < tileColumns; tileX += 1) { + const tileIndex = getNativeTiledOverlayTileIndex(tileX, tileY, tileColumns); + this.extractNativeTiledOverlayTileInto( + scratchTile, + frame, + this.config.width, + this.config.height, + tileX, + tileY, + ); + const previous = previousTiles[tileIndex]; + if ( + previous !== null && + areNativeStaticLayoutOverlayFramesEqual(previous, scratchTile) + ) { + continue; + } + if ( + tiledPayloadOffset + NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE >= + maxTiledPayloadBytes + ) { + tiledAbandoned = true; + rawFallbackReason = "payload-bytes-exceed-raw"; + tiledPayloadBuffers.length = 0; + staticTiles.length = 0; + frameDeltas.length = 0; + previousTiles.length = 0; + break; + } + const tileCopy = scratchTile.slice(); + const record: NativeTiledOverlayTileRecord = { + tileIndex, + byteOffset: tiledPayloadOffset, + byteLength: NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE, + }; + tiledPayloadBuffers.push(tileCopy); + tiledPayloadOffset += NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE; + previousTiles[tileIndex] = tileCopy; + if (frameIndex === 0) { + staticTiles.push(record); + } else { + changedTiles.push(record); + } + } + if (tiledAbandoned) { + break; + } + } + if (tiledAbandoned) { + continue; + } + if (frameIndex > 0 && changedTiles.length > 0) { + if ( + changedTiles.length > + tileCount * NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION + ) { + tiledAbandoned = true; + rawFallbackReason = "dense-frame-delta"; + tiledPayloadBuffers.length = 0; + staticTiles.length = 0; + frameDeltas.length = 0; + previousTiles.length = 0; + continue; + } + frameDeltas.push({ frameIndex, changedTiles }); + } + } + + if (runFrame !== null) { + await writeRawOverlayChunk(runStartFrameIndex, 1, runFrame); + } + + let rawClosed: Awaited>; + try { + rawClosed = await api.closeExportStream(activeRawStreamId); + } catch (error) { + throw new Error( + `overlay-stream-close: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (!rawClosed.success || !rawClosed.tempPath) { + throw new Error( + `overlay-stream-close: ${rawClosed.error ?? "Native overlay export stream did not finalize"}`, + ); + } + rawTempPath = rawClosed.tempPath; + const rawExpectedBytes = frameByteSize * rawWrittenFrameCount; + if (rawClosed.bytesWritten !== rawExpectedBytes) { + throw new Error( + `overlay-stream-truncated: expected ${rawExpectedBytes} bytes, stream wrote ${rawClosed.bytesWritten}`, + ); + } + + if (!tiledAbandoned) { + const tileLayer: NativeTiledOverlayLayerDescriptor = { + id: "native-effects", + order: 0, + x: 0, + y: 0, + width: this.config.width, + height: this.config.height, + frameRate: this.config.frameRate, + durationSec, + frameCount: totalFrames, + tileSize: NATIVE_TILED_OVERLAY_TILE_SIZE, + pixelFormat: NATIVE_TILED_OVERLAY_PIXEL_FORMAT, + payloadPath: "", + payloadByteLength: tiledPayloadOffset, + staticTiles, + frameDeltas, + }; + const finalFallbackReason = resolveNativeTiledOverlayRawFallbackReason(tileLayer); + if (finalFallbackReason) { + tiledAbandoned = true; + rawFallbackReason = finalFallbackReason; + tiledPayloadBuffers.length = 0; + staticTiles.length = 0; + frameDeltas.length = 0; + previousTiles.length = 0; + } else { + let tiledStream: Awaited> | null = null; + try { + tiledStream = await api.openExportStream({ extension: "tiledrgba" }); + if ( + !tiledStream.success || + !tiledStream.streamId || + !tiledStream.tempPath + ) { + throw new Error( + tiledStream.error ?? + "Tiled overlay export stream could not be opened", + ); + } + const activeTiledStreamId = tiledStream.streamId; + for ( + let bufferIndex = 0; + bufferIndex < tiledPayloadBuffers.length; + bufferIndex += 1 + ) { + const offset = bufferIndex * NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE; + const result = await api.writeExportStreamChunk( + activeTiledStreamId, + offset, + tiledPayloadBuffers[bufferIndex]!, + ); + if (!result.success) { + throw new Error( + result.error ?? "Failed to write tiled overlay tile", + ); + } + } + const tiledClosed = await api.closeExportStream(activeTiledStreamId); + if (!tiledClosed.success || !tiledClosed.tempPath) { + throw new Error( + tiledClosed.error ?? "Tiled overlay export stream did not finalize", + ); + } + const tiledExpectedBytes = + tiledPayloadBuffers.length * NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE; + if (tiledClosed.bytesWritten !== tiledExpectedBytes) { + throw new Error( + `tiled-overlay-stream-truncated: expected ${tiledExpectedBytes} bytes, stream wrote ${tiledClosed.bytesWritten}`, + ); + } + tileLayer.payloadPath = tiledClosed.tempPath; + if (rawTempPath) { + await api.discardExportedTemp(rawTempPath).catch(() => undefined); + } + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([tileLayer]), + rawFallbackReason: null, + }; + } catch (error) { + if (tiledStream?.streamId) { + await api + .closeExportStream(tiledStream.streamId, { abort: true }) + .catch(() => undefined); + } + throw error; + } + } + } + + const rawLayer: NativeStaticLayoutOverlayLayer = { + id: "native-effects", + order: 0, + path: rawTempPath, + x: 0, + y: 0, + width: this.config.width, + height: this.config.height, + frameRate: this.config.frameRate, + durationSec, + frameCount: totalFrames, + ...(rawWrittenFrameCount < totalFrames + ? { effectiveFrameCount: rawWrittenFrameCount } + : {}), + pixelFormat: "rgba", + }; + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([rawLayer]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const stage = + message.startsWith("overlay-") || message.startsWith("tiled-overlay-") + ? message.split(":", 1)[0] + : "overlay-preparation"; + this.recordNativeStaticLayoutOverlayFailure(stage, message); + if (rawStreamId) { + try { + await api.closeExportStream(rawStreamId, { abort: true }); + } catch { + // best-effort cleanup + } + } + if (rawTempPath) { + try { + await api.discardExportedTemp(rawTempPath); + } catch { + // best-effort cleanup + } + } + console.warn("[VideoExporter] Native overlay preparation failed", { + stage, + message, + durationSec, + totalFrames, + frameByteSize, + rawWrittenFrameCount, + tiledPayloadOffset, + cancelled: this.cancelled, + }); + return null; + } finally { + try { + renderer.destroy(); + } catch { + // Cleanup is best-effort after the native overlay stream closes. + } + } + } + + private logNativeStaticLayoutPreparationStage( + stage: string, + startedAtMs: number, + extra: Record = {}, + ): void { + const elapsedMs = Math.round(this.getNowMs() - startedAtMs); + console.info(formatLogTs(), "[VideoExporter] Native static layout preparation stage", { + stage, + elapsedMs, + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", + route: this.canUseNativeGpuStaticLayout() ? "nvidia-cuda-compositor" : "static-layout", + ...extra, + }); + } + + private recordNativeStaticLayoutOverlayFailure(stage: string, message: string): void { + this.nativeStaticLayoutOverlayFailure = { stage, message }; + } + + private async tryExportNativeStaticLayout( + videoInfo: DecodedVideoInfo, + audioPlan: NativeAudioPlan, + effectiveDuration: number, + totalFrames: number, + ): Promise { + const skipReason = this.getNativeStaticLayoutSkipReason( + audioPlan, + videoInfo, + effectiveDuration, + ); + const skipReasons = skipReason + ? this.getNativeStaticLayoutSkipReasons(audioPlan, videoInfo, effectiveDuration) + : []; + if (skipReason) { + this.nativeStaticLayoutSkipReason = skipReason; + this.nativeStaticLayoutSkipReasons = skipReasons; + console.info(formatLogTs(), "[VideoExporter] Native static layout skipped", { + route: "native-static-layout", + fallbackRoute: "breeze-stream-or-raw-frame", + reason: skipReason, + reasons: skipReasons, + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + experimentalNativeExport: this.config.experimentalNativeExport === true, + experimentalNvidiaCudaExport: this.config.experimentalNvidiaCudaExport === true, + audioMode: audioPlan.audioMode, + zoomRegions: this.config.zoomRegions?.length ?? 0, + speedRegions: this.config.speedRegions?.length ?? 0, + audioRegions: this.config.audioRegions?.length ?? 0, + annotationRegions: this.config.annotationRegions?.length ?? 0, + hasFrame: Boolean(this.config.frame), + backgroundBlur: this.config.backgroundBlur, + hasCursorOverlay: + this.config.showCursor === true && + (this.config.cursorTelemetry?.length ?? 0) > 0, + }); + return null; + } + + // Emit the initial "preparing" signal before the potentially long + // audio/background/cursor/overlay preparation begins, and identify the + // NVIDIA CUDA compositor as the selected route when it is eligible so the + // first progress never shows a stale WebGPU/Breeze/libx264 backend during + // CUDA preparation. + this.encodeBackend = "ffmpeg"; + this.encoderName = this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : this.config.experimentalNativeExport === true && this.getRuntimePlatform() === "win32" + ? "windows-native-compositor" + : "static-layout-h264-nvenc"; + this.exportStartTimeMs = this.getNowMs(); + this.lastProgressSampleTimeMs = this.exportStartTimeMs; + this.lastProgressSampleFrame = 0; + this.reportProgress(0, totalFrames, "preparing"); + + let preparationStageStartedAt = this.getNowMs(); + const sourcePath = this.getNativeVideoSourcePath(); + const audioOptions = await this.getNativeStaticLayoutAudioOptions(audioPlan, totalFrames); + this.logNativeStaticLayoutPreparationStage("audio", preparationStageStartedAt, { + audioMode: audioPlan.audioMode, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + }); + preparationStageStartedAt = this.getNowMs(); + if (!sourcePath || !audioOptions) { + this.nativeStaticLayoutSkipReason = !sourcePath + ? "missing-source-path" + : "missing-audio-options"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; + return null; + } + const background = await this.resolveNativeStaticLayoutBackground(); + this.logNativeStaticLayoutPreparationStage("background", preparationStageStartedAt, { + backgroundColor: background?.backgroundColor ?? null, + hasBackgroundImage: Boolean(background?.backgroundImagePath), + backgroundSkipReason: this.nativeStaticLayoutBackgroundSkipReason ?? null, + }); + preparationStageStartedAt = this.getNowMs(); + if (!background) { + this.nativeStaticLayoutSkipReason = + this.nativeStaticLayoutBackgroundSkipReason ?? "unsupported-background"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; + return null; + } + + const layout = computePaddedLayout({ + width: this.config.width, + height: this.config.height, + padding: this.config.padding ?? 0, + cropRegion: this.config.cropRegion, + videoWidth: videoInfo.width, + videoHeight: videoInfo.height, + }); + const contentSize = roundNativeStaticLayoutContentSize({ + width: layout.croppedDisplayWidth, + height: layout.croppedDisplayHeight, + }); + const contentWidth = contentSize.width; + const contentHeight = contentSize.height; + if ( + contentWidth > this.config.width || + contentHeight > this.config.height || + !Number.isFinite(effectiveDuration) || + effectiveDuration <= 0 + ) { + this.nativeStaticLayoutSkipReason = "invalid-layout-or-duration"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; + await this.cleanupNativeStaticLayoutBackground(background); + return null; + } + + const offsetX = Math.round(layout.centerOffsetX); + const offsetY = Math.round(layout.centerOffsetY); + const sourceCrop = this.isDefaultCropRegion() + ? null + : this.getNativeStaticLayoutSourceCrop(videoInfo); + const borderRadius = scalePreviewBorderRadius( + this.config.width, + this.config.height, + this.config.borderRadius ?? 0, + ); + const shadowIntensity = this.config.showShadow + ? Math.min(1, Math.max(0, this.config.shadowIntensity)) + : 0; + const webcamOverlay = this.getNativeStaticLayoutWebcamOverlay(); + const webcamNativeOwned = this.canUseNativeWebcamOwnership(); + const cursorTelemetry = this.getNativeStaticLayoutCursorTelemetry(); + const zoomTelemetry = this.getNativeStaticLayoutZoomTelemetry( + layout, + totalFrames, + cursorTelemetry, + ); + const needsTimelineMap = this.shouldUseNativeStaticLayoutTimelineMap( + videoInfo, + effectiveDuration, + ); + const timelineSegments = needsTimelineMap + ? this.buildNativeStaticLayoutVideoTimelineSegments(videoInfo) + : undefined; + if (needsTimelineMap && !timelineSegments?.length) { + this.nativeStaticLayoutSkipReason = + (this.config.speedRegions ?? []).length > 0 + ? "invalid-native-speed-timeline" + : "invalid-native-trim-timeline"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; + await this.cleanupNativeStaticLayoutBackground(background); + return null; + } + const needsOverlayLayers = this.hasNativeStaticLayoutOverlayContent(); + const wantsNativeCursorOwnership = + needsOverlayLayers && this.canUseNativeCursorAtlasOwnership(); + const cursorAtlas = + cursorTelemetry && + cursorTelemetry.length > 0 && + (!needsOverlayLayers || wantsNativeCursorOwnership) + ? await buildNativeCursorAtlas(this.config.cursorStyle ?? "tahoe").catch( + (error) => { + console.warn("[VideoExporter] Native cursor atlas unavailable", error); + return null; + }, + ) + : null; + this.logNativeStaticLayoutPreparationStage("cursor-atlas", preparationStageStartedAt, { + wantsNativeCursorOwnership, + cursorAtlasBuilt: Boolean(cursorAtlas), + atlasEntries: cursorAtlas?.entries.length ?? 0, + }); + preparationStageStartedAt = this.getNowMs(); + const cursorAtlasOwnedByNative = wantsNativeCursorOwnership && Boolean(cursorAtlas); + if (cursorAtlasOwnedByNative) { + console.info("[VideoExporter] Native cursor atlas owns the overlay cursor", { + cursorStyle: this.config.cursorStyle ?? "tahoe", + atlasWidth: cursorAtlas?.width, + atlasHeight: cursorAtlas?.height, + atlasEntries: cursorAtlas?.entries.length, + cursorTelemetrySamples: cursorTelemetry?.length, + }); + } + // The native cursor atlas is required when the cursor is NOT baked into + // the transparent overlay sidecar. Without overlay layers the cursor is + // always native-owned, so a missing atlas skips the route. With overlay + // layers the cursor is baked into the sidecar unless the CUDA compositor + // owns it (cursorAtlasOwnedByNative), so a missing atlas only falls back + // to the baked sidecar and never skips the route. + if ( + shouldSkipForMissingCursorAtlas({ + needsOverlayLayers, + hasCursorTelemetry: Boolean(cursorTelemetry && cursorTelemetry.length > 0), + hasCursorAtlas: Boolean(cursorAtlas), + }) + ) { + this.nativeStaticLayoutSkipReason = "cursor-atlas-unavailable"; + this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; + await this.cleanupNativeStaticLayoutBackground(background); + return null; + } + const fastLaneEligibility = this.getNativeStaticLayoutFastLaneEligibility( + audioPlan, + cursorAtlasOwnedByNative, + webcamNativeOwned, + ); + const useFastLane = fastLaneEligibility.eligible; + let overlayPreparation: NativeStaticLayoutOverlayPreparationResult | null = null; + if (useFastLane) { + // Deterministic no-browser-overlay fast lane: with no captions, + // annotations, or frame pixels (and the webcam owned natively by the CUDA + // compositor when enabled) and the cursor either disabled or owned + // natively by the CUDA compositor, the sidecar is provably empty, so + // skip renderer init, per-frame canvas capture, and overlay sidecar + // creation and start the native export as early as safely allowed. + overlayPreparation = { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + console.info(formatLogTs(), "[VideoExporter] Native static layout fast lane selected", { + route: "nvidia-cuda-compositor", + skipReasons: fastLaneEligibility.skipReasons, + cursorDisabled: + this.config.showCursor !== true || + (this.config.cursorTelemetry?.length ?? 0) === 0, + cursorNativeOwnershipActive: Boolean(cursorAtlasOwnedByNative), + webcamNativeOwned: Boolean(webcamNativeOwned), + audioMode: audioPlan.audioMode, + preparedOverlayLayers: overlayPreparation.overlayLayers.length, + preparedTiledOverlayLayers: overlayPreparation.tiledOverlayLayers.length, + }); + } else { + overlayPreparation = await this.prepareNativeStaticLayoutOverlay( + videoInfo, + effectiveDuration, + totalFrames, + cursorAtlasOwnedByNative, + webcamNativeOwned, + (renderProgress) => + this.reportProgress(0, totalFrames, "preparing", renderProgress), + ); + } + const overlayLayers = overlayPreparation?.overlayLayers ?? []; + const tiledOverlayLayers = overlayPreparation?.tiledOverlayLayers ?? []; + this.logNativeStaticLayoutPreparationStage("overlay", preparationStageStartedAt, { + mode: useFastLane + ? "fast-lane" + : !overlayPreparation + ? "failed" + : tiledOverlayLayers.length > 0 + ? "tiled-sidecar" + : overlayLayers.some(isCursorSpriteOverlayLayer) + ? "cursor-sprite" + : overlayLayers.length > 0 + ? "raw-sidecar" + : "empty", + overlayLayerCount: overlayLayers.length, + tiledOverlayLayerCount: tiledOverlayLayers.length, + rawFallbackReason: overlayPreparation?.rawFallbackReason ?? null, + overlayFailure: this.nativeStaticLayoutOverlayFailure, + webcamNativeOwned: Boolean(webcamNativeOwned), + }); + preparationStageStartedAt = this.getNowMs(); + if (needsOverlayLayers && !overlayPreparation) { + this.nativeStaticLayoutSkipReason = "native-overlay-preparation-failed"; + const overlayFailure = this.nativeStaticLayoutOverlayFailure; + this.nativeStaticLayoutSkipReasons = overlayFailure + ? [ + this.nativeStaticLayoutSkipReason, + `overlay-stage:${overlayFailure.stage}`, + `overlay-error:${overlayFailure.message}`, + ] + : [this.nativeStaticLayoutSkipReason]; + console.warn( + formatLogTs(), + "[VideoExporter] Native static layout skipped: overlay preparation failed", + { + reason: this.nativeStaticLayoutSkipReason, + reasons: this.nativeStaticLayoutSkipReasons, + failure: overlayFailure, + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", + }, + ); + await this.cleanupNativeStaticLayoutBackground(background); + return null; + } + const overlayTempPath = + tiledOverlayLayers[0]?.payloadPath ?? overlayLayers[0]?.path ?? null; + const startedAt = this.getNowMs(); + const sessionId = `recordly-static-layout-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}`; + const previousEncodeBackend = this.encodeBackend; + const previousEncoderName = this.encoderName; + const restoreEncoderState = () => { + this.encodeBackend = previousEncodeBackend; + this.encoderName = previousEncoderName; + }; + this.exportStartTimeMs = startedAt; this.lastThroughputLogTimeMs = startedAt; this.lastProgressSampleTimeMs = startedAt; @@ -2370,13 +4041,15 @@ export class ModernVideoExporter { this.nativeStaticLayoutSkipReason = null; this.nativeStaticLayoutSkipReasons = []; this.nativeStaticLayoutAverageFps = null; + this.nativeStaticLayoutFpsSource = null; this.encodeBackend = "ffmpeg"; const runtimePlatform = typeof navigator !== "undefined" ? normalizeLightningRuntimePlatform(navigator.userAgent) : "unknown"; - this.encoderName = - this.config.experimentalNativeExport === true && runtimePlatform === "win32" + this.encoderName = this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : this.config.experimentalNativeExport === true && runtimePlatform === "win32" ? "windows-native-compositor" : "static-layout-h264-nvenc"; this.reportProgress(0, totalFrames, "preparing"); @@ -2411,6 +4084,7 @@ export class ModernVideoExporter { rawNativePercentage <= 3) ) { this.nativeStaticLayoutAverageFps = null; + this.nativeStaticLayoutFpsSource = null; this.processedFrameCount = 0; this.reportProgress(0, totalFrames, "preparing"); return; @@ -2440,7 +4114,7 @@ export class ModernVideoExporter { maxExtractingFrame, Math.max(this.processedFrameCount, nativeCurrentFrame), ); - this.nativeStaticLayoutAverageFps = + const nativeMeasuredFps = progress.stage === "finalizing" ? null : typeof progress.instantFps === "number" && @@ -2452,6 +4126,26 @@ export class ModernVideoExporter { progress.averageFps > 0 ? progress.averageFps : null; + const estimatedFps = + progress.stage === "finalizing" || nativeMeasuredFps !== null + ? null + : typeof progress.estimatedFps === "number" && + Number.isFinite(progress.estimatedFps) && + progress.estimatedFps > 0 + ? progress.estimatedFps + : null; + if (estimatedFps !== null) { + // Preparation-inclusive estimate; never presented as measured encode speed. + this.nativeStaticLayoutFpsSource = "estimated"; + console.warn( + formatLogTs(), + "[VideoExporter] Native encode FPS not reported yet; using preparation-inclusive estimate", + { backend: progress.backend, estimatedFps }, + ); + } else if (nativeMeasuredFps !== null) { + this.nativeStaticLayoutFpsSource = "native"; + } + this.nativeStaticLayoutAverageFps = nativeMeasuredFps; this.processedFrameCount = currentFrame; if (progress.stage === "finalizing" || nativeFramesComplete) { this.reportFinalizingProgress(totalFrames, nativeFinalizingProgress); @@ -2461,8 +4155,13 @@ export class ModernVideoExporter { }, ); + const requestedVideoCodec = this.config.exportVideoCodec ?? "h264"; + const requestedEncoderPreference = this.config.exportEncoderPreference ?? "auto"; try { - const result = await window.electronAPI.nativeStaticLayoutExport({ + // The IPC surface type predates native cursor ownership; the extra + // cursorAtlasOwned field rides through the structured clone into the + // main-process NativeStaticLayoutExportOptions where it is consumed. + const nativeStaticLayoutOptions = { sessionId, inputPath: sourcePath, width: this.config.width, @@ -2470,6 +4169,8 @@ export class ModernVideoExporter { frameRate: this.config.frameRate, bitrate: this.config.bitrate, encodingMode: this.config.encodingMode ?? "balanced", + videoCodec: requestedVideoCodec, + encoderPreference: requestedEncoderPreference, durationSec: effectiveDuration, contentWidth, contentHeight, @@ -2484,7 +4185,17 @@ export class ModernVideoExporter { backgroundBlurPx: Math.max(0, (this.config.backgroundBlur ?? 0) * 3), borderRadius, shadowIntensity, - webcamInputPath: webcamOverlay?.inputPath ?? null, + // When the webcam is native-owned the renderer excluded it from the + // overlay sidecar, so webcamInputPath must reach the CUDA compositor + // even when a cursor-sprite (or baked-cursor) overlay layer is present. + // Mixed baked content (captions/annotations/frame) never sets + // webcamNativeOwned, so the existing baked-webcam contract (no + // webcamInputPath alongside sidecar pixels) is preserved. + webcamInputPath: webcamNativeOwned + ? (webcamOverlay?.inputPath ?? null) + : overlayLayers.length || tiledOverlayLayers.length + ? null + : (webcamOverlay?.inputPath ?? null), webcamLeft: webcamOverlay?.left, webcamTop: webcamOverlay?.top, webcamSize: webcamOverlay?.size, @@ -2492,11 +4203,24 @@ export class ModernVideoExporter { webcamShadowIntensity: webcamOverlay?.shadowIntensity, webcamMirror: webcamOverlay?.mirror, webcamTimeOffsetMs: webcamOverlay?.timeOffsetMs, + // True only when the CUDA compositor owns the webcam: the overlay + // sidecar excluded webcam pixels and the native webcam overlay must + // draw them (never double-render a baked webcam). + webcamNativeOwned: webcamNativeOwned || undefined, cursorTelemetry, cursorSize: this.getNativeStaticLayoutCursorSize(contentWidth), cursorAtlasPngDataUrl: cursorAtlas?.dataUrl ?? null, cursorAtlasEntries: cursorAtlas?.entries, + // True only when the CUDA compositor owns the cursor: the overlay + // sidecar excluded cursor pixels and the native atlas must draw them. + cursorAtlasOwned: cursorAtlasOwnedByNative || undefined, + overlayLayers: overlayLayers.length ? overlayLayers : undefined, + tiledOverlayLayers: tiledOverlayLayers.length ? tiledOverlayLayers : undefined, zoomTelemetry, + temporalBlur: getTemporalMotionBlurConfig(this.config.zoomTemporalMotionBlur, { + sampleCount: this.config.zoomMotionBlurSampleCount, + shutterFraction: this.config.zoomMotionBlurShutterFraction, + }), timelineSegments, chunkDurationSec: STATIC_LAYOUT_CHUNK_DURATION_SEC, experimentalWindowsGpuCompositor: this.config.experimentalNativeExport === true, @@ -2505,8 +4229,19 @@ export class ModernVideoExporter { ...audioOptions, outputDurationSec: effectiveDuration, }, + }; + const ipcHandoffStartedAt = this.getNowMs(); + const result = + await window.electronAPI.nativeStaticLayoutExport(nativeStaticLayoutOptions); + this.logNativeStaticLayoutPreparationStage("ipc-handoff", ipcHandoffStartedAt, { + route: result.route ?? null, + success: result.success, + requestedVideoCodec, + requestedEncoderPreference, + requestedRoute: this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : "static-layout", }); - if (this.cancelled) { return { success: false, @@ -2516,16 +4251,176 @@ export class ModernVideoExporter { } if (!result.success || !result.tempPath) { - console.warn("[VideoExporter] Native static layout export unavailable", { - error: result.error, - }); + const exportError = + typeof result.error === "string" && result.error.trim() + ? result.error.trim() + : "unknown-native-static-layout-export-error"; + console.warn( + formatLogTs(), + "[VideoExporter] Native static layout export unavailable", + { + error: exportError, + }, + ); + // Surface the real IPC/helper failure instead of a generic + // "route unavailable" when strict HEVC Hardware later refuses the + // renderer raw fallback. The strict error carries this detail so CUDA + // export failures stay diagnosable end-to-end. + this.lastNativeExportError = exportError; + this.nativeStaticLayoutSkipReasons = [ + "native-ipc-export-failed", + `native-error:${exportError}`, + ]; + restoreEncoderState(); + return null; + } + + const isStrictHevcHardware = this.requiresStrictNativeCudaRoute(); + const acceptedHevcNativeRoute = isStrictHevcHardware + ? result.route === "nvidia-cuda-compositor" + : HEVC_NATIVE_STATIC_LAYOUT_ROUTES.has(result.route ?? ""); + if (requestedVideoCodec === "hevc" && !acceptedHevcNativeRoute) { + const routeSkipReason = "unsupported-native-hevc-route"; + console.warn( + "[VideoExporter] Rejecting HEVC native static-layout result from a non-CUDA route", + { route: result.route, isStrictHevcHardware }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs for + // HEVC); discard it before falling back so it is not left on disk for + // the whole session. Best-effort: cleanup must never override the + // intended skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); + restoreEncoderState(); + return null; + } + + const hasSpatialZoomMotionBlur = (this.config.zoomMotionBlur ?? 0) > 0.0005; + const hasTemporalZoomMotionBlur = (this.config.zoomTemporalMotionBlur ?? 0) > 0.0005; + if ( + shouldRejectNativeStaticLayoutResultForEffectPreservation({ + hasSpatialZoomMotionBlur, + hasTemporalMotionBlur: hasTemporalZoomMotionBlur, + hasOverlayContent: this.hasNativeStaticLayoutOverlayContent(), + route: result.route, + }) + ) { + // The generalized CUDA compositor applies spatial zoom blur before + // alpha-compositing the transparent overlay sidecars and implements + // temporal zoom motion blur from the resolved sample plan. The FFmpeg + // effectful overlay route and the D3D11 helper cannot preserve these + // effects; reject so the renderer raw-frame fallback keeps them instead + // of silently dropping them. + const routeSkipReason = "unsupported-motion-blur-on-overlay-route"; + console.warn( + "[VideoExporter] Rejecting native static-layout result that cannot preserve zoom motion blur", + { route: result.route }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs for + // HEVC); discard it before falling back so it is not left on disk for + // the whole session. Best-effort: cleanup must never override the + // intended skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); + restoreEncoderState(); + return null; + } + // A cursor-sprite layer is only composited by the generalized NVIDIA + // CUDA compositor. If the actual route is anything else (FFmpeg effectful + // overlay or D3D11 helper) it would silently drop the cursor, so reject + // and let the renderer raw-frame fallback keep it. + const hasCursorSpriteLayer = overlayLayers.some((layer) => + isCursorSpriteOverlayLayer(layer), + ); + if (hasCursorSpriteLayer && result.route !== "nvidia-cuda-compositor") { + const routeSkipReason = "unsupported-cursor-sprite-route"; + console.warn( + "[VideoExporter] Rejecting native static-layout result that cannot compose the cursor sprite", + { route: result.route, layerCount: overlayLayers.length }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs); + // discard it before falling back so it is not left on disk for the + // whole session. Best-effort: cleanup must never override the intended + // skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); + restoreEncoderState(); + return null; + } + // A native-owned webcam is only drawn by the generalized NVIDIA CUDA + // compositor (the sidecar excluded webcam pixels). Any other route would + // silently drop the webcam, so reject and let the renderer raw-frame + // fallback keep it instead. Strict HEVC Hardware already refuses non-CUDA + // routes; this guard is the explicit observable invariant for webcam + // ownership on every codec/preference combination. + if (webcamNativeOwned && result.route !== "nvidia-cuda-compositor") { + const routeSkipReason = "unsupported-native-webcam-route"; + console.warn( + "[VideoExporter] Rejecting native static-layout result that cannot draw the native-owned webcam", + { route: result.route, webcamNativeOwned }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs); + // discard it before falling back so it is not left on disk for the + // whole session. Best-effort: cleanup must never override the intended + // skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); restoreEncoderState(); return null; } + console.info(formatLogTs(), "[VideoExporter] Native static layout selected", { + route: result.route, + encoderName: result.encoderName, + exportVideoCodec: requestedVideoCodec, + exportEncoderPreference: requestedEncoderPreference, + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + experimentalNativeExport: this.config.experimentalNativeExport === true, + experimentalNvidiaCudaExport: this.config.experimentalNvidiaCudaExport === true, + hasOverlayLayers: this.hasNativeStaticLayoutOverlayContent(), + webcamNativeOwned: Boolean(webcamNativeOwned), + temporalBlurSamples: + getTemporalMotionBlurConfig(this.config.zoomTemporalMotionBlur, { + sampleCount: this.config.zoomMotionBlurSampleCount, + shutterFraction: this.config.zoomMotionBlurShutterFraction, + })?.sampleCount ?? null, + }); + if (result.route === "cuda-overlay" && this.hasNativeStaticLayoutOverlayContent()) { + // Effectful overlay composition runs after a CUDA hwdownload and is + // performed by FFmpeg's CPU alpha overlay filters. This is required to + // alpha-compose RGBA sidecars, but it is the expected throughput + // bottleneck on this route and should not be misreported as GPU encode + // speed in the FPS diagnostics. + console.info( + "[VideoExporter] Native overlay route uses CPU alpha overlay composition", + { + route: result.route, + note: "encode FPS reflects CPU-overlay-limited throughput, not raw NVENC speed", + }, + ); + } const elapsedMs = this.getNowMs() - startedAt; - this.encoderName = result.encoderName ?? "static-layout-h264-nvenc"; + this.encoderName = + result.encoderName ?? + (result.route && requestedVideoCodec === "hevc" + ? result.route + : requestedVideoCodec === "hevc" + ? "static-layout-hevc" + : "static-layout-h264-nvenc"); this.nativeStaticLayoutAverageFps = null; + this.nativeStaticLayoutFpsSource = null; this.processedFrameCount = totalFrames; this.decodeLoopTimeMs = result.metrics?.chunkExecMs ?? elapsedMs; this.finalizationTimeMs = Math.max(0, elapsedMs - this.decodeLoopTimeMs); @@ -2566,13 +4461,27 @@ export class ModernVideoExporter { }; } - console.warn("[VideoExporter] Native static layout export failed; falling back", error); + console.warn( + formatLogTs(), + "[VideoExporter] Native static layout export failed; falling back", + error, + ); + const failureMessage = error instanceof Error ? error.message : String(error); + this.lastNativeExportError = failureMessage; this.nativeStaticLayoutSkipReason = "native-static-runtime-failed"; this.nativeStaticLayoutSkipReasons = [this.nativeStaticLayoutSkipReason]; restoreEncoderState(); return null; } finally { unsubscribeNativeProgress?.(); + // The static-layout attempt is over (success, skip, or runtime failure). + // Clear native-measured FPS so a raw renderer fallback can never present + // stale native encode speed as its own throughput. + this.nativeStaticLayoutAverageFps = null; + this.nativeStaticLayoutFpsSource = null; + if (overlayTempPath && typeof window !== "undefined") { + await window.electronAPI?.discardExportedTemp?.(overlayTempPath); + } await this.cleanupNativeStaticLayoutBackground(background); if (this.nativeStaticLayoutSessionId === sessionId) { this.nativeStaticLayoutSessionId = null; @@ -2650,6 +4559,7 @@ export class ModernVideoExporter { } this.nativeExportSessionId = result.sessionId; + this.nativeRawFrameMode = false; this.lastNativeExportError = null; this.encodeBackend = "ffmpeg"; this.encoderName = "h264-stream-copy"; @@ -2702,11 +4612,199 @@ export class ModernVideoExporter { return true; } + private canUseNativeGpuStaticLayout(): boolean { + return ( + this.config.exportVideoCodec === "hevc" && + this.config.exportEncoderPreference !== "cpu" && + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true + ); + } + + // Strict HEVC Hardware policy: the generalized NVIDIA CUDA compositor is the + // ONLY acceptable route. The export must never silently fall back to the + // renderer raw frame path (WebGPU/WebGL -> FFmpeg hevc_nvenc), Breeze, or CPU + // when the CUDA route cannot run; it hard-fails with an actionable error. + private requiresStrictNativeCudaRoute(): boolean { + return ( + this.config.exportVideoCodec === "hevc" && + this.config.exportEncoderPreference === "hardware" + ); + } + + private buildStrictNativeCudaHardwareError(reason: string): Error { + const overlayDetail = this.nativeStaticLayoutOverlayFailure + ? ` (${this.nativeStaticLayoutOverlayFailure.stage}: ${this.nativeStaticLayoutOverlayFailure.message})` + : ""; + const message = [ + "HEVC Hardware export requires the NVIDIA CUDA compositor.", + `Native CUDA route did not run: ${reason}${overlayDetail}.`, + "The export was stopped instead of falling back to renderer raw frames (WebGPU/Breeze) or CPU.", + "The NVIDIA CUDA compositor backend is mandatory for H.265 + Hardware. Make sure the CUDA compositor is available (NVIDIA GPU with current drivers and the bundled compositor helper), or switch the encoder preference to Auto.", + "noCpuFallback:true", + ].join(" "); + const error = new Error(message); + (error as Error & { noCpuFallback?: boolean }).noCpuFallback = true; + return error; + } + + private requiresNativeRawFrame(): boolean { + return ( + this.config.exportVideoCodec === "hevc" || + (this.config.exportEncoderPreference !== undefined && + this.config.exportEncoderPreference !== "auto") + ); + } + + private shouldForceNativeRawFrame(): boolean { + // Strict HEVC Hardware forbids the renderer raw frame path entirely; the + // native static-layout CUDA compositor is mandatory and any failure must + // hard-fail instead of falling back. + if (this.requiresStrictNativeCudaRoute()) { + return false; + } + // HEVC Auto/Hardware gets one native CUDA static-layout attempt when the + // renderer was given an eligible GPU route. CPU and explicit H.264 encoder + // preferences remain direct rawvideo paths; H.264 + Auto is unchanged. + return this.requiresNativeRawFrame() && !this.canUseNativeGpuStaticLayout(); + } + + private async tryStartNativeVideoExportRawFrame(): Promise { + this.lastNativeExportError = null; + + if (typeof window === "undefined" || !window.electronAPI?.nativeVideoExportStart) { + this.lastNativeExportError = `${NATIVE_EXPORT_ENGINE_NAME} export is not available in this build.`; + return false; + } + + if (this.config.width % 2 !== 0 || this.config.height % 2 !== 0) { + this.lastNativeExportError = `${NATIVE_EXPORT_ENGINE_NAME} export requires even output dimensions (${this.config.width}x${this.config.height}).`; + return false; + } + + const videoCodec = this.config.exportVideoCodec ?? "h264"; + const encoderPreference = this.config.exportEncoderPreference ?? "auto"; + const result = await window.electronAPI.nativeVideoExportStart({ + width: this.config.width, + height: this.config.height, + frameRate: this.config.frameRate, + bitrate: this.config.bitrate, + encodingMode: this.config.encodingMode ?? "balanced", + inputMode: "rawvideo", + videoCodec, + encoderPreference, + }); + + if (!result.success || !result.sessionId) { + this.lastNativeExportError = + result.error ?? + `${NATIVE_EXPORT_ENGINE_NAME} ${videoCodec.toUpperCase()} raw-frame export could not be started on this system.`; + console.warn( + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} raw-frame export unavailable`, + result.error, + ); + return false; + } + + this.nativeExportSessionId = result.sessionId; + this.nativeRawFrameMode = true; + this.lastNativeExportError = null; + await this.negotiateNativeRawFrameTransport(result.sessionId); + this.encodeBackend = "ffmpeg"; + this.encoderName = + result.encoderName ?? + (encoderPreference === "hardware" + ? `${videoCodec.toUpperCase()} hardware` + : encoderPreference === "cpu" + ? videoCodec === "hevc" + ? "libx265" + : "libx264" + : videoCodec === "hevc" + ? "hevc-auto" + : "h264-auto"); + this.pendingNativeWriteChunks = []; + this.pendingNativeWriteBytes = 0; + + console.log(`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} raw-frame session ready`, { + sessionId: result.sessionId, + videoCodec, + encoderPreference, + encoderName: this.encoderName, + }); + return true; + } + + private async negotiateNativeRawFrameTransport(sessionId: string): Promise { + this.nativeTransportMode = "cloned-ipc"; + this.nativeTransportFallbackReason = null; + if ( + typeof window === "undefined" || + typeof window.electronAPI?.nativeVideoExportOpenFrameChannel !== "function" || + typeof window.electronAPI?.nativeVideoExportWriteFrameViaChannel !== "function" + ) { + this.nativeTransportFallbackReason = + "Transferable native frame channel API is unavailable"; + return; + } + + try { + const result = await window.electronAPI.nativeVideoExportOpenFrameChannel(sessionId); + if (result.success) { + this.nativeTransportMode = "transferable-stream"; + return; + } + this.nativeTransportFallbackReason = + result.error ?? "Transferable native frame channel negotiation failed"; + } catch (error) { + this.nativeTransportFallbackReason = + error instanceof Error ? error.message : String(error); + } + console.warn( + `[VideoExporter] Falling back to cloned native raw-frame IPC transport: ${this.nativeTransportFallbackReason}`, + ); + } + + private configureNativeRawFrameBackpressure(): void { + if (!this.nativeRawFrameMode || !this.backpressureProfile) { + return; + } + + const rawLimits = getNativeRawFrameBackpressureLimits({ + width: this.config.width, + height: this.config.height, + profile: this.backpressureProfile, + transportMode: this.nativeTransportMode ?? "cloned-ipc", + maxInFlightFrames: this.config.maxInFlightNativeRawFrames, + maxInFlightBytes: this.config.maxInFlightNativeRawBytes, + }); + this.maxNativeRawWriteFrames = rawLimits.maxInFlightFrames; + this.maxNativeRawWriteBytes = rawLimits.maxInFlightBytes; + this.nativeRawBackpressure = new NativeRawFrameBackpressureQueue( + rawLimits.maxInFlightBytes, + rawLimits.maxInFlightFrames, + ); + } + + private recordNativeWriteError(error: Error): void { + if (!this.nativeWriteError) { + this.nativeWriteError = error; + } + if (!this.cancelled && !this.nativeEncoderError) { + this.nativeEncoderError = error; + } + this.nativeRawBackpressure?.fail(error); + this.notifyEncodeCapacityAvailable(); + } + private async encodeRenderedFrameNative( timestamp: number, frameDuration: number, frameIndex: number, ): Promise { + if (this.nativeRawFrameMode) { + await this.encodeRenderedFrameNativeRaw(timestamp); + return; + } if (!this.nativeH264Encoder || !this.nativeExportSessionId) { if (this.cancelled) return; throw new Error(`${NATIVE_EXPORT_ENGINE_NAME} export session is not active`); @@ -2733,6 +4831,87 @@ export class ModernVideoExporter { frame.close(); } + private async encodeRenderedFrameNativeRaw(timestamp: number): Promise { + const sessionId = this.nativeExportSessionId; + if (!sessionId) { + if (this.cancelled) return; + throw new Error(`${NATIVE_EXPORT_ENGINE_NAME} export session is not active`); + } + if (this.nativeEncoderError) throw this.nativeEncoderError; + const frameByteSize = getNativeRawFrameByteSize(this.config.width, this.config.height); + const rawBackpressure = this.nativeRawBackpressure; + if (!rawBackpressure) { + throw new Error( + `${NATIVE_EXPORT_ENGINE_NAME} raw-frame backpressure is not configured`, + ); + } + try { + await rawBackpressure.waitForCapacity(frameByteSize); + } catch (error) { + if (this.cancelled) return; + throw error; + } + if (this.cancelled) return; + + const canvas = this.renderer!.getCanvas(); + const captureStartedAt = this.getNowMs(); + // Flip rows vertically: buildNativeVideoExportArgs applies an FFmpeg vflip for + // rawvideo input, so we counter-rotate before writing the RGBA frame. + const rawFrame = await captureCanvasFrameForNativeExport(canvas, timestamp, true); + this.nativeCaptureTimeMs += this.getNowMs() - captureStartedAt; + if (this.cancelled) return; + + rawBackpressure.reserve(rawFrame.byteLength); + this.peakNativeWriteInFlightBytes = Math.max( + this.peakNativeWriteInFlightBytes, + rawBackpressure.currentInFlightBytes, + ); + const writeStartedAt = this.getNowMs(); + let latencyRecorded = false; + const recordAckLatency = () => { + if (latencyRecorded) { + return; + } + latencyRecorded = true; + const latencyMs = Math.max(0, this.getNowMs() - writeStartedAt); + this.nativeWriteTimeMs += latencyMs; + this.nativeWriteAckTimeMs += latencyMs; + this.nativeFrameTransportTimeMs += latencyMs; + }; + let writeRequest: Promise<{ success: boolean; error?: string }>; + try { + writeRequest = + this.nativeTransportMode === "transferable-stream" + ? window.electronAPI.nativeVideoExportWriteFrameViaChannel(sessionId, rawFrame) + : window.electronAPI.nativeVideoExportWriteFrame(sessionId, rawFrame); + } catch (error) { + recordAckLatency(); + rawBackpressure.release(rawFrame.byteLength); + const resolvedError = error instanceof Error ? error : new Error(String(error)); + this.recordNativeWriteError(resolvedError); + throw resolvedError; + } + this.nativeRawBytesSubmitted += rawFrame.byteLength; + this.nativeRawFramesSubmitted += 1; + + const writePromise = writeRequest + .then((writeResult) => { + recordAckLatency(); + if (!writeResult.success) { + throw new Error( + writeResult.error || + "Failed to write a raw video frame to the native encoder", + ); + } + }) + .catch((error: unknown) => { + recordAckLatency(); + const resolvedError = error instanceof Error ? error : new Error(String(error)); + this.recordNativeWriteError(resolvedError); + }); + this.trackNativeRawWritePromise(writePromise, rawFrame.byteLength); + } + private async finishNativeVideoExport(audioPlan: NativeAudioPlan): Promise { if (!this.nativeExportSessionId) { return { @@ -3051,9 +5230,21 @@ export class ModernVideoExporter { const chunks = this.pendingNativeWriteChunks; this.pendingNativeWriteChunks = []; this.pendingNativeWriteBytes = 0; + const writeStartedAt = this.getNowMs(); + let latencyRecorded = false; + const recordAckLatency = () => { + if (latencyRecorded) { + return; + } + latencyRecorded = true; + const latencyMs = Math.max(0, this.getNowMs() - writeStartedAt); + this.nativeWriteTimeMs += latencyMs; + this.nativeWriteAckTimeMs += latencyMs; + }; const writePromise = window.electronAPI .nativeVideoExportWriteFrames(sessionId, chunks) .then((writeResult) => { + recordAckLatency(); if (!writeResult.success && !this.cancelled) { throw new Error( writeResult.error || "Failed to write H.264 chunks to native encoder", @@ -3061,15 +5252,9 @@ export class ModernVideoExporter { } }) .catch((error) => { - if (!this.cancelled) { - const resolvedError = error instanceof Error ? error : new Error(String(error)); - if (!this.nativeEncoderError) { - this.nativeEncoderError = resolvedError; - } - if (!this.nativeWriteError) { - this.nativeWriteError = resolvedError; - } - } + recordAckLatency(); + const resolvedError = error instanceof Error ? error : new Error(String(error)); + this.recordNativeWriteError(resolvedError); throw error; }); @@ -3102,6 +5287,29 @@ export class ModernVideoExporter { renderProgress?: number, audioProgress?: number, ) { + // Suppress repeated identical "preparing" start signals (0 frames, no render + // or audio progress) during a single export so the renderer/UI is not + // spammed with identical progress resets. The first signal per total frame + // count is still delivered and progress semantics are unchanged. + const isIdenticalPreparingSignal = + phase === "preparing" && + currentFrame === 0 && + renderProgress === undefined && + audioProgress === undefined; + if (isIdenticalPreparingSignal && this.lastPreparingTotalFrames === totalFrames) { + return; + } + if (isIdenticalPreparingSignal) { + this.lastPreparingTotalFrames = totalFrames; + } + if (phase !== "preparing") { + // A non-preparing progress event ends the current preparing phase; reset + // the watermark so a later preparing phase that reuses the same total + // frame count still delivers its first signal instead of being suppressed + // against a stale total. + this.lastPreparingTotalFrames = null; + } + const nowMs = this.getNowMs(); const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001); const averageRenderFps = currentFrame / elapsedSeconds; @@ -3149,6 +5357,7 @@ export class ModernVideoExporter { averageRenderFps: Number(averageRenderFps.toFixed(1)), sampleRenderFps: Number(sampleRenderFps.toFixed(1)), displayedRenderFps: Number(displayedRenderFps.toFixed(1)), + fpsSource: this.nativeStaticLayoutFpsSource ?? undefined, renderBackend: this.renderBackend ?? undefined, encodeBackend: this.encodeBackend ?? undefined, encoderName: this.encoderName ?? undefined, @@ -3156,7 +5365,8 @@ export class ModernVideoExporter { pendingEncodeQueue: this.encodeQueue, encodeBacklog: this.getCurrentEncodeBacklog(), peakEncodeQueueSize: this.peakEncodeQueueSize, - nativeWriteInFlight: this.nativeWritePromises.size, + nativeWriteInFlight: + this.nativeWritePromises.size + this.nativeRawWritePromises.size, peakNativeWriteInFlight: this.peakNativeWriteInFlight, averageFrameCallbackMs: Number( (this.frameCallbackTimeMs / safeFrameCount).toFixed(3), @@ -3189,6 +5399,7 @@ export class ModernVideoExporter { percentage, estimatedTimeRemaining, renderFps: displayedRenderFps, + fpsSource: this.nativeStaticLayoutFpsSource ?? undefined, renderBackend: this.renderBackend ?? undefined, encodeBackend: this.encodeBackend ?? undefined, encoderName: this.encoderName ?? undefined, @@ -3224,6 +5435,21 @@ export class ModernVideoExporter { peakNativeWriteInFlight: this.peakNativeWriteInFlight, nativeCaptureMs: this.nativeCaptureTimeMs, nativeWriteMs: this.nativeWriteTimeMs, + nativeWriteAckMs: this.nativeWriteAckTimeMs, + nativeRawBytesSubmitted: + this.nativeRawFramesSubmitted > 0 ? this.nativeRawBytesSubmitted : undefined, + nativeTransportMode: this.nativeTransportMode ?? undefined, + nativeTransportFallbackReason: this.nativeTransportFallbackReason ?? undefined, + averageNativeFrameTransportMs: + this.nativeRawFramesSubmitted > 0 + ? this.nativeFrameTransportTimeMs / this.nativeRawFramesSubmitted + : undefined, + averageNativeWriteAckMs: + this.nativeRawFramesSubmitted > 0 + ? this.nativeWriteAckTimeMs / this.nativeRawFramesSubmitted + : undefined, + peakNativeWriteInFlightBytes: + this.nativeRawFramesSubmitted > 0 ? this.peakNativeWriteInFlightBytes : undefined, finalizationMs: this.finalizationTimeMs, frameCount: this.processedFrameCount, renderBackend: this.renderBackend ?? undefined, @@ -3262,12 +5488,36 @@ export class ModernVideoExporter { this.nativeWritePromises.add(writePromise); this.peakNativeWriteInFlight = Math.max( this.peakNativeWriteInFlight, - this.nativeWritePromises.size, + this.nativeWritePromises.size + this.nativeRawWritePromises.size, ); - void writePromise.finally(() => { - this.nativeWritePromises.delete(writePromise); - }); + void writePromise.then( + () => this.nativeWritePromises.delete(writePromise), + () => this.nativeWritePromises.delete(writePromise), + ); + } + + private trackNativeRawWritePromise(writePromise: Promise, frameByteSize: number): void { + const rawBackpressure = this.nativeRawBackpressure; + if (!rawBackpressure) { + return; + } + this.nativeRawWritePromises.add(writePromise); + this.peakNativeWriteInFlight = Math.max( + this.peakNativeWriteInFlight, + this.nativeWritePromises.size + this.nativeRawWritePromises.size, + ); + this.peakNativeWriteInFlightBytes = Math.max( + this.peakNativeWriteInFlightBytes, + rawBackpressure.currentInFlightBytes, + ); + + const settle = () => { + this.nativeRawWritePromises.delete(writePromise); + rawBackpressure.release(frameByteSize); + this.notifyEncodeCapacityAvailable(); + }; + void writePromise.then(settle, settle); } private async awaitOldestNativeWrite(): Promise { @@ -3283,9 +5533,25 @@ export class ModernVideoExporter { } } + private async awaitOldestNativeRawWrite(): Promise { + const oldestWritePromise = this.nativeRawWritePromises.values().next().value; + if (!oldestWritePromise) { + return; + } + + await oldestWritePromise; + if (this.nativeWriteError) { + throw this.nativeWriteError; + } + } + private async awaitPendingNativeWrites(): Promise { - while (this.nativeWritePromises.size > 0) { - await this.awaitOldestNativeWrite(); + while (this.nativeWritePromises.size > 0 || this.nativeRawWritePromises.size > 0) { + if (this.nativeWritePromises.size > 0) { + await this.awaitOldestNativeWrite(); + } else { + await this.awaitOldestNativeRawWrite(); + } } if (this.nativeWriteError) { @@ -3502,6 +5768,8 @@ export class ModernVideoExporter { cancel(): void { this.cancelled = true; + this.nativeRawBackpressure?.fail(new Error("Native raw-frame export was cancelled")); + this.notifyEncodeCapacityAvailable(); if (this.streamingDecoder) { this.streamingDecoder.cancel(); } @@ -3580,6 +5848,11 @@ export class ModernVideoExporter { this.peakNativeWriteInFlight = 0; this.nativeCaptureTimeMs = 0; this.nativeWriteTimeMs = 0; + this.nativeWriteAckTimeMs = 0; + this.nativeFrameTransportTimeMs = 0; + this.nativeRawBytesSubmitted = 0; + this.nativeRawFramesSubmitted = 0; + this.peakNativeWriteInFlightBytes = 0; this.finalizationTimeMs = 0; this.finalizationStageMs = {}; this.effectiveDurationSec = 0; @@ -3591,7 +5864,14 @@ export class ModernVideoExporter { this.lastProgressSampleTimeMs = 0; this.lastProgressSampleFrame = 0; this.displayedRenderFps = 0; + this.lastPreparingTotalFrames = null; this.nativeWritePromises = new Set(); + this.nativeRawWritePromises = new Set(); + this.nativeRawBackpressure = null; + this.maxNativeRawWriteFrames = 1; + this.maxNativeRawWriteBytes = 0; + this.nativeTransportMode = null; + this.nativeTransportFallbackReason = null; this.nativeWriteError = null; this.pendingNativeWriteChunks = []; this.pendingNativeWriteBytes = 0; @@ -3604,7 +5884,9 @@ export class ModernVideoExporter { this.encodeBackend = null; this.encoderName = null; this.nativeStaticLayoutAverageFps = null; + this.nativeStaticLayoutFpsSource = null; this.backpressureProfile = null; + this.nativeRawFrameMode = false; this.lastNativeExportError = null; } } diff --git a/src/lib/exporter/nativeFrameCapture.ts b/src/lib/exporter/nativeFrameCapture.ts index 5d69f6978..4ee1b136a 100644 --- a/src/lib/exporter/nativeFrameCapture.ts +++ b/src/lib/exporter/nativeFrameCapture.ts @@ -57,25 +57,10 @@ function captureCanvasFrameWithReadback( return new Uint8Array(imageData.data); } -function flipRgbaRowsInPlace(buffer: Uint8Array, width: number, height: number): void { - const rowByteLength = width * RGBA_BYTES_PER_PIXEL; - const scratchRow = new Uint8Array(rowByteLength); - const halfRows = Math.floor(height / 2); - - for (let rowIndex = 0; rowIndex < halfRows; rowIndex += 1) { - const topOffset = rowIndex * rowByteLength; - const bottomOffset = (height - rowIndex - 1) * rowByteLength; - - scratchRow.set(buffer.subarray(topOffset, topOffset + rowByteLength)); - buffer.copyWithin(topOffset, bottomOffset, bottomOffset + rowByteLength); - buffer.set(scratchRow, bottomOffset); - } -} - export async function captureCanvasFrameForNativeExport( canvas: HTMLCanvasElement, timestamp: number, - flipVertical = false, + _flipVertical = false, targetWidth?: number, targetHeight?: number, ): Promise { @@ -91,9 +76,6 @@ export async function captureCanvasFrameForNativeExport( format: "RGBA", layout: getRgbaLayout(outWidth), }); - if (flipVertical) { - flipRgbaRowsInPlace(buffer, outWidth, outHeight); - } nativeFrameCaptureMode = "video-frame-rgba"; return buffer; } catch (error) { @@ -110,9 +92,5 @@ export async function captureCanvasFrameForNativeExport( } } - const buffer = captureCanvasFrameWithReadback(canvas, outWidth, outHeight); - if (flipVertical) { - flipRgbaRowsInPlace(buffer, outWidth, outHeight); - } - return buffer; + return captureCanvasFrameWithReadback(canvas, outWidth, outHeight); } diff --git a/src/lib/exporter/nativeStaticLayoutOverlays.test.ts b/src/lib/exporter/nativeStaticLayoutOverlays.test.ts new file mode 100644 index 000000000..b7bdbc8a1 --- /dev/null +++ b/src/lib/exporter/nativeStaticLayoutOverlays.test.ts @@ -0,0 +1,727 @@ +import { describe, expect, it } from "vitest"; +import type { + NativeStaticLayoutOverlayLayer, + NativeTiledOverlayLayerDescriptor, + NativeTiledOverlayStorageDescriptor, +} from "./nativeStaticLayoutOverlays"; +import { + areNativeStaticLayoutOverlayFramesEqual, + getNativeStaticLayoutOverlayFrameByteSize, + getNativeTiledOverlayEffectiveFrameCount, + getNativeTiledOverlayTileColumns, + getNativeTiledOverlayTileCount, + getNativeTiledOverlayTileIndex, + getNativeTiledOverlayTileRows, + getNativeTiledOverlayUploadedTileCount, + isNativeTiledOverlayTilePayloadTransparent, + NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION, + NATIVE_TILED_OVERLAY_MIN_TILE_COUNT, + NATIVE_TILED_OVERLAY_PIXEL_FORMAT, + NATIVE_TILED_OVERLAY_STORAGE_VERSION, + NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE, + NATIVE_TILED_OVERLAY_TILE_SIZE, + resolveNativeTiledOverlayMetrics, + resolveNativeTiledOverlayRawFallbackReason, + sortNativeStaticLayoutOverlayLayers, + sortNativeTiledOverlayLayers, + validateNativeStaticLayoutOverlayLayer, + validateNativeTiledOverlayLayerDescriptor, + validateNativeTiledOverlayStorageDescriptor, +} from "./nativeStaticLayoutOverlays"; + +const layer = ( + overrides: Partial = {}, +): NativeStaticLayoutOverlayLayer => ({ + id: "overlay", + order: 0, + path: "C:/Temp/overlay.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + ...overrides, +}); + +describe("native static-layout overlay layers", () => { + it("calculates RGBA frame size", () => { + expect(getNativeStaticLayoutOverlayFrameByteSize(1920, 1080)).toBe(1920 * 1080 * 4); + }); + + it("detects byte-identical RGBA overlay frames", () => { + const frameA = new Uint8Array([1, 2, 3, 4, 5]); + const frameACopy = new Uint8Array([1, 2, 3, 4, 5]); + const frameB = new Uint8Array([1, 2, 3, 4, 6]); + const shortFrame = new Uint8Array([1, 2, 3, 4]); + + expect(areNativeStaticLayoutOverlayFramesEqual(frameA, frameA)).toBe(true); + expect(areNativeStaticLayoutOverlayFramesEqual(frameA, frameACopy)).toBe(true); + expect(areNativeStaticLayoutOverlayFramesEqual(frameA, frameB)).toBe(false); + expect(areNativeStaticLayoutOverlayFramesEqual(frameA, shortFrame)).toBe(false); + expect(areNativeStaticLayoutOverlayFramesEqual(shortFrame, frameA)).toBe(false); + }); + + it("rejects layers with an invalid effective frame count", () => { + const options = { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 2, + frameRate: 30, + }; + expect( + validateNativeStaticLayoutOverlayLayer(layer({ effectiveFrameCount: 1 }), options), + ).toBeNull(); + expect( + validateNativeStaticLayoutOverlayLayer(layer({ effectiveFrameCount: 60 }), options), + ).toBeNull(); + expect( + validateNativeStaticLayoutOverlayLayer(layer({ effectiveFrameCount: 0 }), options), + ).toBe("overlay layer overlay has an invalid effective frame count"); + expect( + validateNativeStaticLayoutOverlayLayer(layer({ effectiveFrameCount: 61 }), options), + ).toBe("overlay layer overlay has an invalid effective frame count"); + expect( + validateNativeStaticLayoutOverlayLayer(layer({ effectiveFrameCount: 1.5 }), options), + ).toBe("overlay layer overlay has an invalid effective frame count"); + }); + + it("sorts layers by z-order and then id", () => { + expect( + sortNativeStaticLayoutOverlayLayers([ + layer({ id: "b", order: 1 }), + layer({ id: "z", order: 0 }), + layer({ id: "a", order: 0 }), + ]).map((entry) => entry.id), + ).toEqual(["a", "z", "b"]); + }); + + it("rejects layers that do not match the output timeline", () => { + expect( + validateNativeStaticLayoutOverlayLayer(layer({ width: 1921 }), { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 2, + frameRate: 30, + }), + ).toBe("overlay layer overlay has invalid output bounds"); + expect( + validateNativeStaticLayoutOverlayLayer(layer({ frameRate: 24 }), { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 2, + frameRate: 30, + }), + ).toContain("incompatible frame rate"); + }); +}); + +// Tiled/delta overlay storage contract (sparse overlay optimization). Fixed +// 128x128 lossless raw RGBA tiles: staticTiles define the full initial layer +// state emitted once; frameDeltas carry per-frame changed tiles. Every payload +// region is written exactly once and unchanged tiles are referenced, so sparse +// 4K overlays never duplicate unchanged pixels. + +const TILED_TILE_BYTE_SIZE = 128 * 128 * 4; +const TILED_LAYER_WIDTH = 384; +const TILED_LAYER_HEIGHT = 256; +const TILED_LAYER_TILE_COUNT = 3 * 2; // ceil(384/128) * ceil(256/128) + +function tiledTileRecord( + tileIndex: number, + byteOffset: number, + overrides: Partial<{ byteLength: number; byteOffset: number }> = {}, +) { + return { tileIndex, byteOffset, byteLength: TILED_TILE_BYTE_SIZE, ...overrides }; +} + +function staticTilesFor(tileCount = TILED_LAYER_TILE_COUNT) { + return Array.from({ length: tileCount }, (_, tileIndex) => + tiledTileRecord(tileIndex, tileIndex * TILED_TILE_BYTE_SIZE), + ); +} + +function tiledLayer( + overrides: Partial = {}, +): NativeTiledOverlayLayerDescriptor { + return { + id: "tiled-effects", + order: 0, + x: 0, + y: 0, + width: TILED_LAYER_WIDTH, + height: TILED_LAYER_HEIGHT, + frameRate: 30, + durationSec: 2, + frameCount: 60, + tileSize: NATIVE_TILED_OVERLAY_TILE_SIZE, + pixelFormat: NATIVE_TILED_OVERLAY_PIXEL_FORMAT, + payloadPath: "C:/Temp/tiled-overlay.bin", + payloadByteLength: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE, + staticTiles: staticTilesFor(), + frameDeltas: [], + ...overrides, + }; +} + +function tiledStorageDescriptor( + overrides: Partial = {}, +): NativeTiledOverlayStorageDescriptor { + return { + version: NATIVE_TILED_OVERLAY_STORAGE_VERSION, + outputWidth: 1920, + outputHeight: 1080, + frameRate: 30, + durationSec: 2, + layers: [tiledLayer()], + ...overrides, + }; +} + +const tiledTimelineOptions = { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 2, + frameRate: 30, +}; + +describe("tiled overlay storage constants and geometry", () => { + it("fixes the 128px lossless raw RGBA tile contract", () => { + expect(NATIVE_TILED_OVERLAY_STORAGE_VERSION).toBe(1); + expect(NATIVE_TILED_OVERLAY_TILE_SIZE).toBe(128); + expect(NATIVE_TILED_OVERLAY_PIXEL_FORMAT).toBe("rgba"); + expect(NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE).toBe(128 * 128 * 4); + expect(NATIVE_TILED_OVERLAY_MIN_TILE_COUNT).toBeGreaterThan(1); + expect(NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION).toBeGreaterThan(0); + }); + + it("computes tile columns, rows, count, and row-major indices", () => { + expect(getNativeTiledOverlayTileColumns(384)).toBe(3); + expect(getNativeTiledOverlayTileRows(256)).toBe(2); + expect(getNativeTiledOverlayTileCount(384, 256)).toBe(6); + expect(getNativeTiledOverlayTileCount(1920, 1080)).toBe(15 * 9); + expect(getNativeTiledOverlayTileIndex(2, 1, 3)).toBe(5); + expect(getNativeTiledOverlayTileCount(0, 0)).toBe(1); + }); + + it("sorts tiled layers by z-order and then id like the raw contract", () => { + expect( + sortNativeTiledOverlayLayers([ + tiledLayer({ id: "b", order: 1 }), + tiledLayer({ id: "z", order: 0 }), + tiledLayer({ id: "a", order: 0 }), + ]).map((entry) => entry.id), + ).toEqual(["a", "z", "b"]); + }); +}); + +describe("tiled overlay identical frames and dynamic deltas", () => { + it("accepts an identical-frames layer with a static base only", () => { + expect( + validateNativeTiledOverlayLayerDescriptor(tiledLayer(), tiledTimelineOptions), + ).toBeNull(); + expect(getNativeTiledOverlayEffectiveFrameCount(tiledLayer())).toBe(1); + expect(getNativeTiledOverlayUploadedTileCount(tiledLayer())).toBe(TILED_LAYER_TILE_COUNT); + expect(resolveNativeTiledOverlayMetrics(tiledLayer())).toEqual({ + changedTileCount: 0, + uploadedTileCount: TILED_LAYER_TILE_COUNT, + uploadedTileBytes: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE, + cachedTileCount: TILED_LAYER_TILE_COUNT * 60 - TILED_LAYER_TILE_COUNT, + }); + }); + + it("accepts moving/dynamic content described by ascending changed tile deltas", () => { + const nextOffset = TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE; + const layer = tiledLayer({ + payloadByteLength: nextOffset + 4 * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { frameIndex: 10, changedTiles: [tiledTileRecord(1, nextOffset)] }, + { + frameIndex: 20, + changedTiles: [ + tiledTileRecord(4, nextOffset + TILED_TILE_BYTE_SIZE), + tiledTileRecord(5, nextOffset + 2 * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 30, + changedTiles: [tiledTileRecord(2, nextOffset + 3 * TILED_TILE_BYTE_SIZE)], + }, + ], + }); + expect(validateNativeTiledOverlayLayerDescriptor(layer, tiledTimelineOptions)).toBeNull(); + expect(getNativeTiledOverlayEffectiveFrameCount(layer)).toBe(4); + expect(resolveNativeTiledOverlayMetrics(layer)).toEqual({ + changedTileCount: 4, + uploadedTileCount: 10, + uploadedTileBytes: 10 * TILED_TILE_BYTE_SIZE, + cachedTileCount: TILED_LAYER_TILE_COUNT * 60 - 10, + }); + }); + + it("allows identical frames to repeat state between deltas", () => { + const layer = tiledLayer({ + payloadByteLength: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE + TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 5, + changedTiles: [ + tiledTileRecord(0, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + expect(validateNativeTiledOverlayLayerDescriptor(layer, tiledTimelineOptions)).toBeNull(); + }); +}); + +describe("tiled overlay transparent tile payloads", () => { + it("detects fully transparent (all-zero) tile payload regions", () => { + const payload = new Uint8Array(2 * TILED_TILE_BYTE_SIZE); + expect(isNativeTiledOverlayTilePayloadTransparent(payload, tiledTileRecord(0, 0))).toBe( + true, + ); + payload[TILED_TILE_BYTE_SIZE + 7] = 1; + expect( + isNativeTiledOverlayTilePayloadTransparent( + payload, + tiledTileRecord(1, TILED_TILE_BYTE_SIZE), + ), + ).toBe(false); + }); + + it("never treats out-of-bounds tile regions as transparent", () => { + const payload = new Uint8Array(TILED_TILE_BYTE_SIZE); + expect( + isNativeTiledOverlayTilePayloadTransparent( + payload, + tiledTileRecord(0, TILED_TILE_BYTE_SIZE), + ), + ).toBe(false); + expect(isNativeTiledOverlayTilePayloadTransparent(payload, tiledTileRecord(0, 1))).toBe( + false, + ); + }); +}); + +describe("tiled overlay invalid metadata and bytes", () => { + it("rejects non-RGBA pixel formats and non-128px tiles", () => { + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ pixelFormat: "yuv420p" }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects must use RGBA tiles"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ tileSize: 64 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects must use 128px tiles"); + }); + + it("rejects missing id/payload path and malformed geometry/timeline", () => { + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ id: " " }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer requires an id and payload path"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ payloadPath: "" }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer requires an id and payload path"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ width: 1921 }), + tiledTimelineOptions, + ), + ).toContain("invalid output bounds"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ frameRate: 24 }), + tiledTimelineOptions, + ), + ).toContain("incompatible frame rate"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ durationSec: 3 }), + tiledTimelineOptions, + ), + ).toContain("incompatible duration"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ frameCount: 59 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects does not contain enough frames"); + }); + + it("rejects missing static/delta arrays and invalid payload byte length", () => { + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ staticTiles: undefined as never }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects requires a static tile base"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ frameDeltas: undefined as never }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects requires frame delta records"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ payloadByteLength: -1 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects has an invalid payload byte length"); + }); + + it("rejects invalid tile payload ranges and out-of-bounds tile indices", () => { + const invalidRange = tiledTileRecord(0, 0, { + byteLength: TILED_TILE_BYTE_SIZE - 1, + }); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ + staticTiles: [invalidRange, ...staticTilesFor().slice(1)], + }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects has an invalid tile payload range"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ + staticTiles: [ + tiledTileRecord(0, 0, { byteOffset: -4 }), + ...staticTilesFor().slice(1), + ], + }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects has an invalid tile payload range"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ payloadByteLength: TILED_TILE_BYTE_SIZE }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects has an invalid tile payload range"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ + staticTiles: [tiledTileRecord(6, 0), ...staticTilesFor().slice(1)], + }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects references an out-of-bounds tile"); + }); + + it("rejects duplicate static tiles and an incomplete static tile base", () => { + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ + staticTiles: [ + tiledTileRecord(0, 0), + tiledTileRecord(0, TILED_TILE_BYTE_SIZE), + ...staticTilesFor().slice(2), + ], + }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects emits duplicate static tile 0"); + expect( + validateNativeTiledOverlayLayerDescriptor( + tiledLayer({ staticTiles: staticTilesFor().slice(0, 5) }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay layer tiled-effects does not fully define the static tile base"); + }); + + it("rejects duplicate tiles inside a frame delta", () => { + const layer = tiledLayer({ + payloadByteLength: (TILED_LAYER_TILE_COUNT + 1) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tiledTileRecord(0, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + tiledTileRecord(0, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + expect(validateNativeTiledOverlayLayerDescriptor(layer, tiledTimelineOptions)).toBe( + "tiled overlay layer tiled-effects repeats tile 0 within a frame delta", + ); + }); + + it("rejects unsorted, duplicate, and out-of-range delta frame indices", () => { + const base = (frameIndices: number[]) => + tiledLayer({ + payloadByteLength: + (TILED_LAYER_TILE_COUNT + frameIndices.length) * TILED_TILE_BYTE_SIZE, + frameDeltas: frameIndices.map((frameIndex, index) => ({ + frameIndex, + changedTiles: [ + tiledTileRecord(0, (TILED_LAYER_TILE_COUNT + index) * TILED_TILE_BYTE_SIZE), + ], + })), + }); + expect( + validateNativeTiledOverlayLayerDescriptor(base([20, 10]), tiledTimelineOptions), + ).toBe("tiled overlay layer tiled-effects has unsorted or duplicate delta frame indices"); + expect( + validateNativeTiledOverlayLayerDescriptor(base([10, 10]), tiledTimelineOptions), + ).toBe("tiled overlay layer tiled-effects has unsorted or duplicate delta frame indices"); + expect(validateNativeTiledOverlayLayerDescriptor(base([60]), tiledTimelineOptions)).toBe( + "tiled overlay layer tiled-effects has an invalid delta frame index", + ); + }); + + it("rejects more state versions than the logical frame count", () => { + // 1 output frame (1/30s at 30fps) cannot host 3 state versions (static + // base + deltas at frames 0 and 1) when frameCount is only 2. + const shortTimeline = { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 1 / 30, + frameRate: 30, + }; + const layer = tiledLayer({ + frameCount: 2, + durationSec: 1 / 30, + payloadByteLength: (TILED_LAYER_TILE_COUNT + 2) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tiledTileRecord(0, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 1, + changedTiles: [ + tiledTileRecord(0, (TILED_LAYER_TILE_COUNT + 1) * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + expect(validateNativeTiledOverlayLayerDescriptor(layer, shortTimeline)).toBe( + "tiled overlay layer tiled-effects has more state versions than frames", + ); + }); + + it("rejects duplicate payload byte ranges across the stream", () => { + const layer = tiledLayer({ + payloadByteLength: TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE + TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [ + tiledTileRecord(0, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 1, + changedTiles: [ + tiledTileRecord(1, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + expect(validateNativeTiledOverlayLayerDescriptor(layer, tiledTimelineOptions)).toBe( + "tiled overlay layer tiled-effects duplicates tile payload bytes", + ); + }); +}); + +describe("tiled overlay storage descriptor", () => { + it("validates version, output dimensions, frame rate, and duration", () => { + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor(), + tiledTimelineOptions, + ), + ).toBeNull(); + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor({ version: 2 }), + tiledTimelineOptions, + ), + ).toBe("unsupported tiled overlay storage version 2"); + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor({ outputWidth: 1280 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay storage dimensions do not match the output"); + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor({ frameRate: 24 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay storage frame rate does not match the output"); + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor({ durationSec: 4 }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay storage duration does not match the output"); + expect( + validateNativeTiledOverlayStorageDescriptor( + tiledStorageDescriptor({ layers: undefined as never }), + tiledTimelineOptions, + ), + ).toBe("tiled overlay storage requires a layers array"); + }); + + it("requires layers sorted by order then id with unique identities", () => { + const unsorted = tiledStorageDescriptor({ + layers: [tiledLayer({ id: "b", order: 1 }), tiledLayer({ id: "a", order: 0 })], + }); + expect(validateNativeTiledOverlayStorageDescriptor(unsorted, tiledTimelineOptions)).toBe( + "tiled overlay layers must be sorted by order then id", + ); + const duplicateId = tiledStorageDescriptor({ + layers: [tiledLayer({ id: "a", order: 0 }), tiledLayer({ id: "a", order: 0 })], + }); + expect(validateNativeTiledOverlayStorageDescriptor(duplicateId, tiledTimelineOptions)).toBe( + "tiled overlay layers must be sorted by order then id", + ); + }); +}); + +describe("tiled overlay raw fallback heuristic", () => { + it("keeps sparse eligible layers on the tiled path", () => { + const layer = tiledLayer({ + payloadByteLength: (TILED_LAYER_TILE_COUNT + 3) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 10, + changedTiles: [ + tiledTileRecord(1, TILED_LAYER_TILE_COUNT * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 20, + changedTiles: [ + tiledTileRecord(4, (TILED_LAYER_TILE_COUNT + 1) * TILED_TILE_BYTE_SIZE), + ], + }, + { + frameIndex: 30, + changedTiles: [ + tiledTileRecord(2, (TILED_LAYER_TILE_COUNT + 2) * TILED_TILE_BYTE_SIZE), + ], + }, + ], + }); + expect(resolveNativeTiledOverlayRawFallbackReason(layer)).toBeNull(); + }); + + it("falls back to raw for layers smaller than the minimum tile count", () => { + const smallLayer = tiledLayer({ + width: 256, + height: 128, + staticTiles: staticTilesFor(2), + }); + expect(resolveNativeTiledOverlayRawFallbackReason(smallLayer)).toBe("small-layer"); + }); + + it("falls back to raw when a single frame changes a dense tile fraction", () => { + const layer = tiledLayer({ + payloadByteLength: (TILED_LAYER_TILE_COUNT + 4) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [0, 1, 2, 3].map((tileIndex, index) => + tiledTileRecord( + tileIndex, + (TILED_LAYER_TILE_COUNT + index) * TILED_TILE_BYTE_SIZE, + ), + ), + }, + ], + }); + expect(resolveNativeTiledOverlayRawFallbackReason(layer)).toBe("dense-frame-delta"); + }); + + it("falls back to raw when the payload no longer beats the raw sidecar", () => { + const layer = tiledLayer({ + frameCount: 2, + payloadByteLength: (TILED_LAYER_TILE_COUNT + 3) * TILED_TILE_BYTE_SIZE, + frameDeltas: [ + { + frameIndex: 0, + changedTiles: [0, 1, 2].map((tileIndex, index) => + tiledTileRecord( + tileIndex, + (TILED_LAYER_TILE_COUNT + index) * TILED_TILE_BYTE_SIZE, + ), + ), + }, + ], + }); + expect(resolveNativeTiledOverlayRawFallbackReason(layer)).toBe("payload-bytes-exceed-raw"); + }); +}); + +describe("raw + tiled overlay compatibility", () => { + it("keeps the legacy raw RGBA contract fully untouched", () => { + const raw = layer({ effectiveFrameCount: 1 }); + expect( + validateNativeStaticLayoutOverlayLayer(raw, { + outputWidth: 1920, + outputHeight: 1080, + durationSec: 2, + frameRate: 30, + }), + ).toBeNull(); + expect(getNativeStaticLayoutOverlayFrameByteSize(raw.width, raw.height)).toBe( + raw.width * raw.height * 4, + ); + expect( + areNativeStaticLayoutOverlayFramesEqual( + new Uint8Array([1, 2, 3, 4]), + new Uint8Array([1, 2, 3, 4]), + ), + ).toBe(true); + expect( + sortNativeStaticLayoutOverlayLayers([ + layer({ id: "b", order: 1 }), + layer({ id: "a", order: 0 }), + ]).map((entry) => entry.id), + ).toEqual(["a", "b"]); + }); + + it("tiled validation rejects raw-shaped manifests and vice versa", () => { + const rawShaped = { + id: "raw-layer", + order: 0, + path: "overlay.rgba", + x: 0, + y: 0, + width: 384, + height: 256, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + } as unknown as NativeTiledOverlayLayerDescriptor; + expect(validateNativeTiledOverlayLayerDescriptor(rawShaped, tiledTimelineOptions)).toBe( + "tiled overlay layer requires an id and payload path", + ); + + const tiledAsRaw = tiledLayer() as unknown as { + path?: string; + effectiveFrameCount?: number; + }; + expect("path" in tiledAsRaw).toBe(false); + expect(tiledAsRaw.effectiveFrameCount).toBeUndefined(); + }); +}); diff --git a/src/lib/exporter/nativeStaticLayoutOverlays.ts b/src/lib/exporter/nativeStaticLayoutOverlays.ts new file mode 100644 index 000000000..6f4d9f95f --- /dev/null +++ b/src/lib/exporter/nativeStaticLayoutOverlays.ts @@ -0,0 +1,549 @@ +export const NATIVE_STATIC_LAYOUT_OVERLAY_PIXEL_FORMAT = "rgba" as const; + +export type NativeStaticLayoutOverlayLayer = { + id: string; + order: number; + path: string; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + /** + * Physical frames present in the sidecar when renderer-side deduplication + * truncated an identical suffix (1 <= effectiveFrameCount <= frameCount). + * Native readers must clamp/repeat the last written frame for indices + * [effectiveFrameCount, frameCount). Absent when the layer is fully dynamic + * (every frame differs), so readers without dedup support behave unchanged. + */ + effectiveFrameCount?: number; + pixelFormat: typeof NATIVE_STATIC_LAYOUT_OVERLAY_PIXEL_FORMAT; +}; + +export function getNativeStaticLayoutOverlayFrameByteSize(width: number, height: number): number { + return Math.max(0, Math.round(width)) * Math.max(0, Math.round(height)) * 4; +} + +export function areNativeStaticLayoutOverlayFramesEqual( + left: Uint8Array, + right: Uint8Array, +): boolean { + if (left === right) { + return true; + } + if (left.byteLength !== right.byteLength) { + return false; + } + for (let index = 0; index < left.byteLength; index += 1) { + if (left[index] !== right[index]) { + return false; + } + } + return true; +} + +export function sortNativeStaticLayoutOverlayLayers( + layers: readonly NativeStaticLayoutOverlayLayer[], +): NativeStaticLayoutOverlayLayer[] { + return [...layers].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); +} + +// -------------------------------------------------------------------------- +// Tiled/delta overlay storage contract (sparse overlay optimization). +// +// A tiled layer stores lossless raw RGBA tiles (fixed 128x128) once and later +// references unchanged tile state instead of duplicating full 4K frames. The +// payload stream is a bounded raw RGBA byte blob referenced by the descriptor; +// every payload region is written exactly once. Sidecar/session data produced +// from this contract is never persisted. Legacy callers keep the raw full-frame +// manifest above untouched. +// -------------------------------------------------------------------------- + +export const NATIVE_TILED_OVERLAY_STORAGE_VERSION = 1 as const; +export const NATIVE_TILED_OVERLAY_TILE_SIZE = 128 as const; +export const NATIVE_TILED_OVERLAY_PIXEL_FORMAT = "rgba" as const; +export const NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE = + NATIVE_TILED_OVERLAY_TILE_SIZE * NATIVE_TILED_OVERLAY_TILE_SIZE * 4; + +// Conservative tiled-vs-raw density/size heuristics. When any threshold is +// exceeded the renderer must keep the legacy raw full-frame sidecar instead of +// a tiled stream; the decision is observable through rawFallbackReason. +export const NATIVE_TILED_OVERLAY_MIN_TILE_COUNT = 4; +export const NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION = 0.5; +export const NATIVE_TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION = 0.7; + +export type NativeTiledOverlayTileRecord = { + /** Row-major tile index within the layer (tileY * tilesPerRow + tileX). */ + tileIndex: number; + /** Byte offset of the lossless raw RGBA tile payload in the payload stream. */ + byteOffset: number; + /** Tile payload length; always tileSize^2 * 4 for raw RGBA tiles. */ + byteLength: number; +}; + +export type NativeTiledOverlayStaticTileRecord = NativeTiledOverlayTileRecord; + +export type NativeTiledOverlayFrameDelta = { + /** 0-based output frame index this delta takes effect at (ascending, unique). */ + frameIndex: number; + /** + * Tiles whose state changed at this frame relative to the previous frame. + * Empty means an identical frame: no payload is written and all tiles keep + * their previously uploaded state. + */ + changedTiles: readonly NativeTiledOverlayTileRecord[]; +}; + +export type NativeTiledOverlayLayerDescriptor = { + id: string; + order: number; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + /** Logical output frame count (ceil(durationSec * frameRate)). */ + frameCount: number; + tileSize: typeof NATIVE_TILED_OVERLAY_TILE_SIZE; + pixelFormat: typeof NATIVE_TILED_OVERLAY_PIXEL_FORMAT; + /** Bounded reference to the lossless raw RGBA tile payload stream. */ + payloadPath: string; + payloadByteLength: number; + /** + * Initial tile state emitted once before any delta. Must contain every tile + * of the layer exactly once so output frame 0 is fully defined. + */ + staticTiles: readonly NativeTiledOverlayStaticTileRecord[]; + /** + * Per-frame changed tile records over the output timeline. Static tiles are + * emitted once and unchanged tiles are referenced (never re-uploaded), so a + * sparse 4K overlay does not duplicate unchanged pixels. Deltas must be + * sorted ascending by frameIndex with unique indices in [0, frameCount). + */ + frameDeltas: readonly NativeTiledOverlayFrameDelta[]; +}; + +export type NativeTiledOverlayStorageDescriptor = { + version: typeof NATIVE_TILED_OVERLAY_STORAGE_VERSION; + outputWidth: number; + outputHeight: number; + frameRate: number; + durationSec: number; + layers: readonly NativeTiledOverlayLayerDescriptor[]; +}; + +export type NativeTiledOverlayRawFallbackReason = + | "small-layer" + | "dense-frame-delta" + | "payload-bytes-exceed-raw"; + +export function getNativeTiledOverlayTileColumns(width: number): number { + return Math.max(1, Math.ceil(Math.max(0, width) / NATIVE_TILED_OVERLAY_TILE_SIZE)); +} + +export function getNativeTiledOverlayTileRows(height: number): number { + return Math.max(1, Math.ceil(Math.max(0, height) / NATIVE_TILED_OVERLAY_TILE_SIZE)); +} + +export function getNativeTiledOverlayTileCount(width: number, height: number): number { + return getNativeTiledOverlayTileColumns(width) * getNativeTiledOverlayTileRows(height); +} + +export function getNativeTiledOverlayTileIndex( + tileX: number, + tileY: number, + tileColumns: number, +): number { + return tileY * tileColumns + tileX; +} + +/** + * Distinct state versions of a tiled layer: the static base emitted once plus + * one version per frame delta. Never exceeds frameCount for a valid layer. + */ +export function getNativeTiledOverlayEffectiveFrameCount( + layer: NativeTiledOverlayLayerDescriptor, +): number { + return 1 + layer.frameDeltas.length; +} + +export function getNativeTiledOverlayUploadedTileCount( + layer: NativeTiledOverlayLayerDescriptor, +): number { + let count = layer.staticTiles.length; + for (const delta of layer.frameDeltas) { + count += delta.changedTiles.length; + } + return count; +} + +export type NativeTiledOverlayMetrics = { + /** Tile payloads written across all frame deltas (excludes the static base). */ + changedTileCount: number; + /** Tile payloads uploaded once (static base + all changed tiles). */ + uploadedTileCount: number; + /** Bytes uploaded: uploadedTileCount * tileSize^2 * 4. */ + uploadedTileBytes: number; + /** + * Tile-state lookups served from previously uploaded payloads across the + * full output timeline (diagnostic only; never claims zero-copy). + */ + cachedTileCount: number; +}; + +export function resolveNativeTiledOverlayMetrics( + layer: NativeTiledOverlayLayerDescriptor, +): NativeTiledOverlayMetrics { + const changedTileCount = layer.frameDeltas.reduce( + (total, delta) => total + delta.changedTiles.length, + 0, + ); + const uploadedTileCount = layer.staticTiles.length + changedTileCount; + const tileCount = getNativeTiledOverlayTileCount(layer.width, layer.height); + return { + changedTileCount, + uploadedTileCount, + uploadedTileBytes: uploadedTileCount * NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE, + cachedTileCount: Math.max(0, tileCount * layer.frameCount - uploadedTileCount), + }; +} + +/** + * Conservative tiled-vs-raw eligibility heuristic. Returns null when the tiled + * representation is eligible and a reason string when the layer is dense, too + * small, or the payload no longer beats the raw full-frame sidecar. The reason + * is observable through the rawFallbackReason metric so a silent raw fallback + * is never indistinguishable from a tiled export. + */ +export function resolveNativeTiledOverlayRawFallbackReason( + layer: NativeTiledOverlayLayerDescriptor, +): NativeTiledOverlayRawFallbackReason | null { + const tileCount = getNativeTiledOverlayTileCount(layer.width, layer.height); + if (tileCount < NATIVE_TILED_OVERLAY_MIN_TILE_COUNT) { + return "small-layer"; + } + for (const delta of layer.frameDeltas) { + if ( + delta.changedTiles.length > + tileCount * NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION + ) { + return "dense-frame-delta"; + } + } + const { uploadedTileBytes } = resolveNativeTiledOverlayMetrics(layer); + const rawPhysicalBytes = layer.width * layer.height * 4 * layer.frameCount; + if (uploadedTileBytes >= rawPhysicalBytes * NATIVE_TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION) { + return "payload-bytes-exceed-raw"; + } + return null; +} + +/** + * True when the referenced tile payload region is fully transparent (all zero + * alpha/color bytes). Out-of-bounds regions are never transparent. + */ +export function isNativeTiledOverlayTilePayloadTransparent( + payload: Uint8Array, + tile: NativeTiledOverlayTileRecord, +): boolean { + if ( + !Number.isSafeInteger(tile.byteOffset) || + !Number.isSafeInteger(tile.byteLength) || + tile.byteOffset < 0 || + tile.byteLength <= 0 || + tile.byteOffset + tile.byteLength > payload.byteLength + ) { + return false; + } + for (let index = tile.byteOffset; index < tile.byteOffset + tile.byteLength; index += 1) { + if (payload[index] !== 0) { + return false; + } + } + return true; +} + +function validateTiledOverlayTileRecord( + record: NativeTiledOverlayTileRecord, + layerId: string, + tileCount: number, + payloadByteLength: number, +): string | null { + if ( + !Number.isSafeInteger(record.tileIndex) || + record.tileIndex < 0 || + record.tileIndex >= tileCount + ) { + return `tiled overlay layer ${layerId} references an out-of-bounds tile`; + } + if ( + !Number.isSafeInteger(record.byteOffset) || + record.byteOffset < 0 || + record.byteLength !== NATIVE_TILED_OVERLAY_TILE_BYTE_SIZE || + record.byteOffset + record.byteLength > payloadByteLength + ) { + return `tiled overlay layer ${layerId} has an invalid tile payload range`; + } + return null; +} + +export function validateNativeTiledOverlayLayerDescriptor( + layer: NativeTiledOverlayLayerDescriptor, + options: { outputWidth: number; outputHeight: number; durationSec: number; frameRate: number }, +): string | null { + if ( + typeof layer.id !== "string" || + !layer.id.trim() || + typeof layer.payloadPath !== "string" || + !layer.payloadPath.trim() + ) { + return "tiled overlay layer requires an id and payload path"; + } + if (layer.pixelFormat !== NATIVE_TILED_OVERLAY_PIXEL_FORMAT) { + return `tiled overlay layer ${layer.id} must use RGBA tiles`; + } + if (layer.tileSize !== NATIVE_TILED_OVERLAY_TILE_SIZE) { + return `tiled overlay layer ${layer.id} must use ${NATIVE_TILED_OVERLAY_TILE_SIZE}px tiles`; + } + if (!Number.isSafeInteger(layer.order) || layer.order < 0) { + return `tiled overlay layer ${layer.id} has an invalid order`; + } + if ( + !Number.isSafeInteger(layer.x) || + !Number.isSafeInteger(layer.y) || + !Number.isSafeInteger(layer.width) || + !Number.isSafeInteger(layer.height) || + layer.width <= 0 || + layer.height <= 0 || + layer.x < 0 || + layer.y < 0 || + layer.x + layer.width > options.outputWidth || + layer.y + layer.height > options.outputHeight + ) { + return `tiled overlay layer ${layer.id} has invalid output bounds`; + } + if ( + !Number.isFinite(layer.frameRate) || + layer.frameRate <= 0 || + Math.abs(layer.frameRate - options.frameRate) > 0.01 + ) { + return `tiled overlay layer ${layer.id} has an incompatible frame rate`; + } + if ( + !Number.isFinite(layer.durationSec) || + layer.durationSec <= 0 || + Math.abs(layer.durationSec - options.durationSec) > 1 / options.frameRate + ) { + return `tiled overlay layer ${layer.id} has an incompatible duration`; + } + const expectedFrameCount = Math.ceil(layer.durationSec * layer.frameRate); + if (!Number.isSafeInteger(layer.frameCount) || layer.frameCount < expectedFrameCount) { + return `tiled overlay layer ${layer.id} does not contain enough frames`; + } + if (!Number.isSafeInteger(layer.payloadByteLength) || layer.payloadByteLength < 0) { + return `tiled overlay layer ${layer.id} has an invalid payload byte length`; + } + if (!Array.isArray(layer.staticTiles)) { + return `tiled overlay layer ${layer.id} requires a static tile base`; + } + if (!Array.isArray(layer.frameDeltas)) { + return `tiled overlay layer ${layer.id} requires frame delta records`; + } + + const tileCount = getNativeTiledOverlayTileCount(layer.width, layer.height); + const seenStaticTiles = new Set(); + const seenPayloadRanges = new Set(); + const checkPayloadRange = (record: NativeTiledOverlayTileRecord): string | null => { + const rangeKey = `${record.byteOffset}:${record.byteLength}`; + if (seenPayloadRanges.has(rangeKey)) { + return `tiled overlay layer ${layer.id} duplicates tile payload bytes`; + } + seenPayloadRanges.add(rangeKey); + return null; + }; + for (const record of layer.staticTiles) { + const issue = validateTiledOverlayTileRecord( + record, + layer.id, + tileCount, + layer.payloadByteLength, + ); + if (issue) { + return issue; + } + if (seenStaticTiles.has(record.tileIndex)) { + return `tiled overlay layer ${layer.id} emits duplicate static tile ${record.tileIndex}`; + } + seenStaticTiles.add(record.tileIndex); + const rangeIssue = checkPayloadRange(record); + if (rangeIssue) { + return rangeIssue; + } + } + if (seenStaticTiles.size !== tileCount) { + return `tiled overlay layer ${layer.id} does not fully define the static tile base`; + } + + let previousFrameIndex = -1; + for (const delta of layer.frameDeltas) { + if ( + !Number.isSafeInteger(delta.frameIndex) || + delta.frameIndex < 0 || + delta.frameIndex >= layer.frameCount + ) { + return `tiled overlay layer ${layer.id} has an invalid delta frame index`; + } + if (delta.frameIndex <= previousFrameIndex) { + return `tiled overlay layer ${layer.id} has unsorted or duplicate delta frame indices`; + } + previousFrameIndex = delta.frameIndex; + const seenDeltaTiles = new Set(); + for (const record of delta.changedTiles) { + const issue = validateTiledOverlayTileRecord( + record, + layer.id, + tileCount, + layer.payloadByteLength, + ); + if (issue) { + return issue; + } + if (seenDeltaTiles.has(record.tileIndex)) { + return `tiled overlay layer ${layer.id} repeats tile ${record.tileIndex} within a frame delta`; + } + seenDeltaTiles.add(record.tileIndex); + const rangeIssue = checkPayloadRange(record); + if (rangeIssue) { + return rangeIssue; + } + } + } + if (getNativeTiledOverlayEffectiveFrameCount(layer) > layer.frameCount) { + return `tiled overlay layer ${layer.id} has more state versions than frames`; + } + return null; +} + +export function sortNativeTiledOverlayLayers( + layers: readonly NativeTiledOverlayLayerDescriptor[], +): NativeTiledOverlayLayerDescriptor[] { + return [...layers].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); +} + +export function validateNativeTiledOverlayStorageDescriptor( + descriptor: NativeTiledOverlayStorageDescriptor, + options: { outputWidth: number; outputHeight: number; frameRate: number; durationSec: number }, +): string | null { + if (descriptor.version !== NATIVE_TILED_OVERLAY_STORAGE_VERSION) { + return `unsupported tiled overlay storage version ${descriptor.version}`; + } + if ( + descriptor.outputWidth !== options.outputWidth || + descriptor.outputHeight !== options.outputHeight + ) { + return "tiled overlay storage dimensions do not match the output"; + } + if ( + !Number.isFinite(descriptor.frameRate) || + descriptor.frameRate <= 0 || + Math.abs(descriptor.frameRate - options.frameRate) > 0.01 + ) { + return "tiled overlay storage frame rate does not match the output"; + } + if ( + !Number.isFinite(descriptor.durationSec) || + descriptor.durationSec <= 0 || + Math.abs(descriptor.durationSec - options.durationSec) > 1 / options.frameRate + ) { + return "tiled overlay storage duration does not match the output"; + } + if (!Array.isArray(descriptor.layers)) { + return "tiled overlay storage requires a layers array"; + } + let previousOrder = -1; + let previousId = ""; + for (const layer of descriptor.layers) { + const issue = validateNativeTiledOverlayLayerDescriptor(layer, { + outputWidth: descriptor.outputWidth, + outputHeight: descriptor.outputHeight, + durationSec: descriptor.durationSec, + frameRate: descriptor.frameRate, + }); + if (issue) { + return issue; + } + if ( + layer.order < previousOrder || + (layer.order === previousOrder && layer.id <= previousId) + ) { + return "tiled overlay layers must be sorted by order then id"; + } + previousOrder = layer.order; + previousId = layer.id; + } + return null; +} + +export function validateNativeStaticLayoutOverlayLayer( + layer: NativeStaticLayoutOverlayLayer, + options: { outputWidth: number; outputHeight: number; durationSec: number; frameRate: number }, +): string | null { + if (!layer.id.trim() || !layer.path.trim()) { + return "overlay layer requires an id and path"; + } + if (layer.pixelFormat !== NATIVE_STATIC_LAYOUT_OVERLAY_PIXEL_FORMAT) { + return `overlay layer ${layer.id} must use RGBA pixels`; + } + if (!Number.isSafeInteger(layer.order) || layer.order < 0) { + return `overlay layer ${layer.id} has an invalid order`; + } + if ( + !Number.isSafeInteger(layer.x) || + !Number.isSafeInteger(layer.y) || + !Number.isSafeInteger(layer.width) || + !Number.isSafeInteger(layer.height) || + layer.width <= 0 || + layer.height <= 0 || + layer.x < 0 || + layer.y < 0 || + layer.x + layer.width > options.outputWidth || + layer.y + layer.height > options.outputHeight + ) { + return `overlay layer ${layer.id} has invalid output bounds`; + } + if ( + !Number.isFinite(layer.frameRate) || + layer.frameRate <= 0 || + Math.abs(layer.frameRate - options.frameRate) > 0.01 + ) { + return `overlay layer ${layer.id} has an incompatible frame rate`; + } + if ( + !Number.isFinite(layer.durationSec) || + layer.durationSec <= 0 || + Math.abs(layer.durationSec - options.durationSec) > 1 / options.frameRate + ) { + return `overlay layer ${layer.id} has an incompatible duration`; + } + const expectedFrameCount = Math.ceil(layer.durationSec * layer.frameRate); + if (!Number.isSafeInteger(layer.frameCount) || layer.frameCount < expectedFrameCount) { + return `overlay layer ${layer.id} does not contain enough frames`; + } + if (layer.effectiveFrameCount !== undefined) { + if ( + !Number.isSafeInteger(layer.effectiveFrameCount) || + layer.effectiveFrameCount < 1 || + layer.effectiveFrameCount > layer.frameCount + ) { + return `overlay layer ${layer.id} has an invalid effective frame count`; + } + } + return null; +} diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index f474f9988..7ce1812ad 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -9,10 +9,16 @@ export interface ExportConfig { preferredRenderBackend?: ExportRenderBackend; experimentalNativeExport?: boolean; experimentalNvidiaCudaExport?: boolean; + exportVideoCodec?: ExportVideoCodec; + exportEncoderPreference?: ExportEncoderPreference; + exportBitrateMode?: ExportBitrateMode; + exportBitrateMbps?: number; maxEncodeQueue?: number; maxDecodeQueue?: number; maxPendingFrames?: number; maxInFlightNativeWrites?: number; + maxInFlightNativeRawFrames?: number; + maxInFlightNativeRawBytes?: number; sourceAudioFallbackStartDelayMsByPath?: Record; } @@ -27,6 +33,10 @@ export interface ExportProgress { percentage: number; estimatedTimeRemaining: number; // in seconds renderFps?: number; + /** "native" = measured encode FPS from the native helper; "estimated" = + * preparation-inclusive wall-clock estimate that must not be read as encode + * speed. */ + fpsSource?: "native" | "estimated"; renderBackend?: ExportRenderBackend; encodeBackend?: ExportEncodeBackend; encoderName?: string; @@ -113,6 +123,8 @@ export interface ExportFfmpegAudioMuxBreakdown { }>; } +export type ExportNativeTransportMode = "transferable-stream" | "cloned-ipc"; + export interface ExportMetrics { totalElapsedMs: number; metadataLoadMs?: number; @@ -126,7 +138,15 @@ export interface ExportMetrics { peakEncodeQueueSize?: number; peakNativeWriteInFlight?: number; nativeCaptureMs?: number; + /** Total renderer-side native write promise/ACK completion time. */ nativeWriteMs?: number; + nativeWriteAckMs?: number; + nativeRawBytesSubmitted?: number; + nativeTransportMode?: ExportNativeTransportMode; + nativeTransportFallbackReason?: string; + averageNativeFrameTransportMs?: number; + averageNativeWriteAckMs?: number; + peakNativeWriteInFlightBytes?: number; finalizationMs?: number; frameCount?: number; renderBackend?: ExportRenderBackend; @@ -170,6 +190,12 @@ export interface VideoFrameData { duration: number; // in microseconds } +export type ExportVideoCodec = "h264" | "hevc"; + +export type ExportEncoderPreference = "auto" | "hardware" | "cpu"; + +export type ExportBitrateMode = "auto" | "custom"; + export type ExportEncodingMode = "fast" | "balanced" | "quality"; export type ExportQuality = "medium" | "good" | "high" | "source"; @@ -200,10 +226,20 @@ export interface ExportSettings { mp4FrameRate?: ExportMp4FrameRate; backendPreference?: ExportBackendPreference; pipelineModel?: ExportPipelineModel; + exportVideoCodec?: ExportVideoCodec; + exportEncoderPreference?: ExportEncoderPreference; + exportBitrateMode?: ExportBitrateMode; + exportBitrateMbps?: number; // GIF settings gifConfig?: GifExportConfig; } +export const EXPORT_BITRATE_MIN_MBPS = 1; +export const EXPORT_BITRATE_MAX_MBPS = 200; +export const EXPORT_BITRATE_H264_MAX_MBPS = 105; +export const EXPORT_BITRATE_HEVC_MAX_MBPS = 70; +export const EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS = 20; + export const MP4_FRAME_RATES: readonly ExportMp4FrameRate[] = [24, 30, 60] as const; export function isValidMp4FrameRate(rate: number): rate is ExportMp4FrameRate {